From 228ebda5869385170a3cc121089e075de61403bb Mon Sep 17 00:00:00 2001 From: shivam Date: Mon, 3 Apr 2023 20:18:03 +0530 Subject: [PATCH] Move argparse code into separate argparse modules --- kcidb/__init__.py | 34 ++-- kcidb/argparse.py | 298 ++++++++++++++++++++++++++++++++ kcidb/db/__init__.py | 264 +++------------------------- kcidb/db/argparse.py | 230 ++++++++++++++++++++++++ kcidb/misc.py | 273 ----------------------------- kcidb/monitor/spool/__init__.py | 3 +- kcidb/mq/__init__.py | 193 ++------------------- kcidb/mq/argparse.py | 171 ++++++++++++++++++ kcidb/oo/__init__.py | 46 +---- kcidb/oo/argparse.py | 49 ++++++ kcidb/orm/__init__.py | 58 ------- kcidb/orm/argparse.py | 93 ++++++++++ kcidb/orm/query.py | 27 --- kcidb/tests/__init__.py | 4 +- 14 files changed, 902 insertions(+), 841 deletions(-) create mode 100644 kcidb/argparse.py create mode 100644 kcidb/db/argparse.py create mode 100644 kcidb/mq/argparse.py create mode 100644 kcidb/oo/argparse.py create mode 100644 kcidb/orm/argparse.py diff --git a/kcidb/__init__.py b/kcidb/__init__.py index 1b6846f4..4b114d15 100644 --- a/kcidb/__init__.py +++ b/kcidb/__init__.py @@ -5,7 +5,7 @@ import logging from kcidb.misc import LIGHT_ASSERTS # Silence flake8 "imported but unused" warning -from kcidb import io, db, mq, orm, oo, monitor, tests, unittest, misc # noqa +from kcidb import io, db, mq, orm, oo, monitor, tests, unittest, misc, argparse # noqa # Module's logger @@ -184,7 +184,7 @@ def submit_main(): sys.excepthook = misc.log_and_print_excepthook description = \ 'kcidb-submit - Submit Kernel CI reports, print submission IDs' - parser = misc.ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) parser.add_argument( '-p', '--project', help='ID of the Google Cloud project containing the message queue', @@ -214,8 +214,9 @@ def query_main(): sys.excepthook = misc.log_and_print_excepthook description = \ "kcidb-query - Query Kernel CI reports" - parser = db.QueryArgumentParser(driver_types=db.DRIVER_TYPES, - description=description) + parser = db.argparse.QueryArgumentParser( + driver_types=db.DRIVER_TYPES, + description=description) args = parser.parse_args() client = Client(database=args.database) query_iter = client.query_iter( @@ -237,8 +238,8 @@ def schema_main(): """Execute the kcidb-schema command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-schema - Output current or older I/O JSON schema' - parser = misc.OutputArgumentParser(description=description) - misc.argparse_schema_add_args(parser, "output") + parser = argparse.OutputArgumentParser(description=description) + argparse.schema_add_args(parser, "output") args = parser.parse_args() misc.json_dump(args.schema_version.json, sys.stdout, indent=args.indent, seq=args.seq) @@ -248,8 +249,8 @@ def validate_main(): """Execute the kcidb-validate command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-validate - Validate I/O JSON data' - parser = misc.OutputArgumentParser(description=description) - misc.argparse_schema_add_args(parser, "validate against") + parser = argparse.OutputArgumentParser(description=description) + argparse.schema_add_args(parser, "validate against") args = parser.parse_args() misc.json_dump_stream( ( @@ -264,8 +265,8 @@ def upgrade_main(): """Execute the kcidb-upgrade command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-upgrade - Upgrade I/O JSON data to current schema' - parser = misc.OutputArgumentParser(description=description) - misc.argparse_schema_add_args(parser, "upgrade") + parser = argparse.OutputArgumentParser(description=description) + argparse.schema_add_args(parser, "upgrade") args = parser.parse_args() misc.json_dump_stream( ( @@ -280,7 +281,7 @@ def count_main(): """Execute the kcidb-count command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-count - Count number of objects in I/O JSON data' - parser = misc.ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) parser.parse_args() for data in misc.json_load_stream_fd(sys.stdin.fileno()): @@ -292,7 +293,7 @@ def merge_main(): """Execute the kcidb-merge command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-merge - Upgrade and merge I/O data sets' - parser = misc.OutputArgumentParser(description=description) + parser = argparse.OutputArgumentParser(description=description) args = parser.parse_args() sources = [ @@ -312,7 +313,8 @@ def notify_main(): """Execute the kcidb-notify command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-notify - Generate notifications for specified objects' - parser = oo.ArgumentParser(database="json", description=description) + parser = oo.argparse.ArgumentParser(database="json", + description=description) args = parser.parse_args() oo_client = oo.Client(db.Client(args.database)) pattern_set = set() @@ -332,9 +334,9 @@ def ingest_main(): sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-ingest - Load data into a (new) database and ' \ 'generate notifications for new and modified objects' - parser = db.ArgumentParser(driver_types=db.DRIVER_TYPES, - database="sqlite::memory:", - description=description) + parser = db.argparse.ArgumentParser(driver_types=db.DRIVER_TYPES, + database="sqlite::memory:", + description=description) args = parser.parse_args() db_client = db.Client(args.database) diff --git a/kcidb/argparse.py b/kcidb/argparse.py new file mode 100644 index 00000000..91343d3c --- /dev/null +++ b/kcidb/argparse.py @@ -0,0 +1,298 @@ +"""KCIDB general command-line argument parsing""" + +import math +import re +import os +import argparse +import logging +import dateutil.parser +try: # Python 3.9 + from importlib import metadata +except ImportError: # Python 3.6 + import importlib_metadata as metadata +import kcidb.io as io +from kcidb.misc import logging_setup + + +# A dictionary of names of logging levels and their values +LOGGING_LEVEL_MAP = { + name: value + for name, value in logging.__dict__.items() + if name.isalpha() and name.isupper() and isinstance(value, int) and value +} +# Logging level disabling all logging +LOGGING_LEVEL_MAP["NONE"] = max(LOGGING_LEVEL_MAP.values()) + 1 +# Sort levels highest->lowest +# I don't see it, pylint: disable=unnecessary-comprehension +LOGGING_LEVEL_MAP = { + k: v + for k, v in sorted(LOGGING_LEVEL_MAP.items(), + key=lambda i: i[1], reverse=True) +} + +# Check light assertions only, if True +LIGHT_ASSERTS = not os.environ.get("KCIDB_HEAVY_ASSERTS", "") + + +class ArgumentParser(argparse.ArgumentParser): + """ + KCIDB command-line argument parser handling common arguments. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding common arguments. + + Args: + args: Positional arguments to initialize ArgumentParser with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + self.add_argument( + '--version', + action='version', + version=f"Version {metadata.version('kcidb')}" + ) + self.add_argument( + '-l', '--log-level', + metavar="LEVEL", + default="NONE", + choices=LOGGING_LEVEL_MAP.keys(), + help='Limit logging to LEVEL (%(choices)s). Default is NONE.' + ) + + def parse_args(self, args=None, namespace=None): + """ + Parse arguments, including common ones, apply ones affecting global + state. + + Args: + args: List of strings to parse. The default is taken from + sys.argv. + namespace: An object to take the attributes. The default is a new + empty argparse.Namespace object. + + Returns: + Namespace populated with arguments. + """ + args = super().parse_args(args=args, namespace=namespace) + logging.basicConfig() + logging_setup(LOGGING_LEVEL_MAP[args.log_level]) + return args + + +def non_negative_int(string): + """ + Parse a non-negative integer out of a string. + Matches the argparse type function interface. + + Args: + string: The string to parse. + + Returns: + The non-negative integer parsed out of the string. + + Raises: + argparse.ArgumentTypeError: the string wasn't representing a + non-negative integer. + """ + if not re.fullmatch("[0-9]+", string): + raise argparse.ArgumentTypeError( + f'{repr(string)} is not a positive integer, nor zero' + ) + return int(string) + + +def non_negative_int_or_inf(string): + """ + Parse a non-negative integer or a positive infinity out of a string. + Matches the argparse type function interface. + + Args: + string: The string to parse. + + Returns: + The non-negative integer, or positive infinity parsed out of the + string. + + Raises: + argparse.ArgumentTypeError: the string wasn't representing a + non-negative integer or infinity. + """ + try: + value = float(string) + if value != math.inf: + value = non_negative_int(string) + except (ValueError, argparse.ArgumentTypeError) as exc: + raise argparse.ArgumentTypeError( + f'{repr(string)} is not zero, a positive integer, or infinity' + ) from exc + return value + + +def iso_timestamp(string): + """ + Parse an ISO-8601 timestamp out of a string, assuming local timezone, if + not specified. Matches the argparse type function interface. + + Args: + string: The string to parse. + + Returns: + The timestamp parsed out of the string. + + Raises: + argparse.ArgumentTypeError: the string wasn't representing an ISO-8601 + timestamp. + """ + try: + value = dateutil.parser.isoparse(string) + if value.tzinfo is None: + value = value.astimezone() + except ValueError as exc: + raise argparse.ArgumentTypeError( + f'{repr(string)} is not an ISO-8601 timestamp' + ) from exc + return value + + +def version(string): + """ + Parse a version string into a tuple with major and minor version numbers + (both non-negative integers). Matches the argparse type function + interface. + + Args: + string: The string representing the version to parse. + + Returns: + A tuple containing major and minor version numbers. + + Raises: + ValueError if the string was not representing a version. + """ + match = re.fullmatch(r"([0-9]+)\.([0-9]+)", string) + if not match: + raise argparse.ArgumentTypeError( + f"Invalid version: {string!r}" + ) + return int(match.group(1)), int(match.group(2)) + + +def output_add_args(parser): + """ + Add JSON output arguments to a command-line argument parser. + + Args: + parser: The parser to add arguments to. + """ + parser.add_argument( + '--indent', + metavar="NUMBER", + type=non_negative_int, + help='Pretty-print JSON using NUMBER of spaces for indenting. ' + 'Print single-line if zero. Default is 4.', + default=4, + required=False + ) + parser.add_argument( + '--seq', + help='Prefix JSON output with the RS character, to match ' + 'RFC 7464 and "application/json-seq" media type.', + action='store_true' + ) + + +class OutputArgumentParser(ArgumentParser): + """ + Command-line argument parser for tools outputting JSON. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding JSON output arguments. + + Args: + args: Positional arguments to initialize ArgumentParser with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + output_add_args(self) + + +class SplitOutputArgumentParser(OutputArgumentParser): + """ + Command-line argument parser for tools supporting split-report output. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding split-report output arguments. + + Args: + args: Positional arguments to initialize ArgumentParser with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + self.add_argument( + '-o', '--objects-per-report', + metavar="NUMBER", + type=non_negative_int, + help='Put maximum NUMBER of objects into each output ' + 'report, or all, if zero. Default is zero.', + default=0, + required=False + ) + + +def schema_add_args(parser, version_verb): + """ + Add schema selection arguments to a command-line argument parser. + + Args: + parser: The parser to add arguments to. + version_verb: The verb to apply to the schema in the version + argument description. + """ + def schema_version(string): + """ + Lookup a schema version object using a major version number string. + Matches the argparse type function interface. + + Args: + string: The string representing the major version number of the + schema to lookup. + + Returns: + The looked up schema version object. + + Raises: + ValueError if the string was not representing a positive integer + number, or the schema with the supplied major version number was + not found. + """ + if not re.fullmatch("0*[1-9][0-9]*", string): + raise argparse.ArgumentTypeError( + f"Invalid major version number: {string!r}" + ) + major = int(string) + # It's OK, pylint: disable=redefined-outer-name + version = io.SCHEMA + while version and version.major != major: + version = version.previous + if version is None: + raise argparse.ArgumentTypeError( + f"No schema version found for major number {major}" + ) + return version + + parser.add_argument( + 'schema_version', + metavar="SCHEMA_VERSION", + type=schema_version, + help=f"{version_verb.capitalize()} the schema with the specified " + f"major version. Default is the current schema version " + f"(currently {io.SCHEMA.major}).", + nargs='?', + default=io.SCHEMA + ) diff --git a/kcidb/db/__init__.py b/kcidb/db/__init__.py index fc3da2a0..f402d66f 100644 --- a/kcidb/db/__init__.py +++ b/kcidb/db/__init__.py @@ -2,14 +2,14 @@ import sys import logging -import argparse import datetime import kcidb.io as io import kcidb.orm import kcidb.misc from kcidb.misc import LIGHT_ASSERTS from kcidb.db import abstract, schematic, mux, \ - bigquery, postgresql, sqlite, json, null, misc # noqa: F401 + bigquery, postgresql, sqlite, json, null, misc, argparse # noqa: F401 +import kcidb.argparse # Module's logger LOGGER = logging.getLogger(__name__) @@ -375,237 +375,13 @@ def load(self, data): self.driver.load(data) -class DBHelpAction(argparse.Action): - """Argparse action outputting database string help and exiting.""" - def __init__(self, - option_strings, - dest=argparse.SUPPRESS, - default=argparse.SUPPRESS, - help=None): - super().__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - print("KCIDB has several database drivers for both actual and " - "virtual databases.\n" - "You can specify a particular driver to use, and its " - "parameters, using the\n" - "-d/--database option.\n" - "\n" - "The format of the option value is [:], " - "where is the\n" - "name of the driver, and is a (sometimes optional) " - "driver-specific\n" - "parameter string.\n" - "\n" - "For example, \"-d bigquery:kernelci-production.kcidb_01\" " - "requests the use of\n" - "the \"bigquery\" database driver with the parameter string\n" - "\"kernelci-production.kcidb_01\", from which the driver " - "extracts the Google\n" - "Cloud project \"kernelci-production\" and the dataset " - "\"kcidb_01\" to connect to.\n" - "\n" - "Available drivers and format of their parameter strings " - "follow.\n") - for name, driver in parser.driver_types.items(): - print(f"\n{name!r} driver\n" + - "-" * (len(name) + 9) + "\n" + - driver.get_doc()) - parser.exit() - - -def argparse_add_args(parser, database=None): - """ - Add common database arguments to an argument parser. - - Args: - parser: The parser to add arguments to. - database: The default database specification to use. - """ - assert database is None or isinstance(database, str) - parser.add_argument( - '-d', '--database', - help=("Specify DATABASE to use, formatted as :. " + - "Use --database-help for more details." + - ("" if database is None - else f"Default is {database!r}.")), - default=database, - required=(database is None) - ) - parser.add_argument( - '--database-help', - action=DBHelpAction, - help='Print documentation on database specification strings and exit.' - ) - - -class ArgumentParser(kcidb.misc.ArgumentParser): - """ - Command-line argument parser with common database arguments added. - """ - - def __init__(self, driver_types, *args, database=None, **kwargs): - """ - Initialize the parser, adding common database arguments. - - Args: - driver_types: A dictionary of known driver names and types - args: Positional arguments to initialize - ArgumentParser with. - database: The default database specification to use, - or None to make database specification required. - kwargs: Keyword arguments to initialize ArgumentParser - with. - """ - assert isinstance(driver_types, dict) - assert all(isinstance(key, str) for key in driver_types.keys()) - assert all(isinstance(value, type) for value in driver_types.values()) - self.driver_types = driver_types - super().__init__(*args, **kwargs) - argparse_add_args(self, database=database) - - -class OutputArgumentParser(kcidb.misc.OutputArgumentParser): - """ - Command-line argument parser for tools outputting JSON, - with common database arguments added. - """ - - def __init__(self, driver_types, *args, database=None, **kwargs): - """ - Initialize the parser, adding JSON output arguments. - - Args: - driver_types: A dictionary of known driver names and types - args: Positional arguments to initialize - ArgumentParser with. - database: The default database specification to use, - or None to make database specification required. - kwargs: Keyword arguments to initialize ArgumentParser - with. - """ - assert isinstance(driver_types, dict) - assert all(isinstance(key, str) for key in driver_types.keys()) - assert all(isinstance(value, type) for value in driver_types.values()) - self.driver_types = driver_types - super().__init__(*args, **kwargs) - argparse_add_args(self, database=database) - - -class SplitOutputArgumentParser(kcidb.misc.SplitOutputArgumentParser): - """ - Command-line argument parser for tools outputting split-report streams, - with common database arguments added. - """ - - def __init__(self, driver_types, *args, database=None, **kwargs): - """ - Initialize the parser, adding split-report output arguments. - - Args: - driver_types: A dictionary of known driver names and types - args: Positional arguments to initialize - ArgumentParser with. - database: The default database specification to use, - or None to make database specification required. - kwargs: Keyword arguments to initialize ArgumentParser - with. - """ - assert isinstance(driver_types, dict) - assert all(isinstance(key, str) for key in driver_types.keys()) - assert all(isinstance(value, type) for value in driver_types.values()) - self.driver_types = driver_types - super().__init__(*args, **kwargs) - argparse_add_args(self, database=database) - - -# No, it's OK, pylint: disable=too-many-ancestors -class QueryArgumentParser(SplitOutputArgumentParser): - """ - Command-line argument parser with common database query arguments added. - """ - - def __init__(self, driver_types, *args, **kwargs): - """ - Initialize the parser, adding common database query arguments. - - Args: - driver_types: A dictionary of known driver names and types - args: Positional arguments to initialize - ArgumentParser with. - kwargs: Keyword arguments to initialize ArgumentParser - with. - """ - assert isinstance(driver_types, dict) - assert all(isinstance(key, str) for key in driver_types.keys()) - assert all(isinstance(value, type) for value in driver_types.values()) - self.driver_types = driver_types - super().__init__(self.driver_types, *args, **kwargs) - - self.add_argument( - '-c', '--checkout-id', - metavar="ID", - default=[], - help='ID of a checkout to match', - dest="checkout_ids", - action='append', - ) - self.add_argument( - '-b', '--build-id', - metavar="ID", - default=[], - help='ID of a build to match', - dest="build_ids", - action='append', - ) - self.add_argument( - '-t', '--test-id', - metavar="ID", - default=[], - help='ID of a test to match', - dest="test_ids", - action='append', - ) - self.add_argument( - '-i', '--issue-id', - metavar="ID", - default=[], - help='ID of an issue to match', - dest="issue_ids", - action='append', - ) - self.add_argument( - '-n', '--incident-id', - metavar="ID", - default=[], - help='ID of an incident to match', - dest="incident_ids", - action='append', - ) - self.add_argument( - '--parents', - help='Match parents of matching objects', - action='store_true' - ) - self.add_argument( - '--children', - help='Match children of matching objects', - action='store_true' - ) - - def dump_main(): """Execute the kcidb-db-dump command-line tool""" sys.excepthook = kcidb.misc.log_and_print_excepthook description = \ 'kcidb-db-dump - Dump all data from Kernel CI report database' - parser = SplitOutputArgumentParser(driver_types=DRIVER_TYPES, - description=description) + parser = argparse.SplitOutputArgumentParser(driver_types=DRIVER_TYPES, + description=description) args = parser.parse_args() client = Client(args.database) if not client.is_initialized(): @@ -621,8 +397,8 @@ def query_main(): sys.excepthook = kcidb.misc.log_and_print_excepthook description = \ "kcidb-db-query - Query objects from Kernel CI report database" - parser = QueryArgumentParser(driver_types=DRIVER_TYPES, - description=description) + parser = argparse.QueryArgumentParser(driver_types=DRIVER_TYPES, + description=description) args = parser.parse_args() client = Client(args.database) if not client.is_initialized(): @@ -647,8 +423,8 @@ def load_main(): sys.excepthook = kcidb.misc.log_and_print_excepthook description = \ 'kcidb-db-load - Load reports into Kernel CI report database' - parser = ArgumentParser(driver_types=DRIVER_TYPES, - description=description) + parser = argparse.ArgumentParser(driver_types=DRIVER_TYPES, + description=description) args = parser.parse_args() client = Client(args.database) if not client.is_initialized(): @@ -664,8 +440,8 @@ def schemas_main(): sys.excepthook = kcidb.misc.log_and_print_excepthook description = 'kcidb-db-schemas - List available database schemas ' \ '(as : )' - parser = ArgumentParser(driver_types=DRIVER_TYPES, - description=description) + parser = argparse.ArgumentParser(driver_types=DRIVER_TYPES, + description=description) args = parser.parse_args() client = Client(args.database) curr_version = client.get_schema()[0] if client.is_initialized() else None @@ -687,8 +463,8 @@ def init_main(): """Execute the kcidb-db-init command-line tool""" sys.excepthook = kcidb.misc.log_and_print_excepthook description = 'kcidb-db-init - Initialize a Kernel CI report database' - parser = ArgumentParser(driver_types=DRIVER_TYPES, - description=description) + parser = argparse.ArgumentParser(driver_types=DRIVER_TYPES, + description=description) parser.add_argument( '--ignore-initialized', help='Do not fail if the database is already initialized.', @@ -700,7 +476,7 @@ def init_main(): help="Specify database schema VERSION to initialize to " "(a first column value from kcidb-db-schemas output). " "Default is the latest version.", - type=kcidb.misc.version + type=kcidb.argparse.version ) args = parser.parse_args() client = Client(args.database) @@ -720,8 +496,8 @@ def upgrade_main(): """Execute the kcidb-db-upgrade command-line tool""" sys.excepthook = kcidb.misc.log_and_print_excepthook description = 'kcidb-db-upgrade - Upgrade database schema' - parser = ArgumentParser(driver_types=DRIVER_TYPES, - description=description) + parser = argparse.ArgumentParser(driver_types=DRIVER_TYPES, + description=description) parser.add_argument( '-s', '--schema', metavar="VERSION", @@ -731,7 +507,7 @@ def upgrade_main(): "Increases in the major number introduce " "backwards-incompatible changes, in the minor - " "backwards-compatible.", - type=kcidb.misc.version + type=kcidb.argparse.version ) args = parser.parse_args() client = Client(args.database) @@ -757,8 +533,8 @@ def cleanup_main(): """Execute the kcidb-db-cleanup command-line tool""" sys.excepthook = kcidb.misc.log_and_print_excepthook description = 'kcidb-db-cleanup - Cleanup a Kernel CI report database' - parser = ArgumentParser(driver_types=DRIVER_TYPES, - description=description) + parser = argparse.ArgumentParser(driver_types=DRIVER_TYPES, + description=description) parser.add_argument( '--ignore-not-initialized', help='Do not fail if the database is not initialized.', @@ -787,8 +563,8 @@ def empty_main(): sys.excepthook = kcidb.misc.log_and_print_excepthook description = 'kcidb-db-empty - Remove all data from a ' \ 'Kernel CI report database' - parser = ArgumentParser(driver_types=DRIVER_TYPES, - description=description) + parser = argparse.ArgumentParser(driver_types=DRIVER_TYPES, + description=description) args = parser.parse_args() client = Client(args.database) if client.is_initialized(): diff --git a/kcidb/db/argparse.py b/kcidb/db/argparse.py new file mode 100644 index 00000000..2de3ef05 --- /dev/null +++ b/kcidb/db/argparse.py @@ -0,0 +1,230 @@ +""""KCIDB database-management command-line argument parsing""" + +import argparse +import kcidb.orm +import kcidb.misc +import kcidb.argparse + + +class DBHelpAction(argparse.Action): + """Argparse action outputting database string help and exiting.""" + def __init__(self, + option_strings, + dest=argparse.SUPPRESS, + default=argparse.SUPPRESS, + help=None): + super().__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + print("KCIDB has several database drivers for both actual and " + "virtual databases.\n" + "You can specify a particular driver to use, and its " + "parameters, using the\n" + "-d/--database option.\n" + "\n" + "The format of the option value is [:], " + "where is the\n" + "name of the driver, and is a (sometimes optional) " + "driver-specific\n" + "parameter string.\n" + "\n" + "For example, \"-d bigquery:kernelci-production.kcidb_01\" " + "requests the use of\n" + "the \"bigquery\" database driver with the parameter string\n" + "\"kernelci-production.kcidb_01\", from which the driver " + "extracts the Google\n" + "Cloud project \"kernelci-production\" and the dataset " + "\"kcidb_01\" to connect to.\n" + "\n" + "Available drivers and format of their parameter strings " + "follow.\n") + for name, driver in parser.driver_types.items(): + print(f"\n{name!r} driver\n" + + "-" * (len(name) + 9) + "\n" + + driver.get_doc()) + parser.exit() + + +def add_args(parser, database=None): + """ + Add common database arguments to an argument parser. + + Args: + parser: The parser to add arguments to. + database: The default database specification to use. + """ + assert database is None or isinstance(database, str) + parser.add_argument( + '-d', '--database', + help=("Specify DATABASE to use, formatted as :. " + + "Use --database-help for more details." + + ("" if database is None + else f"Default is {database!r}.")), + default=database, + required=(database is None) + ) + parser.add_argument( + '--database-help', + action=DBHelpAction, + help='Print documentation on database specification strings and exit.' + ) + + +class ArgumentParser(kcidb.argparse.ArgumentParser): + """ + Command-line argument parser with common database arguments added. + """ + + def __init__(self, driver_types, *args, database=None, **kwargs): + """ + Initialize the parser, adding common database arguments. + + Args: + driver_types: A dictionary of known driver names and types + args: Positional arguments to initialize + ArgumentParser with. + database: The default database specification to use, + or None to make database specification required. + kwargs: Keyword arguments to initialize ArgumentParser + with. + """ + assert isinstance(driver_types, dict) + assert all(isinstance(key, str) for key in driver_types.keys()) + assert all(isinstance(value, type) for value in driver_types.values()) + self.driver_types = driver_types + super().__init__(*args, **kwargs) + add_args(self, database=database) + + +class OutputArgumentParser(kcidb.argparse.OutputArgumentParser): + """ + Command-line argument parser for tools outputting JSON, + with common database arguments added. + """ + + def __init__(self, driver_types, *args, database=None, **kwargs): + """ + Initialize the parser, adding JSON output arguments. + + Args: + driver_types: A dictionary of known driver names and types + args: Positional arguments to initialize + ArgumentParser with. + database: The default database specification to use, + or None to make database specification required. + kwargs: Keyword arguments to initialize ArgumentParser + with. + """ + assert isinstance(driver_types, dict) + assert all(isinstance(key, str) for key in driver_types.keys()) + assert all(isinstance(value, type) for value in driver_types.values()) + self.driver_types = driver_types + super().__init__(*args, **kwargs) + add_args(self, database=database) + + +class SplitOutputArgumentParser(kcidb.argparse.SplitOutputArgumentParser): + """ + Command-line argument parser for tools outputting split-report streams, + with common database arguments added. + """ + + def __init__(self, driver_types, *args, database=None, **kwargs): + """ + Initialize the parser, adding split-report output arguments. + + Args: + driver_types: A dictionary of known driver names and types + args: Positional arguments to initialize + ArgumentParser with. + database: The default database specification to use, + or None to make database specification required. + kwargs: Keyword arguments to initialize ArgumentParser + with. + """ + assert isinstance(driver_types, dict) + assert all(isinstance(key, str) for key in driver_types.keys()) + assert all(isinstance(value, type) for value in driver_types.values()) + self.driver_types = driver_types + super().__init__(*args, **kwargs) + add_args(self, database=database) + + +# No, it's OK, pylint: disable=too-many-ancestors +class QueryArgumentParser(SplitOutputArgumentParser): + """ + Command-line argument parser with common database query arguments added. + """ + + def __init__(self, driver_types, *args, **kwargs): + """ + Initialize the parser, adding common database query arguments. + + Args: + driver_types: A dictionary of known driver names and types + args: Positional arguments to initialize + ArgumentParser with. + kwargs: Keyword arguments to initialize ArgumentParser + with. + """ + assert isinstance(driver_types, dict) + assert all(isinstance(key, str) for key in driver_types.keys()) + assert all(isinstance(value, type) for value in driver_types.values()) + self.driver_types = driver_types + super().__init__(self.driver_types, *args, **kwargs) + + self.add_argument( + '-c', '--checkout-id', + metavar="ID", + default=[], + help='ID of a checkout to match', + dest="checkout_ids", + action='append', + ) + self.add_argument( + '-b', '--build-id', + metavar="ID", + default=[], + help='ID of a build to match', + dest="build_ids", + action='append', + ) + self.add_argument( + '-t', '--test-id', + metavar="ID", + default=[], + help='ID of a test to match', + dest="test_ids", + action='append', + ) + self.add_argument( + '-i', '--issue-id', + metavar="ID", + default=[], + help='ID of an issue to match', + dest="issue_ids", + action='append', + ) + self.add_argument( + '-n', '--incident-id', + metavar="ID", + default=[], + help='ID of an incident to match', + dest="incident_ids", + action='append', + ) + self.add_argument( + '--parents', + help='Match parents of matching objects', + action='store_true' + ) + self.add_argument( + '--children', + help='Match children of matching objects', + action='store_true' + ) diff --git a/kcidb/misc.py b/kcidb/misc.py index 1fd2dd69..9f984a36 100644 --- a/kcidb/misc.py +++ b/kcidb/misc.py @@ -1,25 +1,16 @@ """Kernel CI reporting - misc definitions""" -import math -import re import os import atexit import tempfile import sys import traceback import itertools -import argparse import logging import json from textwrap import indent -import dateutil.parser -try: # Python 3.9 - from importlib import metadata -except ImportError: # Python 3.6 - import importlib_metadata as metadata from google.cloud import secretmanager import jq -import kcidb.io as io # Module's logger LOGGER = logging.getLogger(__name__) @@ -102,270 +93,6 @@ def log_and_print_excepthook(type, value, tb): print(format_exception_stack(value), file=sys.stderr) -class ArgumentParser(argparse.ArgumentParser): - """ - KCIDB command-line argument parser handling common arguments. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding common arguments. - - Args: - args: Positional arguments to initialize ArgumentParser with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - self.add_argument( - '--version', - action='version', - version=f"Version {metadata.version('kcidb')}" - ) - self.add_argument( - '-l', '--log-level', - metavar="LEVEL", - default="NONE", - choices=LOGGING_LEVEL_MAP.keys(), - help='Limit logging to LEVEL (%(choices)s). Default is NONE.' - ) - - def parse_args(self, args=None, namespace=None): - """ - Parse arguments, including common ones, apply ones affecting global - state. - - Args: - args: List of strings to parse. The default is taken from - sys.argv. - namespace: An object to take the attributes. The default is a new - empty argparse.Namespace object. - - Returns: - Namespace populated with arguments. - """ - args = super().parse_args(args=args, namespace=namespace) - logging.basicConfig() - logging_setup(LOGGING_LEVEL_MAP[args.log_level]) - return args - - -def non_negative_int(string): - """ - Parse a non-negative integer out of a string. - Matches the argparse type function interface. - - Args: - string: The string to parse. - - Returns: - The non-negative integer parsed out of the string. - - Raises: - argparse.ArgumentTypeError: the string wasn't representing a - non-negative integer. - """ - if not re.fullmatch("[0-9]+", string): - raise argparse.ArgumentTypeError( - f'{repr(string)} is not a positive integer, nor zero' - ) - return int(string) - - -def non_negative_int_or_inf(string): - """ - Parse a non-negative integer or a positive infinity out of a string. - Matches the argparse type function interface. - - Args: - string: The string to parse. - - Returns: - The non-negative integer, or positive infinity parsed out of the - string. - - Raises: - argparse.ArgumentTypeError: the string wasn't representing a - non-negative integer or infinity. - """ - try: - value = float(string) - if value != math.inf: - value = non_negative_int(string) - except (ValueError, argparse.ArgumentTypeError) as exc: - raise argparse.ArgumentTypeError( - f'{repr(string)} is not zero, a positive integer, or infinity' - ) from exc - return value - - -def iso_timestamp(string): - """ - Parse an ISO-8601 timestamp out of a string, assuming local timezone, if - not specified. Matches the argparse type function interface. - - Args: - string: The string to parse. - - Returns: - The timestamp parsed out of the string. - - Raises: - argparse.ArgumentTypeError: the string wasn't representing an ISO-8601 - timestamp. - """ - try: - value = dateutil.parser.isoparse(string) - if value.tzinfo is None: - value = value.astimezone() - except ValueError as exc: - raise argparse.ArgumentTypeError( - f'{repr(string)} is not an ISO-8601 timestamp' - ) from exc - return value - - -def version(string): - """ - Parse a version string into a tuple with major and minor version numbers - (both non-negative integers). Matches the argparse type function - interface. - - Args: - string: The string representing the version to parse. - - Returns: - A tuple containing major and minor version numbers. - - Raises: - ValueError if the string was not representing a version. - """ - match = re.fullmatch(r"([0-9]+)\.([0-9]+)", string) - if not match: - raise argparse.ArgumentTypeError( - f"Invalid version: {string!r}" - ) - return int(match.group(1)), int(match.group(2)) - - -def argparse_output_add_args(parser): - """ - Add JSON output arguments to a command-line argument parser. - - Args: - parser: The parser to add arguments to. - """ - parser.add_argument( - '--indent', - metavar="NUMBER", - type=non_negative_int, - help='Pretty-print JSON using NUMBER of spaces for indenting. ' - 'Print single-line if zero. Default is 4.', - default=4, - required=False - ) - parser.add_argument( - '--seq', - help='Prefix JSON output with the RS character, to match ' - 'RFC 7464 and "application/json-seq" media type.', - action='store_true' - ) - - -class OutputArgumentParser(ArgumentParser): - """ - Command-line argument parser for tools outputting JSON. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding JSON output arguments. - - Args: - args: Positional arguments to initialize ArgumentParser with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - argparse_output_add_args(self) - - -class SplitOutputArgumentParser(OutputArgumentParser): - """ - Command-line argument parser for tools supporting split-report output. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding split-report output arguments. - - Args: - args: Positional arguments to initialize ArgumentParser with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - self.add_argument( - '-o', '--objects-per-report', - metavar="NUMBER", - type=non_negative_int, - help='Put maximum NUMBER of objects into each output ' - 'report, or all, if zero. Default is zero.', - default=0, - required=False - ) - - -def argparse_schema_add_args(parser, version_verb): - """ - Add schema selection arguments to a command-line argument parser. - - Args: - parser: The parser to add arguments to. - version_verb: The verb to apply to the schema in the version - argument description. - """ - def schema_version(string): - """ - Lookup a schema version object using a major version number string. - Matches the argparse type function interface. - - Args: - string: The string representing the major version number of the - schema to lookup. - - Returns: - The looked up schema version object. - - Raises: - ValueError if the string was not representing a positive integer - number, or the schema with the supplied major version number was - not found. - """ - if not re.fullmatch("0*[1-9][0-9]*", string): - raise argparse.ArgumentTypeError( - f"Invalid major version number: {string!r}" - ) - major = int(string) - # It's OK, pylint: disable=redefined-outer-name - version = io.SCHEMA - while version and version.major != major: - version = version.previous - if version is None: - raise argparse.ArgumentTypeError( - f"No schema version found for major number {major}" - ) - return version - - parser.add_argument( - 'schema_version', - metavar="SCHEMA_VERSION", - type=schema_version, - help=f"{version_verb.capitalize()} the schema with the specified " - f"major version. Default is the current schema version " - f"(currently {io.SCHEMA.major}).", - nargs='?', - default=io.SCHEMA - ) - - def json_load_stream_fd(stream_fd, chunk_size=4*1024*1024): """ Load a series of JSON values from a stream file descriptor. diff --git a/kcidb/monitor/spool/__init__.py b/kcidb/monitor/spool/__init__.py index c531bb80..b7551430 100644 --- a/kcidb/monitor/spool/__init__.py +++ b/kcidb/monitor/spool/__init__.py @@ -12,9 +12,10 @@ import email import email.policy from google.cloud import firestore -from kcidb.misc import ArgumentParser, iso_timestamp, log_and_print_excepthook +from kcidb.misc import log_and_print_excepthook from kcidb.monitor.misc import is_valid_firestore_id from kcidb.monitor.output import Notification +from kcidb.argparse import ArgumentParser, iso_timestamp # Because we like the "id" name # pylint: disable=invalid-name diff --git a/kcidb/mq/__init__.py b/kcidb/mq/__init__.py index d26e6997..10533e85 100644 --- a/kcidb/mq/__init__.py +++ b/kcidb/mq/__init__.py @@ -6,8 +6,6 @@ import logging import threading import sys -import argparse -import email import email.message import email.policy from abc import ABC, abstractmethod @@ -17,7 +15,9 @@ import kcidb.orm from kcidb import misc from kcidb.misc import LIGHT_ASSERTS - +import kcidb.argparse +from kcidb.mq import argparse +import kcidb.orm.argparse # Module's logger LOGGER = logging.getLogger(__name__) @@ -598,178 +598,14 @@ def __init__(self, *args, **kwargs): self.parser = email.parser.Parser(policy=email.policy.SMTPUTF8) -def argparse_add_args(parser): - """ - Add common message queue arguments to an argument parser. - - Args: - parser: The parser to add arguments to. - """ - parser.add_argument( - '-p', '--project', - help='ID of the Google Cloud project with the message queue', - required=True - ) - parser.add_argument( - '-t', '--topic', - help='Name of the message queue topic', - required=True - ) - - -class ArgumentParser(misc.ArgumentParser): - """ - Command-line argument parser with common message queue arguments added. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding common message queue arguments. - - Args: - args: Positional arguments to initialize ArgumentParser - with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - argparse_add_args(self) - - -def argparse_publisher_add_args(parser, data_name): - """ - Add message queue publisher arguments to an argument parser. - - Args: - parser: The parser to add arguments to. - data_name: Name of the message queue data. - """ - argparse_add_args(parser) - subparsers = parser.add_subparsers(dest="command", - title="Available commands", - metavar="COMMAND", - parser_class=argparse.ArgumentParser) - subparsers.required = True - parser.subparsers = {} - description = f"Initialize {data_name} publisher setup" - parser.subparsers["init"] = subparsers.add_parser( - name="init", help=description, description=description - ) - description = \ - f"Publish {data_name} from standard input, print publishing IDs" - parser.subparsers["publish"] = subparsers.add_parser( - name="publish", help=description, description=description - ) - description = f"Cleanup {data_name} publisher setup" - parser.subparsers["cleanup"] = subparsers.add_parser( - name="cleanup", help=description, description=description - ) - - -class PublisherArgumentParser(misc.ArgumentParser): - """ - Command-line argument parser with common message queue arguments added. - """ - - def __init__(self, data_name, *args, **kwargs): - """ - Initialize the parser, adding common message queue arguments. - - Args: - data_name: Name of the message queue data. - args: Positional arguments to initialize ArgumentParser - with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - self.subparsers = {} - argparse_publisher_add_args(self, data_name) - - -def argparse_subscriber_add_args(parser, data_name): - """ - Add message queue subscriber arguments to an argument parser. - - Args: - parser: The parser to add arguments to. - data_name: Name of the message queue data. - """ - argparse_add_args(parser) - parser.add_argument( - '-s', '--subscription', - help='Name of the subscription', - required=True - ) - subparsers = parser.add_subparsers(dest="command", - title="Available commands", - metavar="COMMAND", - parser_class=argparse.ArgumentParser) - subparsers.required = True - parser.subparsers = {} - - description = f"Initialize {data_name} subscriber setup" - parser.subparsers["init"] = subparsers.add_parser( - name="init", help=description, description=description - ) - - description = \ - f"Pull {data_name} with a subscriber, print to standard output" - parser.subparsers["pull"] = subparsers.add_parser( - name="pull", help=description, description=description - ) - parser.subparsers["pull"].add_argument( - '--timeout', - metavar="SECONDS", - type=float, - help='Wait the specified number of SECONDS for a message, ' - 'or "inf" for infinity. Default is "inf".', - default=math.inf, - required=False - ) - parser.subparsers["pull"].add_argument( - '-m', - '--messages', - metavar="NUMBER", - type=misc.non_negative_int_or_inf, - help='Pull maximum NUMBER of messages, or "inf" for infinity. ' - 'Default is 1.', - default=1, - required=False - ) - - description = f"Cleanup {data_name} subscriber setup" - parser.subparsers["cleanup"] = subparsers.add_parser( - name="cleanup", help=description, description=description - ) - - -class SubscriberArgumentParser(misc.ArgumentParser): - """ - Command-line argument parser with message queue subscriber arguments - added. - """ - - def __init__(self, data_name, *args, **kwargs): - """ - Initialize the parser, adding message queue subscriber arguments. - - Args: - data_name: Name of the message queue data. - args: Positional arguments to initialize ArgumentParser - with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - self.subparsers = {} - argparse_subscriber_add_args(self, data_name) - - def io_publisher_main(): """Execute the kcidb-mq-io-publisher command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = \ 'kcidb-mq-io-publisher - ' \ 'Kernel CI I/O data publisher management tool' - parser = PublisherArgumentParser("I/O data", description=description) + parser = argparse.PublisherArgumentParser("I/O data", + description=description) args = parser.parse_args() publisher = IOPublisher(args.project, args.topic) if args.command == "init": @@ -793,8 +629,9 @@ def io_subscriber_main(): description = \ 'kcidb-mq-io-subscriber - ' \ 'Kernel CI I/O data subscriber management tool' - parser = SubscriberArgumentParser("I/O data", description=description) - misc.argparse_output_add_args(parser.subparsers["pull"]) + parser = argparse.SubscriberArgumentParser("I/O data", + description=description) + kcidb.argparse.output_add_args(parser.subparsers["pull"]) args = parser.parse_args() subscriber = IOSubscriber(args.project, args.topic, args.subscription) if args.command == "init": @@ -815,10 +652,11 @@ def pattern_publisher_main(): description = \ 'kcidb-mq-pattern-publisher - ' \ 'Kernel CI ORM pattern publisher management tool' - parser = PublisherArgumentParser("ORM patterns", description=description) + parser = argparse.PublisherArgumentParser("ORM patterns", + description=description) parser.subparsers["publish"].add_argument( '--pattern-help', - action=kcidb.orm.query.PatternHelpAction, + action=kcidb.orm.argparse.PatternHelpAction, help='Print pattern string documentation and exit.' ) args = parser.parse_args() @@ -846,7 +684,8 @@ def pattern_subscriber_main(): description = \ 'kcidb-mq-pattern-subscriber - ' \ 'Kernel CI ORM pattern subscriber management tool' - parser = SubscriberArgumentParser("ORM patterns", description=description) + parser = argparse.SubscriberArgumentParser("ORM patterns", + description=description) args = parser.parse_args() subscriber = ORMPatternSubscriber(args.project, args.topic, args.subscription) @@ -870,7 +709,8 @@ def email_publisher_main(): description = \ 'kcidb-mq-email-publisher - ' \ 'Kernel CI email queue publisher management tool' - parser = PublisherArgumentParser("email", description=description) + parser = argparse.PublisherArgumentParser("email", + description=description) args = parser.parse_args() publisher = EmailPublisher(args.project, args.topic) if args.command == "init": @@ -888,7 +728,8 @@ def email_subscriber_main(): description = \ 'kcidb-mq-email-subscriber - ' \ 'Kernel CI email queue subscriber management tool' - parser = SubscriberArgumentParser("email", description=description) + parser = argparse.SubscriberArgumentParser("email", + description=description) args = parser.parse_args() subscriber = EmailSubscriber(args.project, args.topic, args.subscription) if args.command == "init": diff --git a/kcidb/mq/argparse.py b/kcidb/mq/argparse.py new file mode 100644 index 00000000..3401d08c --- /dev/null +++ b/kcidb/mq/argparse.py @@ -0,0 +1,171 @@ +""""KCIDB message queue-management command-line argument parsing""" + +import math +import argparse +import kcidb.argparse +import kcidb.orm + + +def add_args(parser): + """ + Add common message queue arguments to an argument parser. + + Args: + parser: The parser to add arguments to. + """ + parser.add_argument( + '-p', '--project', + help='ID of the Google Cloud project with the message queue', + required=True + ) + parser.add_argument( + '-t', '--topic', + help='Name of the message queue topic', + required=True + ) + + +class ArgumentParser(kcidb.argparse.ArgumentParser): + """ + Command-line argument parser with common message queue arguments added. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding common message queue arguments. + + Args: + args: Positional arguments to initialize ArgumentParser + with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + add_args(self) + + +def publisher_add_args(parser, data_name): + """ + Add message queue publisher arguments to an argument parser. + + Args: + parser: The parser to add arguments to. + data_name: Name of the message queue data. + """ + add_args(parser) + subparsers = parser.add_subparsers(dest="command", + title="Available commands", + metavar="COMMAND", + parser_class=argparse.ArgumentParser) + subparsers.required = True + parser.subparsers = {} + description = f"Initialize {data_name} publisher setup" + parser.subparsers["init"] = subparsers.add_parser( + name="init", help=description, description=description + ) + description = \ + f"Publish {data_name} from standard input, print publishing IDs" + parser.subparsers["publish"] = subparsers.add_parser( + name="publish", help=description, description=description + ) + description = f"Cleanup {data_name} publisher setup" + parser.subparsers["cleanup"] = subparsers.add_parser( + name="cleanup", help=description, description=description + ) + + +class PublisherArgumentParser(kcidb.argparse.ArgumentParser): + """ + Command-line argument parser with common message queue arguments added. + """ + + def __init__(self, data_name, *args, **kwargs): + """ + Initialize the parser, adding common message queue arguments. + + Args: + data_name: Name of the message queue data. + args: Positional arguments to initialize ArgumentParser + with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + self.subparsers = {} + publisher_add_args(self, data_name) + + +def subscriber_add_args(parser, data_name): + """ + Add message queue subscriber arguments to an argument parser. + + Args: + parser: The parser to add arguments to. + data_name: Name of the message queue data. + """ + add_args(parser) + parser.add_argument( + '-s', '--subscription', + help='Name of the subscription', + required=True + ) + subparsers = parser.add_subparsers(dest="command", + title="Available commands", + metavar="COMMAND", + parser_class=argparse.ArgumentParser) + subparsers.required = True + parser.subparsers = {} + + description = f"Initialize {data_name} subscriber setup" + parser.subparsers["init"] = subparsers.add_parser( + name="init", help=description, description=description + ) + + description = \ + f"Pull {data_name} with a subscriber, print to standard output" + parser.subparsers["pull"] = subparsers.add_parser( + name="pull", help=description, description=description + ) + parser.subparsers["pull"].add_argument( + '--timeout', + metavar="SECONDS", + type=float, + help='Wait the specified number of SECONDS for a message, ' + 'or "inf" for infinity. Default is "inf".', + default=math.inf, + required=False + ) + parser.subparsers["pull"].add_argument( + '-m', + '--messages', + metavar="NUMBER", + type=kcidb.argparse.non_negative_int_or_inf, + help='Pull maximum NUMBER of messages, or "inf" for infinity. ' + 'Default is 1.', + default=1, + required=False + ) + + description = f"Cleanup {data_name} subscriber setup" + parser.subparsers["cleanup"] = subparsers.add_parser( + name="cleanup", help=description, description=description + ) + + +class SubscriberArgumentParser(kcidb.argparse.ArgumentParser): + """ + Command-line argument parser with message queue subscriber arguments + added. + """ + + def __init__(self, data_name, *args, **kwargs): + """ + Initialize the parser, adding message queue subscriber arguments. + + Args: + data_name: Name of the message queue data. + args: Positional arguments to initialize ArgumentParser + with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + self.subparsers = {} + subscriber_add_args(self, data_name) diff --git a/kcidb/oo/__init__.py b/kcidb/oo/__init__.py index 5b7fac0e..5ac4d94e 100644 --- a/kcidb/oo/__init__.py +++ b/kcidb/oo/__init__.py @@ -11,6 +11,7 @@ from kcidb.orm import Source from kcidb.orm.query import Pattern from kcidb.orm.data import Type, SCHEMA +from kcidb.oo import argparse class Object: @@ -779,56 +780,13 @@ def reset_cache(self): self.cache.reset() -class ArgumentParser(kcidb.misc.ArgumentParser): - """ - Command-line argument parser with common OO arguments added. - """ - - def __init__(self, *args, database=None, **kwargs): - """ - Initialize the parser, adding common OO arguments. - - Args: - args: Positional arguments to initialize ArgumentParser - with. - database: The default database specification to use, or None to - make database specification required. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - kcidb.db.argparse_add_args(self, database=database) - kcidb.orm.argparse_add_args(self) - - -class OutputArgumentParser(kcidb.misc.OutputArgumentParser): - """ - Command-line argument parser for tools outputting JSON, - with common OO arguments added. - """ - - def __init__(self, *args, database=None, **kwargs): - """ - Initialize the parser, adding JSON output arguments. - - Args: - args: Positional arguments to initialize ArgumentParser - with. - database: The default database specification to use, or None to - make database specification required. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - kcidb.db.argparse_add_args(self, database=database) - kcidb.orm.argparse_add_args(self) - - def query_main(): """Execute the kcidb-oo-query command-line tool""" sys.excepthook = kcidb.misc.log_and_print_excepthook description = \ "kcidb-oo-query - Query object-oriented data from " \ "Kernel CI report database" - parser = OutputArgumentParser(description=description) + parser = argparse.OutputArgumentParser(description=description) args = parser.parse_args() db_client = kcidb.db.Client(args.database) pattern_set = set() diff --git a/kcidb/oo/argparse.py b/kcidb/oo/argparse.py new file mode 100644 index 00000000..44276698 --- /dev/null +++ b/kcidb/oo/argparse.py @@ -0,0 +1,49 @@ +""""KCIDB object-oriented (OO) data representation-management + command-line argument parsing + """ + +import kcidb.db.argparse +import kcidb.orm.argparse + + +class ArgumentParser(kcidb.argparse.ArgumentParser): + """ + Command-line argument parser with common OO arguments added. + """ + + def __init__(self, *args, database=None, **kwargs): + """ + Initialize the parser, adding common OO arguments. + + Args: + args: Positional arguments to initialize ArgumentParser + with. + database: The default database specification to use, or None to + make database specification required. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + kcidb.db.argparse.add_args(self, database=database) + kcidb.orm.argparse.add_args(self) + + +class OutputArgumentParser(kcidb.argparse.OutputArgumentParser): + """ + Command-line argument parser for tools outputting JSON, + with common OO arguments added. + """ + + def __init__(self, *args, database=None, **kwargs): + """ + Initialize the parser, adding JSON output arguments. + + Args: + args: Positional arguments to initialize ArgumentParser + with. + database: The default database specification to use, or None to + make database specification required. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + kcidb.db.argparse.add_args(self, database=database) + kcidb.orm.argparse.add_args(self) diff --git a/kcidb/orm/__init__.py b/kcidb/orm/__init__.py index 110d545f..ad59c358 100644 --- a/kcidb/orm/__init__.py +++ b/kcidb/orm/__init__.py @@ -5,7 +5,6 @@ import logging from abc import ABC, abstractmethod -import kcidb.misc from kcidb.misc import LIGHT_ASSERTS from kcidb.orm import query, data @@ -271,60 +270,3 @@ def oo_query(self, pattern_set): } assert LIGHT_ASSERTS or data.SCHEMA.is_valid(response) return response - - -def argparse_add_args(parser): - """ - Add common ORM arguments to an argument parser. - - Args: - The parser to add arguments to. - """ - parser.add_argument( - "pattern_strings", - nargs="*", - default=[], - metavar="PATTERN", - help="Object-matching pattern. " - "See pattern documentation with --pattern-help.", - ) - parser.add_argument( - "--pattern-help", - action=query.PatternHelpAction, - help="Print pattern string documentation and exit.", - ) - - -class ArgumentParser(kcidb.misc.ArgumentParser): - """ - Command-line argument parser with common ORM arguments added. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding common ORM arguments. - - Args: - args: Positional arguments to initialize ArgumentParser with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - argparse_add_args(self) - - -class OutputArgumentParser(kcidb.misc.OutputArgumentParser): - """ - Command-line argument parser for tools outputting JSON, - with common ORM arguments added. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding JSON output arguments. - - Args: - args: Positional arguments to initialize ArgumentParser with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - argparse_add_args(self) diff --git a/kcidb/orm/argparse.py b/kcidb/orm/argparse.py new file mode 100644 index 00000000..8d67fa56 --- /dev/null +++ b/kcidb/orm/argparse.py @@ -0,0 +1,93 @@ +"""KCIDB object-relational mapping (ORM)-management + command-line argument parsing + """ + +import argparse +import kcidb.misc +import kcidb.argparse +from kcidb.orm.query import Pattern + +# We'll get to it, pylint: disable=too-many-lines + + +class PatternHelpAction(argparse.Action): + """Argparse action outputting pattern string help and exiting.""" + def __init__(self, + option_strings, + dest=argparse.SUPPRESS, + default=argparse.SUPPRESS, + help=None): + super().__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + print( + Pattern.STRING_DOC + + "\n" + + "NOTE: Specifying object ID lists separately is not " + "supported using\n" + " command-line tools. " + "Only inline ID lists are supported.\n" + ) + parser.exit() + + +def add_args(parser): + """ + Add common ORM arguments to an argument parser. + + Args: + The parser to add arguments to. + """ + parser.add_argument( + 'pattern_strings', + nargs='*', + default=[], + metavar='PATTERN', + help='Object-matching pattern. ' + 'See pattern documentation with --pattern-help.' + ) + parser.add_argument( + '--pattern-help', + action=PatternHelpAction, + help='Print pattern string documentation and exit.' + ) + + +class ArgumentParser(kcidb.argparse.ArgumentParser): + """ + Command-line argument parser with common ORM arguments added. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding common ORM arguments. + + Args: + args: Positional arguments to initialize ArgumentParser with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + add_args(self) + + +class OutputArgumentParser(kcidb.argparse.OutputArgumentParser): + """ + Command-line argument parser for tools outputting JSON, + with common ORM arguments added. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding JSON output arguments. + + Args: + args: Positional arguments to initialize ArgumentParser with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + add_args(self) diff --git a/kcidb/orm/query.py b/kcidb/orm/query.py index a37951d9..e010c5a6 100644 --- a/kcidb/orm/query.py +++ b/kcidb/orm/query.py @@ -4,7 +4,6 @@ """ import re -import argparse import textwrap from kcidb.orm.data import Schema, SCHEMA, Type @@ -800,29 +799,3 @@ def from_io(io_data, schema=None, max_objs=0): }) ) return pattern_set - - -class PatternHelpAction(argparse.Action): - """Argparse action outputting pattern string help and exiting.""" - def __init__(self, - option_strings, - dest=argparse.SUPPRESS, - default=argparse.SUPPRESS, - help=None): - super().__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - print( - Pattern.STRING_DOC + - "\n" + - "NOTE: Specifying object ID lists separately is not " - "supported using\n" - " command-line tools. " - "Only inline ID lists are supported.\n" - ) - parser.exit() diff --git a/kcidb/tests/__init__.py b/kcidb/tests/__init__.py index 2eccb111..1955d267 100644 --- a/kcidb/tests/__init__.py +++ b/kcidb/tests/__init__.py @@ -3,7 +3,7 @@ import sys import yaml import requests -from kcidb import misc +from kcidb import misc, argparse from kcidb.tests import schema @@ -11,7 +11,7 @@ def validate_main(): """Execute the kcidb-tests-validate command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-tests-validate - Validate test catalog YAML' - parser = misc.ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) parser.add_argument( "-u", "--urls", action='store_true',