Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move argparse code into separate argparse modules #379

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 19 additions & 16 deletions kcidb/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
import logging
from kcidb.misc import LIGHT_ASSERTS
# Silence flake8 "imported but unused" warning
from kcidb import io, db, mq, orm, oo, monitor, tests, unittest, misc # noqa
from kcidb import io, db, mq, orm, oo, monitor, tests, unittest, misc, \
argparse # noqa


# Module's logger
Expand Down Expand Up @@ -184,7 +185,7 @@ def submit_main():
sys.excepthook = misc.log_and_print_excepthook
description = \
'kcidb-submit - Submit Kernel CI reports, print submission IDs'
parser = misc.ArgumentParser(description=description)
parser = argparse.ArgumentParser(description=description)
parser.add_argument(
'-p', '--project',
help='ID of the Google Cloud project containing the message queue',
Expand Down Expand Up @@ -214,8 +215,9 @@ def query_main():
sys.excepthook = misc.log_and_print_excepthook
description = \
"kcidb-query - Query Kernel CI reports"
parser = db.QueryArgumentParser(driver_types=db.DRIVER_TYPES,
description=description)
parser = db.argparse.QueryArgumentParser(
driver_types=db.DRIVER_TYPES,
description=description)
args = parser.parse_args()
client = Client(database=args.database)
query_iter = client.query_iter(
Expand All @@ -237,8 +239,8 @@ def schema_main():
"""Execute the kcidb-schema command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-schema - Output current or older I/O JSON schema'
parser = misc.OutputArgumentParser(description=description)
misc.argparse_schema_add_args(parser, "output")
parser = argparse.OutputArgumentParser(description=description)
argparse.schema_add_args(parser, "output")
args = parser.parse_args()
misc.json_dump(args.schema_version.json, sys.stdout, indent=args.indent,
seq=args.seq)
Expand All @@ -248,8 +250,8 @@ def validate_main():
"""Execute the kcidb-validate command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-validate - Validate I/O JSON data'
parser = misc.OutputArgumentParser(description=description)
misc.argparse_schema_add_args(parser, "validate against")
parser = argparse.OutputArgumentParser(description=description)
argparse.schema_add_args(parser, "validate against")
args = parser.parse_args()
misc.json_dump_stream(
(
Expand All @@ -264,8 +266,8 @@ def upgrade_main():
"""Execute the kcidb-upgrade command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-upgrade - Upgrade I/O JSON data to current schema'
parser = misc.OutputArgumentParser(description=description)
misc.argparse_schema_add_args(parser, "upgrade")
parser = argparse.OutputArgumentParser(description=description)
argparse.schema_add_args(parser, "upgrade")
args = parser.parse_args()
misc.json_dump_stream(
(
Expand All @@ -280,7 +282,7 @@ def count_main():
"""Execute the kcidb-count command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-count - Count number of objects in I/O JSON data'
parser = misc.ArgumentParser(description=description)
parser = argparse.ArgumentParser(description=description)
parser.parse_args()

for data in misc.json_load_stream_fd(sys.stdin.fileno()):
Expand All @@ -292,7 +294,7 @@ def merge_main():
"""Execute the kcidb-merge command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-merge - Upgrade and merge I/O data sets'
parser = misc.OutputArgumentParser(description=description)
parser = argparse.OutputArgumentParser(description=description)
args = parser.parse_args()

sources = [
Expand All @@ -312,7 +314,8 @@ def notify_main():
"""Execute the kcidb-notify command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-notify - Generate notifications for specified objects'
parser = oo.ArgumentParser(database="json", description=description)
parser = oo.argparse.ArgumentParser(database="json",
description=description)
args = parser.parse_args()
oo_client = oo.Client(db.Client(args.database))
pattern_set = set()
Expand All @@ -332,9 +335,9 @@ def ingest_main():
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-ingest - Load data into a (new) database and ' \
'generate notifications for new and modified objects'
parser = db.ArgumentParser(driver_types=db.DRIVER_TYPES,
database="sqlite::memory:",
description=description)
parser = db.argparse.ArgumentParser(driver_types=db.DRIVER_TYPES,
database="sqlite::memory:",
description=description)
args = parser.parse_args()

db_client = db.Client(args.database)
Expand Down
282 changes: 282 additions & 0 deletions kcidb/argparse.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,282 @@
"""KCIDB general command-line argument parsing"""

import math
import re
import os
import argparse
import logging
import dateutil.parser
try: # Python 3.9
from importlib import metadata
except ImportError: # Python 3.6
import importlib_metadata as metadata
import kcidb.io as io
from kcidb.misc import logging_setup, LOGGING_LEVEL_MAP


# Check light assertions only, if True
LIGHT_ASSERTS = not os.environ.get("KCIDB_HEAVY_ASSERTS", "")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should also stay in kcidb.misc, not get copied here. Import it from there and use like other modules do. E.g. kcidb.oo.

Generally, you should avoid duplicating code and definitions.



class ArgumentParser(argparse.ArgumentParser):
"""
KCIDB command-line argument parser handling common arguments.
"""

def __init__(self, *args, **kwargs):
"""
Initialize the parser, adding common arguments.

Args:
args: Positional arguments to initialize ArgumentParser with.
kwargs: Keyword arguments to initialize ArgumentParser with.
"""
super().__init__(*args, **kwargs)
self.add_argument(
'--version',
action='version',
version=f"Version {metadata.version('kcidb')}"
)
self.add_argument(
'-l', '--log-level',
metavar="LEVEL",
default="NONE",
choices=LOGGING_LEVEL_MAP.keys(),
help='Limit logging to LEVEL (%(choices)s). Default is NONE.'
)

def parse_args(self, args=None, namespace=None):
"""
Parse arguments, including common ones, apply ones affecting global
state.

Args:
args: List of strings to parse. The default is taken from
sys.argv.
namespace: An object to take the attributes. The default is a new
empty argparse.Namespace object.

Returns:
Namespace populated with arguments.
"""
args = super().parse_args(args=args, namespace=namespace)
logging.basicConfig()
logging_setup(LOGGING_LEVEL_MAP[args.log_level])
return args


def non_negative_int(string):
"""
Parse a non-negative integer out of a string.
Matches the argparse type function interface.

Args:
string: The string to parse.

Returns:
The non-negative integer parsed out of the string.

Raises:
argparse.ArgumentTypeError: the string wasn't representing a
non-negative integer.
"""
if not re.fullmatch("[0-9]+", string):
raise argparse.ArgumentTypeError(
f'{repr(string)} is not a positive integer, nor zero'
)
return int(string)


def non_negative_int_or_inf(string):
"""
Parse a non-negative integer or a positive infinity out of a string.
Matches the argparse type function interface.

Args:
string: The string to parse.

Returns:
The non-negative integer, or positive infinity parsed out of the
string.

Raises:
argparse.ArgumentTypeError: the string wasn't representing a
non-negative integer or infinity.
"""
try:
value = float(string)
if value != math.inf:
value = non_negative_int(string)
except (ValueError, argparse.ArgumentTypeError) as exc:
raise argparse.ArgumentTypeError(
f'{repr(string)} is not zero, a positive integer, or infinity'
) from exc
return value


def iso_timestamp(string):
"""
Parse an ISO-8601 timestamp out of a string, assuming local timezone, if
not specified. Matches the argparse type function interface.

Args:
string: The string to parse.

Returns:
The timestamp parsed out of the string.

Raises:
argparse.ArgumentTypeError: the string wasn't representing an ISO-8601
timestamp.
"""
try:
value = dateutil.parser.isoparse(string)
if value.tzinfo is None:
value = value.astimezone()
except ValueError as exc:
raise argparse.ArgumentTypeError(
f'{repr(string)} is not an ISO-8601 timestamp'
) from exc
return value


def version(string):
"""
Parse a version string into a tuple with major and minor version numbers
(both non-negative integers). Matches the argparse type function
interface.

Args:
string: The string representing the version to parse.

Returns:
A tuple containing major and minor version numbers.

Raises:
ValueError if the string was not representing a version.
"""
match = re.fullmatch(r"([0-9]+)\.([0-9]+)", string)
if not match:
raise argparse.ArgumentTypeError(
f"Invalid version: {string!r}"
)
return int(match.group(1)), int(match.group(2))


def output_add_args(parser):
"""
Add JSON output arguments to a command-line argument parser.

Args:
parser: The parser to add arguments to.
"""
parser.add_argument(
'--indent',
metavar="NUMBER",
type=non_negative_int,
help='Pretty-print JSON using NUMBER of spaces for indenting. '
'Print single-line if zero. Default is 4.',
default=4,
required=False
)
parser.add_argument(
'--seq',
help='Prefix JSON output with the RS character, to match '
'RFC 7464 and "application/json-seq" media type.',
action='store_true'
)


class OutputArgumentParser(ArgumentParser):
"""
Command-line argument parser for tools outputting JSON.
"""

def __init__(self, *args, **kwargs):
"""
Initialize the parser, adding JSON output arguments.

Args:
args: Positional arguments to initialize ArgumentParser with.
kwargs: Keyword arguments to initialize ArgumentParser with.
"""
super().__init__(*args, **kwargs)
output_add_args(self)


class SplitOutputArgumentParser(OutputArgumentParser):
"""
Command-line argument parser for tools supporting split-report output.
"""

def __init__(self, *args, **kwargs):
"""
Initialize the parser, adding split-report output arguments.

Args:
args: Positional arguments to initialize ArgumentParser with.
kwargs: Keyword arguments to initialize ArgumentParser with.
"""
super().__init__(*args, **kwargs)
self.add_argument(
'-o', '--objects-per-report',
metavar="NUMBER",
type=non_negative_int,
help='Put maximum NUMBER of objects into each output '
'report, or all, if zero. Default is zero.',
default=0,
required=False
)


def schema_add_args(parser, version_verb):
"""
Add schema selection arguments to a command-line argument parser.

Args:
parser: The parser to add arguments to.
version_verb: The verb to apply to the schema in the version
argument description.
"""
def schema_version(string):
"""
Lookup a schema version object using a major version number string.
Matches the argparse type function interface.

Args:
string: The string representing the major version number of the
schema to lookup.

Returns:
The looked up schema version object.

Raises:
ValueError if the string was not representing a positive integer
number, or the schema with the supplied major version number was
not found.
"""
if not re.fullmatch("0*[1-9][0-9]*", string):
raise argparse.ArgumentTypeError(
f"Invalid major version number: {string!r}"
)
major = int(string)
# It's OK, pylint: disable=redefined-outer-name
version = io.SCHEMA
while version and version.major != major:
version = version.previous
if version is None:
raise argparse.ArgumentTypeError(
f"No schema version found for major number {major}"
)
return version

parser.add_argument(
'schema_version',
metavar="SCHEMA_VERSION",
type=schema_version,
help=f"{version_verb.capitalize()} the schema with the specified "
f"major version. Default is the current schema version "
f"(currently {io.SCHEMA.major}).",
nargs='?',
default=io.SCHEMA
)
Loading