diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4683879e..a6ab9133 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: trailing-whitespace exclude: testing/baselines @@ -10,16 +10,9 @@ repos: exclude: testing/baselines - id: check-yaml - id: check-added-large-files -- repo: https://github.com/asottile/pyupgrade - rev: v3.15.1 - hooks: - - id: pyupgrade - args: ["--py37-plus"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.0 + rev: v0.3.7 hooks: - id: ruff args: [--fix] - id: ruff-format - -exclude: doc/ext/sphinxarg diff --git a/doc/conf.py b/doc/conf.py index 38eb944a..349f5c9f 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -50,7 +50,7 @@ # General information about the project. project = "Zeek Package Manager" -copyright = "2019, The Zeek Project" +copyright = "2019, The Zeek Project" # noqa: A001 author = "The Zeek Project" # The version info for the project you're documenting, acts as replacement for diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index 08fe1282..e26ce391 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -1,14 +1,16 @@ -from argparse import ArgumentParser import os +from argparse import ArgumentParser +from collections.abc import Callable +from typing import ClassVar from docutils import nodes -from docutils.statemachine import StringList from docutils.parsers.rst.directives import flag, unchanged +from docutils.statemachine import StringList try: # Removed as of Sphinx 1.7 from sphinx.util.compat import Directive -except ImportError as err: +except ImportError: from docutils.parsers.rst import Directive from sphinx.util.nodes import nested_parse_with_titles @@ -18,7 +20,7 @@ def map_nested_definitions(nested_content): if nested_content is None: - raise Exception('Nested content should be iterable, not null') + raise Exception("Nested content should be iterable, not null") # build definition dictionary definitions = {} for item in nested_content: @@ -29,15 +31,18 @@ def map_nested_definitions(nested_content): continue if not len(subitem.children) > 0: continue - classifier = '@after' + classifier = "@after" idx = subitem.first_child_matching_class(nodes.classifier) if idx is not None: ci = subitem[idx] if len(ci.children) > 0: classifier = ci.children[0].astext() if classifier is not None and classifier not in ( - '@replace', '@before', '@after'): - raise Exception('Unknown classifier: %s' % classifier) + "@replace", + "@before", + "@after", + ): + raise Exception("Unknown classifier: %s" % classifier) idx = subitem.first_child_matching_class(nodes.term) if idx is not None: ch = subitem[idx] @@ -46,7 +51,7 @@ def map_nested_definitions(nested_content): idx = subitem.first_child_matching_class(nodes.definition) if idx is not None: def_node = subitem[idx] - def_node.attributes['classifier'] = classifier + def_node.attributes["classifier"] = classifier definitions[term] = def_node return definitions @@ -54,121 +59,163 @@ def map_nested_definitions(nested_content): def print_arg_list(data, nested_content): definitions = map_nested_definitions(nested_content) items = [] - if 'args' in data: - for arg in data['args']: - my_def = [nodes.paragraph(text=arg['help'])] if arg['help'] else [] - name = arg['name'] + if "args" in data: + for arg in data["args"]: + my_def = [nodes.paragraph(text=arg["help"])] if arg["help"] else [] + name = arg["name"] my_def = apply_definition(definitions, my_def, name) if len(my_def) == 0: - my_def.append(nodes.paragraph(text='Undocumented')) - if 'choices' in arg: - my_def.append(nodes.paragraph( - text=('Possible choices: %s' % ', '.join([str(c) for c in arg['choices']])))) + my_def.append(nodes.paragraph(text="Undocumented")) + if "choices" in arg: + my_def.append( + nodes.paragraph( + text=( + "Possible choices: %s" + % ", ".join([str(c) for c in arg["choices"]]) + ), + ), + ) argname = name - if arg['metavar']: - argname = arg['metavar'] + if arg["metavar"]: + argname = arg["metavar"] items.append( nodes.option_list_item( - '', nodes.option_group('', - nodes.option('', nodes.option_string(text=argname))), - nodes.description('', *my_def))) - return nodes.option_list('', *items) if items else None + "", + nodes.option_group( + "", + nodes.option("", nodes.option_string(text=argname)), + ), + nodes.description("", *my_def), + ), + ) + return nodes.option_list("", *items) if items else None def print_opt_list(data, nested_content): definitions = map_nested_definitions(nested_content) items = [] - if 'options' in data: - for opt in data['options']: + if "options" in data: + for opt in data["options"]: names = [] - my_def = [nodes.paragraph(text=opt['help'])] if opt['help'] else [] - for name in opt['name']: + my_def = [nodes.paragraph(text=opt["help"])] if opt["help"] else [] + for name in opt["name"]: option_declaration = [nodes.option_string(text=name)] - if opt['default'] is not None \ - and opt['default'] != '==SUPPRESS==': + if opt["default"] is not None and opt["default"] != "==SUPPRESS==": option_declaration += nodes.option_argument( - '', text='=' + str(opt['default'])) - names.append(nodes.option('', *option_declaration)) + "", + text="=" + str(opt["default"]), + ) + names.append(nodes.option("", *option_declaration)) my_def = apply_definition(definitions, my_def, name) if len(my_def) == 0: - my_def.append(nodes.paragraph(text='Undocumented')) - if 'choices' in opt: - my_def.append(nodes.paragraph( - text=('Possible choices: %s' % ', '.join([str(c) for c in opt['choices']])))) + my_def.append(nodes.paragraph(text="Undocumented")) + if "choices" in opt: + my_def.append( + nodes.paragraph( + text=( + "Possible choices: %s" + % ", ".join([str(c) for c in opt["choices"]]) + ), + ), + ) items.append( nodes.option_list_item( - '', nodes.option_group('', *names), - nodes.description('', *my_def))) - return nodes.option_list('', *items) if items else None + "", + nodes.option_group("", *names), + nodes.description("", *my_def), + ), + ) + return nodes.option_list("", *items) if items else None def print_command_args_and_opts(arg_list, opt_list, sub_list=None): items = [] if arg_list: - items.append(nodes.definition_list_item( - '', nodes.term(text='Positional arguments:'), - nodes.definition('', arg_list))) + items.append( + nodes.definition_list_item( + "", + nodes.term(text="Positional arguments:"), + nodes.definition("", arg_list), + ), + ) if opt_list: - items.append(nodes.definition_list_item( - '', nodes.term(text='Options:'), - nodes.definition('', opt_list))) + items.append( + nodes.definition_list_item( + "", + nodes.term(text="Options:"), + nodes.definition("", opt_list), + ), + ) if sub_list and len(sub_list): - items.append(nodes.definition_list_item( - '', nodes.term(text='Sub-commands:'), - nodes.definition('', sub_list))) - return nodes.definition_list('', *items) + items.append( + nodes.definition_list_item( + "", + nodes.term(text="Sub-commands:"), + nodes.definition("", sub_list), + ), + ) + return nodes.definition_list("", *items) def apply_definition(definitions, my_def, name): if name in definitions: definition = definitions[name] - classifier = definition['classifier'] - if classifier == '@replace': + classifier = definition["classifier"] + if classifier == "@replace": return definition.children - if classifier == '@after': + if classifier == "@after": return my_def + definition.children - if classifier == '@before': + if classifier == "@before": return definition.children + my_def - raise Exception('Unknown classifier: %s' % classifier) + raise Exception("Unknown classifier: %s" % classifier) return my_def def print_subcommand_list(data, nested_content): definitions = map_nested_definitions(nested_content) items = [] - if 'children' in data: - for child in data['children']: - my_def = [nodes.paragraph( - text=child['help'])] if child['help'] else [] - name = child['name'] + if "children" in data: + for child in data["children"]: + my_def = [nodes.paragraph(text=child["help"])] if child["help"] else [] + name = child["name"] my_def = apply_definition(definitions, my_def, name) if len(my_def) == 0: - my_def.append(nodes.paragraph(text='Undocumented')) - if 'description' in child: - my_def.append(nodes.paragraph(text=child['description'])) - my_def.append(nodes.literal_block(text=child['usage'])) - my_def.append(print_command_args_and_opts( - print_arg_list(child, nested_content), - print_opt_list(child, nested_content), - print_subcommand_list(child, nested_content) - )) + my_def.append(nodes.paragraph(text="Undocumented")) + if "description" in child: + my_def.append(nodes.paragraph(text=child["description"])) + my_def.append(nodes.literal_block(text=child["usage"])) + my_def.append( + print_command_args_and_opts( + print_arg_list(child, nested_content), + print_opt_list(child, nested_content), + print_subcommand_list(child, nested_content), + ), + ) items.append( nodes.definition_list_item( - '', - nodes.term('', '', nodes.strong(text=name)), - nodes.definition('', *my_def) - ) + "", + nodes.term("", "", nodes.strong(text=name)), + nodes.definition("", *my_def), + ), ) - return nodes.definition_list('', *items) + return nodes.definition_list("", *items) class ArgParseDirective(Directive): has_content = True - option_spec = dict(module=unchanged, func=unchanged, ref=unchanged, - prog=unchanged, path=unchanged, nodefault=flag, - manpage=unchanged, nosubcommands=unchanged, passparser=flag) + option_spec: ClassVar[dict[str, Callable[[str | None], str | None]]] = { + "module": unchanged, + "func": unchanged, + "ref": unchanged, + "prog": unchanged, + "path": unchanged, + "nodefault": flag, + "manpage": unchanged, + "nosubcommands": unchanged, + "passparser": flag, + } def _construct_manpage_specific_structure(self, parser_info): """ @@ -183,39 +230,43 @@ def _construct_manpage_specific_structure(self, parser_info): """ # SYNOPSIS section synopsis_section = nodes.section( - '', - nodes.title(text='Synopsis'), + "", + nodes.title(text="Synopsis"), nodes.literal_block(text=parser_info["bare_usage"]), - ids=['synopsis-section']) + ids=["synopsis-section"], + ) # DESCRIPTION section description_section = nodes.section( - '', - nodes.title(text='Description'), - nodes.paragraph(text=parser_info.get( - 'description', parser_info.get( - 'help', "undocumented").capitalize())), - ids=['description-section']) - nested_parse_with_titles( - self.state, self.content, description_section) - if parser_info.get('epilog'): + "", + nodes.title(text="Description"), + nodes.paragraph( + text=parser_info.get( + "description", + parser_info.get("help", "undocumented").capitalize(), + ), + ), + ids=["description-section"], + ) + nested_parse_with_titles(self.state, self.content, description_section) + if parser_info.get("epilog"): # TODO: do whatever sphinx does to understand ReST inside # docstrings magically imported from other places. The nested # parse method invoked above seem to be able to do this but # I haven't found a way to do it for arbitrary text - description_section += nodes.paragraph( - text=parser_info['epilog']) + description_section += nodes.paragraph(text=parser_info["epilog"]) # OPTIONS section options_section = nodes.section( - '', - nodes.title(text='Options'), - ids=['options-section']) - if 'args' in parser_info: + "", + nodes.title(text="Options"), + ids=["options-section"], + ) + if "args" in parser_info: options_section += nodes.paragraph() - options_section += nodes.subtitle(text='Positional arguments:') + options_section += nodes.subtitle(text="Positional arguments:") options_section += self._format_positional_arguments(parser_info) - if 'options' in parser_info: + if "options" in parser_info: options_section += nodes.paragraph() - options_section += nodes.subtitle(text='Optional arguments:') + options_section += nodes.subtitle(text="Optional arguments:") options_section += self._format_optional_arguments(parser_info) items = [ # NOTE: we cannot generate NAME ourselves. It is generated by @@ -228,94 +279,107 @@ def _construct_manpage_specific_structure(self, parser_info): ] if len(options_section.children) > 1: items.append(options_section) - if 'nosubcommands' not in self.options: + if "nosubcommands" not in self.options: # SUBCOMMANDS section (non-standard) subcommands_section = nodes.section( - '', - nodes.title(text='Sub-Commands'), - ids=['subcommands-section']) - if 'children' in parser_info: + "", + nodes.title(text="Sub-Commands"), + ids=["subcommands-section"], + ) + if "children" in parser_info: subcommands_section += self._format_subcommands(parser_info) if len(subcommands_section) > 1: items.append(subcommands_section) if os.getenv("INCLUDE_DEBUG_SECTION"): import json + # DEBUG section (non-standard) debug_section = nodes.section( - '', + "", nodes.title(text="Argparse + Sphinx Debugging"), - nodes.literal_block(text=json.dumps(parser_info, indent=' ')), - ids=['debug-section']) + nodes.literal_block(text=json.dumps(parser_info, indent=" ")), + ids=["debug-section"], + ) items.append(debug_section) return items def _format_positional_arguments(self, parser_info): - assert 'args' in parser_info + assert "args" in parser_info items = [] - for arg in parser_info['args']: + for arg in parser_info["args"]: arg_items = [] - if arg['help']: - arg_items.append(nodes.paragraph(text=arg['help'])) + if arg["help"]: + arg_items.append(nodes.paragraph(text=arg["help"])) else: - arg_items.append(nodes.paragraph(text='Undocumented')) - if 'choices' in arg: + arg_items.append(nodes.paragraph(text="Undocumented")) + if "choices" in arg: arg_items.append( nodes.paragraph( - text='Possible choices: ' + ', '.join(arg['choices']))) + text="Possible choices: " + ", ".join(arg["choices"]), + ), + ) items.append( nodes.option_list_item( - '', + "", nodes.option_group( - '', nodes.option( - '', nodes.option_string(text=arg['metavar']) - ) + "", + nodes.option("", nodes.option_string(text=arg["metavar"])), ), - nodes.description('', *arg_items))) - return nodes.option_list('', *items) + nodes.description("", *arg_items), + ), + ) + return nodes.option_list("", *items) def _format_optional_arguments(self, parser_info): - assert 'options' in parser_info + assert "options" in parser_info items = [] - for opt in parser_info['options']: + for opt in parser_info["options"]: names = [] opt_items = [] - for name in opt['name']: + for name in opt["name"]: option_declaration = [nodes.option_string(text=name)] - if opt['default'] is not None \ - and opt['default'] != '==SUPPRESS==': + if opt["default"] is not None and opt["default"] != "==SUPPRESS==": option_declaration += nodes.option_argument( - '', text='=' + str(opt['default'])) - names.append(nodes.option('', *option_declaration)) - if opt['help']: - opt_items.append(nodes.paragraph(text=opt['help'])) + "", + text="=" + str(opt["default"]), + ) + names.append(nodes.option("", *option_declaration)) + if opt["help"]: + opt_items.append(nodes.paragraph(text=opt["help"])) else: - opt_items.append(nodes.paragraph(text='Undocumented')) - if 'choices' in opt: + opt_items.append(nodes.paragraph(text="Undocumented")) + if "choices" in opt: opt_items.append( nodes.paragraph( - text='Possible choices: ' + ', '.join(opt['choices']))) + text="Possible choices: " + ", ".join(opt["choices"]), + ), + ) items.append( nodes.option_list_item( - '', nodes.option_group('', *names), - nodes.description('', *opt_items))) - return nodes.option_list('', *items) + "", + nodes.option_group("", *names), + nodes.description("", *opt_items), + ), + ) + return nodes.option_list("", *items) def _format_subcommands(self, parser_info): - assert 'children' in parser_info + assert "children" in parser_info items = [] - for subcmd in parser_info['children']: + for subcmd in parser_info["children"]: subcmd_items = [] - if subcmd['help']: - subcmd_items.append(nodes.paragraph(text=subcmd['help'])) + if subcmd["help"]: + subcmd_items.append(nodes.paragraph(text=subcmd["help"])) else: - subcmd_items.append(nodes.paragraph(text='Undocumented')) + subcmd_items.append(nodes.paragraph(text="Undocumented")) items.append( nodes.definition_list_item( - '', - nodes.term('', '', nodes.strong( - text=subcmd['bare_usage'])), - nodes.definition('', *subcmd_items))) - return nodes.definition_list('', *items) + "", + nodes.term("", "", nodes.strong(text=subcmd["bare_usage"])), + nodes.definition("", *subcmd_items), + ), + ) + return nodes.definition_list("", *items) def _nested_parse_paragraph(self, text): content = nodes.paragraph() @@ -323,98 +387,100 @@ def _nested_parse_paragraph(self, text): return content def run(self): - if 'module' in self.options and 'func' in self.options: - module_name = self.options['module'] - attr_name = self.options['func'] - elif 'ref' in self.options: - _parts = self.options['ref'].split('.') - module_name = '.'.join(_parts[0:-1]) + if "module" in self.options and "func" in self.options: + module_name = self.options["module"] + attr_name = self.options["func"] + elif "ref" in self.options: + _parts = self.options["ref"].split(".") + module_name = ".".join(_parts[0:-1]) attr_name = _parts[-1] else: - raise self.error( - ':module: and :func: should be specified, or :ref:') + raise self.error(":module: and :func: should be specified, or :ref:") mod = __import__(module_name, globals(), locals(), [attr_name]) file_dependency = mod.__file__ - if file_dependency.endswith('.pyc'): + if file_dependency.endswith(".pyc"): file_dependency = file_dependency[:-1] env = self.state.document.settings.env - if not hasattr(env, 'argparse_usages'): + if not hasattr(env, "argparse_usages"): env.argparse_usages = [] - env.argparse_usages.append({ - 'docname': env.docname, - 'lineno': self.lineno, - 'dependency_file': file_dependency, - 'dependency_mtime': os.stat(file_dependency).st_mtime, - }) + env.argparse_usages.append( + { + "docname": env.docname, + "lineno": self.lineno, + "dependency_file": file_dependency, + "dependency_mtime": os.stat(file_dependency).st_mtime, + }, + ) if not hasattr(mod, attr_name): - raise self.error(( - 'Module "%s" has no attribute "%s"\n' - 'Incorrect argparse :module: or :func: values?' - ) % (module_name, attr_name)) + raise self.error( + f'Module "{module_name}" has no attribute "{attr_name}"\n' + "Incorrect argparse :module: or :func: values?", + ) func = getattr(mod, attr_name) if isinstance(func, ArgumentParser): parser = func - elif 'passparser' in self.options: + elif "passparser" in self.options: parser = ArgumentParser() func(parser) else: parser = func() - if 'path' not in self.options: - self.options['path'] = '' - path = str(self.options['path']) - if 'prog' in self.options: - parser.prog = self.options['prog'] - result = parse_parser( - parser, skip_default_values='nodefault' in self.options) + if "path" not in self.options: + self.options["path"] = "" + path = str(self.options["path"]) + if "prog" in self.options: + parser.prog = self.options["prog"] + result = parse_parser(parser, skip_default_values="nodefault" in self.options) result = parser_navigate(result, path) - if 'manpage' in self.options: + if "manpage" in self.options: return self._construct_manpage_specific_structure(result) nested_content = nodes.paragraph() - self.state.nested_parse( - self.content, self.content_offset, nested_content) + self.state.nested_parse(self.content, self.content_offset, nested_content) nested_content = nested_content.children items = [] # add common content between for item in nested_content: if not isinstance(item, nodes.definition_list): items.append(item) - if 'description' in result: - items.append(self._nested_parse_paragraph(result['description'])) - items.append(nodes.literal_block(text=result['usage'])) + if "description" in result: + items.append(self._nested_parse_paragraph(result["description"])) + items.append(nodes.literal_block(text=result["usage"])) - if 'nosubcommands' in self.options: + if "nosubcommands" in self.options: subcommands = None else: subcommands = print_subcommand_list(result, nested_content) - items.append(print_command_args_and_opts( - print_arg_list(result, nested_content), - print_opt_list(result, nested_content), - subcommands - )) - if 'epilog' in result: - items.append(self._nested_parse_paragraph(result['epilog'])) + items.append( + print_command_args_and_opts( + print_arg_list(result, nested_content), + print_opt_list(result, nested_content), + subcommands, + ), + ) + if "epilog" in result: + items.append(self._nested_parse_paragraph(result["epilog"])) return items def env_get_outdated_hook(app, env, added, changed, removed): from sphinx.util import logging + logger = logging.getLogger(__name__) rval = set() - if not hasattr(env, 'argparse_usages'): + if not hasattr(env, "argparse_usages"): return [] for usage in env.argparse_usages: - docname = usage['docname'] - dep_file = usage['dependency_file'] - dep_mtime = usage['dependency_mtime'] + docname = usage["docname"] + dep_file = usage["dependency_file"] + dep_mtime = usage["dependency_mtime"] current_mtime = os.stat(dep_file).st_mtime if current_mtime > dep_mtime and docname not in removed: @@ -422,21 +488,23 @@ def env_get_outdated_hook(app, env, added, changed, removed): for docname in rval: from sphinx.util.console import blue - msg = blue('found outdated argparse doc: {0}'.format(docname)) + + msg = blue(f"found outdated argparse doc: {docname}") logger.info(msg) return list(rval) def env_purge_doc_hook(app, env, docname): - if not hasattr(env, 'argparse_usages'): + if not hasattr(env, "argparse_usages"): return env.argparse_usages = [ - usage for usage in env.argparse_usages if usage['docname'] != docname] + usage for usage in env.argparse_usages if usage["docname"] != docname + ] def setup(app): - app.add_directive('argparse', ArgParseDirective) - app.connect('env-get-outdated', env_get_outdated_hook) - app.connect('env-purge-doc', env_purge_doc_hook) + app.add_directive("argparse", ArgParseDirective) + app.connect("env-get-outdated", env_get_outdated_hook) + app.connect("env-purge-doc", env_purge_doc_hook) diff --git a/doc/ext/sphinxarg/parser.py b/doc/ext/sphinxarg/parser.py index 79a541ad..85a94aa6 100644 --- a/doc/ext/sphinxarg/parser.py +++ b/doc/ext/sphinxarg/parser.py @@ -1,33 +1,35 @@ -from argparse import _HelpAction, _SubParsersAction import re +from argparse import _HelpAction, _SubParsersAction -class NavigationException(Exception): +class NavigationError(Exception): pass def parser_navigate(parser_result, path, current_path=None): if isinstance(path, str): - if path == '': + if path == "": return parser_result - path = re.split(r'\s+', path) + path = re.split(r"\s+", path) current_path = current_path or [] if len(path) == 0: return parser_result - if 'children' not in parser_result: - raise NavigationException( - 'Current parser have no children elements. (path: %s)' % - ' '.join(current_path)) + if "children" not in parser_result: + raise NavigationError( + "Current parser have no children elements. (path: %s)" + % " ".join(current_path), + ) next_hop = path.pop(0) - for child in parser_result['children']: + for child in parser_result["children"]: # identifer is only used for aliased subcommands - identifier = child['identifier'] if 'identifier' in child else child['name'] + identifier = child["identifier"] if "identifier" in child else child["name"] if identifier == next_hop: current_path.append(next_hop) return parser_navigate(child, path, current_path) - raise NavigationException( - 'Current parser have no children element with name: %s (path: %s)' % ( - next_hop, ' '.join(current_path))) + raise NavigationError( + f"Current parser have no children element with name: {next_hop} (path: %s)" + % " ".join(current_path), + ) def _try_add_parser_attribute(data, parser, attribname): @@ -46,21 +48,25 @@ def _format_usage_without_prefix(parser): the 'usage: ' prefix. """ fmt = parser._get_formatter() - fmt.add_usage(parser.usage, parser._actions, - parser._mutually_exclusive_groups, prefix='') + fmt.add_usage( + parser.usage, + parser._actions, + parser._mutually_exclusive_groups, + prefix="", + ) return fmt.format_help().strip() def parse_parser(parser, data=None, **kwargs): if data is None: data = { - 'name': '', - 'usage': parser.format_usage().strip(), - 'bare_usage': _format_usage_without_prefix(parser), - 'prog': parser.prog, + "name": "", + "usage": parser.format_usage().strip(), + "bare_usage": _format_usage_without_prefix(parser), + "prog": parser.prog, } - _try_add_parser_attribute(data, parser, 'description') - _try_add_parser_attribute(data, parser, 'epilog') + _try_add_parser_attribute(data, parser, "description") + _try_add_parser_attribute(data, parser, "epilog") for action in parser._get_positional_actions(): if isinstance(action, _HelpAction): continue @@ -84,44 +90,45 @@ def parse_parser(parser, data=None, **kwargs): if name in subsection_alias_names: continue subalias = subsection_alias[subaction] - subaction.prog = '%s %s' % (parser.prog, name) + subaction.prog = f"{parser.prog} {name}" subdata = { - 'name': name if not subalias else - '%s (%s)' % (name, ', '.join(subalias)), - 'help': helps.get(name, ''), - 'usage': subaction.format_usage().strip(), - 'bare_usage': _format_usage_without_prefix(subaction), + "name": name + if not subalias + else "{} ({})".format(name, ", ".join(subalias)), + "help": helps.get(name, ""), + "usage": subaction.format_usage().strip(), + "bare_usage": _format_usage_without_prefix(subaction), } if subalias: - subdata['identifier'] = name + subdata["identifier"] = name parse_parser(subaction, subdata, **kwargs) - data.setdefault('children', []).append(subdata) + data.setdefault("children", []).append(subdata) continue - if 'args' not in data: - data['args'] = [] + if "args" not in data: + data["args"] = [] arg = { - 'name': action.dest, - 'help': action.help or '', - 'metavar': action.metavar + "name": action.dest, + "help": action.help or "", + "metavar": action.metavar, } if action.choices: - arg['choices'] = action.choices - data['args'].append(arg) - show_defaults = ( - ('skip_default_values' not in kwargs) - or (kwargs['skip_default_values'] is False)) + arg["choices"] = action.choices + data["args"].append(arg) + show_defaults = ("skip_default_values" not in kwargs) or ( + kwargs["skip_default_values"] is False + ) for action in parser._get_optional_actions(): if isinstance(action, _HelpAction): continue - if 'options' not in data: - data['options'] = [] + if "options" not in data: + data["options"] = [] option = { - 'name': action.option_strings, - 'default': action.default if show_defaults else '==SUPPRESS==', - 'help': action.help or '' + "name": action.option_strings, + "default": action.default if show_defaults else "==SUPPRESS==", + "help": action.help or "", } if action.choices: - option['choices'] = action.choices - if "==SUPPRESS==" not in option['help']: - data['options'].append(option) + option["choices"] = action.choices + if "==SUPPRESS==" not in option["help"]: + data["options"].append(option) return data diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..ef3c1549 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,68 @@ +[build-system] +requires = [ + "setuptools", +] + +[project] +name = "zkg" +dynamic = ["version"] +description = "The Zeek Package Manager" +readme = "README" + +requires-python = ">= 3.9" + +keywords = [ + "zeek", + "zeekctl", + "zeekcontrol", + "package", + "manager", + "scripts", + "plugins", + "security", +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "License :: OSI Approved :: University of Illinois/NCSA Open Source License", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python :: 3", + "Topic :: System :: Networking :: Monitoring", + "Topic :: Utilities", +] + +# NOTE: Keep `requirements.txt` in sync which we currently use e.g., for RTD. +dependencies = [ + "GitPython>=3.1.43", + "semantic_version>=2.10.0", +] + +[project.optional-dependencies] +dev = [ + "btest>=1.1", + "Sphinx>=7.2.6", + "sphinx_rtd_theme>=2.0.0", +] + +[project.license] +file = "COPYING" + +[project.urls] +Homepage = "https://docs.zeek.org/projects/package-manager" +Repository = "https://github.com/zeek/package-manager" + +[[project.maintainers]] +name = "The Zeek Project" +email = "info@zeek.org" + +[tool.setuptools] +packages = ["zeekpkg"] +script-files = ["zkg"] + +[tool.distutils.bdist_wheel] +universal = true + +[tool.ruff.lint] +select = ["A", "B", "C4", "COM", "F", "I", "ISC", "N", "RUF", "UP"] diff --git a/requirements.txt b/requirements.txt index 8006be9a..4cfdbc41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ +# NOTE: Dependencies should go into `pyproject.toml`. We keep this file to e.g., support RTD. + # Requirements for general zkg usage -GitPython -semantic_version -btest +GitPython>3.1.43 +semantic_version>2.10.0 # Requirements for development (e.g. building docs) -Sphinx>=3.0 -sphinx_rtd_theme +btest>=1.1 +Sphinx>=7.2.6 +sphinx_rtd_theme>=2.0.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 2ce3b7db..00000000 --- a/setup.cfg +++ /dev/null @@ -1,14 +0,0 @@ -[bdist_wheel] -universal = 1 - -[flake8] -max_line_length = 100 -# E203: whitespace before ':' (black / flake8 disagreement) -# W503: line break before binary operator (black / flake8 disagreement) -ignore=E203,W503 -# E402: module level import not at top of file -# F405: may be undefined, or defined from star imports -# F403: from .manager import *' used; unable to detect undefined names -per-file-ignores = - zeekpkg/__init__.py: F405,F403,E402 - zkg: E402 diff --git a/setup.py b/setup.py index 066db7c4..18306008 100644 --- a/setup.py +++ b/setup.py @@ -1,38 +1,15 @@ import pathlib -from setuptools import setup -install_requires = ["gitpython", "semantic_version", "btest"] +from setuptools import setup def version(): return pathlib.Path("VERSION").read_text().replace("-", ".dev", 1).strip() -def long_description(): - return pathlib.Path("README").read_text() - - setup( - name="zkg", version=version(), - description="The Zeek Package Manager", - long_description=long_description(), - license="University of Illinois/NCSA Open Source License", - keywords="zeek zeekctl zeekcontrol package manager scripts plugins security", - maintainer="The Zeek Project", - maintainer_email="info@zeek.org", - url="https://github.com/zeek/package-manager", - scripts=["zkg"], - packages=["zeekpkg"], - install_requires=install_requires, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "License :: OSI Approved :: University of Illinois/NCSA Open Source License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS :: MacOS X", - "Programming Language :: Python :: 3", - "Topic :: System :: Networking :: Monitoring", - "Topic :: Utilities", + data_files=[ + ("output_dir", ["VERSION"]), ], ) diff --git a/testing/templates/foo/__init__.py b/testing/templates/foo/__init__.py index 777d836c..11c74aa3 100644 --- a/testing/templates/foo/__init__.py +++ b/testing/templates/foo/__init__.py @@ -23,15 +23,21 @@ def contentdir(self): def needed_user_vars(self): return ["readme"] + def validate(self, tmpl): + pass + class Template(zeekpkg.template.Template): def define_user_vars(self): return [ zeekpkg.uservar.UserVar( - "name", desc='the name of the package, e.g. "FooBar"' + "name", + desc='the name of the package, e.g. "FooBar"', ), zeekpkg.uservar.UserVar( - "readme", desc="Content of the README file", val="This is a README." + "readme", + desc="Content of the README file", + val="This is a README.", ), ] @@ -48,3 +54,6 @@ def package(self): def features(self): return [Readme()] + + def validate(self, tmpl): + pass diff --git a/zeekpkg/__init__.py b/zeekpkg/__init__.py index 7adbc3fb..2dd7fcdb 100644 --- a/zeekpkg/__init__.py +++ b/zeekpkg/__init__.py @@ -15,7 +15,7 @@ LOG = logging.getLogger(__name__) LOG.addHandler(logging.NullHandler()) -from .manager import * # noqa: E402, F403 -from .package import * # noqa: E402, F403 -from .source import * # noqa: E402, F403 -from .uservar import * # noqa: E402, F403 +from .manager import * # noqa: F403 +from .package import * # noqa: F403 +from .source import * # noqa: F403 +from .uservar import * # noqa: F403 diff --git a/zeekpkg/_util.py b/zeekpkg/_util.py index 0b967a6d..e7f96a44 100644 --- a/zeekpkg/_util.py +++ b/zeekpkg/_util.py @@ -126,7 +126,11 @@ def git_clone(git_url, dst_path, shallow=False): if shallow: try: git.Git().clone( - git_url, dst_path, "--no-single-branch", recursive=True, depth=1 + git_url, + dst_path, + "--no-single-branch", + recursive=True, + depth=1, ) except git.exc.GitCommandError: if not git_url.startswith(".") and not git_url.startswith("/"): diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index 7f7f21b3..a78a99a3 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -14,68 +14,65 @@ import subprocess import sys import tarfile - -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse +from collections import deque +from urllib.parse import urlparse import git import semantic_version as semver -from collections import deque - +from . import ( + LOG, + __version__, +) from ._util import ( - make_dir, - delete_path, - make_symlink, + configparser_section_dict, copy_over_path, + delete_path, + find_program, get_zeek_info, - git_default_branch, + get_zeek_version, git_checkout, git_clone, + git_default_branch, git_pull, git_version_tags, is_sha1, - get_zeek_version, - std_encoding, - find_program, - read_zeek_config_line, + make_dir, + make_symlink, normalize_version_tag, - configparser_section_dict, + read_zeek_config_line, safe_tarfile_extractall, + std_encoding, ) -from .source import AGGREGATE_DATA_FILE, Source from .package import ( - BUILTIN_SOURCE, BUILTIN_SCHEME, - METADATA_FILENAME, + BUILTIN_SOURCE, LEGACY_METADATA_FILENAME, - TRACKING_METHOD_VERSION, - TRACKING_METHOD_BRANCH, - TRACKING_METHOD_COMMIT, - PLUGIN_MAGIC_FILE, - PLUGIN_MAGIC_FILE_DISABLED, LEGACY_PLUGIN_MAGIC_FILE, LEGACY_PLUGIN_MAGIC_FILE_DISABLED, - name_from_path, - aliases, - canonical_url, - is_valid_name as is_valid_package_name, - make_builtin_package, + METADATA_FILENAME, + PLUGIN_MAGIC_FILE, + PLUGIN_MAGIC_FILE_DISABLED, + TRACKING_METHOD_BRANCH, + TRACKING_METHOD_COMMIT, + TRACKING_METHOD_VERSION, + InstalledPackage, Package, PackageInfo, PackageStatus, - InstalledPackage, PackageVersion, + aliases, + canonical_url, + make_builtin_package, + name_from_path, +) +from .package import ( + is_valid_name as is_valid_package_name, ) +from .source import AGGREGATE_DATA_FILE, Source from .uservar import ( UserVar, ) -from . import ( - __version__, - LOG, -) class Stage: @@ -307,7 +304,8 @@ def __init__( # Place all Zeek built-in packages into installed packages. for info in self.discover_builtin_packages(): self.installed_pkgs[info.package.name] = InstalledPackage( - package=info.package, status=info.status + package=info.package, + status=info.status, ) refresh_bin_dir = False # whether we need to updates link in bin_dir @@ -347,7 +345,7 @@ def __init__( refresh_bin_dir = True if prev_bin_dir and os.path.realpath(prev_bin_dir) != os.path.realpath( - self.bin_dir + self.bin_dir, ): LOG.info("relocating bin_dir %s -> %s", prev_bin_dir, self.bin_dir) need_manifest_update = True @@ -423,7 +421,8 @@ def _write_plugin_magic(self, ipkg): ] for path_enabled, path_disabled in zip( - magic_paths_enabled, magic_paths_disabled + magic_paths_enabled, + magic_paths_disabled, ): if ipkg.status.is_loaded: if path_disabled.exists(): @@ -492,7 +491,7 @@ def _write_manifest(self): { "package_dict": installed_pkg.package.__dict__, "status_dict": installed_pkg.status.__dict__, - } + }, ) data = { @@ -545,8 +544,8 @@ def add_source(self, name, git_url): LOG.debug('duplicate source "%s"', name) return True - return "source already exists with different URL: {}".format( - existing_source.git_url + return ( + f"source already exists with different URL: {existing_source.git_url}" ) clone_path = os.path.join(self.source_clonedir, name) @@ -620,7 +619,9 @@ def discover_builtin_packages(self): try: build_info_str = subprocess.check_output( - [zeek_executable, "--build-info"], stderr=subprocess.DEVNULL, timeout=10 + [zeek_executable, "--build-info"], + stderr=subprocess.DEVNULL, + timeout=10, ) build_info = json.loads(build_info_str) except subprocess.CalledProcessError: @@ -666,7 +667,7 @@ def discover_builtin_packages(self): name=name, current_version=version, current_hash=commit, - ) + ), ) return self._builtin_packages @@ -834,7 +835,7 @@ def save_temporary_config_files(self, installed_pkg): if not os.path.isfile(config_file_path): LOG.info( - "package '%s' claims config file at '%s'," " but it does not exist", + "package '%s' claims config file at '%s', but it does not exist", pkg_name, config_file, ) @@ -882,7 +883,7 @@ def modified_config_files(self, installed_pkg): if not os.path.isfile(their_config_file_path): LOG.info( - "package '%s' claims config file at '%s'," " but it does not exist", + "package '%s' claims config file at '%s', but it does not exist", pkg_name, config_file, ) @@ -890,12 +891,13 @@ def modified_config_files(self, installed_pkg): if config_file.startswith(plugin_dir): our_config_file_path = os.path.join( - plugin_install_dir, config_file[len(plugin_dir) :] + plugin_install_dir, + config_file[len(plugin_dir) :], ) if not os.path.isfile(our_config_file_path): LOG.info( - "package '%s' config file '%s' not found" " in plugin_dir: %s", + "package '%s' config file '%s' not found in plugin_dir: %s", pkg_name, config_file, our_config_file_path, @@ -903,12 +905,13 @@ def modified_config_files(self, installed_pkg): continue elif config_file.startswith(script_dir): our_config_file_path = os.path.join( - script_install_dir, config_file[len(script_dir) :] + script_install_dir, + config_file[len(script_dir) :], ) if not os.path.isfile(our_config_file_path): LOG.info( - "package '%s' config file '%s' not found" " in script_dir: %s", + "package '%s' config file '%s' not found in script_dir: %s", pkg_name, config_file, our_config_file_path, @@ -977,9 +980,9 @@ class SourceAggregationResults: the failure. """ - def __init__(self, refresh_error="", package_issues=[]): + def __init__(self, refresh_error="", package_issues=None): self.refresh_error = refresh_error - self.package_issues = package_issues + self.package_issues = package_issues if package_issues else [] def aggregate_source(self, name, push=False): """Pull latest git info from a package source and aggregate metadata. @@ -1046,7 +1049,8 @@ def _refresh_source(self, name, aggregate=False, push=False): aggregate_file = os.path.join(source.clone.working_dir, AGGREGATE_DATA_FILE) agg_file_ours = os.path.join(self.scratch_dir, AGGREGATE_DATA_FILE) agg_file_their_orig = os.path.join( - self.scratch_dir, AGGREGATE_DATA_FILE + ".orig" + self.scratch_dir, + AGGREGATE_DATA_FILE + ".orig", ) delete_path(agg_file_ours) @@ -1067,7 +1071,7 @@ def _refresh_source(self, name, aggregate=False, push=False): except git.exc.GitCommandError as error: LOG.error("failed to pull source %s: %s", name, error) return self.SourceAggregationResults( - f"failed to pull from remote source: {error}" + f"failed to pull from remote source: {error}", ) if os.path.isfile(agg_file_ours): @@ -1079,7 +1083,7 @@ def _refresh_source(self, name, aggregate=False, push=False): # Their file hasn't changed, use ours. shutil.copy2(agg_file_ours, aggregate_file) LOG.debug( - "aggegrate file in source unchanged, restore local one" + "aggegrate file in source unchanged, restore local one", ) else: # Their file changed, use theirs. @@ -1123,7 +1127,9 @@ def _refresh_source(self, name, aggregate=False, push=False): clone = git_clone(url, clonepath, shallow=True) except git.exc.GitCommandError as error: LOG.warn( - "failed to clone %s, skipping aggregation: %s", url, error + "failed to clone %s, skipping aggregation: %s", + url, + error, ) aggregation_issues.append((url, repr(error))) continue @@ -1145,8 +1151,8 @@ def _refresh_source(self, name, aggregate=False, push=False): url, error, ) - msg = 'failed to checkout branch/version "{}": {}'.format( - version, repr(error) + msg = ( + f'failed to checkout branch/version "{version}": {error!r}' ) aggregation_issues.append((url, msg)) continue @@ -1154,7 +1160,8 @@ def _refresh_source(self, name, aggregate=False, push=False): metadata_file = _pick_metadata_file(clone.working_dir) metadata_parser = configparser.ConfigParser(interpolation=None) invalid_reason = _parse_package_metadata( - metadata_parser, metadata_file + metadata_parser, + metadata_file, ) if invalid_reason: @@ -1184,7 +1191,8 @@ def _refresh_source(self, name, aggregate=False, push=False): agg_adds.append(qualified_name) else: prev_meta = configparser_section_dict( - prev_parser, qualified_name + prev_parser, + qualified_name, ) new_meta = configparser_section_dict(parser, qualified_name) if prev_meta != new_meta: @@ -1211,7 +1219,7 @@ def _refresh_source(self, name, aggregate=False, push=False): if push: if os.path.isfile( - os.path.join(source.clone.working_dir, AGGREGATE_DATA_FILE) + os.path.join(source.clone.working_dir, AGGREGATE_DATA_FILE), ): source.clone.git.add(AGGREGATE_DATA_FILE) @@ -1223,7 +1231,9 @@ def _refresh_source(self, name, aggregate=False, push=False): # why one would use zkg for this as opposed to git # itself. source.clone.git.commit( - "--no-verify", "--message", "Update aggregated metadata." + "--no-verify", + "--message", + "Update aggregated metadata.", ) LOG.info('committed package source "%s" metadata update', name) @@ -1243,7 +1253,8 @@ def refresh_installed_packages(self): for ipkg in self.installed_packages(): if ipkg.is_builtin(): LOG.debug( - 'skipping refresh of built-in package "%s"', ipkg.package.name + 'skipping refresh of built-in package "%s"', + ipkg.package.name, ) continue @@ -1261,7 +1272,9 @@ def refresh_installed_packages(self): ) ipkg.status.is_outdated = _is_clone_outdated( - clone, ipkg.status.current_version, ipkg.status.tracking_method + clone, + ipkg.status.current_version, + ipkg.status.tracking_method, ) self._write_manifest() @@ -1356,8 +1369,8 @@ def remove(self, pkg_path): for alias in pkg_to_remove.aliases(): delete_path(os.path.join(self.zeekpath(), alias)) - for exec in self._get_executables(pkg_to_remove.metadata): - link = os.path.join(self.bin_dir, os.path.basename(exec)) + for exe in self._get_executables(pkg_to_remove.metadata): + link = os.path.join(self.bin_dir, os.path.basename(exe)) if os.path.islink(link): try: LOG.debug("removing link %s", link) @@ -1474,7 +1487,9 @@ def load(self, pkg_path): return "" pkg_load_script = os.path.join( - self.script_dir, ipkg.package.name, "__load__.zeek" + self.script_dir, + ipkg.package.name, + "__load__.zeek", ) if not os.path.exists(pkg_load_script) and not self.has_plugin(ipkg): @@ -1520,7 +1535,7 @@ def restore_loaded_package_states(self, saved_state): self._write_autoloader() self._write_manifest() - def load_with_dependencies(self, pkg_name, visited=set()): + def load_with_dependencies(self, pkg_name, visited=None): """Mark dependent (but previously installed) packages as being "loaded". Args: @@ -1533,6 +1548,9 @@ def load_with_dependencies(self, pkg_name, visited=set()): it was marked as loaded or else an explanation of why the loading failed. """ + if visited is None: + visited = set() + ipkg = self.find_installed_package(pkg_name) # skip loading a package if it is not installed. @@ -1592,17 +1610,17 @@ def list_depender_pkgs(self, pkg_path): item = queue.popleft() for _pkg_name in pkg_dependencies: - pkg_dependees = {_pkg for _pkg in pkg_dependencies.get(_pkg_name)} + pkg_dependees = set(pkg_dependencies.get(_pkg_name)) if item in pkg_dependees: # check if there is a cyclic dependency if _pkg_name == pkg_name: - return sorted([pkg for pkg in depender_packages] + [pkg_name]) + return sorted([*list(depender_packages), [pkg_name]]) queue.append(_pkg_name) depender_packages.add(_pkg_name) - return sorted([pkg for pkg in depender_packages]) + return sorted(depender_packages) def unload_with_unused_dependers(self, pkg_name): """Unmark dependent (but previously installed packages) as being "loaded". @@ -1688,10 +1706,8 @@ def _has_all_dependers_unloaded(item, dependers): errors.append( ( item, - "Package is in use by other packages --- {}.".format( - dep_listing[:-2] - ), - ) + f"Package is in use by other packages --- {dep_listing[:-2]}.", + ), ) return errors @@ -1776,7 +1792,9 @@ def bundle_info(self, bundle_file): for git_url, version in manifest: package = Package( - git_url=git_url, name=git_url.split("/")[-1], canonical=True + git_url=git_url, + name=git_url.split("/")[-1], + canonical=True, ) pkg_path = os.path.join(bundle_dir, package.name) LOG.debug('getting info for bundled package "%s"', package.name) @@ -1843,7 +1861,9 @@ def info(self, pkg_path, version="", prefer_installed=True): return self._info(package, status, version) except git.exc.GitCommandError as error: LOG.info( - 'getting info on "%s": invalid git repo path: %s', pkg_path, error + 'getting info on "%s": invalid git repo path: %s', + pkg_path, + error, ) LOG.info('getting info on "%s": matched no source package', pkg_path) @@ -1861,11 +1881,11 @@ def info(self, pkg_path, version="", prefer_installed=True): pkg_path, matches_string, ) - reason = str.format( - '"{}" matches multiple packages, try a more' " specific name from: {}", - pkg_path, - matches_string, + reason = ( + f'"{pkg_path}" matches multiple packages, ' + f"try a more specific name from: {matches_string}" ) + return PackageInfo(invalid_reason=reason, status=status) package = matches[0] @@ -1978,21 +1998,20 @@ def __init__(self, name): self.info = None self.requested_version = None # (tracking method, version) self.installed_version = None # (tracking method, version) - self.dependers = dict() # name -> version, name needs self at version - self.dependees = dict() # name -> version, self needs name at version + self.dependers = {} # name -> version, name needs self at version + self.dependees = {} # name -> version, self needs name at version self.is_suggestion = False def __str__(self): - return str.format( - "{}\n\trequested: {}\n\tinstalled: {}\n\tdependers: {}\n\tsuggestion: {}", - self.name, - self.requested_version, - self.installed_version, - self.dependers, - self.is_suggestion, + return ( + f"{self.name}\n\t" + f"requested: {self.requested_version}\n\t" + f"installed: {self.installed_version}\n\t" + f"dependers: {self.dependers}\n\t" + f"suggestion: {self.is_suggestion}" ) - graph = dict() # Node.name -> Node, nodes store edges + graph = {} # Node.name -> Node, nodes store edges requests = [] # List of Node, just for requested packages # 1. Try to make nodes for everything in the dependency graph... @@ -2024,7 +2043,7 @@ def __str__(self): if dd is None: return ( - str.format('package "{}" has malformed "depends" field', node.name), + f'package "{node.name}" has malformed "depends" field', [], ) @@ -2033,9 +2052,7 @@ def __str__(self): if not ignore_suggestions: if ds is None: return ( - str.format( - 'package "{}" has malformed "suggests" field', node.name - ), + f'package "{node.name}" has malformed "suggests" field', [], ) @@ -2068,12 +2085,7 @@ def __str__(self): if info.invalid_reason: return ( - str.format( - 'package "{}" has invalid dependency "{}": {}', - node.name, - dep_name, - info.invalid_reason, - ), + f'package "{node.name}" has invalid dependency "{dep_name}": {info.invalid_reason}', [], ) @@ -2111,7 +2123,8 @@ def __str__(self): if zeek_version: node = Node("zeek") node.installed_version = PackageVersion( - TRACKING_METHOD_VERSION, zeek_version + TRACKING_METHOD_VERSION, + zeek_version, ) graph["zeek"] = node else: @@ -2119,7 +2132,8 @@ def __str__(self): node = Node("zkg") node.installed_version = PackageVersion( - TRACKING_METHOD_VERSION, __version__ + TRACKING_METHOD_VERSION, + __version__, ) graph["zkg"] = node @@ -2151,7 +2165,7 @@ def __str__(self): if dd is None: return ( - str.format('package "{}" has malformed "depends" field', node.name), + f'package "{node.name}" has malformed "depends" field', [], ) @@ -2160,9 +2174,7 @@ def __str__(self): if not ignore_suggestions: if ds is None: return ( - str.format( - 'package "{}" has malformed "suggests" field', node.name - ), + f'package "{node.name}" has malformed "suggests" field', [], ) @@ -2224,7 +2236,7 @@ def __str__(self): # A new package nothing depends on -- odd? new_pkgs.append( - (node.info, node.info.best_version(), node.is_suggestion) + (node.info, node.info.best_version(), node.is_suggestion), ) continue @@ -2234,15 +2246,8 @@ def __str__(self): msg, fullfills = node.requested_version.fullfills(version_spec) if not fullfills: return ( - str.format( - 'unsatisfiable dependency: requested "{}" ({}),' - ' but "{}" requires {} ({})', - node.name, - node.requested_version.version, - depender_name, - version_spec, - msg, - ), + f'unsatisfiable dependency: requested "{node.name}" ({node.requested_version.version}),' + f' but "{depender_name}" requires {version_spec} ({msg})', new_pkgs, ) @@ -2254,15 +2259,8 @@ def __str__(self): msg, fullfills = node.installed_version.fullfills(version_spec) if not fullfills: return ( - str.format( - 'unsatisfiable dependency: "{}" ({}) is installed,' - ' but "{}" requires {} ({})', - node.name, - node.installed_version.version, - depender_name, - version_spec, - msg, - ), + f'unsatisfiable dependency: "{node.name}" ({node.installed_version.version}) is installed,' + f' but "{depender_name}" requires {version_spec} ({msg})', new_pkgs, ) else: @@ -2272,14 +2270,10 @@ def __str__(self): need_version = False def no_best_version_string(node): - rval = str.format( - '"{}" has no version satisfying dependencies:\n', node.name - ) + rval = f'"{node.name}" has no version satisfying dependencies:\n' for depender_name, version_spec in node.dependers.items(): - rval += str.format( - '\t"{}" requires: "{}"\n', depender_name, version_spec - ) + rval += f'\t"{depender_name}" requires: "{version_spec}"\n' return rval @@ -2295,7 +2289,7 @@ def no_best_version_string(node): if need_branch: branch_name = None - for depender_name, version_spec in node.dependers.items(): + for _, version_spec in node.dependers.items(): if version_spec == "*": continue @@ -2322,11 +2316,7 @@ def no_best_version_string(node): semver_spec = semver.Spec(version_spec) except ValueError: return ( - str.format( - 'package "{}" has invalid semver spec: {}', - depender_name, - version_spec, - ), + f'package "{depender_name}" has invalid semver spec: {version_spec}', new_pkgs, ) @@ -2488,7 +2478,9 @@ def unbundle(self, bundle_file): for git_url, version in manifest: package = Package( - git_url=git_url, name=git_url.split("/")[-1], canonical=True + git_url=git_url, + name=git_url.split("/")[-1], + canonical=True, ) # Prepare the clonepath with the contents from the bundle. @@ -2508,7 +2500,7 @@ def unbundle(self, bundle_file): # # Possible reasons are built-in packages on the source system missing # on the destination system or usage of --nodeps when creating the bundle. - for git_url, version in manifest: + for git_url, _ in manifest: deps = self.get_installed_package_dependencies(git_url) if deps is None: LOG.warning('package "%s" not installed?', git_url) @@ -2600,7 +2592,9 @@ def test(self, pkg_path, version="", test_dependencies=False): # staging area. for info, version in reversed(pkgs): LOG.debug( - 'preparing "%s" for testing: version %s', info.package.name, version + 'preparing "%s" for testing: version %s', + info.package.name, + version, ) clonepath = os.path.join(stage.clone_dir, info.package.name) @@ -2626,9 +2620,7 @@ def test(self, pkg_path, version="", test_dependencies=False): except git.exc.GitCommandError as error: LOG.warning("failed to checkout git repo version: %s", error) return ( - str.format( - "failed to checkout {} of {}", version, info.package.git_url - ), + f"failed to checkout {version} of {info.package.git_url}", False, stage.state_dir, ) @@ -2644,11 +2636,12 @@ def test(self, pkg_path, version="", test_dependencies=False): else: test_pkgs = [(pkg_info, version)] - for info, version in reversed(test_pkgs): + for info, _ in reversed(test_pkgs): LOG.info('testing "%s"', package) # Interpolate the test command: metadata, invalid_reason = self._interpolate_package_metadata( - info.metadata, stage + info.metadata, + stage, ) if invalid_reason: return (invalid_reason, False, stage.state_dir) @@ -2747,7 +2740,9 @@ def _stage(self, package, version, clone, stage, env=None): build_command = metadata.get("build_command", "") if build_command: LOG.debug( - 'building "%s": running build_command: %s', package, build_command + 'building "%s": running build_command: %s', + package, + build_command, ) bufsize = 4096 build = subprocess.Popen( @@ -2765,7 +2760,9 @@ def _stage(self, package, version, clone, stage, env=None): with open(buildlog, "wb") as f: LOG.info( - 'installing "%s": writing build log: %s', package, buildlog + 'installing "%s": writing build log: %s', + package, + buildlog, ) f.write("=== STDERR ===\n".encode(std_encoding(sys.stderr))) @@ -2807,22 +2804,22 @@ def _stage(self, package, version, clone, stage, env=None): script_dir_dst = os.path.join(stage.script_dir, package.name) if not os.path.exists(script_dir_src): - return str.format( - "package's 'script_dir' does not exist: {}", pkg_script_dir - ) + return f"package's 'script_dir' does not exist: {pkg_script_dir}" pkgload = os.path.join(script_dir_src, "__load__.zeek") if os.path.isfile(pkgload): try: symlink_path = os.path.join( - os.path.dirname(stage.script_dir), package.name + os.path.dirname(stage.script_dir), + package.name, ) make_symlink(os.path.join("packages", package.name), symlink_path) for alias in aliases(metadata): symlink_path = os.path.join( - os.path.dirname(stage.script_dir), alias + os.path.dirname(stage.script_dir), + alias, ) make_symlink(os.path.join("packages", package.name), symlink_path) @@ -2832,17 +2829,18 @@ def _stage(self, package, version, clone, stage, env=None): return error error = _copy_package_dir( - package, "script_dir", script_dir_src, script_dir_dst, self.scratch_dir + package, + "script_dir", + script_dir_src, + script_dir_dst, + self.scratch_dir, ) if error: return error else: if "script_dir" in metadata: - return str.format( - "no __load__.zeek file found" " in package's 'script_dir' : {}", - pkg_script_dir, - ) + return f"no __load__.zeek file found in package's 'script_dir' : {pkg_script_dir}" else: LOG.warning( 'installing "%s": no __load__.zeek in implicit' @@ -2864,12 +2862,14 @@ def _stage(self, package, version, clone, stage, env=None): if pkg_plugin_dir != "build": # It's common for a package to not have build directory for # plugins, so don't error out in that case, just log it. - return str.format( - "package's 'plugin_dir' does not exist: {}", pkg_plugin_dir - ) + return f"package's 'plugin_dir' does not exist: {pkg_plugin_dir}" error = _copy_package_dir( - package, "plugin_dir", plugin_dir_src, plugin_dir_dst, self.scratch_dir + package, + "plugin_dir", + plugin_dir_src, + plugin_dir_dst, + self.scratch_dir, ) if error: @@ -2879,10 +2879,10 @@ def _stage(self, package, version, clone, stage, env=None): for p in self._get_executables(metadata): full_path = os.path.join(clone.working_dir, p) if not os.path.isfile(full_path): - return str.format("executable '{}' is missing", p) + return f"executable '{p}' is missing" if not os.access(full_path, os.X_OK): - return str.format("file '{}' is not executable", p) + return f"file '{p}' is not executable" if stage.bin_dir is not None: make_symlink( @@ -2933,11 +2933,7 @@ def install(self, pkg_path, version=""): pkg_path, conflict, ) - return str.format( - 'package with name "{}" ({}) is already installed', - conflict.name, - conflict, - ) + return f'package with name "{conflict.name}" ({conflict}) is already installed' matches = self.match_source_packages(pkg_path) @@ -2958,10 +2954,10 @@ def install(self, pkg_path, version=""): pkg_path, matches_string, ) - return str.format( - '"{}" matches multiple packages, try a more' " specific name from: {}", - pkg_path, - matches_string, + + return ( + f'"{pkg_path}" matches multiple packages, ' + f"try a more specific name from: {matches_string}" ) try: @@ -3055,7 +3051,9 @@ def _install(self, package, version, use_existing_clone=False): status.tracking_method = TRACKING_METHOD_BRANCH else: LOG.info( - 'branch "%s" not in available branches: %s', version, branches + 'branch "%s" not in available branches: %s', + version, + branches, ) return f'no such branch or version tag: "{version}"' @@ -3145,10 +3143,10 @@ def _interpolate_package_metadata(self, metadata, stage): # the currently installed packages. def _refresh_bin_dir(self, bin_dir, prev_bin_dir=None): for ipkg in self.installed_pkgs.values(): - for exec in self._get_executables(ipkg.package.metadata): + for exe in self._get_executables(ipkg.package.metadata): # Put symlinks in place that are missing in current directory - src = os.path.join(self.package_clonedir, ipkg.package.name, exec) - dst = os.path.join(bin_dir, os.path.basename(exec)) + src = os.path.join(self.package_clonedir, ipkg.package.name, exe) + dst = os.path.join(bin_dir, os.path.basename(exe)) if ( not os.path.exists(dst) @@ -3164,8 +3162,8 @@ def _refresh_bin_dir(self, bin_dir, prev_bin_dir=None): # coming with any of the currently installed package. def _clear_bin_dir(self, bin_dir): for ipkg in self.installed_pkgs.values(): - for exec in self._get_executables(ipkg.package.metadata): - old = os.path.join(bin_dir, os.path.basename(exec)) + for exe in self._get_executables(ipkg.package.metadata): + old = os.path.join(bin_dir, os.path.basename(exe)) if os.path.islink(old): try: os.unlink(old) @@ -3195,7 +3193,7 @@ def _is_version_outdated(clone, version): def _is_branch_outdated(clone, branch): - it = clone.iter_commits("{0}..origin/{0}".format(branch)) + it = clone.iter_commits(f"{branch}..origin/{branch}") num_commits_behind = sum(1 for c in it) return num_commits_behind > 0 @@ -3316,8 +3314,8 @@ def _parse_package_metadata(parser, metadata_file): """Return string explaining why metadata is invalid, or '' if valid.""" if not parser.read(metadata_file): LOG.warning("%s: missing metadata file", metadata_file) - return "missing {} (or {}) metadata file".format( - METADATA_FILENAME, LEGACY_METADATA_FILENAME + return ( + f"missing {METADATA_FILENAME} (or {LEGACY_METADATA_FILENAME}) metadata file" ) if not parser.has_section("package"): diff --git a/zeekpkg/package.py b/zeekpkg/package.py index eb1663a8..3f7a3a35 100644 --- a/zeekpkg/package.py +++ b/zeekpkg/package.py @@ -5,13 +5,13 @@ import os import re - from functools import total_ordering +from typing import Optional import semantic_version as semver -from .uservar import UserVar from ._util import find_sentence_end, normalize_version_tag +from .uservar import UserVar #: The name of files used by packages to store their metadata. METADATA_FILENAME = "zkg.meta" @@ -135,9 +135,9 @@ def dependencies(metadata_dict, field="depends"): number of values), then None is returned. """ if field not in metadata_dict: - return dict() + return {} - rval = dict() + rval = {} depends = metadata_dict[field] parts = depends.split() keys = parts[::2] @@ -244,7 +244,8 @@ def fullfills(self, version_spec): Does the current version fullfill version_spec? """ return PackageVersion( - self.status.tracking_method, self.status.current_version + self.status.tracking_method, + self.status.current_version, ).fullfills(version_spec) @@ -588,7 +589,12 @@ def matches_path(self, path): return path == self.git_url -def make_builtin_package(*, name: str, current_version: str, current_hash: str = None): +def make_builtin_package( + *, + name: str, + current_version: str, + current_hash: Optional[str] = None, +): """ Given ``name``, ``version`` and ``commit`` as found in Zeek's ``zkg.provides`` entry, construct a :class:`PackageInfo` instance representing the built-in diff --git a/zeekpkg/source.py b/zeekpkg/source.py index 3b07909c..0fa76bab 100644 --- a/zeekpkg/source.py +++ b/zeekpkg/source.py @@ -7,13 +7,14 @@ """ import configparser -import git import os import shutil +import git + from . import LOG -from .package import name_from_path, Package -from ._util import git_default_branch, git_checkout, git_clone +from ._util import git_checkout, git_clone, git_default_branch +from .package import Package, name_from_path #: The name of package index files. INDEX_FILENAME = "zkg.index" @@ -132,7 +133,7 @@ def packages(self): metadata = {} if parser.has_section(agg_key): - metadata = {key: value for key, value in parser.items(agg_key)} + metadata = dict(parser.items(agg_key)) package = Package( git_url=url, diff --git a/zeekpkg/template.py b/zeekpkg/template.py index 7e841cf2..6e723de1 100644 --- a/zeekpkg/template.py +++ b/zeekpkg/template.py @@ -4,20 +4,16 @@ import abc import configparser -import re import os +import re import shutil -import semantic_version as semver import git +import semantic_version as semver from . import ( - __version__, LOG, -) -from .package import ( - METADATA_FILENAME, - name_from_path, + __version__, ) from ._util import ( delete_path, @@ -25,11 +21,15 @@ git_clone, git_default_branch, git_pull, - git_version_tags, git_remote_urls, + git_version_tags, load_source, make_dir, ) +from .package import ( + METADATA_FILENAME, + name_from_path, +) API_VERSION = "1.1.0" @@ -113,7 +113,9 @@ def load(config, template, version=None): # zkg state folder's clone space and support version # requests. template_clonedir = os.path.join( - config.get("paths", "state_dir"), "clones", "template" + config.get("paths", "state_dir"), + "clones", + "template", ) templatedir = os.path.join(template_clonedir, name_from_path(template)) make_dir(template_clonedir) @@ -153,11 +155,7 @@ def load(config, template, version=None): try: git_checkout(repo, version) except git.exc.GitCommandError as error: - msg = ( - 'failed to checkout branch/version "{}" of template {}: {}'.format( - version, template, error - ) - ) + msg = f'failed to checkout branch/version "{version}" of template {template}: {error}' LOG.warn(msg) raise GitError(msg) from error @@ -171,9 +169,7 @@ def load(config, template, version=None): except TypeError: pass # Not on a branch, do nothing except git.exc.GitCommandError as error: - msg = 'failed to update branch "{}" of template {}: {}'.format( - version, template, error - ) + msg = f'failed to update branch "{version}" of template {template}: {error}' LOG.warning(msg) raise GitError(msg) from error @@ -186,7 +182,7 @@ def load(config, template, version=None): if not hasattr(mod, "TEMPLATE_API_VERSION"): msg = "template{} does not indicate its API version".format( - " version " + version if version else "" + " version " + version if version else "", ) LOG.error(msg) raise LoadError(msg) @@ -194,14 +190,13 @@ def load(config, template, version=None): # The above guards against absence of TEMPLATE_API_VERSION, so # appease pylint for the rest of this function while we access # it. - # pylint: disable=no-member try: is_compat = Template.is_api_compatible(mod.TEMPLATE_API_VERSION) except ValueError: raise LoadError( - f'API version string "{mod.TEMPLATE_API_VERSION}" is invalid' - ) + f'API version string "{mod.TEMPLATE_API_VERSION}" is invalid', + ) from None if not is_compat: msg = "template{} API version is incompatible with zkg ({} vs {})".format( @@ -316,7 +311,7 @@ def package(self): """ return None - def features(self): # pylint: disable=no-self-use + def features(self): """Provides any additional features templates supported. If the template provides extra features, return each as an @@ -425,7 +420,7 @@ def info(self): res["versions"] = [] res["has_repo"] = False - pkg = self.package() # pylint: disable=assignment-from-none + pkg = self.package() uvars = self.define_user_vars() feature_names = [] res["user_vars"] = {} @@ -444,7 +439,8 @@ def info(self): res["user_vars"][uvar_name]["used_by"].append("package") except KeyError: LOG.warning( - 'Package requires undefined user var "%s", skipping', uvar_name + 'Package requires undefined user var "%s", skipping', + uvar_name, ) for feature in self.features(): @@ -516,6 +512,7 @@ def do_validate(self, tmpl): for feature in self._features: feature.validate(tmpl) + @abc.abstractmethod def validate(self, tmpl): """Validation of template configuration for this component. @@ -532,6 +529,7 @@ def validate(self, tmpl): zeekpkg.template.InputError when failing validation. """ + pass def do_instantiate(self, tmpl, packagedir, use_force=False): """Main driver for instantiating template content. @@ -675,7 +673,7 @@ def _walk(self, tmpl): yield in_file, out_path, out_file, out_content - def _replace(self, tmpl, content): # pylint: disable=no-self-use + def _replace(self, tmpl, content): """Helper for content substitution. Args: @@ -758,7 +756,7 @@ def _update_metadata(self, tmpl): config.remove_section(section) config.add_section(section) - for uvar in tmpl._get_user_vars(): # pylint: disable=protected-access + for uvar in tmpl._get_user_vars(): if uvar.val() is not None: config.set(section, uvar.name(), uvar.val()) @@ -795,10 +793,10 @@ def _git_init(self, tmpl): ver_info += " (" + ver_sha[:8] + ")" repo.index.commit( - """Initial commit. + f"""Initial commit. -zkg {} created this package from template "{}" -using {}{}.""".format(__version__, tmpl.name(), ver_info, features_info) +zkg {__version__} created this package from template "{tmpl.name()}" +using {ver_info}{features_info}.""", ) diff --git a/zeekpkg/uservar.py b/zeekpkg/uservar.py index 6e455535..dcdbd000 100644 --- a/zeekpkg/uservar.py +++ b/zeekpkg/uservar.py @@ -106,9 +106,7 @@ def resolve(self, name, config, user_var_args=None, force=False): if source: print( - '"{}" will use value of "{}" ({}) from {}: {}'.format( - name, self._name, self._desc, source, val - ) + f'"{name}" will use value of "{self._name}" ({self._desc}) from {source}: {val}', ) self._val = val return val @@ -137,7 +135,7 @@ def parse_arg(arg): return UserVar(name, val=val) except ValueError as error: raise ValueError( - f'invalid user var argument "{arg}", must be NAME=VAR' + f'invalid user var argument "{arg}", must be NAME=VAR', ) from error @staticmethod diff --git a/zkg b/zkg index 5c18609b..b65f58e2 100755 --- a/zkg +++ b/zkg @@ -15,12 +15,11 @@ import shutil import subprocess import sys import threading - from collections import OrderedDict try: import git - import semantic_version # noqa # pylint: disable=unused-import + import semantic_version # noqa: F401 except ImportError as error: print( "error: zkg failed to import one or more dependencies:\n" @@ -33,7 +32,7 @@ except ImportError as error: " pip3 install GitPython semantic-version\n" "\n" "Also check the following exception output for possible alternate explanations:\n\n" - "{}: {}".format(type(error).__name__, error), + f"{type(error).__name__}: {error}", file=sys.stderr, ) sys.exit(1) @@ -41,7 +40,7 @@ except ImportError as error: try: # Argcomplete provides command-line completion for users of argparse. # We support it if available, but don't complain when it isn't. - import argcomplete # pylint: disable=import-error + import argcomplete except ImportError: pass @@ -85,27 +84,26 @@ ZKG_DEFAULT_SOURCE = "https://github.com/zeek/packages" # The default package template ZKG_DEFAULT_TEMPLATE = "https://github.com/zeek/package-template" -from zeekpkg._util import ( # noqa: E402 +import zeekpkg +from zeekpkg._util import ( delete_path, - make_dir, find_program, + make_dir, read_zeek_config_line, std_encoding, ) -from zeekpkg.package import ( # noqa: E402 +from zeekpkg.package import ( BUILTIN_SCHEME, TRACKING_METHOD_VERSION, ) -from zeekpkg.template import ( # noqa: E402 +from zeekpkg.template import ( LoadError, Template, ) -from zeekpkg.uservar import ( # noqa: E402 +from zeekpkg.uservar import ( UserVar, ) -import zeekpkg # noqa: E402 - def confirmation_prompt(prompt, default_to_yes=True): yes = {"y", "ye", "yes"} @@ -139,21 +137,21 @@ def prompt_for_user_vars(manager, config, configfile, args, pkg_infos): requested_user_vars = info.user_vars() if requested_user_vars is None: - print_error(str.format('error: malformed user_vars in "{}"', name)) + print_error(f'error: malformed user_vars in "{name}"') sys.exit(1) for uvar in requested_user_vars: try: answers[uvar.name()] = uvar.resolve( - name, config, args.user_var, args.force + name, + config, + args.user_var, + args.force, ) except ValueError: print_error( - str.format( - 'error: could not determine value of user variable "{}",' - " provide via environment or --user-var", - uvar.name(), - ) + f'error: could not determine value of user variable "{uvar.name()}",' + " provide via environment or --user-var", ) sys.exit(1) @@ -267,10 +265,16 @@ def create_config(args, configfile): state_dir = get_option(config, "paths", "state_dir", os.path.join(def_state_dir)) script_dir = get_option( - config, "paths", "script_dir", os.path.join(state_dir, "script_dir") + config, + "paths", + "script_dir", + os.path.join(state_dir, "script_dir"), ) plugin_dir = get_option( - config, "paths", "plugin_dir", os.path.join(state_dir, "plugin_dir") + config, + "paths", + "plugin_dir", + os.path.join(state_dir, "plugin_dir"), ) bin_dir = get_option(config, "paths", "bin_dir", os.path.join(state_dir, "bin")) zeek_dist = get_option(config, "paths", "zeek_dist", "") @@ -293,13 +297,9 @@ def create_config(args, configfile): for key, value in config.items("paths"): if value and not os.path.isabs(value): print_error( - str.format( - "error: invalid config file value for key" - ' "{}" in section [paths]: "{}" is not' - " an absolute path", - key, - value, - ) + "error: invalid config file value for key" + f' "{key}" in section [paths]: "{value}" is not' + " an absolute path", ) sys.exit(1) @@ -400,14 +400,12 @@ def create_manager(args, config): check_permission(script_dir), check_permission(plugin_dir), check_permission(bin_dir), - ] + ], ) if permissions_trouble and not args.user: print_error( - "Consider the --user flag to manage zkg state via {}/config".format( - home_config_dir() - ) + f"Consider the --user flag to manage zkg state via {home_config_dir()}/config", ) sys.exit(1) @@ -433,9 +431,7 @@ def create_manager(args, config): if error: print_error( - str.format( - 'warning: skipped using package source named "{}": {}', key, error - ) + f'warning: skipped using package source named "{key}": {error}', ) return manager @@ -542,9 +538,7 @@ def cmd_test(manager, args, config, configfile): if package_info.invalid_reason: print_error( - str.format( - 'error: invalid package "{}": {}', name, package_info.invalid_reason - ) + f'error: invalid package "{name}": {package_info.invalid_reason}', ) sys.exit(1) @@ -559,37 +553,33 @@ def cmd_test(manager, args, config, configfile): name = info.package.qualified_name() if "test_command" not in info.metadata: - print(str.format("{}: no test_command found in metadata, skipping", name)) + print(f"{name}: no test_command found in metadata, skipping") continue error_msg, passed, test_dir = manager.test( - name, version, test_dependencies=True + name, + version, + test_dependencies=True, ) if error_msg: all_passed = False - print_error( - str.format('error: failed to run tests for "{}": {}', name, error_msg) - ) + print_error(f'error: failed to run tests for "{name}": {error_msg}') continue if passed: - print(str.format("{}: all tests passed", name)) + print(f"{name}: all tests passed") else: all_passed = False clone_dir = os.path.join( - os.path.join(test_dir, "clones"), info.package.name + os.path.join(test_dir, "clones"), + info.package.name, ) print_error( - str.format( - 'error: package "{}" tests failed, inspect' - " contents of {} for details, especially" - ' any "zkg.test_command.{{stderr,stdout}}"' - " files within {}", - name, - test_dir, - clone_dir, - ) + f'error: package "{name}" tests failed, inspect' + f" contents of {test_dir} for details, especially" + ' any "zkg.test_command.{{stderr,stdout}}"' + f" files within {clone_dir}", ) if not all_passed: @@ -618,9 +608,7 @@ def cmd_install(manager, args, config, configfile): if package_info.invalid_reason: print_error( - str.format( - 'error: invalid package "{}": {}', name, package_info.invalid_reason - ) + f'error: invalid package "{name}": {package_info.invalid_reason}', ) sys.exit(1) @@ -639,7 +627,8 @@ def cmd_install(manager, args, config, configfile): for info, version, _ in package_infos ] invalid_reason, new_pkgs = manager.validate_dependencies( - to_validate, ignore_suggestions=args.nosuggestions + to_validate, + ignore_suggestions=args.nosuggestions, ) if invalid_reason: @@ -660,7 +649,8 @@ def cmd_install(manager, args, config, configfile): dependency_listing = "" for info, version, suggested in sorted( - new_pkgs, key=lambda x: x[0].package.name + new_pkgs, + key=lambda x: x[0].package.name, ): name = info.package.qualified_name() dependency_listing += f" {name} ({version})" @@ -681,9 +671,7 @@ def cmd_install(manager, args, config, configfile): extdeps = info.dependencies(field="external_depends") if extdeps is None: - extdep_listing += " from {} ({}):\n \n".format( - name, version - ) + extdep_listing += f" from {name} ({version}):\n \n" continue if extdeps: @@ -696,7 +684,7 @@ def cmd_install(manager, args, config, configfile): print( "Verify the following REQUIRED external dependencies:\n" "(Ensure their installation on all relevant systems before" - " proceeding):" + " proceeding):", ) print(extdep_listing) @@ -706,7 +694,11 @@ def cmd_install(manager, args, config, configfile): package_infos += new_pkgs prompt_for_user_vars( - manager, config, configfile, args, [info for info, _, _ in package_infos] + manager, + config, + configfile, + args, + [info for info, _, _ in package_infos], ) if not args.skiptests: @@ -717,7 +709,7 @@ def cmd_install(manager, args, config, configfile): if "test_command" not in info.metadata: zeekpkg.LOG.info( - f'Skipping unit tests for "{name}": no test_command in metadata' + f'Skipping unit tests for "{name}": no test_command in metadata', ) continue @@ -727,22 +719,22 @@ def cmd_install(manager, args, config, configfile): # well fail without them. If the user wants --nodeps and the tests # fail because of it, they'll also need to say --skiptests. error, passed, test_dir = manager.test( - name, version, test_dependencies=True + name, + version, + test_dependencies=True, ) if error: - error_msg = str.format("failed to run tests for {}: {}", name, error) + error_msg = f"failed to run tests for {name}: {error}" elif not passed: clone_dir = os.path.join( - os.path.join(test_dir, "clones"), info.package.name + os.path.join(test_dir, "clones"), + info.package.name, ) - error_msg = str.format( - '"{}" tests failed, inspect contents of' - " {} for details, especially any" + error_msg = ( + f'"{name}" tests failed, inspect contents of' + f" {test_dir} for details, especially any" ' "zkg.test_command.{{stderr,stdout}}"' - " files within {}", - name, - test_dir, - clone_dir, + f" files within {clone_dir}" ) if error_msg: @@ -752,7 +744,8 @@ def cmd_install(manager, args, config, configfile): sys.exit(1) if not confirmation_prompt( - "Proceed to install anyway?", default_to_yes=False + "Proceed to install anyway?", + default_to_yes=False, ): return @@ -816,12 +809,12 @@ def cmd_install(manager, args, config, configfile): if not args.nodeps: # Now load runtime dependencies after all dependencies and suggested # packages have been installed and loaded. - for info, ver, _ in sorted(orig_pkgs, key=lambda x: x[0].package.name): + for info, _, _ in sorted(orig_pkgs, key=lambda x: x[0].package.name): _listing, saved_state = "", manager.loaded_package_states() name = info.package.qualified_name() load_error = manager.load_with_dependencies( - zeekpkg.package.name_from_path(name) + zeekpkg.package.name_from_path(name), ) for _name, _error in load_error: @@ -834,14 +827,14 @@ def cmd_install(manager, args, config, configfile): if dep_listing: print( "The following installed packages were additionally " - "loaded to satisfy runtime dependencies" + "loaded to satisfy runtime dependencies", ) print(dep_listing) else: print( "The following installed packages could NOT be loaded " - 'to satisfy runtime dependencies for "{}"'.format(name) + f'to satisfy runtime dependencies for "{name}"', ) print(_listing) manager.restore_loaded_package_states(saved_state) @@ -849,7 +842,7 @@ def cmd_install(manager, args, config, configfile): if installs_failed: print_error( "error: incomplete installation, the follow packages" - " failed to be installed:" + " failed to be installed:", ) for n, v in installs_failed: @@ -889,11 +882,7 @@ def cmd_bundle(manager, args, config, configfile): info = manager.info(name, version=version, prefer_installed=False) if info.invalid_reason: - print_error( - str.format( - 'error: invalid package "{}": {}', name, info.invalid_reason - ) - ) + print_error(f'error: invalid package "{name}": {info.invalid_reason}') sys.exit(1) if not version: @@ -907,7 +896,7 @@ def cmd_bundle(manager, args, config, configfile): version, False, False, - ) + ), ) if not args.nodeps: @@ -929,7 +918,7 @@ def cmd_bundle(manager, args, config, configfile): version, True, suggested, - ) + ), ) else: prefer_existing_clones = True @@ -942,7 +931,7 @@ def cmd_bundle(manager, args, config, configfile): ipkg.status.current_version, False, False, - ) + ), ) if not packages_to_bundle: @@ -965,11 +954,7 @@ def cmd_bundle(manager, args, config, configfile): package_listing += "\n" - print( - "The following packages will be BUNDLED into {}:".format( - args.bundle_filename - ) - ) + print(f"The following packages will be BUNDLED into {args.bundle_filename}:") print(package_listing) if not confirmation_prompt("Proceed?"): @@ -977,7 +962,9 @@ def cmd_bundle(manager, args, config, configfile): git_urls = [(git_url, version) for _, git_url, version, _, _ in packages_to_bundle] error = manager.bundle( - args.bundle_filename, git_urls, prefer_existing_clones=prefer_existing_clones + args.bundle_filename, + git_urls, + prefer_existing_clones=prefer_existing_clones, ) if error: @@ -1002,13 +989,11 @@ def cmd_unbundle(manager, args, config, configfile): print_error(f"error: failed to unbundle {args.bundle_filename}: {error}") sys.exit(1) - for git_url, version, pkg_info in bundle_info: + for git_url, _, pkg_info in bundle_info: if pkg_info.invalid_reason: name = pkg_info.package.qualified_name() print_error( - "error: bundle {} contains invalid package {} ({}): {}".format( - args.bundle_filename, git_url, name, pkg_info.invalid_reason - ) + f"error: bundle {args.bundle_filename} contains invalid package {git_url} ({name}): {pkg_info.invalid_reason}", ) sys.exit(1) @@ -1049,9 +1034,7 @@ def cmd_unbundle(manager, args, config, configfile): extdeps = info.dependencies(field="external_depends") if extdeps is None: - extdep_listing += " from {} ({}):\n \n".format( - name, version - ) + extdep_listing += f" from {name} ({version}):\n \n" continue if extdeps: @@ -1064,7 +1047,7 @@ def cmd_unbundle(manager, args, config, configfile): print( "Verify the following REQUIRED external dependencies:\n" "(Ensure their installation on all relevant systems before" - " proceeding):" + " proceeding):", ) print(extdep_listing) @@ -1072,7 +1055,11 @@ def cmd_unbundle(manager, args, config, configfile): return prompt_for_user_vars( - manager, config, configfile, args, [info for _, _, info in bundle_info] + manager, + config, + configfile, + args, + [info for _, _, info in bundle_info], ) error = manager.unbundle(args.bundle_filename) @@ -1259,7 +1246,7 @@ def cmd_refresh(manager, args, config, configfile): if args.fail_on_aggregate_problems and not args.aggregate: print_error( "warning: --fail-on-aggregate-problems without --aggregate" - " has no effect." + " has no effect.", ) had_failure = False @@ -1301,7 +1288,7 @@ def cmd_refresh(manager, args, config, configfile): if aggregation_issues: print( "\tWARNING: Metadata aggregated, but excludes the " - "following packages due to described problems:" + "following packages due to described problems:", ) for url, issue in aggregation_issues: @@ -1314,10 +1301,10 @@ def cmd_refresh(manager, args, config, configfile): if args.push: print("\tPushed aggregated metadata") - outdated_before = {i for i in outdated(manager)} + outdated_before = set(outdated(manager)) print("Refresh installed packages") manager.refresh_installed_packages() - outdated_after = {i for i in outdated(manager)} + outdated_after = set(outdated(manager)) if outdated_before == outdated_after: print("\tNo new outdated packages") @@ -1379,13 +1366,13 @@ def cmd_upgrade(manager, args, config, configfile): name = ipkg.package.git_url info = manager.info( - name, version=ipkg.status.current_version, prefer_installed=False + name, + version=ipkg.status.current_version, + prefer_installed=False, ) if info.invalid_reason: - print_error( - str.format('error: invalid package "{}": {}', name, info.invalid_reason) - ) + print_error(f'error: invalid package "{name}": {info.invalid_reason}') sys.exit(1) next_version = ipkg.status.current_version @@ -1409,7 +1396,8 @@ def cmd_upgrade(manager, args, config, configfile): for info, next_version, _ in outdated_packages ] invalid_reason, new_pkgs = manager.validate_dependencies( - to_validate, ignore_suggestions=args.nosuggestions + to_validate, + ignore_suggestions=args.nosuggestions, ) if invalid_reason: @@ -1444,9 +1432,7 @@ def cmd_upgrade(manager, args, config, configfile): extdeps = info.dependencies(field="external_depends") if extdeps is None: - extdep_listing += " from {} ({}):\n \n".format( - name, version - ) + extdep_listing += f" from {name} ({version}):\n \n" continue if extdeps: @@ -1459,7 +1445,7 @@ def cmd_upgrade(manager, args, config, configfile): print( "Verify the following REQUIRED external dependencies:\n" "(Ensure their installation on all relevant systems before" - " proceeding):" + " proceeding):", ) print(extdep_listing) @@ -1467,7 +1453,11 @@ def cmd_upgrade(manager, args, config, configfile): return prompt_for_user_vars( - manager, config, configfile, args, [info for info, _, _ in allpkgs] + manager, + config, + configfile, + args, + [info for info, _, _ in allpkgs], ) if not args.skiptests: @@ -1479,13 +1469,14 @@ def cmd_upgrade(manager, args, config, configfile): next_info = manager.info(name, version=version, prefer_installed=False) if next_info.invalid_reason: print_error( - f'error: invalid package "{name}": {next_info.invalid_reason}' + f'error: invalid package "{name}": {next_info.invalid_reason}', ) sys.exit(1) if "test_command" not in next_info.metadata: zeekpkg.LOG.info( - 'Skipping unit tests for "%s": no test_command in metadata', name + 'Skipping unit tests for "%s": no test_command in metadata', + name, ) continue @@ -1495,23 +1486,23 @@ def cmd_upgrade(manager, args, config, configfile): # might well fail without them. If the user wants --nodeps and the # tests fail because of it, they'll also need to say --skiptests. error, passed, test_dir = manager.test( - name, version, test_dependencies=True + name, + version, + test_dependencies=True, ) if error: - error_msg = str.format("failed to run tests for {}: {}", name, error) + error_msg = f"failed to run tests for {name}: {error}" elif not passed: clone_dir = os.path.join( - os.path.join(test_dir, "clones"), info.package.name + os.path.join(test_dir, "clones"), + info.package.name, ) - error_msg = str.format( - '"{}" tests failed, inspect contents of' - " {} for details, especially any" + error_msg = ( + f'"{name}" tests failed, inspect contents of' + f" {test_dir} for details, especially any" ' "zkg.test_command.{{stderr,stdout}}"' - " files within {}", - name, - test_dir, - clone_dir, + f" files within {clone_dir}" ) if error_msg: @@ -1521,7 +1512,8 @@ def cmd_upgrade(manager, args, config, configfile): sys.exit(1) if not confirmation_prompt( - "Proceed to install anyway?", default_to_yes=False + "Proceed to install anyway?", + default_to_yes=False, ): return @@ -1548,7 +1540,7 @@ def cmd_upgrade(manager, args, config, configfile): had_failure = False - for info, next_version, _ in outdated_packages: + for info, _, _ in outdated_packages: name = info.package.qualified_name() if not manager.match_source_packages(name): @@ -1620,7 +1612,7 @@ def cmd_load(manager, args, config, configfile): dep_error_listing, load_error = "", False loaded_dep_list = manager.load_with_dependencies( - zeekpkg.package.name_from_path(name) + zeekpkg.package.name_from_path(name), ) for _name, _error in loaded_dep_list: @@ -1634,7 +1626,7 @@ def cmd_load(manager, args, config, configfile): if dep_listing: print( "The following installed packages were additionally loaded to satisfy" - ' runtime dependencies for "{}".'.format(name) + f' runtime dependencies for "{name}".', ) print(dep_listing) @@ -1644,9 +1636,7 @@ def cmd_load(manager, args, config, configfile): if not args.nodeps: if dep_error_listing: print( - 'The following installed dependencies could not be loaded for "{}".'.format( - name - ) + f'The following installed dependencies could not be loaded for "{name}".', ) print(dep_error_listing) manager.restore_loaded_package_states(saved_state) @@ -1760,9 +1750,7 @@ def cmd_pin(manager, args, config, configfile): if ipkg: print( - 'Pinned "{}" at version: {} ({})'.format( - name, ipkg.status.current_version, ipkg.status.current_hash - ) + f'Pinned "{name}" at version: {ipkg.status.current_version} ({ipkg.status.current_hash})', ) else: had_failure = True @@ -1793,9 +1781,7 @@ def cmd_unpin(manager, args, config, configfile): if ipkg: print( - 'Unpinned "{}" from version: {} ({})'.format( - name, ipkg.status.current_version, ipkg.status.current_hash - ) + f'Unpinned "{name}" from version: {ipkg.status.current_version} ({ipkg.status.current_hash})', ) else: had_failure = True @@ -1806,7 +1792,7 @@ def cmd_unpin(manager, args, config, configfile): def _get_filtered_packages(manager, category): - pkg_dict = dict() + pkg_dict = {} for ipkg in manager.installed_packages(): pkg_dict[ipkg.package.qualified_name()] = ipkg @@ -1937,14 +1923,15 @@ def cmd_info(manager, args, config, configfile): sys.exit(1) # Dictionary for storing package info to output as JSON - pkginfo = dict() + pkginfo = {} had_invalid_package = False if len(args.package) == 1: try: package_names = [] for pkg_name, info in _get_filtered_packages( - manager, args.package[0] + manager, + args.package[0], ).items(): if info.is_builtin() and not args.include_builtin: continue @@ -1957,15 +1944,17 @@ def cmd_info(manager, args, config, configfile): for name in package_names: info = manager.info( - name, version=args.version, prefer_installed=(not args.nolocal) + name, + version=args.version, + prefer_installed=(not args.nolocal), ) if info.package: name = info.package.qualified_name() if args.json: - pkginfo[name] = dict() - pkginfo[name]["metadata"] = dict() + pkginfo[name] = {} + pkginfo[name]["metadata"] = {} else: print(f'"{name}" info:') @@ -1987,7 +1976,7 @@ def cmd_info(manager, args, config, configfile): if info.status: if args.json: - pkginfo[name]["install_status"] = dict() + pkginfo[name]["install_status"] = {} for key, value in sorted(info.status.__dict__.items()): pkginfo[name]["install_status"][key] = value @@ -2000,7 +1989,7 @@ def cmd_info(manager, args, config, configfile): if args.json: if info.metadata_file: pkginfo[name]["metadata_file"] = info.metadata_file - pkginfo[name]["metadata"][info.metadata_version] = dict() + pkginfo[name]["metadata"][info.metadata_version] = {} else: if info.metadata_file: print(f"\tmetadata file: {info.metadata_file}") @@ -2012,7 +2001,8 @@ def cmd_info(manager, args, config, configfile): else: if args.json: _fill_metadata_version( - pkginfo[name]["metadata"][info.metadata_version], info.metadata + pkginfo[name]["metadata"][info.metadata_version], + info.metadata, ) else: for key, value in sorted(info.metadata.items()): @@ -2026,9 +2016,11 @@ def cmd_info(manager, args, config, configfile): # Skip the version that was already processed if vers != info.metadata_version: info2 = manager.info( - name, vers, prefer_installed=(not args.nolocal) + name, + vers, + prefer_installed=(not args.nolocal), ) - pkginfo[name]["metadata"][info2.metadata_version] = dict() + pkginfo[name]["metadata"][info2.metadata_version] = {} if info2.metadata_file: pkginfo[name]["metadata_file"] = info2.metadata_file _fill_metadata_version( @@ -2064,7 +2056,7 @@ def _fill_metadata_version(pkginfo_name_metadata_version, info_metadata): """ for key, value in info_metadata.items(): if key == "depends" or key == "suggests": - pkginfo_name_metadata_version[key] = dict() + pkginfo_name_metadata_version[key] = {} deps = value.split("\n") for i in range(1, len(deps)): @@ -2179,8 +2171,8 @@ def cmd_env(manager, args, config, configfile): if not pluginpath: pluginpath = line2 - zeekpaths = [p for p in zeekpath.split(":")] if zeekpath else [] - pluginpaths = [p for p in pluginpath.split(":")] if pluginpath else [] + zeekpaths = list(zeekpath.split(":")) if zeekpath else [] + pluginpaths = list(pluginpath.split(":")) if pluginpath else [] zeekpaths.append(manager.zeekpath()) pluginpaths.append(manager.zeek_plugin_path()) @@ -2241,8 +2233,10 @@ def cmd_create(manager, args, config, configfile): print_error( "error: the following features are unknown: {}." ' Template "{}" offers {}.'.format( - unknowns, tmpl.name(), knowns or "no features" - ) + unknowns, + tmpl.name(), + knowns or "no features", + ), ) sys.exit(1) @@ -2255,11 +2249,8 @@ def cmd_create(manager, args, config, configfile): uvar.resolve(tmpl.name(), config, args.user_var, args.force) except ValueError: print_error( - str.format( - 'error: could not determine value of user variable "{}",' - " provide via environment or --user-var", - uvar.name(), - ) + f'error: could not determine value of user variable "{uvar.name()}",' + " provide via environment or --user-var", ) sys.exit(1) @@ -2284,13 +2275,12 @@ def cmd_create(manager, args, config, configfile): try: delete_path(args.packagedir) zeekpkg.LOG.info( - "Removed existing package directory %s", args.packagedir + "Removed existing package directory %s", + args.packagedir, ) except OSError as err: print_error( - "error: could not remove package directory {}: {}".format( - args.packagedir, err - ) + f"error: could not remove package directory {args.packagedir}: {err}", ) sys.exit(1) @@ -2337,7 +2327,7 @@ def cmd_template_info(manager, args, config, configfile): uvar_info["description"], uvar_info["default"] or "no default", ", ".join(uvar_info["used_by"]) or "not used", - ) + ), ) print("versions: " + ", ".join(tmplinfo["versions"])) @@ -2363,14 +2353,14 @@ def top_level_parser(): " ``ZKG_CONFIG_FILE``:\t" "Same as ``--configfile`` option, but has less precedence.\n" " ``ZKG_DEFAULT_SOURCE``:\t" - "The default package source to use (normally {}).\n" + f"The default package source to use (normally {ZKG_DEFAULT_SOURCE}).\n" " ``ZKG_DEFAULT_TEMPLATE``:\t" - "The default package template to use (normally {}).\n".format( - ZKG_DEFAULT_SOURCE, ZKG_DEFAULT_TEMPLATE - ), + f"The default package template to use (normally {ZKG_DEFAULT_TEMPLATE}).\n", ) top_parser.add_argument( - "--version", action="version", version="%(prog)s " + zeekpkg.__version__ + "--version", + action="version", + version="%(prog)s " + zeekpkg.__version__, ) group = top_parser.add_mutually_exclusive_group() @@ -2550,7 +2540,9 @@ def argparser(): " to first terminate that argument list.", ) sub_parser.add_argument( - "--force", action="store_true", help="Skip the confirmation prompt." + "--force", + action="store_true", + help="Skip the confirmation prompt.", ) sub_parser.add_argument( "--nodeps", @@ -2611,7 +2603,9 @@ def argparser(): sub_parser.set_defaults(run_cmd=cmd_remove) sub_parser.add_argument("package", nargs="+", help=pkg_name_help) sub_parser.add_argument( - "--force", action="store_true", help="Skip the confirmation prompt." + "--force", + action="store_true", + help="Skip the confirmation prompt.", ) sub_parser.add_argument( "--nodeps", @@ -2631,7 +2625,9 @@ def argparser(): ) sub_parser.set_defaults(run_cmd=cmd_purge) sub_parser.add_argument( - "--force", action="store_true", help="Skip the confirmation prompt." + "--force", + action="store_true", + help="Skip the confirmation prompt.", ) # refresh @@ -2721,7 +2717,10 @@ def argparser(): ) sub_parser.set_defaults(run_cmd=cmd_load) sub_parser.add_argument( - "package", nargs="+", default=[], help="Name(s) of package(s) to load." + "package", + nargs="+", + default=[], + help="Name(s) of package(s) to load.", ) sub_parser.add_argument( "--nodeps", @@ -2745,7 +2744,9 @@ def argparser(): sub_parser.set_defaults(run_cmd=cmd_unload) sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) sub_parser.add_argument( - "--force", action="store_true", help="Skip the confirmation prompt." + "--force", + action="store_true", + help="Skip the confirmation prompt.", ) sub_parser.add_argument( "--nodeps", @@ -2918,7 +2919,9 @@ def argparser(): ) sub_parser.set_defaults(run_cmd=cmd_autoconfig) sub_parser.add_argument( - "--force", action="store_true", help="Skip any confirmation prompt." + "--force", + action="store_true", + help="Skip any confirmation prompt.", ) # env @@ -3024,7 +3027,8 @@ def main(): args = argparser().parse_args() formatter = logging.Formatter( - "%(asctime)s %(levelname)-8s %(message)s", "%Y-%m-%d %H:%M:%S" + "%(asctime)s %(levelname)-8s %(message)s", + "%Y-%m-%d %H:%M:%S", ) handler = logging.StreamHandler() handler.setFormatter(formatter)