From 56167d250baa33e140ee8e4ed6c2a165ac3a796f Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:53 +0200 Subject: [PATCH 01/20] Centralize project configuration We previously used a mix of `setup.py` and `setup.cfg` which are not well supported by more modern Python developer tools which instead prefer `pyproject.toml`. This meant that developing this project required manually setting up and managing virtualenvs. With this patch we now move most of our configuration over into `pyproject.toml` so especially dev environments can be managed with tools like Hatch[^1] or Rye[^2]. Dev envs managed by these tools are automaticallt discovered by many editors so there is potentially less need to even think about virtualenvs. Now `setup.py` contains the absolute minimum required settings. We still need to keep dynamically generating the package version from the Zeek-style `VERSION` file, and also need to add a tweak to make sure it is distributed in wheels so the installation code reading it has access to it. Since settings in `setup.cfg` can conflict with `pyproject.toml` we delete the file after either moving all setttings to `pyproject.toml` or to tool specific files (flake8 still does not support `pyproject.toml`). For now we keep `requirements.txt` (which updated versions) so we can install dev dependencies, e.g., for RTD. Unfortunately dev dependencies in `pyproject.toml` are still not standardized[^3]. [^1]: hatch.pypa.io [^2]: https://rye-up.com/ [^3]: https://discuss.python.org/t/development-dependencies-in-pyproject-toml/26149 --- setup.cfg => .flake8 | 3 --- pyproject.toml | 63 ++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 12 +++++---- setup.py | 28 ++------------------ 4 files changed, 72 insertions(+), 34 deletions(-) rename setup.cfg => .flake8 (93%) create mode 100644 pyproject.toml diff --git a/setup.cfg b/.flake8 similarity index 93% rename from setup.cfg rename to .flake8 index 2ce3b7db..9d4f1410 100644 --- a/setup.cfg +++ b/.flake8 @@ -1,6 +1,3 @@ -[bdist_wheel] -universal = 1 - [flake8] max_line_length = 100 # E203: whitespace before ':' (black / flake8 disagreement) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..bfe57879 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,63 @@ +[build-system] +requires = [ + "setuptools", +] + +[project] +name = "zkg" +dynamic = ["version"] +description = "The Zeek Package Manager" +readme = "README" + +keywords = [ + "zeek", + "zeekctl", + "zeekcontrol", + "package", + "manager", + "scripts", + "plugins", + "security", +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "License :: OSI Approved :: University of Illinois/NCSA Open Source License", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python :: 3", + "Topic :: System :: Networking :: Monitoring", + "Topic :: Utilities", +] + +# NOTE: Keep `requirements.txt` in sync which we currently use e.g., for RTD. +dependencies = [ + "GitPython>=3.1.43", + "semantic_version>=2.10.0", + "btest>=1.1", +] + +[project.optional-dependencies] +dev = [ + "Sphinx>=7.2.6", + "sphinx_rtd_theme>=2.0.0", +] + +[project.license] +file = "COPYING" + +[project.urls] +Homepage = "https://docs.zeek.org/projects/package-manager" +Repository = "https://github.com/zeek/package-manager" + +[[project.maintainers]] +name = "The Zeek Project" +email = "info@zeek.org" + +[tool.setuptools] +packages = ["zeekpkg"] +script-files = ["zkg"] + +[tool.distutils.bdist_wheel] +universal = true diff --git a/requirements.txt b/requirements.txt index 8006be9a..94ec6fb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ +# NOTE: Dependencies should go into `pyproject.toml`. We keep this file to e.g., support RTD. + # Requirements for general zkg usage -GitPython -semantic_version -btest +GitPython>3.1.43 +semantic_version>2.10.0 +btest>=1.1 # Requirements for development (e.g. building docs) -Sphinx>=3.0 -sphinx_rtd_theme +Sphinx>=7.2.6 +sphinx_rtd_theme>=2.0.0 diff --git a/setup.py b/setup.py index 066db7c4..1ff886e3 100644 --- a/setup.py +++ b/setup.py @@ -1,38 +1,14 @@ import pathlib from setuptools import setup -install_requires = ["gitpython", "semantic_version", "btest"] - def version(): return pathlib.Path("VERSION").read_text().replace("-", ".dev", 1).strip() -def long_description(): - return pathlib.Path("README").read_text() - - setup( - name="zkg", version=version(), - description="The Zeek Package Manager", - long_description=long_description(), - license="University of Illinois/NCSA Open Source License", - keywords="zeek zeekctl zeekcontrol package manager scripts plugins security", - maintainer="The Zeek Project", - maintainer_email="info@zeek.org", - url="https://github.com/zeek/package-manager", - scripts=["zkg"], - packages=["zeekpkg"], - install_requires=install_requires, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "License :: OSI Approved :: University of Illinois/NCSA Open Source License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS :: MacOS X", - "Programming Language :: Python :: 3", - "Topic :: System :: Networking :: Monitoring", - "Topic :: Utilities", + data_files=[ + ("output_dir", ["VERSION"]), ], ) From 6c818994d9cc4ccb7609b6215399c924d9373a35 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:53 +0200 Subject: [PATCH 02/20] Bump pre-commit hooks --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4683879e..8790b876 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: trailing-whitespace exclude: testing/baselines @@ -16,7 +16,7 @@ repos: - id: pyupgrade args: ["--py37-plus"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.0 + rev: v0.3.7 hooks: - id: ruff args: [--fix] From c9f4a8bc84fb6764e170ff717b1d287e1c23cae4 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:54 +0200 Subject: [PATCH 03/20] Stop excluding `doc/ext/sphinxarg` from linting --- .pre-commit-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8790b876..0d1f6c0e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,5 +21,3 @@ repos: - id: ruff args: [--fix] - id: ruff-format - -exclude: doc/ext/sphinxarg From a59a33f7df650847087a5d23f0f933f2a17a3067 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:54 +0200 Subject: [PATCH 04/20] Fix ruff lints in doc modules --- doc/ext/sphinxarg/ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index 08fe1282..506dd667 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -8,7 +8,7 @@ try: # Removed as of Sphinx 1.7 from sphinx.util.compat import Directive -except ImportError as err: +except ImportError: from docutils.parsers.rst import Directive from sphinx.util.nodes import nested_parse_with_titles From 546df9b988d3a50f48067b8a04e2a456bc895c49 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:54 +0200 Subject: [PATCH 05/20] Automatically reformat doc module --- doc/ext/sphinxarg/ext.py | 432 ++++++++++++++++++++---------------- doc/ext/sphinxarg/parser.py | 90 ++++---- 2 files changed, 290 insertions(+), 232 deletions(-) diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index 506dd667..ec0fdc3d 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -18,7 +18,7 @@ def map_nested_definitions(nested_content): if nested_content is None: - raise Exception('Nested content should be iterable, not null') + raise Exception("Nested content should be iterable, not null") # build definition dictionary definitions = {} for item in nested_content: @@ -29,15 +29,18 @@ def map_nested_definitions(nested_content): continue if not len(subitem.children) > 0: continue - classifier = '@after' + classifier = "@after" idx = subitem.first_child_matching_class(nodes.classifier) if idx is not None: ci = subitem[idx] if len(ci.children) > 0: classifier = ci.children[0].astext() if classifier is not None and classifier not in ( - '@replace', '@before', '@after'): - raise Exception('Unknown classifier: %s' % classifier) + "@replace", + "@before", + "@after", + ): + raise Exception("Unknown classifier: %s" % classifier) idx = subitem.first_child_matching_class(nodes.term) if idx is not None: ch = subitem[idx] @@ -46,7 +49,7 @@ def map_nested_definitions(nested_content): idx = subitem.first_child_matching_class(nodes.definition) if idx is not None: def_node = subitem[idx] - def_node.attributes['classifier'] = classifier + def_node.attributes["classifier"] = classifier definitions[term] = def_node return definitions @@ -54,121 +57,155 @@ def map_nested_definitions(nested_content): def print_arg_list(data, nested_content): definitions = map_nested_definitions(nested_content) items = [] - if 'args' in data: - for arg in data['args']: - my_def = [nodes.paragraph(text=arg['help'])] if arg['help'] else [] - name = arg['name'] + if "args" in data: + for arg in data["args"]: + my_def = [nodes.paragraph(text=arg["help"])] if arg["help"] else [] + name = arg["name"] my_def = apply_definition(definitions, my_def, name) if len(my_def) == 0: - my_def.append(nodes.paragraph(text='Undocumented')) - if 'choices' in arg: - my_def.append(nodes.paragraph( - text=('Possible choices: %s' % ', '.join([str(c) for c in arg['choices']])))) + my_def.append(nodes.paragraph(text="Undocumented")) + if "choices" in arg: + my_def.append( + nodes.paragraph( + text=( + "Possible choices: %s" + % ", ".join([str(c) for c in arg["choices"]]) + ) + ) + ) argname = name - if arg['metavar']: - argname = arg['metavar'] + if arg["metavar"]: + argname = arg["metavar"] items.append( nodes.option_list_item( - '', nodes.option_group('', - nodes.option('', nodes.option_string(text=argname))), - nodes.description('', *my_def))) - return nodes.option_list('', *items) if items else None + "", + nodes.option_group( + "", nodes.option("", nodes.option_string(text=argname)) + ), + nodes.description("", *my_def), + ) + ) + return nodes.option_list("", *items) if items else None def print_opt_list(data, nested_content): definitions = map_nested_definitions(nested_content) items = [] - if 'options' in data: - for opt in data['options']: + if "options" in data: + for opt in data["options"]: names = [] - my_def = [nodes.paragraph(text=opt['help'])] if opt['help'] else [] - for name in opt['name']: + my_def = [nodes.paragraph(text=opt["help"])] if opt["help"] else [] + for name in opt["name"]: option_declaration = [nodes.option_string(text=name)] - if opt['default'] is not None \ - and opt['default'] != '==SUPPRESS==': + if opt["default"] is not None and opt["default"] != "==SUPPRESS==": option_declaration += nodes.option_argument( - '', text='=' + str(opt['default'])) - names.append(nodes.option('', *option_declaration)) + "", text="=" + str(opt["default"]) + ) + names.append(nodes.option("", *option_declaration)) my_def = apply_definition(definitions, my_def, name) if len(my_def) == 0: - my_def.append(nodes.paragraph(text='Undocumented')) - if 'choices' in opt: - my_def.append(nodes.paragraph( - text=('Possible choices: %s' % ', '.join([str(c) for c in opt['choices']])))) + my_def.append(nodes.paragraph(text="Undocumented")) + if "choices" in opt: + my_def.append( + nodes.paragraph( + text=( + "Possible choices: %s" + % ", ".join([str(c) for c in opt["choices"]]) + ) + ) + ) items.append( nodes.option_list_item( - '', nodes.option_group('', *names), - nodes.description('', *my_def))) - return nodes.option_list('', *items) if items else None + "", nodes.option_group("", *names), nodes.description("", *my_def) + ) + ) + return nodes.option_list("", *items) if items else None def print_command_args_and_opts(arg_list, opt_list, sub_list=None): items = [] if arg_list: - items.append(nodes.definition_list_item( - '', nodes.term(text='Positional arguments:'), - nodes.definition('', arg_list))) + items.append( + nodes.definition_list_item( + "", + nodes.term(text="Positional arguments:"), + nodes.definition("", arg_list), + ) + ) if opt_list: - items.append(nodes.definition_list_item( - '', nodes.term(text='Options:'), - nodes.definition('', opt_list))) + items.append( + nodes.definition_list_item( + "", nodes.term(text="Options:"), nodes.definition("", opt_list) + ) + ) if sub_list and len(sub_list): - items.append(nodes.definition_list_item( - '', nodes.term(text='Sub-commands:'), - nodes.definition('', sub_list))) - return nodes.definition_list('', *items) + items.append( + nodes.definition_list_item( + "", nodes.term(text="Sub-commands:"), nodes.definition("", sub_list) + ) + ) + return nodes.definition_list("", *items) def apply_definition(definitions, my_def, name): if name in definitions: definition = definitions[name] - classifier = definition['classifier'] - if classifier == '@replace': + classifier = definition["classifier"] + if classifier == "@replace": return definition.children - if classifier == '@after': + if classifier == "@after": return my_def + definition.children - if classifier == '@before': + if classifier == "@before": return definition.children + my_def - raise Exception('Unknown classifier: %s' % classifier) + raise Exception("Unknown classifier: %s" % classifier) return my_def def print_subcommand_list(data, nested_content): definitions = map_nested_definitions(nested_content) items = [] - if 'children' in data: - for child in data['children']: - my_def = [nodes.paragraph( - text=child['help'])] if child['help'] else [] - name = child['name'] + if "children" in data: + for child in data["children"]: + my_def = [nodes.paragraph(text=child["help"])] if child["help"] else [] + name = child["name"] my_def = apply_definition(definitions, my_def, name) if len(my_def) == 0: - my_def.append(nodes.paragraph(text='Undocumented')) - if 'description' in child: - my_def.append(nodes.paragraph(text=child['description'])) - my_def.append(nodes.literal_block(text=child['usage'])) - my_def.append(print_command_args_and_opts( - print_arg_list(child, nested_content), - print_opt_list(child, nested_content), - print_subcommand_list(child, nested_content) - )) + my_def.append(nodes.paragraph(text="Undocumented")) + if "description" in child: + my_def.append(nodes.paragraph(text=child["description"])) + my_def.append(nodes.literal_block(text=child["usage"])) + my_def.append( + print_command_args_and_opts( + print_arg_list(child, nested_content), + print_opt_list(child, nested_content), + print_subcommand_list(child, nested_content), + ) + ) items.append( nodes.definition_list_item( - '', - nodes.term('', '', nodes.strong(text=name)), - nodes.definition('', *my_def) + "", + nodes.term("", "", nodes.strong(text=name)), + nodes.definition("", *my_def), ) ) - return nodes.definition_list('', *items) + return nodes.definition_list("", *items) class ArgParseDirective(Directive): has_content = True - option_spec = dict(module=unchanged, func=unchanged, ref=unchanged, - prog=unchanged, path=unchanged, nodefault=flag, - manpage=unchanged, nosubcommands=unchanged, passparser=flag) + option_spec = dict( + module=unchanged, + func=unchanged, + ref=unchanged, + prog=unchanged, + path=unchanged, + nodefault=flag, + manpage=unchanged, + nosubcommands=unchanged, + passparser=flag, + ) def _construct_manpage_specific_structure(self, parser_info): """ @@ -183,39 +220,40 @@ def _construct_manpage_specific_structure(self, parser_info): """ # SYNOPSIS section synopsis_section = nodes.section( - '', - nodes.title(text='Synopsis'), + "", + nodes.title(text="Synopsis"), nodes.literal_block(text=parser_info["bare_usage"]), - ids=['synopsis-section']) + ids=["synopsis-section"], + ) # DESCRIPTION section description_section = nodes.section( - '', - nodes.title(text='Description'), - nodes.paragraph(text=parser_info.get( - 'description', parser_info.get( - 'help', "undocumented").capitalize())), - ids=['description-section']) - nested_parse_with_titles( - self.state, self.content, description_section) - if parser_info.get('epilog'): + "", + nodes.title(text="Description"), + nodes.paragraph( + text=parser_info.get( + "description", parser_info.get("help", "undocumented").capitalize() + ) + ), + ids=["description-section"], + ) + nested_parse_with_titles(self.state, self.content, description_section) + if parser_info.get("epilog"): # TODO: do whatever sphinx does to understand ReST inside # docstrings magically imported from other places. The nested # parse method invoked above seem to be able to do this but # I haven't found a way to do it for arbitrary text - description_section += nodes.paragraph( - text=parser_info['epilog']) + description_section += nodes.paragraph(text=parser_info["epilog"]) # OPTIONS section options_section = nodes.section( - '', - nodes.title(text='Options'), - ids=['options-section']) - if 'args' in parser_info: + "", nodes.title(text="Options"), ids=["options-section"] + ) + if "args" in parser_info: options_section += nodes.paragraph() - options_section += nodes.subtitle(text='Positional arguments:') + options_section += nodes.subtitle(text="Positional arguments:") options_section += self._format_positional_arguments(parser_info) - if 'options' in parser_info: + if "options" in parser_info: options_section += nodes.paragraph() - options_section += nodes.subtitle(text='Optional arguments:') + options_section += nodes.subtitle(text="Optional arguments:") options_section += self._format_optional_arguments(parser_info) items = [ # NOTE: we cannot generate NAME ourselves. It is generated by @@ -228,94 +266,103 @@ def _construct_manpage_specific_structure(self, parser_info): ] if len(options_section.children) > 1: items.append(options_section) - if 'nosubcommands' not in self.options: + if "nosubcommands" not in self.options: # SUBCOMMANDS section (non-standard) subcommands_section = nodes.section( - '', - nodes.title(text='Sub-Commands'), - ids=['subcommands-section']) - if 'children' in parser_info: + "", nodes.title(text="Sub-Commands"), ids=["subcommands-section"] + ) + if "children" in parser_info: subcommands_section += self._format_subcommands(parser_info) if len(subcommands_section) > 1: items.append(subcommands_section) if os.getenv("INCLUDE_DEBUG_SECTION"): import json + # DEBUG section (non-standard) debug_section = nodes.section( - '', + "", nodes.title(text="Argparse + Sphinx Debugging"), - nodes.literal_block(text=json.dumps(parser_info, indent=' ')), - ids=['debug-section']) + nodes.literal_block(text=json.dumps(parser_info, indent=" ")), + ids=["debug-section"], + ) items.append(debug_section) return items def _format_positional_arguments(self, parser_info): - assert 'args' in parser_info + assert "args" in parser_info items = [] - for arg in parser_info['args']: + for arg in parser_info["args"]: arg_items = [] - if arg['help']: - arg_items.append(nodes.paragraph(text=arg['help'])) + if arg["help"]: + arg_items.append(nodes.paragraph(text=arg["help"])) else: - arg_items.append(nodes.paragraph(text='Undocumented')) - if 'choices' in arg: + arg_items.append(nodes.paragraph(text="Undocumented")) + if "choices" in arg: arg_items.append( nodes.paragraph( - text='Possible choices: ' + ', '.join(arg['choices']))) + text="Possible choices: " + ", ".join(arg["choices"]) + ) + ) items.append( nodes.option_list_item( - '', + "", nodes.option_group( - '', nodes.option( - '', nodes.option_string(text=arg['metavar']) - ) + "", nodes.option("", nodes.option_string(text=arg["metavar"])) ), - nodes.description('', *arg_items))) - return nodes.option_list('', *items) + nodes.description("", *arg_items), + ) + ) + return nodes.option_list("", *items) def _format_optional_arguments(self, parser_info): - assert 'options' in parser_info + assert "options" in parser_info items = [] - for opt in parser_info['options']: + for opt in parser_info["options"]: names = [] opt_items = [] - for name in opt['name']: + for name in opt["name"]: option_declaration = [nodes.option_string(text=name)] - if opt['default'] is not None \ - and opt['default'] != '==SUPPRESS==': + if opt["default"] is not None and opt["default"] != "==SUPPRESS==": option_declaration += nodes.option_argument( - '', text='=' + str(opt['default'])) - names.append(nodes.option('', *option_declaration)) - if opt['help']: - opt_items.append(nodes.paragraph(text=opt['help'])) + "", text="=" + str(opt["default"]) + ) + names.append(nodes.option("", *option_declaration)) + if opt["help"]: + opt_items.append(nodes.paragraph(text=opt["help"])) else: - opt_items.append(nodes.paragraph(text='Undocumented')) - if 'choices' in opt: + opt_items.append(nodes.paragraph(text="Undocumented")) + if "choices" in opt: opt_items.append( nodes.paragraph( - text='Possible choices: ' + ', '.join(opt['choices']))) + text="Possible choices: " + ", ".join(opt["choices"]) + ) + ) items.append( nodes.option_list_item( - '', nodes.option_group('', *names), - nodes.description('', *opt_items))) - return nodes.option_list('', *items) + "", + nodes.option_group("", *names), + nodes.description("", *opt_items), + ) + ) + return nodes.option_list("", *items) def _format_subcommands(self, parser_info): - assert 'children' in parser_info + assert "children" in parser_info items = [] - for subcmd in parser_info['children']: + for subcmd in parser_info["children"]: subcmd_items = [] - if subcmd['help']: - subcmd_items.append(nodes.paragraph(text=subcmd['help'])) + if subcmd["help"]: + subcmd_items.append(nodes.paragraph(text=subcmd["help"])) else: - subcmd_items.append(nodes.paragraph(text='Undocumented')) + subcmd_items.append(nodes.paragraph(text="Undocumented")) items.append( nodes.definition_list_item( - '', - nodes.term('', '', nodes.strong( - text=subcmd['bare_usage'])), - nodes.definition('', *subcmd_items))) - return nodes.definition_list('', *items) + "", + nodes.term("", "", nodes.strong(text=subcmd["bare_usage"])), + nodes.definition("", *subcmd_items), + ) + ) + return nodes.definition_list("", *items) def _nested_parse_paragraph(self, text): content = nodes.paragraph() @@ -323,98 +370,103 @@ def _nested_parse_paragraph(self, text): return content def run(self): - if 'module' in self.options and 'func' in self.options: - module_name = self.options['module'] - attr_name = self.options['func'] - elif 'ref' in self.options: - _parts = self.options['ref'].split('.') - module_name = '.'.join(_parts[0:-1]) + if "module" in self.options and "func" in self.options: + module_name = self.options["module"] + attr_name = self.options["func"] + elif "ref" in self.options: + _parts = self.options["ref"].split(".") + module_name = ".".join(_parts[0:-1]) attr_name = _parts[-1] else: - raise self.error( - ':module: and :func: should be specified, or :ref:') + raise self.error(":module: and :func: should be specified, or :ref:") mod = __import__(module_name, globals(), locals(), [attr_name]) file_dependency = mod.__file__ - if file_dependency.endswith('.pyc'): + if file_dependency.endswith(".pyc"): file_dependency = file_dependency[:-1] env = self.state.document.settings.env - if not hasattr(env, 'argparse_usages'): + if not hasattr(env, "argparse_usages"): env.argparse_usages = [] - env.argparse_usages.append({ - 'docname': env.docname, - 'lineno': self.lineno, - 'dependency_file': file_dependency, - 'dependency_mtime': os.stat(file_dependency).st_mtime, - }) + env.argparse_usages.append( + { + "docname": env.docname, + "lineno": self.lineno, + "dependency_file": file_dependency, + "dependency_mtime": os.stat(file_dependency).st_mtime, + } + ) if not hasattr(mod, attr_name): - raise self.error(( - 'Module "%s" has no attribute "%s"\n' - 'Incorrect argparse :module: or :func: values?' - ) % (module_name, attr_name)) + raise self.error( + ( + 'Module "%s" has no attribute "%s"\n' + "Incorrect argparse :module: or :func: values?" + ) + % (module_name, attr_name) + ) func = getattr(mod, attr_name) if isinstance(func, ArgumentParser): parser = func - elif 'passparser' in self.options: + elif "passparser" in self.options: parser = ArgumentParser() func(parser) else: parser = func() - if 'path' not in self.options: - self.options['path'] = '' - path = str(self.options['path']) - if 'prog' in self.options: - parser.prog = self.options['prog'] - result = parse_parser( - parser, skip_default_values='nodefault' in self.options) + if "path" not in self.options: + self.options["path"] = "" + path = str(self.options["path"]) + if "prog" in self.options: + parser.prog = self.options["prog"] + result = parse_parser(parser, skip_default_values="nodefault" in self.options) result = parser_navigate(result, path) - if 'manpage' in self.options: + if "manpage" in self.options: return self._construct_manpage_specific_structure(result) nested_content = nodes.paragraph() - self.state.nested_parse( - self.content, self.content_offset, nested_content) + self.state.nested_parse(self.content, self.content_offset, nested_content) nested_content = nested_content.children items = [] # add common content between for item in nested_content: if not isinstance(item, nodes.definition_list): items.append(item) - if 'description' in result: - items.append(self._nested_parse_paragraph(result['description'])) - items.append(nodes.literal_block(text=result['usage'])) + if "description" in result: + items.append(self._nested_parse_paragraph(result["description"])) + items.append(nodes.literal_block(text=result["usage"])) - if 'nosubcommands' in self.options: + if "nosubcommands" in self.options: subcommands = None else: subcommands = print_subcommand_list(result, nested_content) - items.append(print_command_args_and_opts( - print_arg_list(result, nested_content), - print_opt_list(result, nested_content), - subcommands - )) - if 'epilog' in result: - items.append(self._nested_parse_paragraph(result['epilog'])) + items.append( + print_command_args_and_opts( + print_arg_list(result, nested_content), + print_opt_list(result, nested_content), + subcommands, + ) + ) + if "epilog" in result: + items.append(self._nested_parse_paragraph(result["epilog"])) return items def env_get_outdated_hook(app, env, added, changed, removed): from sphinx.util import logging + logger = logging.getLogger(__name__) rval = set() - if not hasattr(env, 'argparse_usages'): + if not hasattr(env, "argparse_usages"): return [] for usage in env.argparse_usages: - docname = usage['docname'] - dep_file = usage['dependency_file'] - dep_mtime = usage['dependency_mtime'] + docname = usage["docname"] + dep_file = usage["dependency_file"] + dep_mtime = usage["dependency_mtime"] current_mtime = os.stat(dep_file).st_mtime if current_mtime > dep_mtime and docname not in removed: @@ -422,21 +474,23 @@ def env_get_outdated_hook(app, env, added, changed, removed): for docname in rval: from sphinx.util.console import blue - msg = blue('found outdated argparse doc: {0}'.format(docname)) + + msg = blue("found outdated argparse doc: {0}".format(docname)) logger.info(msg) return list(rval) def env_purge_doc_hook(app, env, docname): - if not hasattr(env, 'argparse_usages'): + if not hasattr(env, "argparse_usages"): return env.argparse_usages = [ - usage for usage in env.argparse_usages if usage['docname'] != docname] + usage for usage in env.argparse_usages if usage["docname"] != docname + ] def setup(app): - app.add_directive('argparse', ArgParseDirective) - app.connect('env-get-outdated', env_get_outdated_hook) - app.connect('env-purge-doc', env_purge_doc_hook) + app.add_directive("argparse", ArgParseDirective) + app.connect("env-get-outdated", env_get_outdated_hook) + app.connect("env-purge-doc", env_purge_doc_hook) diff --git a/doc/ext/sphinxarg/parser.py b/doc/ext/sphinxarg/parser.py index 79a541ad..9261e645 100644 --- a/doc/ext/sphinxarg/parser.py +++ b/doc/ext/sphinxarg/parser.py @@ -8,26 +8,28 @@ class NavigationException(Exception): def parser_navigate(parser_result, path, current_path=None): if isinstance(path, str): - if path == '': + if path == "": return parser_result - path = re.split(r'\s+', path) + path = re.split(r"\s+", path) current_path = current_path or [] if len(path) == 0: return parser_result - if 'children' not in parser_result: + if "children" not in parser_result: raise NavigationException( - 'Current parser have no children elements. (path: %s)' % - ' '.join(current_path)) + "Current parser have no children elements. (path: %s)" + % " ".join(current_path) + ) next_hop = path.pop(0) - for child in parser_result['children']: + for child in parser_result["children"]: # identifer is only used for aliased subcommands - identifier = child['identifier'] if 'identifier' in child else child['name'] + identifier = child["identifier"] if "identifier" in child else child["name"] if identifier == next_hop: current_path.append(next_hop) return parser_navigate(child, path, current_path) raise NavigationException( - 'Current parser have no children element with name: %s (path: %s)' % ( - next_hop, ' '.join(current_path))) + "Current parser have no children element with name: %s (path: %s)" + % (next_hop, " ".join(current_path)) + ) def _try_add_parser_attribute(data, parser, attribname): @@ -46,21 +48,22 @@ def _format_usage_without_prefix(parser): the 'usage: ' prefix. """ fmt = parser._get_formatter() - fmt.add_usage(parser.usage, parser._actions, - parser._mutually_exclusive_groups, prefix='') + fmt.add_usage( + parser.usage, parser._actions, parser._mutually_exclusive_groups, prefix="" + ) return fmt.format_help().strip() def parse_parser(parser, data=None, **kwargs): if data is None: data = { - 'name': '', - 'usage': parser.format_usage().strip(), - 'bare_usage': _format_usage_without_prefix(parser), - 'prog': parser.prog, + "name": "", + "usage": parser.format_usage().strip(), + "bare_usage": _format_usage_without_prefix(parser), + "prog": parser.prog, } - _try_add_parser_attribute(data, parser, 'description') - _try_add_parser_attribute(data, parser, 'epilog') + _try_add_parser_attribute(data, parser, "description") + _try_add_parser_attribute(data, parser, "epilog") for action in parser._get_positional_actions(): if isinstance(action, _HelpAction): continue @@ -84,44 +87,45 @@ def parse_parser(parser, data=None, **kwargs): if name in subsection_alias_names: continue subalias = subsection_alias[subaction] - subaction.prog = '%s %s' % (parser.prog, name) + subaction.prog = "%s %s" % (parser.prog, name) subdata = { - 'name': name if not subalias else - '%s (%s)' % (name, ', '.join(subalias)), - 'help': helps.get(name, ''), - 'usage': subaction.format_usage().strip(), - 'bare_usage': _format_usage_without_prefix(subaction), + "name": name + if not subalias + else "%s (%s)" % (name, ", ".join(subalias)), + "help": helps.get(name, ""), + "usage": subaction.format_usage().strip(), + "bare_usage": _format_usage_without_prefix(subaction), } if subalias: - subdata['identifier'] = name + subdata["identifier"] = name parse_parser(subaction, subdata, **kwargs) - data.setdefault('children', []).append(subdata) + data.setdefault("children", []).append(subdata) continue - if 'args' not in data: - data['args'] = [] + if "args" not in data: + data["args"] = [] arg = { - 'name': action.dest, - 'help': action.help or '', - 'metavar': action.metavar + "name": action.dest, + "help": action.help or "", + "metavar": action.metavar, } if action.choices: - arg['choices'] = action.choices - data['args'].append(arg) - show_defaults = ( - ('skip_default_values' not in kwargs) - or (kwargs['skip_default_values'] is False)) + arg["choices"] = action.choices + data["args"].append(arg) + show_defaults = ("skip_default_values" not in kwargs) or ( + kwargs["skip_default_values"] is False + ) for action in parser._get_optional_actions(): if isinstance(action, _HelpAction): continue - if 'options' not in data: - data['options'] = [] + if "options" not in data: + data["options"] = [] option = { - 'name': action.option_strings, - 'default': action.default if show_defaults else '==SUPPRESS==', - 'help': action.help or '' + "name": action.option_strings, + "default": action.default if show_defaults else "==SUPPRESS==", + "help": action.help or "", } if action.choices: - option['choices'] = action.choices - if "==SUPPRESS==" not in option['help']: - data['options'].append(option) + option["choices"] = action.choices + if "==SUPPRESS==" not in option["help"]: + data["options"].append(option) return data From a78fd59a00f4a299d8129a9a04148796df8d35a2 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:55 +0200 Subject: [PATCH 06/20] Modernize doc module with pyupgrade --- doc/ext/sphinxarg/ext.py | 2 +- doc/ext/sphinxarg/parser.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index ec0fdc3d..234cf712 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -475,7 +475,7 @@ def env_get_outdated_hook(app, env, added, changed, removed): for docname in rval: from sphinx.util.console import blue - msg = blue("found outdated argparse doc: {0}".format(docname)) + msg = blue(f"found outdated argparse doc: {docname}") logger.info(msg) return list(rval) diff --git a/doc/ext/sphinxarg/parser.py b/doc/ext/sphinxarg/parser.py index 9261e645..455eb082 100644 --- a/doc/ext/sphinxarg/parser.py +++ b/doc/ext/sphinxarg/parser.py @@ -87,11 +87,11 @@ def parse_parser(parser, data=None, **kwargs): if name in subsection_alias_names: continue subalias = subsection_alias[subaction] - subaction.prog = "%s %s" % (parser.prog, name) + subaction.prog = f"{parser.prog} {name}" subdata = { "name": name if not subalias - else "%s (%s)" % (name, ", ".join(subalias)), + else "{} ({})".format(name, ", ".join(subalias)), "help": helps.get(name, ""), "usage": subaction.format_usage().strip(), "bare_usage": _format_usage_without_prefix(subaction), From a24124d2f298b306e47d43a8aa0361f98ede9a04 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:55 +0200 Subject: [PATCH 07/20] Move btest to dev dependencies --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bfe57879..3a1ea0dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,11 +35,11 @@ classifiers = [ dependencies = [ "GitPython>=3.1.43", "semantic_version>=2.10.0", - "btest>=1.1", ] [project.optional-dependencies] dev = [ + "btest>=1.1", "Sphinx>=7.2.6", "sphinx_rtd_theme>=2.0.0", ] diff --git a/requirements.txt b/requirements.txt index 94ec6fb1..4cfdbc41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ # Requirements for general zkg usage GitPython>3.1.43 semantic_version>2.10.0 -btest>=1.1 # Requirements for development (e.g. building docs) +btest>=1.1 Sphinx>=7.2.6 sphinx_rtd_theme>=2.0.0 From a7c58d35b0022f78c57ba9a5cf72a4c45722a9a5 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:56 +0200 Subject: [PATCH 08/20] Modernize Python code with ruff --- .pre-commit-config.yaml | 5 ---- doc/ext/sphinxarg/ext.py | 7 ++--- doc/ext/sphinxarg/parser.py | 4 +-- pyproject.toml | 3 +++ zeekpkg/manager.py | 18 +++++-------- zeekpkg/template.py | 16 ++++------- zeekpkg/uservar.py | 4 +-- zkg | 54 +++++++++++-------------------------- 8 files changed, 35 insertions(+), 76 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d1f6c0e..a6ab9133 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,11 +10,6 @@ repos: exclude: testing/baselines - id: check-yaml - id: check-added-large-files -- repo: https://github.com/asottile/pyupgrade - rev: v3.15.1 - hooks: - - id: pyupgrade - args: ["--py37-plus"] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.3.7 hooks: diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index 234cf712..1eb4a5ce 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -401,11 +401,8 @@ def run(self): if not hasattr(mod, attr_name): raise self.error( - ( - 'Module "%s" has no attribute "%s"\n' - "Incorrect argparse :module: or :func: values?" - ) - % (module_name, attr_name) + f'Module "{module_name}" has no attribute "{attr_name}"\n' + "Incorrect argparse :module: or :func: values?" ) func = getattr(mod, attr_name) if isinstance(func, ArgumentParser): diff --git a/doc/ext/sphinxarg/parser.py b/doc/ext/sphinxarg/parser.py index 455eb082..3ad2678a 100644 --- a/doc/ext/sphinxarg/parser.py +++ b/doc/ext/sphinxarg/parser.py @@ -27,8 +27,8 @@ def parser_navigate(parser_result, path, current_path=None): current_path.append(next_hop) return parser_navigate(child, path, current_path) raise NavigationException( - "Current parser have no children element with name: %s (path: %s)" - % (next_hop, " ".join(current_path)) + f"Current parser have no children element with name: {next_hop} (path: %s)" + % " ".join(current_path) ) diff --git a/pyproject.toml b/pyproject.toml index 3a1ea0dc..56c6e056 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,3 +61,6 @@ script-files = ["zkg"] [tool.distutils.bdist_wheel] universal = true + +[tool.ruff.lint] +select = ["UP"] diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index 7f7f21b3..ef89799b 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -545,8 +545,8 @@ def add_source(self, name, git_url): LOG.debug('duplicate source "%s"', name) return True - return "source already exists with different URL: {}".format( - existing_source.git_url + return ( + f"source already exists with different URL: {existing_source.git_url}" ) clone_path = os.path.join(self.source_clonedir, name) @@ -1145,9 +1145,7 @@ def _refresh_source(self, name, aggregate=False, push=False): url, error, ) - msg = 'failed to checkout branch/version "{}": {}'.format( - version, repr(error) - ) + msg = f'failed to checkout branch/version "{version}": {repr(error)}' aggregation_issues.append((url, msg)) continue @@ -1688,9 +1686,7 @@ def _has_all_dependers_unloaded(item, dependers): errors.append( ( item, - "Package is in use by other packages --- {}.".format( - dep_listing[:-2] - ), + f"Package is in use by other packages --- {dep_listing[:-2]}.", ) ) return errors @@ -3195,7 +3191,7 @@ def _is_version_outdated(clone, version): def _is_branch_outdated(clone, branch): - it = clone.iter_commits("{0}..origin/{0}".format(branch)) + it = clone.iter_commits(f"{branch}..origin/{branch}") num_commits_behind = sum(1 for c in it) return num_commits_behind > 0 @@ -3316,8 +3312,8 @@ def _parse_package_metadata(parser, metadata_file): """Return string explaining why metadata is invalid, or '' if valid.""" if not parser.read(metadata_file): LOG.warning("%s: missing metadata file", metadata_file) - return "missing {} (or {}) metadata file".format( - METADATA_FILENAME, LEGACY_METADATA_FILENAME + return ( + f"missing {METADATA_FILENAME} (or {LEGACY_METADATA_FILENAME}) metadata file" ) if not parser.has_section("package"): diff --git a/zeekpkg/template.py b/zeekpkg/template.py index 7e841cf2..c1f08ada 100644 --- a/zeekpkg/template.py +++ b/zeekpkg/template.py @@ -153,11 +153,7 @@ def load(config, template, version=None): try: git_checkout(repo, version) except git.exc.GitCommandError as error: - msg = ( - 'failed to checkout branch/version "{}" of template {}: {}'.format( - version, template, error - ) - ) + msg = f'failed to checkout branch/version "{version}" of template {template}: {error}' LOG.warn(msg) raise GitError(msg) from error @@ -171,9 +167,7 @@ def load(config, template, version=None): except TypeError: pass # Not on a branch, do nothing except git.exc.GitCommandError as error: - msg = 'failed to update branch "{}" of template {}: {}'.format( - version, template, error - ) + msg = f'failed to update branch "{version}" of template {template}: {error}' LOG.warning(msg) raise GitError(msg) from error @@ -795,10 +789,10 @@ def _git_init(self, tmpl): ver_info += " (" + ver_sha[:8] + ")" repo.index.commit( - """Initial commit. + f"""Initial commit. -zkg {} created this package from template "{}" -using {}{}.""".format(__version__, tmpl.name(), ver_info, features_info) +zkg {__version__} created this package from template "{tmpl.name()}" +using {ver_info}{features_info}.""" ) diff --git a/zeekpkg/uservar.py b/zeekpkg/uservar.py index 6e455535..67f4cb4c 100644 --- a/zeekpkg/uservar.py +++ b/zeekpkg/uservar.py @@ -106,9 +106,7 @@ def resolve(self, name, config, user_var_args=None, force=False): if source: print( - '"{}" will use value of "{}" ({}) from {}: {}'.format( - name, self._name, self._desc, source, val - ) + f'"{name}" will use value of "{self._name}" ({self._desc}) from {source}: {val}' ) self._val = val return val diff --git a/zkg b/zkg index 5c18609b..5e0e2617 100755 --- a/zkg +++ b/zkg @@ -33,7 +33,7 @@ except ImportError as error: " pip3 install GitPython semantic-version\n" "\n" "Also check the following exception output for possible alternate explanations:\n\n" - "{}: {}".format(type(error).__name__, error), + f"{type(error).__name__}: {error}", file=sys.stderr, ) sys.exit(1) @@ -405,9 +405,7 @@ def create_manager(args, config): if permissions_trouble and not args.user: print_error( - "Consider the --user flag to manage zkg state via {}/config".format( - home_config_dir() - ) + f"Consider the --user flag to manage zkg state via {home_config_dir()}/config" ) sys.exit(1) @@ -681,9 +679,7 @@ def cmd_install(manager, args, config, configfile): extdeps = info.dependencies(field="external_depends") if extdeps is None: - extdep_listing += " from {} ({}):\n \n".format( - name, version - ) + extdep_listing += f" from {name} ({version}):\n \n" continue if extdeps: @@ -841,7 +837,7 @@ def cmd_install(manager, args, config, configfile): else: print( "The following installed packages could NOT be loaded " - 'to satisfy runtime dependencies for "{}"'.format(name) + f'to satisfy runtime dependencies for "{name}"' ) print(_listing) manager.restore_loaded_package_states(saved_state) @@ -965,11 +961,7 @@ def cmd_bundle(manager, args, config, configfile): package_listing += "\n" - print( - "The following packages will be BUNDLED into {}:".format( - args.bundle_filename - ) - ) + print(f"The following packages will be BUNDLED into {args.bundle_filename}:") print(package_listing) if not confirmation_prompt("Proceed?"): @@ -1006,9 +998,7 @@ def cmd_unbundle(manager, args, config, configfile): if pkg_info.invalid_reason: name = pkg_info.package.qualified_name() print_error( - "error: bundle {} contains invalid package {} ({}): {}".format( - args.bundle_filename, git_url, name, pkg_info.invalid_reason - ) + f"error: bundle {args.bundle_filename} contains invalid package {git_url} ({name}): {pkg_info.invalid_reason}" ) sys.exit(1) @@ -1049,9 +1039,7 @@ def cmd_unbundle(manager, args, config, configfile): extdeps = info.dependencies(field="external_depends") if extdeps is None: - extdep_listing += " from {} ({}):\n \n".format( - name, version - ) + extdep_listing += f" from {name} ({version}):\n \n" continue if extdeps: @@ -1444,9 +1432,7 @@ def cmd_upgrade(manager, args, config, configfile): extdeps = info.dependencies(field="external_depends") if extdeps is None: - extdep_listing += " from {} ({}):\n \n".format( - name, version - ) + extdep_listing += f" from {name} ({version}):\n \n" continue if extdeps: @@ -1634,7 +1620,7 @@ def cmd_load(manager, args, config, configfile): if dep_listing: print( "The following installed packages were additionally loaded to satisfy" - ' runtime dependencies for "{}".'.format(name) + f' runtime dependencies for "{name}".' ) print(dep_listing) @@ -1644,9 +1630,7 @@ def cmd_load(manager, args, config, configfile): if not args.nodeps: if dep_error_listing: print( - 'The following installed dependencies could not be loaded for "{}".'.format( - name - ) + f'The following installed dependencies could not be loaded for "{name}".' ) print(dep_error_listing) manager.restore_loaded_package_states(saved_state) @@ -1760,9 +1744,7 @@ def cmd_pin(manager, args, config, configfile): if ipkg: print( - 'Pinned "{}" at version: {} ({})'.format( - name, ipkg.status.current_version, ipkg.status.current_hash - ) + f'Pinned "{name}" at version: {ipkg.status.current_version} ({ipkg.status.current_hash})' ) else: had_failure = True @@ -1793,9 +1775,7 @@ def cmd_unpin(manager, args, config, configfile): if ipkg: print( - 'Unpinned "{}" from version: {} ({})'.format( - name, ipkg.status.current_version, ipkg.status.current_hash - ) + f'Unpinned "{name}" from version: {ipkg.status.current_version} ({ipkg.status.current_hash})' ) else: had_failure = True @@ -2288,9 +2268,7 @@ def cmd_create(manager, args, config, configfile): ) except OSError as err: print_error( - "error: could not remove package directory {}: {}".format( - args.packagedir, err - ) + f"error: could not remove package directory {args.packagedir}: {err}" ) sys.exit(1) @@ -2363,11 +2341,9 @@ def top_level_parser(): " ``ZKG_CONFIG_FILE``:\t" "Same as ``--configfile`` option, but has less precedence.\n" " ``ZKG_DEFAULT_SOURCE``:\t" - "The default package source to use (normally {}).\n" + f"The default package source to use (normally {ZKG_DEFAULT_SOURCE}).\n" " ``ZKG_DEFAULT_TEMPLATE``:\t" - "The default package template to use (normally {}).\n".format( - ZKG_DEFAULT_SOURCE, ZKG_DEFAULT_TEMPLATE - ), + f"The default package template to use (normally {ZKG_DEFAULT_TEMPLATE}).\n", ) top_parser.add_argument( "--version", action="version", version="%(prog)s " + zeekpkg.__version__ From 9c7714ff680337627d58216d602701966cf003d9 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:56 +0200 Subject: [PATCH 09/20] Fix imports with ruff --- doc/ext/sphinxarg/ext.py | 4 +-- doc/ext/sphinxarg/parser.py | 2 +- pyproject.toml | 2 +- setup.py | 1 + zeekpkg/manager.py | 60 +++++++++++++++++++------------------ zeekpkg/package.py | 3 +- zeekpkg/source.py | 7 +++-- zeekpkg/template.py | 16 +++++----- zkg | 6 ++-- 9 files changed, 51 insertions(+), 50 deletions(-) diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index 1eb4a5ce..b8014b9e 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -1,9 +1,9 @@ -from argparse import ArgumentParser import os +from argparse import ArgumentParser from docutils import nodes -from docutils.statemachine import StringList from docutils.parsers.rst.directives import flag, unchanged +from docutils.statemachine import StringList try: # Removed as of Sphinx 1.7 diff --git a/doc/ext/sphinxarg/parser.py b/doc/ext/sphinxarg/parser.py index 3ad2678a..b98a84c2 100644 --- a/doc/ext/sphinxarg/parser.py +++ b/doc/ext/sphinxarg/parser.py @@ -1,5 +1,5 @@ -from argparse import _HelpAction, _SubParsersAction import re +from argparse import _HelpAction, _SubParsersAction class NavigationException(Exception): diff --git a/pyproject.toml b/pyproject.toml index 56c6e056..8c2e1ed4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,4 +63,4 @@ script-files = ["zkg"] universal = true [tool.ruff.lint] -select = ["UP"] +select = ["I", "UP"] diff --git a/setup.py b/setup.py index 1ff886e3..18306008 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ import pathlib + from setuptools import setup diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index ef89799b..daceaac7 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -20,62 +20,64 @@ except ImportError: from urlparse import urlparse +from collections import deque + import git import semantic_version as semver -from collections import deque - +from . import ( + LOG, + __version__, +) from ._util import ( - make_dir, - delete_path, - make_symlink, + configparser_section_dict, copy_over_path, + delete_path, + find_program, get_zeek_info, - git_default_branch, + get_zeek_version, git_checkout, git_clone, + git_default_branch, git_pull, git_version_tags, is_sha1, - get_zeek_version, - std_encoding, - find_program, - read_zeek_config_line, + make_dir, + make_symlink, normalize_version_tag, - configparser_section_dict, + read_zeek_config_line, safe_tarfile_extractall, + std_encoding, ) -from .source import AGGREGATE_DATA_FILE, Source from .package import ( - BUILTIN_SOURCE, BUILTIN_SCHEME, - METADATA_FILENAME, + BUILTIN_SOURCE, LEGACY_METADATA_FILENAME, - TRACKING_METHOD_VERSION, - TRACKING_METHOD_BRANCH, - TRACKING_METHOD_COMMIT, - PLUGIN_MAGIC_FILE, - PLUGIN_MAGIC_FILE_DISABLED, LEGACY_PLUGIN_MAGIC_FILE, LEGACY_PLUGIN_MAGIC_FILE_DISABLED, - name_from_path, - aliases, - canonical_url, - is_valid_name as is_valid_package_name, - make_builtin_package, + METADATA_FILENAME, + PLUGIN_MAGIC_FILE, + PLUGIN_MAGIC_FILE_DISABLED, + TRACKING_METHOD_BRANCH, + TRACKING_METHOD_COMMIT, + TRACKING_METHOD_VERSION, + InstalledPackage, Package, PackageInfo, PackageStatus, - InstalledPackage, PackageVersion, + aliases, + canonical_url, + make_builtin_package, + name_from_path, ) +from .package import ( + is_valid_name as is_valid_package_name, +) +from .source import AGGREGATE_DATA_FILE, Source from .uservar import ( UserVar, ) -from . import ( - __version__, - LOG, -) class Stage: diff --git a/zeekpkg/package.py b/zeekpkg/package.py index eb1663a8..41cdd233 100644 --- a/zeekpkg/package.py +++ b/zeekpkg/package.py @@ -5,13 +5,12 @@ import os import re - from functools import total_ordering import semantic_version as semver -from .uservar import UserVar from ._util import find_sentence_end, normalize_version_tag +from .uservar import UserVar #: The name of files used by packages to store their metadata. METADATA_FILENAME = "zkg.meta" diff --git a/zeekpkg/source.py b/zeekpkg/source.py index 3b07909c..3de28bd7 100644 --- a/zeekpkg/source.py +++ b/zeekpkg/source.py @@ -7,13 +7,14 @@ """ import configparser -import git import os import shutil +import git + from . import LOG -from .package import name_from_path, Package -from ._util import git_default_branch, git_checkout, git_clone +from ._util import git_checkout, git_clone, git_default_branch +from .package import Package, name_from_path #: The name of package index files. INDEX_FILENAME = "zkg.index" diff --git a/zeekpkg/template.py b/zeekpkg/template.py index c1f08ada..7c287f16 100644 --- a/zeekpkg/template.py +++ b/zeekpkg/template.py @@ -4,20 +4,16 @@ import abc import configparser -import re import os +import re import shutil -import semantic_version as semver import git +import semantic_version as semver from . import ( - __version__, LOG, -) -from .package import ( - METADATA_FILENAME, - name_from_path, + __version__, ) from ._util import ( delete_path, @@ -25,11 +21,15 @@ git_clone, git_default_branch, git_pull, - git_version_tags, git_remote_urls, + git_version_tags, load_source, make_dir, ) +from .package import ( + METADATA_FILENAME, + name_from_path, +) API_VERSION = "1.1.0" diff --git a/zkg b/zkg index 5e0e2617..dafc026c 100755 --- a/zkg +++ b/zkg @@ -15,7 +15,6 @@ import shutil import subprocess import sys import threading - from collections import OrderedDict try: @@ -85,10 +84,11 @@ ZKG_DEFAULT_SOURCE = "https://github.com/zeek/packages" # The default package template ZKG_DEFAULT_TEMPLATE = "https://github.com/zeek/package-template" +import zeekpkg # noqa: E402 from zeekpkg._util import ( # noqa: E402 delete_path, - make_dir, find_program, + make_dir, read_zeek_config_line, std_encoding, ) @@ -104,8 +104,6 @@ from zeekpkg.uservar import ( # noqa: E402 UserVar, ) -import zeekpkg # noqa: E402 - def confirmation_prompt(prompt, default_to_yes=True): yes = {"y", "ye", "yes"} From b09afa49397d6e737f4614675c4279d4505f1fe8 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:57 +0200 Subject: [PATCH 10/20] Enable RUF lint set --- doc/ext/sphinxarg/ext.py | 3 ++- pyproject.toml | 2 +- zeekpkg/__init__.py | 10 +++++----- zeekpkg/package.py | 5 ++++- zeekpkg/template.py | 9 ++++----- zkg | 14 +++++++------- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index b8014b9e..0eec2745 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -1,5 +1,6 @@ import os from argparse import ArgumentParser +from typing import Callable, ClassVar from docutils import nodes from docutils.parsers.rst.directives import flag, unchanged @@ -195,7 +196,7 @@ def print_subcommand_list(data, nested_content): class ArgParseDirective(Directive): has_content = True - option_spec = dict( + option_spec: ClassVar[dict[str, Callable[[str | None], str | None]]] = dict( module=unchanged, func=unchanged, ref=unchanged, diff --git a/pyproject.toml b/pyproject.toml index 8c2e1ed4..8e34e9ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,4 +63,4 @@ script-files = ["zkg"] universal = true [tool.ruff.lint] -select = ["I", "UP"] +select = ["I", "RUF", "UP"] diff --git a/zeekpkg/__init__.py b/zeekpkg/__init__.py index 7adbc3fb..eab82b0c 100644 --- a/zeekpkg/__init__.py +++ b/zeekpkg/__init__.py @@ -10,12 +10,12 @@ import logging __version__ = "3.0.1-8" -__all__ = ["manager", "package", "source", "template", "uservar"] # noqa: F405 +__all__ = ["manager", "package", "source", "template", "uservar"] LOG = logging.getLogger(__name__) LOG.addHandler(logging.NullHandler()) -from .manager import * # noqa: E402, F403 -from .package import * # noqa: E402, F403 -from .source import * # noqa: E402, F403 -from .uservar import * # noqa: E402, F403 +from .manager import * +from .package import * +from .source import * +from .uservar import * diff --git a/zeekpkg/package.py b/zeekpkg/package.py index 41cdd233..f8df59df 100644 --- a/zeekpkg/package.py +++ b/zeekpkg/package.py @@ -6,6 +6,7 @@ import os import re from functools import total_ordering +from typing import Optional import semantic_version as semver @@ -587,7 +588,9 @@ def matches_path(self, path): return path == self.git_url -def make_builtin_package(*, name: str, current_version: str, current_hash: str = None): +def make_builtin_package( + *, name: str, current_version: str, current_hash: Optional[str] = None +): """ Given ``name``, ``version`` and ``commit`` as found in Zeek's ``zkg.provides`` entry, construct a :class:`PackageInfo` instance representing the built-in diff --git a/zeekpkg/template.py b/zeekpkg/template.py index 7c287f16..a588acc1 100644 --- a/zeekpkg/template.py +++ b/zeekpkg/template.py @@ -188,7 +188,6 @@ def load(config, template, version=None): # The above guards against absence of TEMPLATE_API_VERSION, so # appease pylint for the rest of this function while we access # it. - # pylint: disable=no-member try: is_compat = Template.is_api_compatible(mod.TEMPLATE_API_VERSION) @@ -310,7 +309,7 @@ def package(self): """ return None - def features(self): # pylint: disable=no-self-use + def features(self): """Provides any additional features templates supported. If the template provides extra features, return each as an @@ -419,7 +418,7 @@ def info(self): res["versions"] = [] res["has_repo"] = False - pkg = self.package() # pylint: disable=assignment-from-none + pkg = self.package() uvars = self.define_user_vars() feature_names = [] res["user_vars"] = {} @@ -669,7 +668,7 @@ def _walk(self, tmpl): yield in_file, out_path, out_file, out_content - def _replace(self, tmpl, content): # pylint: disable=no-self-use + def _replace(self, tmpl, content): """Helper for content substitution. Args: @@ -752,7 +751,7 @@ def _update_metadata(self, tmpl): config.remove_section(section) config.add_section(section) - for uvar in tmpl._get_user_vars(): # pylint: disable=protected-access + for uvar in tmpl._get_user_vars(): if uvar.val() is not None: config.set(section, uvar.name(), uvar.val()) diff --git a/zkg b/zkg index dafc026c..16d7da8a 100755 --- a/zkg +++ b/zkg @@ -19,7 +19,7 @@ from collections import OrderedDict try: import git - import semantic_version # noqa # pylint: disable=unused-import + import semantic_version except ImportError as error: print( "error: zkg failed to import one or more dependencies:\n" @@ -40,7 +40,7 @@ except ImportError as error: try: # Argcomplete provides command-line completion for users of argparse. # We support it if available, but don't complain when it isn't. - import argcomplete # pylint: disable=import-error + import argcomplete except ImportError: pass @@ -84,23 +84,23 @@ ZKG_DEFAULT_SOURCE = "https://github.com/zeek/packages" # The default package template ZKG_DEFAULT_TEMPLATE = "https://github.com/zeek/package-template" -import zeekpkg # noqa: E402 -from zeekpkg._util import ( # noqa: E402 +import zeekpkg +from zeekpkg._util import ( delete_path, find_program, make_dir, read_zeek_config_line, std_encoding, ) -from zeekpkg.package import ( # noqa: E402 +from zeekpkg.package import ( BUILTIN_SCHEME, TRACKING_METHOD_VERSION, ) -from zeekpkg.template import ( # noqa: E402 +from zeekpkg.template import ( LoadError, Template, ) -from zeekpkg.uservar import ( # noqa: E402 +from zeekpkg.uservar import ( UserVar, ) From ff451b594d36f9e27e941e21dd642d5ac33e8b0a Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:57 +0200 Subject: [PATCH 11/20] Enforce PEP8 naming conventions --- doc/ext/sphinxarg/parser.py | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ext/sphinxarg/parser.py b/doc/ext/sphinxarg/parser.py index b98a84c2..eb6fa432 100644 --- a/doc/ext/sphinxarg/parser.py +++ b/doc/ext/sphinxarg/parser.py @@ -2,7 +2,7 @@ from argparse import _HelpAction, _SubParsersAction -class NavigationException(Exception): +class NavigationError(Exception): pass @@ -15,7 +15,7 @@ def parser_navigate(parser_result, path, current_path=None): if len(path) == 0: return parser_result if "children" not in parser_result: - raise NavigationException( + raise NavigationError( "Current parser have no children elements. (path: %s)" % " ".join(current_path) ) @@ -26,7 +26,7 @@ def parser_navigate(parser_result, path, current_path=None): if identifier == next_hop: current_path.append(next_hop) return parser_navigate(child, path, current_path) - raise NavigationException( + raise NavigationError( f"Current parser have no children element with name: {next_hop} (path: %s)" % " ".join(current_path) ) diff --git a/pyproject.toml b/pyproject.toml index 8e34e9ba..12209ac9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,4 +63,4 @@ script-files = ["zkg"] universal = true [tool.ruff.lint] -select = ["I", "RUF", "UP"] +select = ["I", "N", "RUF", "UP"] From 4eac9ebcbfe4daf129c058baf010be6e0eca6c89 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:57 +0200 Subject: [PATCH 12/20] Fix accidental implicit str concats --- pyproject.toml | 2 +- zeekpkg/manager.py | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 12209ac9..12f11ee8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,4 +63,4 @@ script-files = ["zkg"] universal = true [tool.ruff.lint] -select = ["I", "N", "RUF", "UP"] +select = ["I", "ISC", "N", "RUF", "UP"] diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index daceaac7..2c182379 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -836,7 +836,7 @@ def save_temporary_config_files(self, installed_pkg): if not os.path.isfile(config_file_path): LOG.info( - "package '%s' claims config file at '%s'," " but it does not exist", + "package '%s' claims config file at '%s', but it does not exist", pkg_name, config_file, ) @@ -884,7 +884,7 @@ def modified_config_files(self, installed_pkg): if not os.path.isfile(their_config_file_path): LOG.info( - "package '%s' claims config file at '%s'," " but it does not exist", + "package '%s' claims config file at '%s', but it does not exist", pkg_name, config_file, ) @@ -897,7 +897,7 @@ def modified_config_files(self, installed_pkg): if not os.path.isfile(our_config_file_path): LOG.info( - "package '%s' config file '%s' not found" " in plugin_dir: %s", + "package '%s' config file '%s' not found in plugin_dir: %s", pkg_name, config_file, our_config_file_path, @@ -910,7 +910,7 @@ def modified_config_files(self, installed_pkg): if not os.path.isfile(our_config_file_path): LOG.info( - "package '%s' config file '%s' not found" " in script_dir: %s", + "package '%s' config file '%s' not found in script_dir: %s", pkg_name, config_file, our_config_file_path, @@ -1859,11 +1859,11 @@ def info(self, pkg_path, version="", prefer_installed=True): pkg_path, matches_string, ) - reason = str.format( - '"{}" matches multiple packages, try a more' " specific name from: {}", - pkg_path, - matches_string, + reason = ( + f'"{pkg_path}" matches multiple packages, ' + f"try a more specific name from: {matches_string}" ) + return PackageInfo(invalid_reason=reason, status=status) package = matches[0] @@ -2838,7 +2838,7 @@ def _stage(self, package, version, clone, stage, env=None): else: if "script_dir" in metadata: return str.format( - "no __load__.zeek file found" " in package's 'script_dir' : {}", + "no __load__.zeek file found in package's 'script_dir' : {}", pkg_script_dir, ) else: @@ -2956,10 +2956,10 @@ def install(self, pkg_path, version=""): pkg_path, matches_string, ) - return str.format( - '"{}" matches multiple packages, try a more' " specific name from: {}", - pkg_path, - matches_string, + + return ( + f'"{pkg_path}" matches multiple packages, ' + f"try a more specific name from: {matches_string}" ) try: From f6dee6129f7b405fbcc0a2dbe33efa2b91e1d3dd Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:58 +0200 Subject: [PATCH 13/20] Remove unnecessary comprehensions --- doc/ext/sphinxarg/ext.py | 22 +++++++++++----------- pyproject.toml | 2 +- zeekpkg/manager.py | 12 ++++++------ zeekpkg/package.py | 4 ++-- zeekpkg/source.py | 2 +- zkg | 24 ++++++++++++------------ 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index 0eec2745..576a0fba 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -196,17 +196,17 @@ def print_subcommand_list(data, nested_content): class ArgParseDirective(Directive): has_content = True - option_spec: ClassVar[dict[str, Callable[[str | None], str | None]]] = dict( - module=unchanged, - func=unchanged, - ref=unchanged, - prog=unchanged, - path=unchanged, - nodefault=flag, - manpage=unchanged, - nosubcommands=unchanged, - passparser=flag, - ) + option_spec: ClassVar[dict[str, Callable[[str | None], str | None]]] = { + "module": unchanged, + "func": unchanged, + "ref": unchanged, + "prog": unchanged, + "path": unchanged, + "nodefault": flag, + "manpage": unchanged, + "nosubcommands": unchanged, + "passparser": flag, + } def _construct_manpage_specific_structure(self, parser_info): """ diff --git a/pyproject.toml b/pyproject.toml index 12f11ee8..2d41b6bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,4 +63,4 @@ script-files = ["zkg"] universal = true [tool.ruff.lint] -select = ["I", "ISC", "N", "RUF", "UP"] +select = ["C4", "I", "ISC", "N", "RUF", "UP"] diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index 2c182379..48baac54 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -1592,17 +1592,17 @@ def list_depender_pkgs(self, pkg_path): item = queue.popleft() for _pkg_name in pkg_dependencies: - pkg_dependees = {_pkg for _pkg in pkg_dependencies.get(_pkg_name)} + pkg_dependees = set(pkg_dependencies.get(_pkg_name)) if item in pkg_dependees: # check if there is a cyclic dependency if _pkg_name == pkg_name: - return sorted([pkg for pkg in depender_packages] + [pkg_name]) + return sorted([*list(depender_packages), [pkg_name]]) queue.append(_pkg_name) depender_packages.add(_pkg_name) - return sorted([pkg for pkg in depender_packages]) + return sorted(depender_packages) def unload_with_unused_dependers(self, pkg_name): """Unmark dependent (but previously installed packages) as being "loaded". @@ -1976,8 +1976,8 @@ def __init__(self, name): self.info = None self.requested_version = None # (tracking method, version) self.installed_version = None # (tracking method, version) - self.dependers = dict() # name -> version, name needs self at version - self.dependees = dict() # name -> version, self needs name at version + self.dependers = {} # name -> version, name needs self at version + self.dependees = {} # name -> version, self needs name at version self.is_suggestion = False def __str__(self): @@ -1990,7 +1990,7 @@ def __str__(self): self.is_suggestion, ) - graph = dict() # Node.name -> Node, nodes store edges + graph = {} # Node.name -> Node, nodes store edges requests = [] # List of Node, just for requested packages # 1. Try to make nodes for everything in the dependency graph... diff --git a/zeekpkg/package.py b/zeekpkg/package.py index f8df59df..070f6f13 100644 --- a/zeekpkg/package.py +++ b/zeekpkg/package.py @@ -135,9 +135,9 @@ def dependencies(metadata_dict, field="depends"): number of values), then None is returned. """ if field not in metadata_dict: - return dict() + return {} - rval = dict() + rval = {} depends = metadata_dict[field] parts = depends.split() keys = parts[::2] diff --git a/zeekpkg/source.py b/zeekpkg/source.py index 3de28bd7..0fa76bab 100644 --- a/zeekpkg/source.py +++ b/zeekpkg/source.py @@ -133,7 +133,7 @@ def packages(self): metadata = {} if parser.has_section(agg_key): - metadata = {key: value for key, value in parser.items(agg_key)} + metadata = dict(parser.items(agg_key)) package = Package( git_url=url, diff --git a/zkg b/zkg index 16d7da8a..d4a7ed4d 100755 --- a/zkg +++ b/zkg @@ -1300,10 +1300,10 @@ def cmd_refresh(manager, args, config, configfile): if args.push: print("\tPushed aggregated metadata") - outdated_before = {i for i in outdated(manager)} + outdated_before = set(outdated(manager)) print("Refresh installed packages") manager.refresh_installed_packages() - outdated_after = {i for i in outdated(manager)} + outdated_after = set(outdated(manager)) if outdated_before == outdated_after: print("\tNo new outdated packages") @@ -1784,7 +1784,7 @@ def cmd_unpin(manager, args, config, configfile): def _get_filtered_packages(manager, category): - pkg_dict = dict() + pkg_dict = {} for ipkg in manager.installed_packages(): pkg_dict[ipkg.package.qualified_name()] = ipkg @@ -1915,7 +1915,7 @@ def cmd_info(manager, args, config, configfile): sys.exit(1) # Dictionary for storing package info to output as JSON - pkginfo = dict() + pkginfo = {} had_invalid_package = False if len(args.package) == 1: @@ -1942,8 +1942,8 @@ def cmd_info(manager, args, config, configfile): name = info.package.qualified_name() if args.json: - pkginfo[name] = dict() - pkginfo[name]["metadata"] = dict() + pkginfo[name] = {} + pkginfo[name]["metadata"] = {} else: print(f'"{name}" info:') @@ -1965,7 +1965,7 @@ def cmd_info(manager, args, config, configfile): if info.status: if args.json: - pkginfo[name]["install_status"] = dict() + pkginfo[name]["install_status"] = {} for key, value in sorted(info.status.__dict__.items()): pkginfo[name]["install_status"][key] = value @@ -1978,7 +1978,7 @@ def cmd_info(manager, args, config, configfile): if args.json: if info.metadata_file: pkginfo[name]["metadata_file"] = info.metadata_file - pkginfo[name]["metadata"][info.metadata_version] = dict() + pkginfo[name]["metadata"][info.metadata_version] = {} else: if info.metadata_file: print(f"\tmetadata file: {info.metadata_file}") @@ -2006,7 +2006,7 @@ def cmd_info(manager, args, config, configfile): info2 = manager.info( name, vers, prefer_installed=(not args.nolocal) ) - pkginfo[name]["metadata"][info2.metadata_version] = dict() + pkginfo[name]["metadata"][info2.metadata_version] = {} if info2.metadata_file: pkginfo[name]["metadata_file"] = info2.metadata_file _fill_metadata_version( @@ -2042,7 +2042,7 @@ def _fill_metadata_version(pkginfo_name_metadata_version, info_metadata): """ for key, value in info_metadata.items(): if key == "depends" or key == "suggests": - pkginfo_name_metadata_version[key] = dict() + pkginfo_name_metadata_version[key] = {} deps = value.split("\n") for i in range(1, len(deps)): @@ -2157,8 +2157,8 @@ def cmd_env(manager, args, config, configfile): if not pluginpath: pluginpath = line2 - zeekpaths = [p for p in zeekpath.split(":")] if zeekpath else [] - pluginpaths = [p for p in pluginpath.split(":")] if pluginpath else [] + zeekpaths = list(zeekpath.split(":")) if zeekpath else [] + pluginpaths = list(pluginpath.split(":")) if pluginpath else [] zeekpaths.append(manager.zeekpath()) pluginpaths.append(manager.zeek_plugin_path()) From c8df4e3e8eb45914fbd436b288e7a4240617fc09 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:58 +0200 Subject: [PATCH 14/20] Remove shadowing of builtins --- doc/conf.py | 2 +- pyproject.toml | 2 +- zeekpkg/manager.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 38eb944a..349f5c9f 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -50,7 +50,7 @@ # General information about the project. project = "Zeek Package Manager" -copyright = "2019, The Zeek Project" +copyright = "2019, The Zeek Project" # noqa: A001 author = "The Zeek Project" # The version info for the project you're documenting, acts as replacement for diff --git a/pyproject.toml b/pyproject.toml index 2d41b6bc..3fd8511a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,4 +63,4 @@ script-files = ["zkg"] universal = true [tool.ruff.lint] -select = ["C4", "I", "ISC", "N", "RUF", "UP"] +select = ["A", "C4", "I", "ISC", "N", "RUF", "UP"] diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index 48baac54..bf37108c 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -1356,8 +1356,8 @@ def remove(self, pkg_path): for alias in pkg_to_remove.aliases(): delete_path(os.path.join(self.zeekpath(), alias)) - for exec in self._get_executables(pkg_to_remove.metadata): - link = os.path.join(self.bin_dir, os.path.basename(exec)) + for exe in self._get_executables(pkg_to_remove.metadata): + link = os.path.join(self.bin_dir, os.path.basename(exe)) if os.path.islink(link): try: LOG.debug("removing link %s", link) @@ -3143,10 +3143,10 @@ def _interpolate_package_metadata(self, metadata, stage): # the currently installed packages. def _refresh_bin_dir(self, bin_dir, prev_bin_dir=None): for ipkg in self.installed_pkgs.values(): - for exec in self._get_executables(ipkg.package.metadata): + for exe in self._get_executables(ipkg.package.metadata): # Put symlinks in place that are missing in current directory - src = os.path.join(self.package_clonedir, ipkg.package.name, exec) - dst = os.path.join(bin_dir, os.path.basename(exec)) + src = os.path.join(self.package_clonedir, ipkg.package.name, exe) + dst = os.path.join(bin_dir, os.path.basename(exe)) if ( not os.path.exists(dst) @@ -3162,8 +3162,8 @@ def _refresh_bin_dir(self, bin_dir, prev_bin_dir=None): # coming with any of the currently installed package. def _clear_bin_dir(self, bin_dir): for ipkg in self.installed_pkgs.values(): - for exec in self._get_executables(ipkg.package.metadata): - old = os.path.join(bin_dir, os.path.basename(exec)) + for exe in self._get_executables(ipkg.package.metadata): + old = os.path.join(bin_dir, os.path.basename(exe)) if os.path.islink(old): try: os.unlink(old) From 7d106e275e888b92fd8ce2f9532afa4df3c64fbd Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:59 +0200 Subject: [PATCH 15/20] Enforce trainling commas --- doc/ext/sphinxarg/ext.py | 79 ++++++----- doc/ext/sphinxarg/parser.py | 9 +- pyproject.toml | 2 +- testing/templates/foo/__init__.py | 7 +- zeekpkg/_util.py | 6 +- zeekpkg/manager.py | 142 +++++++++++++------ zeekpkg/package.py | 8 +- zeekpkg/template.py | 13 +- zeekpkg/uservar.py | 4 +- zkg | 223 ++++++++++++++++++++---------- 10 files changed, 329 insertions(+), 164 deletions(-) diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index 576a0fba..adfb9c9a 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -71,8 +71,8 @@ def print_arg_list(data, nested_content): text=( "Possible choices: %s" % ", ".join([str(c) for c in arg["choices"]]) - ) - ) + ), + ), ) argname = name @@ -83,10 +83,11 @@ def print_arg_list(data, nested_content): nodes.option_list_item( "", nodes.option_group( - "", nodes.option("", nodes.option_string(text=argname)) + "", + nodes.option("", nodes.option_string(text=argname)), ), nodes.description("", *my_def), - ) + ), ) return nodes.option_list("", *items) if items else None @@ -102,7 +103,8 @@ def print_opt_list(data, nested_content): option_declaration = [nodes.option_string(text=name)] if opt["default"] is not None and opt["default"] != "==SUPPRESS==": option_declaration += nodes.option_argument( - "", text="=" + str(opt["default"]) + "", + text="=" + str(opt["default"]), ) names.append(nodes.option("", *option_declaration)) my_def = apply_definition(definitions, my_def, name) @@ -114,13 +116,15 @@ def print_opt_list(data, nested_content): text=( "Possible choices: %s" % ", ".join([str(c) for c in opt["choices"]]) - ) - ) + ), + ), ) items.append( nodes.option_list_item( - "", nodes.option_group("", *names), nodes.description("", *my_def) - ) + "", + nodes.option_group("", *names), + nodes.description("", *my_def), + ), ) return nodes.option_list("", *items) if items else None @@ -133,19 +137,23 @@ def print_command_args_and_opts(arg_list, opt_list, sub_list=None): "", nodes.term(text="Positional arguments:"), nodes.definition("", arg_list), - ) + ), ) if opt_list: items.append( nodes.definition_list_item( - "", nodes.term(text="Options:"), nodes.definition("", opt_list) - ) + "", + nodes.term(text="Options:"), + nodes.definition("", opt_list), + ), ) if sub_list and len(sub_list): items.append( nodes.definition_list_item( - "", nodes.term(text="Sub-commands:"), nodes.definition("", sub_list) - ) + "", + nodes.term(text="Sub-commands:"), + nodes.definition("", sub_list), + ), ) return nodes.definition_list("", *items) @@ -182,14 +190,14 @@ def print_subcommand_list(data, nested_content): print_arg_list(child, nested_content), print_opt_list(child, nested_content), print_subcommand_list(child, nested_content), - ) + ), ) items.append( nodes.definition_list_item( "", nodes.term("", "", nodes.strong(text=name)), nodes.definition("", *my_def), - ) + ), ) return nodes.definition_list("", *items) @@ -232,8 +240,9 @@ def _construct_manpage_specific_structure(self, parser_info): nodes.title(text="Description"), nodes.paragraph( text=parser_info.get( - "description", parser_info.get("help", "undocumented").capitalize() - ) + "description", + parser_info.get("help", "undocumented").capitalize(), + ), ), ids=["description-section"], ) @@ -246,7 +255,9 @@ def _construct_manpage_specific_structure(self, parser_info): description_section += nodes.paragraph(text=parser_info["epilog"]) # OPTIONS section options_section = nodes.section( - "", nodes.title(text="Options"), ids=["options-section"] + "", + nodes.title(text="Options"), + ids=["options-section"], ) if "args" in parser_info: options_section += nodes.paragraph() @@ -270,7 +281,9 @@ def _construct_manpage_specific_structure(self, parser_info): if "nosubcommands" not in self.options: # SUBCOMMANDS section (non-standard) subcommands_section = nodes.section( - "", nodes.title(text="Sub-Commands"), ids=["subcommands-section"] + "", + nodes.title(text="Sub-Commands"), + ids=["subcommands-section"], ) if "children" in parser_info: subcommands_section += self._format_subcommands(parser_info) @@ -301,17 +314,18 @@ def _format_positional_arguments(self, parser_info): if "choices" in arg: arg_items.append( nodes.paragraph( - text="Possible choices: " + ", ".join(arg["choices"]) - ) + text="Possible choices: " + ", ".join(arg["choices"]), + ), ) items.append( nodes.option_list_item( "", nodes.option_group( - "", nodes.option("", nodes.option_string(text=arg["metavar"])) + "", + nodes.option("", nodes.option_string(text=arg["metavar"])), ), nodes.description("", *arg_items), - ) + ), ) return nodes.option_list("", *items) @@ -325,7 +339,8 @@ def _format_optional_arguments(self, parser_info): option_declaration = [nodes.option_string(text=name)] if opt["default"] is not None and opt["default"] != "==SUPPRESS==": option_declaration += nodes.option_argument( - "", text="=" + str(opt["default"]) + "", + text="=" + str(opt["default"]), ) names.append(nodes.option("", *option_declaration)) if opt["help"]: @@ -335,15 +350,15 @@ def _format_optional_arguments(self, parser_info): if "choices" in opt: opt_items.append( nodes.paragraph( - text="Possible choices: " + ", ".join(opt["choices"]) - ) + text="Possible choices: " + ", ".join(opt["choices"]), + ), ) items.append( nodes.option_list_item( "", nodes.option_group("", *names), nodes.description("", *opt_items), - ) + ), ) return nodes.option_list("", *items) @@ -361,7 +376,7 @@ def _format_subcommands(self, parser_info): "", nodes.term("", "", nodes.strong(text=subcmd["bare_usage"])), nodes.definition("", *subcmd_items), - ) + ), ) return nodes.definition_list("", *items) @@ -397,13 +412,13 @@ def run(self): "lineno": self.lineno, "dependency_file": file_dependency, "dependency_mtime": os.stat(file_dependency).st_mtime, - } + }, ) if not hasattr(mod, attr_name): raise self.error( f'Module "{module_name}" has no attribute "{attr_name}"\n' - "Incorrect argparse :module: or :func: values?" + "Incorrect argparse :module: or :func: values?", ) func = getattr(mod, attr_name) if isinstance(func, ArgumentParser): @@ -444,7 +459,7 @@ def run(self): print_arg_list(result, nested_content), print_opt_list(result, nested_content), subcommands, - ) + ), ) if "epilog" in result: items.append(self._nested_parse_paragraph(result["epilog"])) diff --git a/doc/ext/sphinxarg/parser.py b/doc/ext/sphinxarg/parser.py index eb6fa432..85a94aa6 100644 --- a/doc/ext/sphinxarg/parser.py +++ b/doc/ext/sphinxarg/parser.py @@ -17,7 +17,7 @@ def parser_navigate(parser_result, path, current_path=None): if "children" not in parser_result: raise NavigationError( "Current parser have no children elements. (path: %s)" - % " ".join(current_path) + % " ".join(current_path), ) next_hop = path.pop(0) for child in parser_result["children"]: @@ -28,7 +28,7 @@ def parser_navigate(parser_result, path, current_path=None): return parser_navigate(child, path, current_path) raise NavigationError( f"Current parser have no children element with name: {next_hop} (path: %s)" - % " ".join(current_path) + % " ".join(current_path), ) @@ -49,7 +49,10 @@ def _format_usage_without_prefix(parser): """ fmt = parser._get_formatter() fmt.add_usage( - parser.usage, parser._actions, parser._mutually_exclusive_groups, prefix="" + parser.usage, + parser._actions, + parser._mutually_exclusive_groups, + prefix="", ) return fmt.format_help().strip() diff --git a/pyproject.toml b/pyproject.toml index 3fd8511a..d014126a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,4 +63,4 @@ script-files = ["zkg"] universal = true [tool.ruff.lint] -select = ["A", "C4", "I", "ISC", "N", "RUF", "UP"] +select = ["A", "C4", "COM", "I", "ISC", "N", "RUF", "UP"] diff --git a/testing/templates/foo/__init__.py b/testing/templates/foo/__init__.py index 777d836c..a2c6c5ad 100644 --- a/testing/templates/foo/__init__.py +++ b/testing/templates/foo/__init__.py @@ -28,10 +28,13 @@ class Template(zeekpkg.template.Template): def define_user_vars(self): return [ zeekpkg.uservar.UserVar( - "name", desc='the name of the package, e.g. "FooBar"' + "name", + desc='the name of the package, e.g. "FooBar"', ), zeekpkg.uservar.UserVar( - "readme", desc="Content of the README file", val="This is a README." + "readme", + desc="Content of the README file", + val="This is a README.", ), ] diff --git a/zeekpkg/_util.py b/zeekpkg/_util.py index 0b967a6d..e7f96a44 100644 --- a/zeekpkg/_util.py +++ b/zeekpkg/_util.py @@ -126,7 +126,11 @@ def git_clone(git_url, dst_path, shallow=False): if shallow: try: git.Git().clone( - git_url, dst_path, "--no-single-branch", recursive=True, depth=1 + git_url, + dst_path, + "--no-single-branch", + recursive=True, + depth=1, ) except git.exc.GitCommandError: if not git_url.startswith(".") and not git_url.startswith("/"): diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index bf37108c..1e012b60 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -309,7 +309,8 @@ def __init__( # Place all Zeek built-in packages into installed packages. for info in self.discover_builtin_packages(): self.installed_pkgs[info.package.name] = InstalledPackage( - package=info.package, status=info.status + package=info.package, + status=info.status, ) refresh_bin_dir = False # whether we need to updates link in bin_dir @@ -349,7 +350,7 @@ def __init__( refresh_bin_dir = True if prev_bin_dir and os.path.realpath(prev_bin_dir) != os.path.realpath( - self.bin_dir + self.bin_dir, ): LOG.info("relocating bin_dir %s -> %s", prev_bin_dir, self.bin_dir) need_manifest_update = True @@ -425,7 +426,8 @@ def _write_plugin_magic(self, ipkg): ] for path_enabled, path_disabled in zip( - magic_paths_enabled, magic_paths_disabled + magic_paths_enabled, + magic_paths_disabled, ): if ipkg.status.is_loaded: if path_disabled.exists(): @@ -494,7 +496,7 @@ def _write_manifest(self): { "package_dict": installed_pkg.package.__dict__, "status_dict": installed_pkg.status.__dict__, - } + }, ) data = { @@ -622,7 +624,9 @@ def discover_builtin_packages(self): try: build_info_str = subprocess.check_output( - [zeek_executable, "--build-info"], stderr=subprocess.DEVNULL, timeout=10 + [zeek_executable, "--build-info"], + stderr=subprocess.DEVNULL, + timeout=10, ) build_info = json.loads(build_info_str) except subprocess.CalledProcessError: @@ -668,7 +672,7 @@ def discover_builtin_packages(self): name=name, current_version=version, current_hash=commit, - ) + ), ) return self._builtin_packages @@ -892,7 +896,8 @@ def modified_config_files(self, installed_pkg): if config_file.startswith(plugin_dir): our_config_file_path = os.path.join( - plugin_install_dir, config_file[len(plugin_dir) :] + plugin_install_dir, + config_file[len(plugin_dir) :], ) if not os.path.isfile(our_config_file_path): @@ -905,7 +910,8 @@ def modified_config_files(self, installed_pkg): continue elif config_file.startswith(script_dir): our_config_file_path = os.path.join( - script_install_dir, config_file[len(script_dir) :] + script_install_dir, + config_file[len(script_dir) :], ) if not os.path.isfile(our_config_file_path): @@ -1048,7 +1054,8 @@ def _refresh_source(self, name, aggregate=False, push=False): aggregate_file = os.path.join(source.clone.working_dir, AGGREGATE_DATA_FILE) agg_file_ours = os.path.join(self.scratch_dir, AGGREGATE_DATA_FILE) agg_file_their_orig = os.path.join( - self.scratch_dir, AGGREGATE_DATA_FILE + ".orig" + self.scratch_dir, + AGGREGATE_DATA_FILE + ".orig", ) delete_path(agg_file_ours) @@ -1069,7 +1076,7 @@ def _refresh_source(self, name, aggregate=False, push=False): except git.exc.GitCommandError as error: LOG.error("failed to pull source %s: %s", name, error) return self.SourceAggregationResults( - f"failed to pull from remote source: {error}" + f"failed to pull from remote source: {error}", ) if os.path.isfile(agg_file_ours): @@ -1081,7 +1088,7 @@ def _refresh_source(self, name, aggregate=False, push=False): # Their file hasn't changed, use ours. shutil.copy2(agg_file_ours, aggregate_file) LOG.debug( - "aggegrate file in source unchanged, restore local one" + "aggegrate file in source unchanged, restore local one", ) else: # Their file changed, use theirs. @@ -1125,7 +1132,9 @@ def _refresh_source(self, name, aggregate=False, push=False): clone = git_clone(url, clonepath, shallow=True) except git.exc.GitCommandError as error: LOG.warn( - "failed to clone %s, skipping aggregation: %s", url, error + "failed to clone %s, skipping aggregation: %s", + url, + error, ) aggregation_issues.append((url, repr(error))) continue @@ -1147,14 +1156,17 @@ def _refresh_source(self, name, aggregate=False, push=False): url, error, ) - msg = f'failed to checkout branch/version "{version}": {repr(error)}' + msg = ( + f'failed to checkout branch/version "{version}": {error!r}' + ) aggregation_issues.append((url, msg)) continue metadata_file = _pick_metadata_file(clone.working_dir) metadata_parser = configparser.ConfigParser(interpolation=None) invalid_reason = _parse_package_metadata( - metadata_parser, metadata_file + metadata_parser, + metadata_file, ) if invalid_reason: @@ -1184,7 +1196,8 @@ def _refresh_source(self, name, aggregate=False, push=False): agg_adds.append(qualified_name) else: prev_meta = configparser_section_dict( - prev_parser, qualified_name + prev_parser, + qualified_name, ) new_meta = configparser_section_dict(parser, qualified_name) if prev_meta != new_meta: @@ -1211,7 +1224,7 @@ def _refresh_source(self, name, aggregate=False, push=False): if push: if os.path.isfile( - os.path.join(source.clone.working_dir, AGGREGATE_DATA_FILE) + os.path.join(source.clone.working_dir, AGGREGATE_DATA_FILE), ): source.clone.git.add(AGGREGATE_DATA_FILE) @@ -1223,7 +1236,9 @@ def _refresh_source(self, name, aggregate=False, push=False): # why one would use zkg for this as opposed to git # itself. source.clone.git.commit( - "--no-verify", "--message", "Update aggregated metadata." + "--no-verify", + "--message", + "Update aggregated metadata.", ) LOG.info('committed package source "%s" metadata update', name) @@ -1243,7 +1258,8 @@ def refresh_installed_packages(self): for ipkg in self.installed_packages(): if ipkg.is_builtin(): LOG.debug( - 'skipping refresh of built-in package "%s"', ipkg.package.name + 'skipping refresh of built-in package "%s"', + ipkg.package.name, ) continue @@ -1261,7 +1277,9 @@ def refresh_installed_packages(self): ) ipkg.status.is_outdated = _is_clone_outdated( - clone, ipkg.status.current_version, ipkg.status.tracking_method + clone, + ipkg.status.current_version, + ipkg.status.tracking_method, ) self._write_manifest() @@ -1474,7 +1492,9 @@ def load(self, pkg_path): return "" pkg_load_script = os.path.join( - self.script_dir, ipkg.package.name, "__load__.zeek" + self.script_dir, + ipkg.package.name, + "__load__.zeek", ) if not os.path.exists(pkg_load_script) and not self.has_plugin(ipkg): @@ -1689,7 +1709,7 @@ def _has_all_dependers_unloaded(item, dependers): ( item, f"Package is in use by other packages --- {dep_listing[:-2]}.", - ) + ), ) return errors @@ -1774,7 +1794,9 @@ def bundle_info(self, bundle_file): for git_url, version in manifest: package = Package( - git_url=git_url, name=git_url.split("/")[-1], canonical=True + git_url=git_url, + name=git_url.split("/")[-1], + canonical=True, ) pkg_path = os.path.join(bundle_dir, package.name) LOG.debug('getting info for bundled package "%s"', package.name) @@ -1841,7 +1863,9 @@ def info(self, pkg_path, version="", prefer_installed=True): return self._info(package, status, version) except git.exc.GitCommandError as error: LOG.info( - 'getting info on "%s": invalid git repo path: %s', pkg_path, error + 'getting info on "%s": invalid git repo path: %s', + pkg_path, + error, ) LOG.info('getting info on "%s": matched no source package', pkg_path) @@ -2032,7 +2056,8 @@ def __str__(self): if ds is None: return ( str.format( - 'package "{}" has malformed "suggests" field', node.name + 'package "{}" has malformed "suggests" field', + node.name, ), [], ) @@ -2109,7 +2134,8 @@ def __str__(self): if zeek_version: node = Node("zeek") node.installed_version = PackageVersion( - TRACKING_METHOD_VERSION, zeek_version + TRACKING_METHOD_VERSION, + zeek_version, ) graph["zeek"] = node else: @@ -2117,7 +2143,8 @@ def __str__(self): node = Node("zkg") node.installed_version = PackageVersion( - TRACKING_METHOD_VERSION, __version__ + TRACKING_METHOD_VERSION, + __version__, ) graph["zkg"] = node @@ -2159,7 +2186,8 @@ def __str__(self): if ds is None: return ( str.format( - 'package "{}" has malformed "suggests" field', node.name + 'package "{}" has malformed "suggests" field', + node.name, ), [], ) @@ -2222,7 +2250,7 @@ def __str__(self): # A new package nothing depends on -- odd? new_pkgs.append( - (node.info, node.info.best_version(), node.is_suggestion) + (node.info, node.info.best_version(), node.is_suggestion), ) continue @@ -2271,12 +2299,15 @@ def __str__(self): def no_best_version_string(node): rval = str.format( - '"{}" has no version satisfying dependencies:\n', node.name + '"{}" has no version satisfying dependencies:\n', + node.name, ) for depender_name, version_spec in node.dependers.items(): rval += str.format( - '\t"{}" requires: "{}"\n', depender_name, version_spec + '\t"{}" requires: "{}"\n', + depender_name, + version_spec, ) return rval @@ -2486,7 +2517,9 @@ def unbundle(self, bundle_file): for git_url, version in manifest: package = Package( - git_url=git_url, name=git_url.split("/")[-1], canonical=True + git_url=git_url, + name=git_url.split("/")[-1], + canonical=True, ) # Prepare the clonepath with the contents from the bundle. @@ -2598,7 +2631,9 @@ def test(self, pkg_path, version="", test_dependencies=False): # staging area. for info, version in reversed(pkgs): LOG.debug( - 'preparing "%s" for testing: version %s', info.package.name, version + 'preparing "%s" for testing: version %s', + info.package.name, + version, ) clonepath = os.path.join(stage.clone_dir, info.package.name) @@ -2625,7 +2660,9 @@ def test(self, pkg_path, version="", test_dependencies=False): LOG.warning("failed to checkout git repo version: %s", error) return ( str.format( - "failed to checkout {} of {}", version, info.package.git_url + "failed to checkout {} of {}", + version, + info.package.git_url, ), False, stage.state_dir, @@ -2646,7 +2683,8 @@ def test(self, pkg_path, version="", test_dependencies=False): LOG.info('testing "%s"', package) # Interpolate the test command: metadata, invalid_reason = self._interpolate_package_metadata( - info.metadata, stage + info.metadata, + stage, ) if invalid_reason: return (invalid_reason, False, stage.state_dir) @@ -2745,7 +2783,9 @@ def _stage(self, package, version, clone, stage, env=None): build_command = metadata.get("build_command", "") if build_command: LOG.debug( - 'building "%s": running build_command: %s', package, build_command + 'building "%s": running build_command: %s', + package, + build_command, ) bufsize = 4096 build = subprocess.Popen( @@ -2763,7 +2803,9 @@ def _stage(self, package, version, clone, stage, env=None): with open(buildlog, "wb") as f: LOG.info( - 'installing "%s": writing build log: %s', package, buildlog + 'installing "%s": writing build log: %s', + package, + buildlog, ) f.write("=== STDERR ===\n".encode(std_encoding(sys.stderr))) @@ -2806,7 +2848,8 @@ def _stage(self, package, version, clone, stage, env=None): if not os.path.exists(script_dir_src): return str.format( - "package's 'script_dir' does not exist: {}", pkg_script_dir + "package's 'script_dir' does not exist: {}", + pkg_script_dir, ) pkgload = os.path.join(script_dir_src, "__load__.zeek") @@ -2814,13 +2857,15 @@ def _stage(self, package, version, clone, stage, env=None): if os.path.isfile(pkgload): try: symlink_path = os.path.join( - os.path.dirname(stage.script_dir), package.name + os.path.dirname(stage.script_dir), + package.name, ) make_symlink(os.path.join("packages", package.name), symlink_path) for alias in aliases(metadata): symlink_path = os.path.join( - os.path.dirname(stage.script_dir), alias + os.path.dirname(stage.script_dir), + alias, ) make_symlink(os.path.join("packages", package.name), symlink_path) @@ -2830,7 +2875,11 @@ def _stage(self, package, version, clone, stage, env=None): return error error = _copy_package_dir( - package, "script_dir", script_dir_src, script_dir_dst, self.scratch_dir + package, + "script_dir", + script_dir_src, + script_dir_dst, + self.scratch_dir, ) if error: @@ -2863,11 +2912,16 @@ def _stage(self, package, version, clone, stage, env=None): # It's common for a package to not have build directory for # plugins, so don't error out in that case, just log it. return str.format( - "package's 'plugin_dir' does not exist: {}", pkg_plugin_dir + "package's 'plugin_dir' does not exist: {}", + pkg_plugin_dir, ) error = _copy_package_dir( - package, "plugin_dir", plugin_dir_src, plugin_dir_dst, self.scratch_dir + package, + "plugin_dir", + plugin_dir_src, + plugin_dir_dst, + self.scratch_dir, ) if error: @@ -3053,7 +3107,9 @@ def _install(self, package, version, use_existing_clone=False): status.tracking_method = TRACKING_METHOD_BRANCH else: LOG.info( - 'branch "%s" not in available branches: %s', version, branches + 'branch "%s" not in available branches: %s', + version, + branches, ) return f'no such branch or version tag: "{version}"' diff --git a/zeekpkg/package.py b/zeekpkg/package.py index 070f6f13..3f7a3a35 100644 --- a/zeekpkg/package.py +++ b/zeekpkg/package.py @@ -244,7 +244,8 @@ def fullfills(self, version_spec): Does the current version fullfill version_spec? """ return PackageVersion( - self.status.tracking_method, self.status.current_version + self.status.tracking_method, + self.status.current_version, ).fullfills(version_spec) @@ -589,7 +590,10 @@ def matches_path(self, path): def make_builtin_package( - *, name: str, current_version: str, current_hash: Optional[str] = None + *, + name: str, + current_version: str, + current_hash: Optional[str] = None, ): """ Given ``name``, ``version`` and ``commit`` as found in Zeek's ``zkg.provides`` diff --git a/zeekpkg/template.py b/zeekpkg/template.py index a588acc1..1eec06c4 100644 --- a/zeekpkg/template.py +++ b/zeekpkg/template.py @@ -113,7 +113,9 @@ def load(config, template, version=None): # zkg state folder's clone space and support version # requests. template_clonedir = os.path.join( - config.get("paths", "state_dir"), "clones", "template" + config.get("paths", "state_dir"), + "clones", + "template", ) templatedir = os.path.join(template_clonedir, name_from_path(template)) make_dir(template_clonedir) @@ -180,7 +182,7 @@ def load(config, template, version=None): if not hasattr(mod, "TEMPLATE_API_VERSION"): msg = "template{} does not indicate its API version".format( - " version " + version if version else "" + " version " + version if version else "", ) LOG.error(msg) raise LoadError(msg) @@ -193,7 +195,7 @@ def load(config, template, version=None): is_compat = Template.is_api_compatible(mod.TEMPLATE_API_VERSION) except ValueError: raise LoadError( - f'API version string "{mod.TEMPLATE_API_VERSION}" is invalid' + f'API version string "{mod.TEMPLATE_API_VERSION}" is invalid', ) if not is_compat: @@ -437,7 +439,8 @@ def info(self): res["user_vars"][uvar_name]["used_by"].append("package") except KeyError: LOG.warning( - 'Package requires undefined user var "%s", skipping', uvar_name + 'Package requires undefined user var "%s", skipping', + uvar_name, ) for feature in self.features(): @@ -791,7 +794,7 @@ def _git_init(self, tmpl): f"""Initial commit. zkg {__version__} created this package from template "{tmpl.name()}" -using {ver_info}{features_info}.""" +using {ver_info}{features_info}.""", ) diff --git a/zeekpkg/uservar.py b/zeekpkg/uservar.py index 67f4cb4c..dcdbd000 100644 --- a/zeekpkg/uservar.py +++ b/zeekpkg/uservar.py @@ -106,7 +106,7 @@ def resolve(self, name, config, user_var_args=None, force=False): if source: print( - f'"{name}" will use value of "{self._name}" ({self._desc}) from {source}: {val}' + f'"{name}" will use value of "{self._name}" ({self._desc}) from {source}: {val}', ) self._val = val return val @@ -135,7 +135,7 @@ def parse_arg(arg): return UserVar(name, val=val) except ValueError as error: raise ValueError( - f'invalid user var argument "{arg}", must be NAME=VAR' + f'invalid user var argument "{arg}", must be NAME=VAR', ) from error @staticmethod diff --git a/zkg b/zkg index d4a7ed4d..44432f40 100755 --- a/zkg +++ b/zkg @@ -143,7 +143,10 @@ def prompt_for_user_vars(manager, config, configfile, args, pkg_infos): for uvar in requested_user_vars: try: answers[uvar.name()] = uvar.resolve( - name, config, args.user_var, args.force + name, + config, + args.user_var, + args.force, ) except ValueError: print_error( @@ -151,7 +154,7 @@ def prompt_for_user_vars(manager, config, configfile, args, pkg_infos): 'error: could not determine value of user variable "{}",' " provide via environment or --user-var", uvar.name(), - ) + ), ) sys.exit(1) @@ -265,10 +268,16 @@ def create_config(args, configfile): state_dir = get_option(config, "paths", "state_dir", os.path.join(def_state_dir)) script_dir = get_option( - config, "paths", "script_dir", os.path.join(state_dir, "script_dir") + config, + "paths", + "script_dir", + os.path.join(state_dir, "script_dir"), ) plugin_dir = get_option( - config, "paths", "plugin_dir", os.path.join(state_dir, "plugin_dir") + config, + "paths", + "plugin_dir", + os.path.join(state_dir, "plugin_dir"), ) bin_dir = get_option(config, "paths", "bin_dir", os.path.join(state_dir, "bin")) zeek_dist = get_option(config, "paths", "zeek_dist", "") @@ -297,7 +306,7 @@ def create_config(args, configfile): " an absolute path", key, value, - ) + ), ) sys.exit(1) @@ -398,12 +407,12 @@ def create_manager(args, config): check_permission(script_dir), check_permission(plugin_dir), check_permission(bin_dir), - ] + ], ) if permissions_trouble and not args.user: print_error( - f"Consider the --user flag to manage zkg state via {home_config_dir()}/config" + f"Consider the --user flag to manage zkg state via {home_config_dir()}/config", ) sys.exit(1) @@ -430,8 +439,10 @@ def create_manager(args, config): if error: print_error( str.format( - 'warning: skipped using package source named "{}": {}', key, error - ) + 'warning: skipped using package source named "{}": {}', + key, + error, + ), ) return manager @@ -539,8 +550,10 @@ def cmd_test(manager, args, config, configfile): if package_info.invalid_reason: print_error( str.format( - 'error: invalid package "{}": {}', name, package_info.invalid_reason - ) + 'error: invalid package "{}": {}', + name, + package_info.invalid_reason, + ), ) sys.exit(1) @@ -559,13 +572,15 @@ def cmd_test(manager, args, config, configfile): continue error_msg, passed, test_dir = manager.test( - name, version, test_dependencies=True + name, + version, + test_dependencies=True, ) if error_msg: all_passed = False print_error( - str.format('error: failed to run tests for "{}": {}', name, error_msg) + str.format('error: failed to run tests for "{}": {}', name, error_msg), ) continue @@ -574,7 +589,8 @@ def cmd_test(manager, args, config, configfile): else: all_passed = False clone_dir = os.path.join( - os.path.join(test_dir, "clones"), info.package.name + os.path.join(test_dir, "clones"), + info.package.name, ) print_error( str.format( @@ -585,7 +601,7 @@ def cmd_test(manager, args, config, configfile): name, test_dir, clone_dir, - ) + ), ) if not all_passed: @@ -615,8 +631,10 @@ def cmd_install(manager, args, config, configfile): if package_info.invalid_reason: print_error( str.format( - 'error: invalid package "{}": {}', name, package_info.invalid_reason - ) + 'error: invalid package "{}": {}', + name, + package_info.invalid_reason, + ), ) sys.exit(1) @@ -635,7 +653,8 @@ def cmd_install(manager, args, config, configfile): for info, version, _ in package_infos ] invalid_reason, new_pkgs = manager.validate_dependencies( - to_validate, ignore_suggestions=args.nosuggestions + to_validate, + ignore_suggestions=args.nosuggestions, ) if invalid_reason: @@ -656,7 +675,8 @@ def cmd_install(manager, args, config, configfile): dependency_listing = "" for info, version, suggested in sorted( - new_pkgs, key=lambda x: x[0].package.name + new_pkgs, + key=lambda x: x[0].package.name, ): name = info.package.qualified_name() dependency_listing += f" {name} ({version})" @@ -690,7 +710,7 @@ def cmd_install(manager, args, config, configfile): print( "Verify the following REQUIRED external dependencies:\n" "(Ensure their installation on all relevant systems before" - " proceeding):" + " proceeding):", ) print(extdep_listing) @@ -700,7 +720,11 @@ def cmd_install(manager, args, config, configfile): package_infos += new_pkgs prompt_for_user_vars( - manager, config, configfile, args, [info for info, _, _ in package_infos] + manager, + config, + configfile, + args, + [info for info, _, _ in package_infos], ) if not args.skiptests: @@ -711,7 +735,7 @@ def cmd_install(manager, args, config, configfile): if "test_command" not in info.metadata: zeekpkg.LOG.info( - f'Skipping unit tests for "{name}": no test_command in metadata' + f'Skipping unit tests for "{name}": no test_command in metadata', ) continue @@ -721,13 +745,16 @@ def cmd_install(manager, args, config, configfile): # well fail without them. If the user wants --nodeps and the tests # fail because of it, they'll also need to say --skiptests. error, passed, test_dir = manager.test( - name, version, test_dependencies=True + name, + version, + test_dependencies=True, ) if error: error_msg = str.format("failed to run tests for {}: {}", name, error) elif not passed: clone_dir = os.path.join( - os.path.join(test_dir, "clones"), info.package.name + os.path.join(test_dir, "clones"), + info.package.name, ) error_msg = str.format( '"{}" tests failed, inspect contents of' @@ -746,7 +773,8 @@ def cmd_install(manager, args, config, configfile): sys.exit(1) if not confirmation_prompt( - "Proceed to install anyway?", default_to_yes=False + "Proceed to install anyway?", + default_to_yes=False, ): return @@ -815,7 +843,7 @@ def cmd_install(manager, args, config, configfile): name = info.package.qualified_name() load_error = manager.load_with_dependencies( - zeekpkg.package.name_from_path(name) + zeekpkg.package.name_from_path(name), ) for _name, _error in load_error: @@ -828,14 +856,14 @@ def cmd_install(manager, args, config, configfile): if dep_listing: print( "The following installed packages were additionally " - "loaded to satisfy runtime dependencies" + "loaded to satisfy runtime dependencies", ) print(dep_listing) else: print( "The following installed packages could NOT be loaded " - f'to satisfy runtime dependencies for "{name}"' + f'to satisfy runtime dependencies for "{name}"', ) print(_listing) manager.restore_loaded_package_states(saved_state) @@ -843,7 +871,7 @@ def cmd_install(manager, args, config, configfile): if installs_failed: print_error( "error: incomplete installation, the follow packages" - " failed to be installed:" + " failed to be installed:", ) for n, v in installs_failed: @@ -885,8 +913,10 @@ def cmd_bundle(manager, args, config, configfile): if info.invalid_reason: print_error( str.format( - 'error: invalid package "{}": {}', name, info.invalid_reason - ) + 'error: invalid package "{}": {}', + name, + info.invalid_reason, + ), ) sys.exit(1) @@ -901,7 +931,7 @@ def cmd_bundle(manager, args, config, configfile): version, False, False, - ) + ), ) if not args.nodeps: @@ -923,7 +953,7 @@ def cmd_bundle(manager, args, config, configfile): version, True, suggested, - ) + ), ) else: prefer_existing_clones = True @@ -936,7 +966,7 @@ def cmd_bundle(manager, args, config, configfile): ipkg.status.current_version, False, False, - ) + ), ) if not packages_to_bundle: @@ -967,7 +997,9 @@ def cmd_bundle(manager, args, config, configfile): git_urls = [(git_url, version) for _, git_url, version, _, _ in packages_to_bundle] error = manager.bundle( - args.bundle_filename, git_urls, prefer_existing_clones=prefer_existing_clones + args.bundle_filename, + git_urls, + prefer_existing_clones=prefer_existing_clones, ) if error: @@ -996,7 +1028,7 @@ def cmd_unbundle(manager, args, config, configfile): if pkg_info.invalid_reason: name = pkg_info.package.qualified_name() print_error( - f"error: bundle {args.bundle_filename} contains invalid package {git_url} ({name}): {pkg_info.invalid_reason}" + f"error: bundle {args.bundle_filename} contains invalid package {git_url} ({name}): {pkg_info.invalid_reason}", ) sys.exit(1) @@ -1050,7 +1082,7 @@ def cmd_unbundle(manager, args, config, configfile): print( "Verify the following REQUIRED external dependencies:\n" "(Ensure their installation on all relevant systems before" - " proceeding):" + " proceeding):", ) print(extdep_listing) @@ -1058,7 +1090,11 @@ def cmd_unbundle(manager, args, config, configfile): return prompt_for_user_vars( - manager, config, configfile, args, [info for _, _, info in bundle_info] + manager, + config, + configfile, + args, + [info for _, _, info in bundle_info], ) error = manager.unbundle(args.bundle_filename) @@ -1245,7 +1281,7 @@ def cmd_refresh(manager, args, config, configfile): if args.fail_on_aggregate_problems and not args.aggregate: print_error( "warning: --fail-on-aggregate-problems without --aggregate" - " has no effect." + " has no effect.", ) had_failure = False @@ -1287,7 +1323,7 @@ def cmd_refresh(manager, args, config, configfile): if aggregation_issues: print( "\tWARNING: Metadata aggregated, but excludes the " - "following packages due to described problems:" + "following packages due to described problems:", ) for url, issue in aggregation_issues: @@ -1365,12 +1401,18 @@ def cmd_upgrade(manager, args, config, configfile): name = ipkg.package.git_url info = manager.info( - name, version=ipkg.status.current_version, prefer_installed=False + name, + version=ipkg.status.current_version, + prefer_installed=False, ) if info.invalid_reason: print_error( - str.format('error: invalid package "{}": {}', name, info.invalid_reason) + str.format( + 'error: invalid package "{}": {}', + name, + info.invalid_reason, + ), ) sys.exit(1) @@ -1395,7 +1437,8 @@ def cmd_upgrade(manager, args, config, configfile): for info, next_version, _ in outdated_packages ] invalid_reason, new_pkgs = manager.validate_dependencies( - to_validate, ignore_suggestions=args.nosuggestions + to_validate, + ignore_suggestions=args.nosuggestions, ) if invalid_reason: @@ -1443,7 +1486,7 @@ def cmd_upgrade(manager, args, config, configfile): print( "Verify the following REQUIRED external dependencies:\n" "(Ensure their installation on all relevant systems before" - " proceeding):" + " proceeding):", ) print(extdep_listing) @@ -1451,7 +1494,11 @@ def cmd_upgrade(manager, args, config, configfile): return prompt_for_user_vars( - manager, config, configfile, args, [info for info, _, _ in allpkgs] + manager, + config, + configfile, + args, + [info for info, _, _ in allpkgs], ) if not args.skiptests: @@ -1463,13 +1510,14 @@ def cmd_upgrade(manager, args, config, configfile): next_info = manager.info(name, version=version, prefer_installed=False) if next_info.invalid_reason: print_error( - f'error: invalid package "{name}": {next_info.invalid_reason}' + f'error: invalid package "{name}": {next_info.invalid_reason}', ) sys.exit(1) if "test_command" not in next_info.metadata: zeekpkg.LOG.info( - 'Skipping unit tests for "%s": no test_command in metadata', name + 'Skipping unit tests for "%s": no test_command in metadata', + name, ) continue @@ -1479,14 +1527,17 @@ def cmd_upgrade(manager, args, config, configfile): # might well fail without them. If the user wants --nodeps and the # tests fail because of it, they'll also need to say --skiptests. error, passed, test_dir = manager.test( - name, version, test_dependencies=True + name, + version, + test_dependencies=True, ) if error: error_msg = str.format("failed to run tests for {}: {}", name, error) elif not passed: clone_dir = os.path.join( - os.path.join(test_dir, "clones"), info.package.name + os.path.join(test_dir, "clones"), + info.package.name, ) error_msg = str.format( '"{}" tests failed, inspect contents of' @@ -1505,7 +1556,8 @@ def cmd_upgrade(manager, args, config, configfile): sys.exit(1) if not confirmation_prompt( - "Proceed to install anyway?", default_to_yes=False + "Proceed to install anyway?", + default_to_yes=False, ): return @@ -1604,7 +1656,7 @@ def cmd_load(manager, args, config, configfile): dep_error_listing, load_error = "", False loaded_dep_list = manager.load_with_dependencies( - zeekpkg.package.name_from_path(name) + zeekpkg.package.name_from_path(name), ) for _name, _error in loaded_dep_list: @@ -1618,7 +1670,7 @@ def cmd_load(manager, args, config, configfile): if dep_listing: print( "The following installed packages were additionally loaded to satisfy" - f' runtime dependencies for "{name}".' + f' runtime dependencies for "{name}".', ) print(dep_listing) @@ -1628,7 +1680,7 @@ def cmd_load(manager, args, config, configfile): if not args.nodeps: if dep_error_listing: print( - f'The following installed dependencies could not be loaded for "{name}".' + f'The following installed dependencies could not be loaded for "{name}".', ) print(dep_error_listing) manager.restore_loaded_package_states(saved_state) @@ -1742,7 +1794,7 @@ def cmd_pin(manager, args, config, configfile): if ipkg: print( - f'Pinned "{name}" at version: {ipkg.status.current_version} ({ipkg.status.current_hash})' + f'Pinned "{name}" at version: {ipkg.status.current_version} ({ipkg.status.current_hash})', ) else: had_failure = True @@ -1773,7 +1825,7 @@ def cmd_unpin(manager, args, config, configfile): if ipkg: print( - f'Unpinned "{name}" from version: {ipkg.status.current_version} ({ipkg.status.current_hash})' + f'Unpinned "{name}" from version: {ipkg.status.current_version} ({ipkg.status.current_hash})', ) else: had_failure = True @@ -1922,7 +1974,8 @@ def cmd_info(manager, args, config, configfile): try: package_names = [] for pkg_name, info in _get_filtered_packages( - manager, args.package[0] + manager, + args.package[0], ).items(): if info.is_builtin() and not args.include_builtin: continue @@ -1935,7 +1988,9 @@ def cmd_info(manager, args, config, configfile): for name in package_names: info = manager.info( - name, version=args.version, prefer_installed=(not args.nolocal) + name, + version=args.version, + prefer_installed=(not args.nolocal), ) if info.package: @@ -1990,7 +2045,8 @@ def cmd_info(manager, args, config, configfile): else: if args.json: _fill_metadata_version( - pkginfo[name]["metadata"][info.metadata_version], info.metadata + pkginfo[name]["metadata"][info.metadata_version], + info.metadata, ) else: for key, value in sorted(info.metadata.items()): @@ -2004,7 +2060,9 @@ def cmd_info(manager, args, config, configfile): # Skip the version that was already processed if vers != info.metadata_version: info2 = manager.info( - name, vers, prefer_installed=(not args.nolocal) + name, + vers, + prefer_installed=(not args.nolocal), ) pkginfo[name]["metadata"][info2.metadata_version] = {} if info2.metadata_file: @@ -2219,8 +2277,10 @@ def cmd_create(manager, args, config, configfile): print_error( "error: the following features are unknown: {}." ' Template "{}" offers {}.'.format( - unknowns, tmpl.name(), knowns or "no features" - ) + unknowns, + tmpl.name(), + knowns or "no features", + ), ) sys.exit(1) @@ -2237,7 +2297,7 @@ def cmd_create(manager, args, config, configfile): 'error: could not determine value of user variable "{}",' " provide via environment or --user-var", uvar.name(), - ) + ), ) sys.exit(1) @@ -2262,11 +2322,12 @@ def cmd_create(manager, args, config, configfile): try: delete_path(args.packagedir) zeekpkg.LOG.info( - "Removed existing package directory %s", args.packagedir + "Removed existing package directory %s", + args.packagedir, ) except OSError as err: print_error( - f"error: could not remove package directory {args.packagedir}: {err}" + f"error: could not remove package directory {args.packagedir}: {err}", ) sys.exit(1) @@ -2313,7 +2374,7 @@ def cmd_template_info(manager, args, config, configfile): uvar_info["description"], uvar_info["default"] or "no default", ", ".join(uvar_info["used_by"]) or "not used", - ) + ), ) print("versions: " + ", ".join(tmplinfo["versions"])) @@ -2344,7 +2405,9 @@ def top_level_parser(): f"The default package template to use (normally {ZKG_DEFAULT_TEMPLATE}).\n", ) top_parser.add_argument( - "--version", action="version", version="%(prog)s " + zeekpkg.__version__ + "--version", + action="version", + version="%(prog)s " + zeekpkg.__version__, ) group = top_parser.add_mutually_exclusive_group() @@ -2524,7 +2587,9 @@ def argparser(): " to first terminate that argument list.", ) sub_parser.add_argument( - "--force", action="store_true", help="Skip the confirmation prompt." + "--force", + action="store_true", + help="Skip the confirmation prompt.", ) sub_parser.add_argument( "--nodeps", @@ -2585,7 +2650,9 @@ def argparser(): sub_parser.set_defaults(run_cmd=cmd_remove) sub_parser.add_argument("package", nargs="+", help=pkg_name_help) sub_parser.add_argument( - "--force", action="store_true", help="Skip the confirmation prompt." + "--force", + action="store_true", + help="Skip the confirmation prompt.", ) sub_parser.add_argument( "--nodeps", @@ -2605,7 +2672,9 @@ def argparser(): ) sub_parser.set_defaults(run_cmd=cmd_purge) sub_parser.add_argument( - "--force", action="store_true", help="Skip the confirmation prompt." + "--force", + action="store_true", + help="Skip the confirmation prompt.", ) # refresh @@ -2695,7 +2764,10 @@ def argparser(): ) sub_parser.set_defaults(run_cmd=cmd_load) sub_parser.add_argument( - "package", nargs="+", default=[], help="Name(s) of package(s) to load." + "package", + nargs="+", + default=[], + help="Name(s) of package(s) to load.", ) sub_parser.add_argument( "--nodeps", @@ -2719,7 +2791,9 @@ def argparser(): sub_parser.set_defaults(run_cmd=cmd_unload) sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) sub_parser.add_argument( - "--force", action="store_true", help="Skip the confirmation prompt." + "--force", + action="store_true", + help="Skip the confirmation prompt.", ) sub_parser.add_argument( "--nodeps", @@ -2892,7 +2966,9 @@ def argparser(): ) sub_parser.set_defaults(run_cmd=cmd_autoconfig) sub_parser.add_argument( - "--force", action="store_true", help="Skip any confirmation prompt." + "--force", + action="store_true", + help="Skip any confirmation prompt.", ) # env @@ -2998,7 +3074,8 @@ def main(): args = argparser().parse_args() formatter = logging.Formatter( - "%(asctime)s %(levelname)-8s %(message)s", "%Y-%m-%d %H:%M:%S" + "%(asctime)s %(levelname)-8s %(message)s", + "%Y-%m-%d %H:%M:%S", ) handler = logging.StreamHandler() handler.setFormatter(formatter) From 6aabe285de2af87cf43cb2f5434a72dd55913bc9 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:09:59 +0200 Subject: [PATCH 16/20] Enforce flake8-bugbear rules --- pyproject.toml | 2 +- testing/templates/foo/__init__.py | 6 ++++++ zeekpkg/manager.py | 15 +++++++++------ zeekpkg/template.py | 4 +++- zkg | 6 +++--- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d014126a..90271e2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,4 +63,4 @@ script-files = ["zkg"] universal = true [tool.ruff.lint] -select = ["A", "C4", "COM", "I", "ISC", "N", "RUF", "UP"] +select = ["A", "B", "C4", "COM", "I", "ISC", "N", "RUF", "UP"] diff --git a/testing/templates/foo/__init__.py b/testing/templates/foo/__init__.py index a2c6c5ad..11c74aa3 100644 --- a/testing/templates/foo/__init__.py +++ b/testing/templates/foo/__init__.py @@ -23,6 +23,9 @@ def contentdir(self): def needed_user_vars(self): return ["readme"] + def validate(self, tmpl): + pass + class Template(zeekpkg.template.Template): def define_user_vars(self): @@ -51,3 +54,6 @@ def package(self): def features(self): return [Readme()] + + def validate(self, tmpl): + pass diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index 1e012b60..c5fd3552 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -985,9 +985,9 @@ class SourceAggregationResults: the failure. """ - def __init__(self, refresh_error="", package_issues=[]): + def __init__(self, refresh_error="", package_issues=None): self.refresh_error = refresh_error - self.package_issues = package_issues + self.package_issues = package_issues if package_issues else [] def aggregate_source(self, name, push=False): """Pull latest git info from a package source and aggregate metadata. @@ -1540,7 +1540,7 @@ def restore_loaded_package_states(self, saved_state): self._write_autoloader() self._write_manifest() - def load_with_dependencies(self, pkg_name, visited=set()): + def load_with_dependencies(self, pkg_name, visited=None): """Mark dependent (but previously installed) packages as being "loaded". Args: @@ -1553,6 +1553,9 @@ def load_with_dependencies(self, pkg_name, visited=set()): it was marked as loaded or else an explanation of why the loading failed. """ + if visited is None: + visited = set() + ipkg = self.find_installed_package(pkg_name) # skip loading a package if it is not installed. @@ -2324,7 +2327,7 @@ def no_best_version_string(node): if need_branch: branch_name = None - for depender_name, version_spec in node.dependers.items(): + for _, version_spec in node.dependers.items(): if version_spec == "*": continue @@ -2539,7 +2542,7 @@ def unbundle(self, bundle_file): # # Possible reasons are built-in packages on the source system missing # on the destination system or usage of --nodeps when creating the bundle. - for git_url, version in manifest: + for git_url, _ in manifest: deps = self.get_installed_package_dependencies(git_url) if deps is None: LOG.warning('package "%s" not installed?', git_url) @@ -2679,7 +2682,7 @@ def test(self, pkg_path, version="", test_dependencies=False): else: test_pkgs = [(pkg_info, version)] - for info, version in reversed(test_pkgs): + for info, _ in reversed(test_pkgs): LOG.info('testing "%s"', package) # Interpolate the test command: metadata, invalid_reason = self._interpolate_package_metadata( diff --git a/zeekpkg/template.py b/zeekpkg/template.py index 1eec06c4..6e723de1 100644 --- a/zeekpkg/template.py +++ b/zeekpkg/template.py @@ -196,7 +196,7 @@ def load(config, template, version=None): except ValueError: raise LoadError( f'API version string "{mod.TEMPLATE_API_VERSION}" is invalid', - ) + ) from None if not is_compat: msg = "template{} API version is incompatible with zkg ({} vs {})".format( @@ -512,6 +512,7 @@ def do_validate(self, tmpl): for feature in self._features: feature.validate(tmpl) + @abc.abstractmethod def validate(self, tmpl): """Validation of template configuration for this component. @@ -528,6 +529,7 @@ def validate(self, tmpl): zeekpkg.template.InputError when failing validation. """ + pass def do_instantiate(self, tmpl, packagedir, use_force=False): """Main driver for instantiating template content. diff --git a/zkg b/zkg index 44432f40..54fdab4f 100755 --- a/zkg +++ b/zkg @@ -838,7 +838,7 @@ def cmd_install(manager, args, config, configfile): if not args.nodeps: # Now load runtime dependencies after all dependencies and suggested # packages have been installed and loaded. - for info, ver, _ in sorted(orig_pkgs, key=lambda x: x[0].package.name): + for info, _, _ in sorted(orig_pkgs, key=lambda x: x[0].package.name): _listing, saved_state = "", manager.loaded_package_states() name = info.package.qualified_name() @@ -1024,7 +1024,7 @@ def cmd_unbundle(manager, args, config, configfile): print_error(f"error: failed to unbundle {args.bundle_filename}: {error}") sys.exit(1) - for git_url, version, pkg_info in bundle_info: + for git_url, _, pkg_info in bundle_info: if pkg_info.invalid_reason: name = pkg_info.package.qualified_name() print_error( @@ -1584,7 +1584,7 @@ def cmd_upgrade(manager, args, config, configfile): had_failure = False - for info, next_version, _ in outdated_packages: + for info, _, _ in outdated_packages: name = info.package.qualified_name() if not manager.match_source_packages(name): From 3adcc3fe91ac0b0122650c4167630a4f322375eb Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:10:00 +0200 Subject: [PATCH 17/20] Replace `str.format` uses with f-strings --- zeekpkg/manager.py | 104 +++++++++++-------------------------------- zkg | 107 +++++++++++++-------------------------------- 2 files changed, 55 insertions(+), 156 deletions(-) diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index c5fd3552..5ef27118 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -2008,13 +2008,12 @@ def __init__(self, name): self.is_suggestion = False def __str__(self): - return str.format( - "{}\n\trequested: {}\n\tinstalled: {}\n\tdependers: {}\n\tsuggestion: {}", - self.name, - self.requested_version, - self.installed_version, - self.dependers, - self.is_suggestion, + return ( + f"{self.name}\n\t" + f"requested: {self.requested_version}\n\t" + f"installed: {self.installed_version}\n\t" + f"dependers: {self.dependers}\n\t" + f"suggestion: {self.is_suggestion}" ) graph = {} # Node.name -> Node, nodes store edges @@ -2049,7 +2048,7 @@ def __str__(self): if dd is None: return ( - str.format('package "{}" has malformed "depends" field', node.name), + f'package "{node.name}" has malformed "depends" field', [], ) @@ -2058,10 +2057,7 @@ def __str__(self): if not ignore_suggestions: if ds is None: return ( - str.format( - 'package "{}" has malformed "suggests" field', - node.name, - ), + f'package "{node.name}" has malformed "suggests" field', [], ) @@ -2094,12 +2090,7 @@ def __str__(self): if info.invalid_reason: return ( - str.format( - 'package "{}" has invalid dependency "{}": {}', - node.name, - dep_name, - info.invalid_reason, - ), + f'package "{node.name}" has invalid dependency "{dep_name}": {info.invalid_reason}', [], ) @@ -2179,7 +2170,7 @@ def __str__(self): if dd is None: return ( - str.format('package "{}" has malformed "depends" field', node.name), + f'package "{node.name}" has malformed "depends" field', [], ) @@ -2188,10 +2179,7 @@ def __str__(self): if not ignore_suggestions: if ds is None: return ( - str.format( - 'package "{}" has malformed "suggests" field', - node.name, - ), + f'package "{node.name}" has malformed "suggests" field', [], ) @@ -2263,15 +2251,8 @@ def __str__(self): msg, fullfills = node.requested_version.fullfills(version_spec) if not fullfills: return ( - str.format( - 'unsatisfiable dependency: requested "{}" ({}),' - ' but "{}" requires {} ({})', - node.name, - node.requested_version.version, - depender_name, - version_spec, - msg, - ), + f'unsatisfiable dependency: requested "{node.name}" ({node.requested_version.version}),' + f' but "{depender_name}" requires {version_spec} ({msg})', new_pkgs, ) @@ -2283,15 +2264,8 @@ def __str__(self): msg, fullfills = node.installed_version.fullfills(version_spec) if not fullfills: return ( - str.format( - 'unsatisfiable dependency: "{}" ({}) is installed,' - ' but "{}" requires {} ({})', - node.name, - node.installed_version.version, - depender_name, - version_spec, - msg, - ), + f'unsatisfiable dependency: "{node.name}" ({node.installed_version.version}) is installed,' + f' but "{depender_name}" requires {version_spec} ({msg})', new_pkgs, ) else: @@ -2301,17 +2275,10 @@ def __str__(self): need_version = False def no_best_version_string(node): - rval = str.format( - '"{}" has no version satisfying dependencies:\n', - node.name, - ) + rval = f'"{node.name}" has no version satisfying dependencies:\n' for depender_name, version_spec in node.dependers.items(): - rval += str.format( - '\t"{}" requires: "{}"\n', - depender_name, - version_spec, - ) + rval += f'\t"{depender_name}" requires: "{version_spec}"\n' return rval @@ -2354,11 +2321,7 @@ def no_best_version_string(node): semver_spec = semver.Spec(version_spec) except ValueError: return ( - str.format( - 'package "{}" has invalid semver spec: {}', - depender_name, - version_spec, - ), + f'package "{depender_name}" has invalid semver spec: {version_spec}', new_pkgs, ) @@ -2662,11 +2625,7 @@ def test(self, pkg_path, version="", test_dependencies=False): except git.exc.GitCommandError as error: LOG.warning("failed to checkout git repo version: %s", error) return ( - str.format( - "failed to checkout {} of {}", - version, - info.package.git_url, - ), + f"failed to checkout {version} of {info.package.git_url}", False, stage.state_dir, ) @@ -2850,10 +2809,7 @@ def _stage(self, package, version, clone, stage, env=None): script_dir_dst = os.path.join(stage.script_dir, package.name) if not os.path.exists(script_dir_src): - return str.format( - "package's 'script_dir' does not exist: {}", - pkg_script_dir, - ) + return f"package's 'script_dir' does not exist: {pkg_script_dir}" pkgload = os.path.join(script_dir_src, "__load__.zeek") @@ -2889,10 +2845,7 @@ def _stage(self, package, version, clone, stage, env=None): return error else: if "script_dir" in metadata: - return str.format( - "no __load__.zeek file found in package's 'script_dir' : {}", - pkg_script_dir, - ) + return f"no __load__.zeek file found in package's 'script_dir' : {pkg_script_dir}" else: LOG.warning( 'installing "%s": no __load__.zeek in implicit' @@ -2914,10 +2867,7 @@ def _stage(self, package, version, clone, stage, env=None): if pkg_plugin_dir != "build": # It's common for a package to not have build directory for # plugins, so don't error out in that case, just log it. - return str.format( - "package's 'plugin_dir' does not exist: {}", - pkg_plugin_dir, - ) + return f"package's 'plugin_dir' does not exist: {pkg_plugin_dir}" error = _copy_package_dir( package, @@ -2934,10 +2884,10 @@ def _stage(self, package, version, clone, stage, env=None): for p in self._get_executables(metadata): full_path = os.path.join(clone.working_dir, p) if not os.path.isfile(full_path): - return str.format("executable '{}' is missing", p) + return f"executable '{p}' is missing" if not os.access(full_path, os.X_OK): - return str.format("file '{}' is not executable", p) + return f"file '{p}' is not executable" if stage.bin_dir is not None: make_symlink( @@ -2988,11 +2938,7 @@ def install(self, pkg_path, version=""): pkg_path, conflict, ) - return str.format( - 'package with name "{}" ({}) is already installed', - conflict.name, - conflict, - ) + return f'package with name "{conflict.name}" ({conflict}) is already installed' matches = self.match_source_packages(pkg_path) diff --git a/zkg b/zkg index 54fdab4f..037b2ff7 100755 --- a/zkg +++ b/zkg @@ -137,7 +137,7 @@ def prompt_for_user_vars(manager, config, configfile, args, pkg_infos): requested_user_vars = info.user_vars() if requested_user_vars is None: - print_error(str.format('error: malformed user_vars in "{}"', name)) + print_error(f'error: malformed user_vars in "{name}"') sys.exit(1) for uvar in requested_user_vars: @@ -150,11 +150,8 @@ def prompt_for_user_vars(manager, config, configfile, args, pkg_infos): ) except ValueError: print_error( - str.format( - 'error: could not determine value of user variable "{}",' - " provide via environment or --user-var", - uvar.name(), - ), + f'error: could not determine value of user variable "{uvar.name()}",' + " provide via environment or --user-var", ) sys.exit(1) @@ -300,13 +297,9 @@ def create_config(args, configfile): for key, value in config.items("paths"): if value and not os.path.isabs(value): print_error( - str.format( - "error: invalid config file value for key" - ' "{}" in section [paths]: "{}" is not' - " an absolute path", - key, - value, - ), + "error: invalid config file value for key" + f' "{key}" in section [paths]: "{value}" is not' + " an absolute path", ) sys.exit(1) @@ -438,11 +431,7 @@ def create_manager(args, config): if error: print_error( - str.format( - 'warning: skipped using package source named "{}": {}', - key, - error, - ), + f'warning: skipped using package source named "{key}": {error}', ) return manager @@ -549,11 +538,7 @@ def cmd_test(manager, args, config, configfile): if package_info.invalid_reason: print_error( - str.format( - 'error: invalid package "{}": {}', - name, - package_info.invalid_reason, - ), + f'error: invalid package "{name}": {package_info.invalid_reason}', ) sys.exit(1) @@ -568,7 +553,7 @@ def cmd_test(manager, args, config, configfile): name = info.package.qualified_name() if "test_command" not in info.metadata: - print(str.format("{}: no test_command found in metadata, skipping", name)) + print(f"{name}: no test_command found in metadata, skipping") continue error_msg, passed, test_dir = manager.test( @@ -579,13 +564,11 @@ def cmd_test(manager, args, config, configfile): if error_msg: all_passed = False - print_error( - str.format('error: failed to run tests for "{}": {}', name, error_msg), - ) + print_error(f'error: failed to run tests for "{name}": {error_msg}') continue if passed: - print(str.format("{}: all tests passed", name)) + print(f"{name}: all tests passed") else: all_passed = False clone_dir = os.path.join( @@ -593,15 +576,10 @@ def cmd_test(manager, args, config, configfile): info.package.name, ) print_error( - str.format( - 'error: package "{}" tests failed, inspect' - " contents of {} for details, especially" - ' any "zkg.test_command.{{stderr,stdout}}"' - " files within {}", - name, - test_dir, - clone_dir, - ), + f'error: package "{name}" tests failed, inspect' + f" contents of {test_dir} for details, especially" + ' any "zkg.test_command.{{stderr,stdout}}"' + f" files within {clone_dir}", ) if not all_passed: @@ -630,11 +608,7 @@ def cmd_install(manager, args, config, configfile): if package_info.invalid_reason: print_error( - str.format( - 'error: invalid package "{}": {}', - name, - package_info.invalid_reason, - ), + f'error: invalid package "{name}": {package_info.invalid_reason}', ) sys.exit(1) @@ -750,20 +724,17 @@ def cmd_install(manager, args, config, configfile): test_dependencies=True, ) if error: - error_msg = str.format("failed to run tests for {}: {}", name, error) + error_msg = f"failed to run tests for {name}: {error}" elif not passed: clone_dir = os.path.join( os.path.join(test_dir, "clones"), info.package.name, ) - error_msg = str.format( - '"{}" tests failed, inspect contents of' - " {} for details, especially any" + error_msg = ( + f'"{name}" tests failed, inspect contents of' + f" {test_dir} for details, especially any" ' "zkg.test_command.{{stderr,stdout}}"' - " files within {}", - name, - test_dir, - clone_dir, + f" files within {clone_dir}" ) if error_msg: @@ -911,13 +882,7 @@ def cmd_bundle(manager, args, config, configfile): info = manager.info(name, version=version, prefer_installed=False) if info.invalid_reason: - print_error( - str.format( - 'error: invalid package "{}": {}', - name, - info.invalid_reason, - ), - ) + print_error(f'error: invalid package "{name}": {info.invalid_reason}') sys.exit(1) if not version: @@ -1407,13 +1372,7 @@ def cmd_upgrade(manager, args, config, configfile): ) if info.invalid_reason: - print_error( - str.format( - 'error: invalid package "{}": {}', - name, - info.invalid_reason, - ), - ) + print_error(f'error: invalid package "{name}": {info.invalid_reason}') sys.exit(1) next_version = ipkg.status.current_version @@ -1533,20 +1492,17 @@ def cmd_upgrade(manager, args, config, configfile): ) if error: - error_msg = str.format("failed to run tests for {}: {}", name, error) + error_msg = f"failed to run tests for {name}: {error}" elif not passed: clone_dir = os.path.join( os.path.join(test_dir, "clones"), info.package.name, ) - error_msg = str.format( - '"{}" tests failed, inspect contents of' - " {} for details, especially any" + error_msg = ( + f'"{name}" tests failed, inspect contents of' + f" {test_dir} for details, especially any" ' "zkg.test_command.{{stderr,stdout}}"' - " files within {}", - name, - test_dir, - clone_dir, + f" files within {clone_dir}" ) if error_msg: @@ -2293,11 +2249,8 @@ def cmd_create(manager, args, config, configfile): uvar.resolve(tmpl.name(), config, args.user_var, args.force) except ValueError: print_error( - str.format( - 'error: could not determine value of user variable "{}",' - " provide via environment or --user-var", - uvar.name(), - ), + f'error: could not determine value of user variable "{uvar.name()}",' + " provide via environment or --user-var", ) sys.exit(1) From d6d221b8dd2ca59fdeb7523070def4175dc50a0e Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:10:00 +0200 Subject: [PATCH 18/20] Enforce pyflakes lint set --- .flake8 | 11 ----------- doc/ext/sphinxarg/ext.py | 3 ++- pyproject.toml | 2 +- zeekpkg/__init__.py | 10 +++++----- zkg | 2 +- 5 files changed, 9 insertions(+), 19 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 9d4f1410..00000000 --- a/.flake8 +++ /dev/null @@ -1,11 +0,0 @@ -[flake8] -max_line_length = 100 -# E203: whitespace before ':' (black / flake8 disagreement) -# W503: line break before binary operator (black / flake8 disagreement) -ignore=E203,W503 -# E402: module level import not at top of file -# F405: may be undefined, or defined from star imports -# F403: from .manager import *' used; unable to detect undefined names -per-file-ignores = - zeekpkg/__init__.py: F405,F403,E402 - zkg: E402 diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index adfb9c9a..e26ce391 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -1,6 +1,7 @@ import os from argparse import ArgumentParser -from typing import Callable, ClassVar +from collections.abc import Callable +from typing import ClassVar from docutils import nodes from docutils.parsers.rst.directives import flag, unchanged diff --git a/pyproject.toml b/pyproject.toml index 90271e2c..d93c532d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,4 +63,4 @@ script-files = ["zkg"] universal = true [tool.ruff.lint] -select = ["A", "B", "C4", "COM", "I", "ISC", "N", "RUF", "UP"] +select = ["A", "B", "C4", "COM", "F", "I", "ISC", "N", "RUF", "UP"] diff --git a/zeekpkg/__init__.py b/zeekpkg/__init__.py index eab82b0c..2dd7fcdb 100644 --- a/zeekpkg/__init__.py +++ b/zeekpkg/__init__.py @@ -10,12 +10,12 @@ import logging __version__ = "3.0.1-8" -__all__ = ["manager", "package", "source", "template", "uservar"] +__all__ = ["manager", "package", "source", "template", "uservar"] # noqa: F405 LOG = logging.getLogger(__name__) LOG.addHandler(logging.NullHandler()) -from .manager import * -from .package import * -from .source import * -from .uservar import * +from .manager import * # noqa: F403 +from .package import * # noqa: F403 +from .source import * # noqa: F403 +from .uservar import * # noqa: F403 diff --git a/zkg b/zkg index 037b2ff7..b65f58e2 100755 --- a/zkg +++ b/zkg @@ -19,7 +19,7 @@ from collections import OrderedDict try: import git - import semantic_version + import semantic_version # noqa: F401 except ImportError as error: print( "error: zkg failed to import one or more dependencies:\n" From 6411a76b94531144af803053ed0f9705524f6dc5 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:10:01 +0200 Subject: [PATCH 19/20] Require at least python-3.9 --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d93c532d..ef3c1549 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,8 @@ dynamic = ["version"] description = "The Zeek Package Manager" readme = "README" +requires-python = ">= 3.9" + keywords = [ "zeek", "zeekctl", From c95a8238e7291fb0385075b98ea711ec114ef896 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Sat, 13 Apr 2024 22:10:01 +0200 Subject: [PATCH 20/20] Remove python2 `urlparse` --- zeekpkg/manager.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index 5ef27118..a78a99a3 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -14,13 +14,8 @@ import subprocess import sys import tarfile - -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - from collections import deque +from urllib.parse import urlparse import git import semantic_version as semver