From 533a8df6fdd420b2bf6739f51c5887a08b6a3368 Mon Sep 17 00:00:00 2001 From: JackAtGaia <86691241+JackAtGaia@users.noreply.github.com> Date: Wed, 23 Feb 2022 14:21:02 -0800 Subject: [PATCH] Gaiaplat 1838 : Applying PyLint and Resolving any warnings from Docker_Dev (#1360) * GAIAPLAT-1838 - Starting with initial GDEV https://gaiaplatform.atlassian.net/browse/GAIAPLAT-1838 * Fixing Pylint Warnings --- .pre-commit-config.yaml | 2 +- dev_tools/docker_dev/gdev/__main__.py | 3 + .../docker_dev/gdev/cmd/gen/_abc/build.py | 78 ++--- dev_tools/docker_dev/gdev/cmd/gen/_abc/cfg.py | 45 +-- .../gdev/cmd/gen/_abc/dockerfile.py | 77 +++-- .../docker_dev/gdev/cmd/gen/_abc/push.py | 4 +- dev_tools/docker_dev/gdev/cmd/gen/_abc/run.py | 33 ++- .../docker_dev/gdev/cmd/gen/_custom/build.py | 3 +- .../docker_dev/gdev/cmd/gen/_custom/cfg.py | 12 +- .../gdev/cmd/gen/_custom/dockerfile.py | 28 +- dev_tools/docker_dev/gdev/cmd/gen/apt/cfg.py | 1 - .../docker_dev/gdev/cmd/gen/apt/dockerfile.py | 6 +- dev_tools/docker_dev/gdev/cmd/gen/env/cfg.py | 1 - .../docker_dev/gdev/cmd/gen/env/dockerfile.py | 10 +- dev_tools/docker_dev/gdev/cmd/gen/gaia/cfg.py | 2 - .../gdev/cmd/gen/gaia/dockerfile.py | 12 +- dev_tools/docker_dev/gdev/cmd/gen/git/cfg.py | 1 - .../docker_dev/gdev/cmd/gen/git/dockerfile.py | 6 +- dev_tools/docker_dev/gdev/cmd/gen/git/run.py | 7 +- dev_tools/docker_dev/gdev/cmd/gen/pip/cfg.py | 1 - .../docker_dev/gdev/cmd/gen/pip/dockerfile.py | 6 +- .../docker_dev/gdev/cmd/gen/pre_run/build.py | 3 - .../docker_dev/gdev/cmd/gen/pre_run/cfg.py | 1 - .../gdev/cmd/gen/pre_run/dockerfile.py | 31 +- dev_tools/docker_dev/gdev/cmd/gen/run/cfg.py | 1 - .../docker_dev/gdev/cmd/gen/run/dockerfile.py | 21 +- dev_tools/docker_dev/gdev/cmd/gen/web/cfg.py | 1 - .../docker_dev/gdev/cmd/gen/web/dockerfile.py | 12 +- dev_tools/docker_dev/gdev/custom/gaia_path.py | 96 ++++++ dev_tools/docker_dev/gdev/custom/pathlib.py | 71 ----- dev_tools/docker_dev/gdev/dependency.py | 54 ++-- dev_tools/docker_dev/gdev/host.py | 41 ++- dev_tools/docker_dev/gdev/main.py | 14 +- dev_tools/docker_dev/gdev/mount.py | 6 +- dev_tools/docker_dev/gdev/options.py | 2 + dev_tools/docker_dev/gdev/parser_structure.py | 10 +- .../gdev/third_party/argcomplete.py | 6 + .../docker_dev/gdev/third_party/atools.py | 7 +- dev_tools/docker_dev/test/gdev_execute.py | 88 ++++-- dev_tools/docker_dev/test/pytest_execute.py | 9 +- dev_tools/docker_dev/test/test_scenarios.py | 280 ++++++++++++------ dev_tools/gdev/gdev/dependency.py | 9 +- 42 files changed, 656 insertions(+), 445 deletions(-) create mode 100644 dev_tools/docker_dev/gdev/custom/gaia_path.py delete mode 100644 dev_tools/docker_dev/gdev/custom/pathlib.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d490ddbcfe7..3e81f199381f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -185,5 +185,5 @@ repos: rev: v2.11.1 hooks: - id: pylint - exclude: ^(dev_tools/gdev/|dev_tools/docker_dev/|production/tools/tests/gaiat/lit.cfg.py|production/tests/python/ps_mem.py) + exclude: ^(dev_tools/gdev/|production/tools/tests/gaiat/lit.cfg.py|production/tests/python/ps_mem.py) args: [--disable=R0801] diff --git a/dev_tools/docker_dev/gdev/__main__.py b/dev_tools/docker_dev/gdev/__main__.py index 9f0ae8226728..e53e7aa275ae 100644 --- a/dev_tools/docker_dev/gdev/__main__.py +++ b/dev_tools/docker_dev/gdev/__main__.py @@ -15,6 +15,9 @@ from gdev.main import DockerDev def main() -> int: + """ + Module main entry point into the application. + """ DockerDev().main() return 0 diff --git a/dev_tools/docker_dev/gdev/cmd/gen/_abc/build.py b/dev_tools/docker_dev/gdev/cmd/gen/_abc/build.py index c57493f1054a..fb4da88f3935 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/_abc/build.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/_abc/build.py @@ -5,17 +5,21 @@ # All rights reserved. ############################################# +""" +Module to help build a docker image from its component pieces. +""" + from __future__ import annotations from abc import ABC, abstractmethod import os -from typing import Awaitable, Callable, Iterable, Mapping, Union +from typing import Iterable, Mapping -from gdev.custom.pathlib import Path +from gdev.custom.gaia_path import GaiaPath from gdev.dependency import Dependency from gdev.host import Host from gdev.third_party.atools import memoize -from .cfg import GenAbcCfg -from .dockerfile import GenAbcDockerfile +from gdev.cmd.gen._abc.cfg import GenAbcCfg +from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile # We require buildkit support for inline caching of multi-stage dockerfiles. It's also way faster @@ -44,15 +48,6 @@ def dockerfile(self) -> GenAbcDockerfile: """ raise NotImplementedError - # TODO: Used? - @memoize - def __get_actual_git_hash(self) -> str: - actual_git_hash = self._get_actual_label_value('GitHash') - - self.log.debug(f'{actual_git_hash = }') - - return actual_git_hash - @memoize def _get_actual_label_value(self, name: str) -> str: """ @@ -72,18 +67,20 @@ def _get_actual_label_value(self, name: str) -> str: @memoize def _get_actual_label_value_by_name(self) -> Mapping[str, str]: """ - Get the hash of an image with the actual label values that are called for by the configuration. + Get the hash of an image with the actual label values that are called + for by the configuration. """ return {'GitHash': self._get_actual_label_value(name='GitHash')} @memoize def get_actual_label_value_by_name(self) -> Mapping[str, str]: """ - Get the hash of an image with the actual label values that are called for by the configuration. + Get the hash of an image with the actual label values that are called + for by the configuration. """ actual_label_value_by_name = self._get_actual_label_value_by_name() - self.log.debug(f'{actual_label_value_by_name = }') + self.log.debug('actual_label_value_by_name = %s', actual_label_value_by_name) return actual_label_value_by_name @memoize @@ -102,7 +99,7 @@ def inner(dockerfile: GenAbcDockerfile) -> Iterable[str]: base_build_names = tuple(inner(self.dockerfile)) - self.log.debug(f'{base_build_names = }') + self.log.debug('base_build_names = %s', base_build_names) return base_build_names @@ -119,7 +116,7 @@ def get_sha(self) -> str: else: sha = '' - self.log.debug(f'{sha = }') + self.log.debug('sha = %s', sha) return sha @@ -131,36 +128,10 @@ def get_tag(self) -> str: tag = f'{self.dockerfile.get_name()}:latest' - self.log.debug(f'{tag = }') + self.log.debug('tag = %s', tag) return tag - # TODO unused? - @memoize - def __get_uncommitted(self) -> str: - seen_dockerfiles = set() - - def inner(dockerfile: GenAbcDockerfile) -> Iterable[Path]: - paths = set() - if dockerfile not in seen_dockerfiles: - seen_dockerfiles.add(dockerfile) - paths.add( Path.repo() / Path(dockerfile.options.target)) - for input_dockerfile in dockerfile.get_input_dockerfiles(): - paths |= inner(input_dockerfile) - return paths - - uncommitted = '\n'.join( - Host.execute_and_get_lines_sync( - f'git status --short ' + ' '.join( - f'{path.parent}' for path in inner(self.dockerfile) - ) - ) - ) - - self.log.debug(f'{uncommitted = }') - - return uncommitted - @memoize def __get_wanted_git_hash(self) -> str: """ @@ -168,7 +139,7 @@ def __get_wanted_git_hash(self) -> str: """ wanted_git_hash = Host.execute_and_get_line_sync('git rev-parse HEAD') - self.log.debug(f'{wanted_git_hash = }') + self.log.debug('wanted_git_hash = %s', wanted_git_hash) return wanted_git_hash @@ -185,7 +156,7 @@ def get_wanted_label_value_by_name(self) -> Mapping[str, str]: """ wanted_label_value_by_name = self._get_wanted_label_value_by_name() - self.log.debug(f'{wanted_label_value_by_name = }') + self.log.debug('wanted_label_value_by_name = %s', wanted_label_value_by_name) return wanted_label_value_by_name @@ -203,8 +174,7 @@ def main(self) -> None: for base_build_name in self.__get_base_build_names()]) cached_images = f"--cache-from {cached_images}" - # TODO query remotely for cached build sources. - self.log.info(f'Creating image "{self.get_tag()}"') + self.log.info('Creating image "%s"', self.get_tag()) Host.execute_sync( f'docker buildx build' f' -f {self.dockerfile.path}' @@ -229,10 +199,13 @@ def main(self) -> None: f' {cached_images}' - f' {Path.repo()}' + f' {GaiaPath.repo()}' ) - Host.execute_sync(f'docker image prune -f') + Host.execute_sync('docker image prune -f') + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def cli_entrypoint(self) -> None: """ @@ -242,7 +215,8 @@ def cli_entrypoint(self) -> None: if not self.options.mixins: build = self else: - from .._custom.build import GenCustomBuild + from gdev.cmd.gen._custom.build import GenCustomBuild build = GenCustomBuild(options=self.options, base_build=self) build.run() + # pylint: enable=import-outside-toplevel diff --git a/dev_tools/docker_dev/gdev/cmd/gen/_abc/cfg.py b/dev_tools/docker_dev/gdev/cmd/gen/_abc/cfg.py index cabd93238da9..5ea9296771d7 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/_abc/cfg.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/_abc/cfg.py @@ -13,11 +13,10 @@ import re from typing import FrozenSet, Iterable, Pattern -from gdev.custom.pathlib import Path +from gdev.custom.gaia_path import GaiaPath from gdev.dependency import Dependency from gdev.third_party.atools import memoize - class GenAbcCfg(Dependency, ABC): """ Class to parse the target gdev.cfg for build rules. @@ -25,12 +24,12 @@ class GenAbcCfg(Dependency, ABC): @property @memoize - def path(self) -> Path: + def path(self) -> GaiaPath: """ Determine the path to the configuration file that we are observing. """ - path = Path.repo() / self.options.target / 'gdev.cfg' - self.log.debug(f'{path = }') + path = GaiaPath.repo() / self.options.target / 'gdev.cfg' + self.log.debug('path = %s', path) return path @property @@ -38,7 +37,7 @@ def section_name(self) -> str: """ Determine the section name in the configuration based on the type of the class. """ - return Path(getfile(type(self))).parent.name.strip('_') + return GaiaPath(getfile(type(self))).parent.name.strip('_') @memoize def __get_begin_pattern(self) -> Pattern: @@ -47,7 +46,7 @@ def __get_begin_pattern(self) -> Pattern: """ begin_pattern = re.compile(fr'^\[{self.section_name}]$') - self.log.debug(f'{begin_pattern = }') + self.log.debug('begin_pattern = %s', begin_pattern) return begin_pattern @memoize @@ -55,22 +54,28 @@ def __get_end_pattern(self) -> Pattern: """ Get the regex pattern that identifies the end of the section. """ - end_pattern = re.compile(fr'^(# .*|)\[.+]$') - self.log.debug(f'{end_pattern = }') + end_pattern = re.compile(r'^(# .*|)\[.+]$') + self.log.debug('end_pattern = %s', end_pattern) return end_pattern + + # pylint: disable=eval-used @staticmethod @memoize - def get_lines(cfg_enables: FrozenSet[str], path: Path) -> Iterable[str]: + def get_lines(cfg_enables: FrozenSet[str], path: GaiaPath) -> Iterable[str]: """ - Get the various lines for the section with the inline notations like `{enable_if('CI_GitHub')}` - replaced with `# enable by setting "{'CI_GitHub'}": `. + Get the various lines for the section with the inline notations like + `{enable_if('CI_GitHub')}` replaced with `# enable by setting "{'CI_GitHub'}": `. """ - return tuple([ + # Per suggestions from https://realpython.com/python-eval-function/: + # - `__locals` field set to an empty dictionary + # - `__globals` field set to contain empty `__builtins__` item + + return tuple(( eval( f'fr""" {line} """', { - 'build_dir': Path.build, + 'build_dir': GaiaPath.build, 'enable_if': lambda enable: '' if enable in cfg_enables else f'# enable by setting "{enable}": ', @@ -89,15 +94,17 @@ def get_lines(cfg_enables: FrozenSet[str], path: Path) -> Iterable[str]: 'enable_if_not_all': lambda *enables: '' if not (set(enables) in cfg_enables) else f'# enable by not setting all of "{sorted(set(enables))}": ', - 'source_dir': Path.source - } + 'source_dir': GaiaPath.source, + "__builtins__": {} + }, {} )[1:-1] for line in (GenAbcCfg.__get_raw_text(path)).splitlines() - ]) + )) + # pylint: enable=eval-used @staticmethod @memoize - def __get_raw_text(path: Path) -> str: + def __get_raw_text(path: GaiaPath) -> str: if not path.is_file(): raise Dependency.Abort(f'File "/{path.context()}" must exist.') @@ -142,7 +149,7 @@ def get_section_lines(self) -> Iterable[str]: section_lines = tuple(section_lines) - self.log.debug(f'{section_lines = }') + self.log.debug('section_lines = %s', section_lines) return section_lines diff --git a/dev_tools/docker_dev/gdev/cmd/gen/_abc/dockerfile.py b/dev_tools/docker_dev/gdev/cmd/gen/_abc/dockerfile.py index fa994084c988..5e4a5dae3756 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/_abc/dockerfile.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/_abc/dockerfile.py @@ -6,7 +6,8 @@ ############################################# """ -Module to create a Dockerfile from the rules in the `gdev.cfg` file in the current working directory. +Module to create a Dockerfile from the rules in the `gdev.cfg` file in the +current working directory. """ from __future__ import annotations @@ -14,15 +15,16 @@ from textwrap import dedent from typing import Iterable -from gdev.custom.pathlib import Path +from gdev.custom.gaia_path import GaiaPath from gdev.dependency import Dependency from gdev.third_party.atools import memoize -from .cfg import GenAbcCfg +from gdev.cmd.gen._abc.cfg import GenAbcCfg class GenAbcDockerfile(Dependency, ABC): """ - Class to create a Dockerfile from the rules in the `gdev.cfg` file in the current working directory. + Class to create a Dockerfile from the rules in the `gdev.cfg` file in the current + working directory. """ @property @@ -34,18 +36,18 @@ def cfg(self) -> GenAbcCfg: raise NotImplementedError @property - def path(self) -> Path: + def path(self) -> GaiaPath: """ Return path where dockerfile is to be written. """ path = ( - Path.repo() + GaiaPath.repo() / '.gdev' / self.options.target / f'{self.cfg.section_name}.dockerfile.gdev' ) - self.log.debug(f'{path = }') + self.log.debug('path = %s', path) return path @@ -98,16 +100,19 @@ def get_base_stages_text(self) -> str: && apt-get clean ''').strip() - self.log.debug(f'{base_stages_text}') + self.log.debug('base_stages_text = %s', base_stages_text) return base_stages_text + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def get_copy_section(self) -> str: """ Return text for the COPY section of the final build stage. """ - from ..pre_run.dockerfile import GenPreRunDockerfile + from gdev.cmd.gen.pre_run.dockerfile import GenPreRunDockerfile seen_dockerfiles = set() # Calculating which build stages to copy from has a tricky problem problems. Docker @@ -138,9 +143,10 @@ def inner(dockerfile: GenAbcDockerfile) -> Iterable[str]: copy_section = '\n'.join(inner(self)) - self.log.debug(f'{copy_section = }') + self.log.debug('copy_section = %s', copy_section) return copy_section + # pylint: enable=import-outside-toplevel @memoize def get_env_section(self) -> str: @@ -148,11 +154,11 @@ def get_env_section(self) -> str: Return text for the ENV section of the final build stage. """ env_section = '' - self.log.debug(f'{env_section = }') + self.log.debug('env_section = %s', env_section) return env_section @memoize - def __get_final_stage_text(self) -> str: + def get_final_stage_text(self) -> str: """ Return the text for the final stage, built up of the individual sections, in order. """ @@ -166,7 +172,7 @@ def __get_final_stage_text(self) -> str: 'ENTRYPOINT [ "/bin/bash" ]', ] if line) - self.log.debug(f'{final_stage_text = }') + self.log.debug('final_stage_text = %s', final_stage_text) return final_stage_text @@ -177,7 +183,7 @@ def get_from_section(self) -> str: """ from_section = f'FROM base AS {self.get_name()}' - self.log.debug(f'{from_section = }') + self.log.debug('from_section = %s', from_section) return from_section @@ -188,7 +194,7 @@ def get_input_dockerfiles(self) -> Iterable[GenAbcDockerfile]: """ input_dockerfiles = tuple() - self.log.debug(f'{input_dockerfiles = }') + self.log.debug('input_dockerfiles = %s', input_dockerfiles) return input_dockerfiles @@ -199,7 +205,7 @@ def get_name(self) -> str: """ name = f'{self.options.target.replace("/", "__")}__{self.cfg.section_name}'.lower() - self.log.debug(f'{name = }') + self.log.debug('name = %s', name) return name @@ -210,10 +216,23 @@ def get_run_section(self) -> str: """ run_section = '' - self.log.debug(f'{run_section = }') + self.log.debug('run_section = %s', run_section) return run_section + + def __get_dockerfile_sections_recursive(self, dockerfile: GenAbcDockerfile, + seen_dockerfiles) -> Iterable[str]: + text_parts = [] + if dockerfile not in seen_dockerfiles: + seen_dockerfiles.add(dockerfile) + for input_dockerfile in dockerfile.get_input_dockerfiles(): + text_parts += self.__get_dockerfile_sections_recursive(input_dockerfile, + seen_dockerfiles) + if dockerfile.get_run_section() or dockerfile is self: + text_parts.append(dockerfile.get_final_stage_text()) + return text_parts + @memoize def get_text(self) -> str: """ @@ -228,19 +247,11 @@ def get_text(self) -> str: # only part of the dockerfile that is affected by this is the copy section for each stage, # as it will need to copy from any of the missing, empty stage's non-empty input stages # instead. - def inner(dockerfile: GenAbcDockerfile) -> Iterable[str]: - text_parts = [] - if dockerfile not in seen_dockerfiles: - seen_dockerfiles.add(dockerfile) - for input_dockerfile in dockerfile.get_input_dockerfiles(): - text_parts += inner(input_dockerfile) - if dockerfile.get_run_section() or dockerfile is self: - text_parts.append(dockerfile.__get_final_stage_text()) - return text_parts - text = '\n'.join([self.get_base_stages_text(), *inner(self)]) + text = '\n'.join([self.get_base_stages_text(), \ + *self.__get_dockerfile_sections_recursive(self, seen_dockerfiles)]) - self.log.debug(f'{text = }') + self.log.debug('text = %s', text) return text @@ -251,7 +262,7 @@ def __get_workdir_section(self) -> str: """ workdir_section = f'WORKDIR {self.cfg.path.parent.image_build()}' - self.log.debug(f'{workdir_section = }') + self.log.debug('workdir_section = %s', workdir_section) return workdir_section @@ -260,9 +271,12 @@ def main(self) -> None: """ Mainline interface. """ - self.log.info(f'Creating dockerfile {self.path}') + self.log.info('Creating dockerfile %s', self.path) self.path.write_text(data=self.get_text()) + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def cli_entrypoint(self) -> None: """ @@ -274,8 +288,9 @@ def cli_entrypoint(self) -> None: if not self.options.mixins: dockerfile = self else: - from .._custom.dockerfile import GenCustomDockerfile + from gdev.cmd.gen._custom.dockerfile import GenCustomDockerfile dockerfile = GenCustomDockerfile(options=self.options, base_dockerfile=self) dockerfile.run() print(dockerfile.get_text()) + # pylint: enable=import-outside-toplevel diff --git a/dev_tools/docker_dev/gdev/cmd/gen/_abc/push.py b/dev_tools/docker_dev/gdev/cmd/gen/_abc/push.py index 81cde22f7580..08edf32d2c84 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/_abc/push.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/_abc/push.py @@ -13,7 +13,7 @@ from gdev.dependency import Dependency from gdev.host import Host from gdev.third_party.atools import memoize -from .build import GenAbcBuild +from gdev.cmd.gen._abc.build import GenAbcBuild class GenAbcPush(Dependency, ABC): @@ -27,7 +27,7 @@ def build(self) -> GenAbcBuild: """ Return the class that will be used to generate the build requirements. """ - raise NotImplemented + raise NotImplementedError @memoize def main(self) -> None: diff --git a/dev_tools/docker_dev/gdev/cmd/gen/_abc/run.py b/dev_tools/docker_dev/gdev/cmd/gen/_abc/run.py index 0495319893ca..de30f460650c 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/_abc/run.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/_abc/run.py @@ -5,16 +5,20 @@ # All rights reserved. ############################################# +""" +Module to create a Docker container from the image build with `gdev build`. +""" + from abc import ABC, abstractmethod import os import shlex import sys -from gdev.custom.pathlib import Path +from gdev.custom.gaia_path import GaiaPath from gdev.dependency import Dependency from gdev.third_party.atools import memoize -from .build import GenAbcBuild from gdev.host import Host +from gdev.cmd.gen._abc.build import GenAbcBuild class GenAbcRun(Dependency, ABC): """ @@ -36,21 +40,21 @@ def _get_flags(self) -> str: """ flags_parts = [ # Remove the container once we exit it. - f'--rm', + '--rm', # Use a minimal init system to allow starting services. - f'--init', + '--init', # Usually the default entrypoint, but override it to be certain. - f'--entrypoint /bin/bash', + '--entrypoint /bin/bash', f'--hostname {self.build.dockerfile.get_name()}', f'--platform linux/{self.options.platform}', - f'--privileged', + '--privileged', # Mount our current repo as /source/. Modifications to source in the container # are reflected on host. - f' --volume {Path.repo()}:{Path.repo().image_source()}' + f' --volume {GaiaPath.repo()}:{GaiaPath.repo().image_source()}' ] # Handle non-TTY environments as well, e.g. TeamCity continuous integration. if sys.stdout.isatty(): @@ -59,7 +63,7 @@ def _get_flags(self) -> str: ports = set(self.options.ports) if {'clion', 'sshd', 'vscode'} & self.options.mixins: ports.add('22') - if (authorized_keys_path := Path.home() / '.ssh' / 'authorized_keys').is_file(): + if (authorized_keys_path := GaiaPath.home() / '.ssh' / 'authorized_keys').is_file(): if {'clion', 'sudo', 'vscode'} & self.options.mixins: flags_parts.append( f'-v {authorized_keys_path.absolute()}:{authorized_keys_path.absolute()}' @@ -92,7 +96,7 @@ def get_flags(self) -> str: """ flags = self._get_flags() - self.log.debug(f'{flags = }') + self.log.debug('flags = %s', flags) return flags @@ -114,7 +118,8 @@ def main(self) -> None: if self.options.mounts: for mount in self.options.mounts: if mount.host_path.exists(): - self.log.info(f'Binding existing host path "{mount.host_path}" into container.') + self.log.info('Binding existing host path "%s" into container.', \ + mount.host_path) else: mount.host_path.mkdir(parents=True) @@ -127,9 +132,12 @@ def main(self) -> None: print(f"[execvpe:{command_to_execute}]") else: command = shlex.split(command_to_execute) - self.log.debug(f'execvpe {command = }') + self.log.debug('execvpe command=%s', command) os.execvpe(command[0], command, os.environ) + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def cli_entrypoint(self) -> None: """ @@ -138,7 +146,8 @@ def cli_entrypoint(self) -> None: if not self.options.mixins: run = self else: - from .._custom.run import GenCustomRun + from gdev.cmd.gen._custom.run import GenCustomRun run = GenCustomRun(options=self.options, base_run=self) run.run() + # pylint: enable=import-outside-toplevel diff --git a/dev_tools/docker_dev/gdev/cmd/gen/_custom/build.py b/dev_tools/docker_dev/gdev/cmd/gen/_custom/build.py index 6f113b5681c3..9774e17f4577 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/_custom/build.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/_custom/build.py @@ -43,7 +43,8 @@ def _get_wanted_label_value_by_name(self) -> Mapping[str, str]: @memoize def _get_actual_label_value_by_name(self) -> Mapping[str, str]: """ - Get the hash of an image with the actual label values that are called for by the configuration. + Get the hash of an image with the actual label values that are called + for by the configuration. """ return { **super()._get_actual_label_value_by_name(), diff --git a/dev_tools/docker_dev/gdev/cmd/gen/_custom/cfg.py b/dev_tools/docker_dev/gdev/cmd/gen/_custom/cfg.py index dacae0247e88..c53695912c94 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/_custom/cfg.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/_custom/cfg.py @@ -10,23 +10,25 @@ """ from typing import Iterable -from gdev.custom.pathlib import Path +from gdev.custom.gaia_path import GaiaPath from gdev.third_party.atools import memoize from .._abc.cfg import GenAbcCfg class GenCustomCfg(GenAbcCfg): """ - Class to provide a subclass of the GenAbcCfg class for the Apt section where MIXINs are used. + Class to provide for a subclass of the GenAbcCfg class for the Apt section where MIXINs + are used. """ @memoize - def get_lines(self) -> Iterable[str]: + def get_mixin_lines(self) -> Iterable[str]: """ Get the various lines for the section, all derived from the needed mixins. """ - lines = tuple([f'{Path.mixin().context() / mixin}' for mixin in sorted(self.options.mixins)]) + lines = tuple((f'{GaiaPath.mixin().context() / mixin}' \ + for mixin in sorted(self.options.mixins))) - self.log.info(f'{lines = }') + self.log.info('lines = %s', lines) return lines diff --git a/dev_tools/docker_dev/gdev/cmd/gen/_custom/dockerfile.py b/dev_tools/docker_dev/gdev/cmd/gen/_custom/dockerfile.py index 3d0702e83161..d83f21e27553 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/_custom/dockerfile.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/_custom/dockerfile.py @@ -13,10 +13,10 @@ from textwrap import dedent from typing import Iterable -from gdev.custom.pathlib import Path +from gdev.custom.gaia_path import GaiaPath from gdev.third_party.atools import memoize +from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile from .cfg import GenCustomCfg -from .._abc.dockerfile import GenAbcDockerfile @dataclass(frozen=True, repr=False) @@ -34,34 +34,42 @@ def cfg(self) -> GenCustomCfg: """ return GenCustomCfg(self.options) + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def get_env_section(self) -> str: """ Return text for the ENV section of the final build stage. """ - from ..pre_run.dockerfile import GenPreRunDockerfile + from gdev.cmd.gen.pre_run.dockerfile import GenPreRunDockerfile env_section = GenPreRunDockerfile(self.options).get_env_section() - self.log.debug(f'{env_section = }') + self.log.debug('env_section = %s', env_section) return env_section + # pylint: enable=import-outside-toplevel + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def get_input_dockerfiles(self) -> Iterable[GenAbcDockerfile]: """ Return dockerfiles that describe build stages that come directly before this one. """ - from ..run.dockerfile import GenRunDockerfile + from gdev.cmd.gen.run.dockerfile import GenRunDockerfile input_dockerfiles = [self.base_dockerfile] - for line in self.cfg.get_lines(): + for line in self.cfg.get_mixin_lines(): input_dockerfiles.append(GenRunDockerfile(replace(self.options, target=line))) input_dockerfiles = tuple(input_dockerfiles) - self.log.debug(f'{input_dockerfiles = }') + self.log.debug('input_dockerfiles = %s', input_dockerfiles) return input_dockerfiles + # pylint: enable=import-outside-toplevel @memoize def get_text(self) -> str: @@ -77,7 +85,7 @@ def get_text(self) -> str: if {'clion', 'sudo', 'vscode'} & self.options.mixins: uid = os.getuid() gid = os.getgid() - home = Path.home() + home = GaiaPath.home() login = os.getlogin() text_parts.append(dedent(fr''' RUN groupadd -r -o -g {gid} {login} \ @@ -86,11 +94,11 @@ def get_text(self) -> str: && chown {login}:{login} {home} \ && echo "{login} ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers \ && touch {home}/.sudo_as_admin_successful \ - && chown -R {login}:{login} {Path.repo().image_build()} + && chown -R {login}:{login} {GaiaPath.repo().image_build()} ''').strip()) text = '\n'.join(text_parts) - self.log.debug(f'{text = }') + self.log.debug('text = %s', text) return text diff --git a/dev_tools/docker_dev/gdev/cmd/gen/apt/cfg.py b/dev_tools/docker_dev/gdev/cmd/gen/apt/cfg.py index d7102cdc24cc..dd4368430c98 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/apt/cfg.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/apt/cfg.py @@ -18,4 +18,3 @@ class GenAptCfg(GenAbcCfg): Note that this class is empty, but kept to maintain a subcommand structure that is equivalent to the other subcommands, such as build. """ - pass diff --git a/dev_tools/docker_dev/gdev/cmd/gen/apt/dockerfile.py b/dev_tools/docker_dev/gdev/cmd/gen/apt/dockerfile.py index 65323222bed1..f91f935fb519 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/apt/dockerfile.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/apt/dockerfile.py @@ -9,8 +9,8 @@ Module to generate the APT section of the dockerfile. """ from gdev.third_party.atools import memoize +from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile from .cfg import GenAptCfg -from .._abc.dockerfile import GenAbcDockerfile class GenAptDockerfile(GenAbcDockerfile): @@ -32,7 +32,7 @@ def get_from_section(self) -> str: """ from_section = f'FROM apt_base AS {self.get_name()}' - self.log.debug(f'{from_section = }') + self.log.debug('from_section = %s', from_section) return from_section @@ -52,6 +52,6 @@ def get_run_section(self) -> str: else: run_section = '' - self.log.debug(f'{run_section = }') + self.log.debug('run_section = %s', run_section) return run_section diff --git a/dev_tools/docker_dev/gdev/cmd/gen/env/cfg.py b/dev_tools/docker_dev/gdev/cmd/gen/env/cfg.py index 4a2a868991ee..e38abfe1e065 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/env/cfg.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/env/cfg.py @@ -18,4 +18,3 @@ class GenEnvCfg(GenAbcCfg): Note that this class is empty, but kept to maintain a subcommand structure that is equivalent to the other subcommands, such as build. """ - pass diff --git a/dev_tools/docker_dev/gdev/cmd/gen/env/dockerfile.py b/dev_tools/docker_dev/gdev/cmd/gen/env/dockerfile.py index c614e65c0f80..21dc4be31b2b 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/env/dockerfile.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/env/dockerfile.py @@ -13,8 +13,8 @@ from typing import Iterable from gdev.third_party.atools import memoize +from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile from .cfg import GenEnvCfg -from .._abc.dockerfile import GenAbcDockerfile class GenEnvDockerfile(GenAbcDockerfile): @@ -29,18 +29,22 @@ def cfg(self) -> GenEnvCfg: """ return GenEnvCfg(self.options) + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def get_input_dockerfiles(self) -> Iterable[GenAbcDockerfile]: """ Return dockerfiles that describe build stages that come directly before this one. """ - from ..gaia.dockerfile import GenGaiaDockerfile + from gdev.cmd.gen.gaia.dockerfile import GenGaiaDockerfile input_dockerfiles = [] for section_line in GenGaiaDockerfile(self.options).cfg.get_section_lines(): input_dockerfiles.append(GenEnvDockerfile(replace(self.options, target=section_line))) input_dockerfiles = tuple(input_dockerfiles) - self.log.debug(f'{input_dockerfiles}') + self.log.debug('input_dockerfiles = %s', input_dockerfiles) return input_dockerfiles + # pylint: enable=import-outside-toplevel diff --git a/dev_tools/docker_dev/gdev/cmd/gen/gaia/cfg.py b/dev_tools/docker_dev/gdev/cmd/gen/gaia/cfg.py index 24fa8f23c75a..55e704d7b376 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/gaia/cfg.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/gaia/cfg.py @@ -8,7 +8,6 @@ """ Module to provide a subclass of the GenAbcCfg class for the Gaia section. """ -from gdev.third_party.atools import memoize from .._abc.cfg import GenAbcCfg @@ -19,4 +18,3 @@ class GenGaiaCfg(GenAbcCfg): Note that this class is empty, but kept to maintain a subcommand structure that is equivalent to the other subcommands, such as build. """ - pass diff --git a/dev_tools/docker_dev/gdev/cmd/gen/gaia/dockerfile.py b/dev_tools/docker_dev/gdev/cmd/gen/gaia/dockerfile.py index 6c7c89ed07dc..46a76052e9f5 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/gaia/dockerfile.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/gaia/dockerfile.py @@ -8,12 +8,13 @@ """ Module to handle the processing of the 'gaia' section of the 'gdev.cfg' file. """ + from dataclasses import replace from typing import Iterable from gdev.third_party.atools import memoize +from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile from .cfg import GenGaiaCfg -from .._abc.dockerfile import GenAbcDockerfile class GenGaiaDockerfile(GenAbcDockerfile): @@ -28,19 +29,22 @@ def cfg(self) -> GenGaiaCfg: """ return GenGaiaCfg(self.options) + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def get_input_dockerfiles(self) -> Iterable[GenAbcDockerfile]: """ Return dockerfiles that describe build stages that come directly before this one. """ - - from ..run.dockerfile import GenRunDockerfile + from gdev.cmd.gen.run.dockerfile import GenRunDockerfile input_dockerfiles = [] for section_line in self.cfg.get_section_lines(): input_dockerfiles.append(GenRunDockerfile(replace(self.options, target=section_line))) input_dockerfiles = tuple(input_dockerfiles) - self.log.debug(f'{input_dockerfiles = }') + self.log.debug('input_dockerfiles = %s', input_dockerfiles) return input_dockerfiles + # pylint: enable=import-outside-toplevel diff --git a/dev_tools/docker_dev/gdev/cmd/gen/git/cfg.py b/dev_tools/docker_dev/gdev/cmd/gen/git/cfg.py index a5338702fe9f..942c71d4b7c7 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/git/cfg.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/git/cfg.py @@ -18,4 +18,3 @@ class GenGitCfg(GenAbcCfg): Note that this class is empty, but kept to maintain a subcommand structure that is equivalent to the other subcommands, such as build. """ - pass diff --git a/dev_tools/docker_dev/gdev/cmd/gen/git/dockerfile.py b/dev_tools/docker_dev/gdev/cmd/gen/git/dockerfile.py index 71437d9edc31..16b08eb2a8e0 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/git/dockerfile.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/git/dockerfile.py @@ -10,8 +10,8 @@ """ from gdev.third_party.atools import memoize +from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile from .cfg import GenGitCfg -from .._abc.dockerfile import GenAbcDockerfile class GenGitDockerfile(GenAbcDockerfile): @@ -33,7 +33,7 @@ def get_from_section(self) -> str: """ from_section = f'FROM git_base AS {self.get_name()}' - self.log.debug(f'{from_section = }') + self.log.debug('from_section = %s', from_section) return from_section @@ -54,6 +54,6 @@ def get_run_section(self) -> str: else: run_section = '' - self.log.debug(f'{run_section = }') + self.log.debug('run_section = %s', run_section) return run_section diff --git a/dev_tools/docker_dev/gdev/cmd/gen/git/run.py b/dev_tools/docker_dev/gdev/cmd/gen/git/run.py index aac5dfc67c17..fd53fa8e0534 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/git/run.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/git/run.py @@ -8,8 +8,8 @@ """ Module to satisfy the run requirements for the GIT section. """ -from .._abc.build import GenAbcBuild -from .._abc.run import GenAbcRun +from gdev.cmd.gen._abc.run import GenAbcRun +from gdev.cmd.gen.git.build import GenGitBuild class GenGitRun(GenAbcRun): @@ -18,9 +18,8 @@ class GenGitRun(GenAbcRun): """ @property - def build(self) -> GenAbcBuild: + def build(self) -> GenGitBuild: """ Return the class that will be used to generate the run requirements. """ - from .build import GenGitBuild return GenGitBuild(self.options) diff --git a/dev_tools/docker_dev/gdev/cmd/gen/pip/cfg.py b/dev_tools/docker_dev/gdev/cmd/gen/pip/cfg.py index 21202a2a3bed..dc073770ebf8 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/pip/cfg.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/pip/cfg.py @@ -18,4 +18,3 @@ class GenPipCfg(GenAbcCfg): Note that this class is empty, but kept to maintain a subcommand structure that is equivalent to the other subcommands, such as build. """ - pass diff --git a/dev_tools/docker_dev/gdev/cmd/gen/pip/dockerfile.py b/dev_tools/docker_dev/gdev/cmd/gen/pip/dockerfile.py index bc31a5d3e5c2..09504f30d51d 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/pip/dockerfile.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/pip/dockerfile.py @@ -10,8 +10,8 @@ """ from gdev.third_party.atools import memoize +from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile from .cfg import GenPipCfg -from .._abc.dockerfile import GenAbcDockerfile class GenPipDockerfile(GenAbcDockerfile): @@ -33,7 +33,7 @@ def get_from_section(self) -> str: """ from_section = f'FROM pip_base AS {self.get_name()}' - self.log.debug(f'{from_section = }') + self.log.debug('from_section = %s', from_section) return from_section @@ -51,6 +51,6 @@ def get_run_section(self) -> str: else: run_section = '' - self.log.debug(f'{run_section = }') + self.log.debug('run_section = %s', run_section) return run_section diff --git a/dev_tools/docker_dev/gdev/cmd/gen/pre_run/build.py b/dev_tools/docker_dev/gdev/cmd/gen/pre_run/build.py index 8f6da0748635..c52e9ed9d6e2 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/pre_run/build.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/pre_run/build.py @@ -8,9 +8,6 @@ """ Module to satisfy the build requirements to generate the dockerfile for the PRERUN section. """ -from typing import Iterable - -from gdev.third_party.atools import memoize from .dockerfile import GenPreRunDockerfile from .._abc.build import GenAbcBuild diff --git a/dev_tools/docker_dev/gdev/cmd/gen/pre_run/cfg.py b/dev_tools/docker_dev/gdev/cmd/gen/pre_run/cfg.py index f5e1404fd26c..12fd5dfba9e2 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/pre_run/cfg.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/pre_run/cfg.py @@ -18,4 +18,3 @@ class GenPreRunCfg(GenAbcCfg): Note that this class is empty, but kept to maintain a subcommand structure that is equivalent to the other subcommands, such as build. """ - pass diff --git a/dev_tools/docker_dev/gdev/cmd/gen/pre_run/dockerfile.py b/dev_tools/docker_dev/gdev/cmd/gen/pre_run/dockerfile.py index 6ff178804521..f31cba195505 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/pre_run/dockerfile.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/pre_run/dockerfile.py @@ -11,9 +11,8 @@ from typing import Iterable from gdev.third_party.atools import memoize +from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile from .cfg import GenPreRunCfg -from .._abc.dockerfile import GenAbcDockerfile - class GenPreRunDockerfile(GenAbcDockerfile): """ @@ -27,12 +26,15 @@ def cfg(self) -> GenPreRunCfg: """ return GenPreRunCfg(self.options) + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def get_env_section(self) -> str: """ Return text for the ENV section of the final build stage. """ - from ..env.dockerfile import GenEnvDockerfile + from gdev.cmd.gen.env.dockerfile import GenEnvDockerfile seen_env_dockerfiles = set() @@ -50,21 +52,25 @@ def inner(env_dockerfile: GenEnvDockerfile) -> Iterable[str]: env_section = '\n'.join(inner(GenEnvDockerfile(self.options))) - self.log.debug(f'{env_section = }') + self.log.debug('env_section = %s', env_section) return env_section + # pylint: enable=import-outside-toplevel + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def get_input_dockerfiles(self) -> Iterable[GenAbcDockerfile]: """ Return dockerfiles that describe build stages that come directly before this one. """ - from ..apt.dockerfile import GenAptDockerfile - from ..env.dockerfile import GenEnvDockerfile - from ..gaia.dockerfile import GenGaiaDockerfile - from ..git.dockerfile import GenGitDockerfile - from ..pip.dockerfile import GenPipDockerfile - from ..web.dockerfile import GenWebDockerfile + from gdev.cmd.gen.env.dockerfile import GenEnvDockerfile + from gdev.cmd.gen.apt.dockerfile import GenAptDockerfile + from gdev.cmd.gen.gaia.dockerfile import GenGaiaDockerfile + from gdev.cmd.gen.git.dockerfile import GenGitDockerfile + from gdev.cmd.gen.pip.dockerfile import GenPipDockerfile + from gdev.cmd.gen.web.dockerfile import GenWebDockerfile input_dockerfiles = tuple([ GenAptDockerfile(self.options), @@ -75,9 +81,10 @@ def get_input_dockerfiles(self) -> Iterable[GenAbcDockerfile]: GenWebDockerfile(self.options), ]) - self.log.debug(f'{input_dockerfiles = }') + self.log.debug('input_dockerfiles = %s', input_dockerfiles) return input_dockerfiles + # pylint: enable=import-outside-toplevel @memoize def get_run_section(self) -> str: @@ -92,6 +99,6 @@ def get_run_section(self) -> str: else: run_section = '' - self.log.debug(f'{run_section = }') + self.log.debug('run_section = %s', run_section) return run_section diff --git a/dev_tools/docker_dev/gdev/cmd/gen/run/cfg.py b/dev_tools/docker_dev/gdev/cmd/gen/run/cfg.py index 8a9e283d19a5..031b3a505435 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/run/cfg.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/run/cfg.py @@ -18,4 +18,3 @@ class GenRunCfg(GenAbcCfg): Note that this class is empty, but kept to maintain a subcommand structure that is equivalent to the other subcommands, such as build. """ - pass diff --git a/dev_tools/docker_dev/gdev/cmd/gen/run/dockerfile.py b/dev_tools/docker_dev/gdev/cmd/gen/run/dockerfile.py index 2a7d10f2a4e6..592f6c75b0e6 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/run/dockerfile.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/run/dockerfile.py @@ -12,9 +12,8 @@ from typing import Iterable from gdev.third_party.atools import memoize +from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile from .cfg import GenRunCfg -from .._abc.dockerfile import GenAbcDockerfile - class GenRunDockerfile(GenAbcDockerfile): """ @@ -29,31 +28,39 @@ def cfg(self) -> GenRunCfg: """ return GenRunCfg(self.options) + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def get_env_section(self) -> str: """ Return text for the ENV section of the final build stage. """ - from ..pre_run.dockerfile import GenPreRunDockerfile + from gdev.cmd.gen.pre_run.dockerfile import GenPreRunDockerfile env_section = GenPreRunDockerfile(self.options).get_env_section() - self.log.debug(f'{env_section = }') + self.log.debug('env_section = %s', env_section) return env_section + # pylint: enable=import-outside-toplevel + # pylint: disable=import-outside-toplevel + # + # Required to resolve cyclical dependency issues. @memoize def get_input_dockerfiles(self) -> Iterable[GenAbcDockerfile]: """ Return dockerfiles that describe build stages that come directly before this one. """ - from ..pre_run.dockerfile import GenPreRunDockerfile + from gdev.cmd.gen.pre_run.dockerfile import GenPreRunDockerfile input_dockerfiles = tuple([GenPreRunDockerfile(self.options)]) - self.log.debug(f'{input_dockerfiles = }') + self.log.debug('input_dockerfiles = %s', input_dockerfiles) return input_dockerfiles + # pylint: enable=import-outside-toplevel @memoize def get_run_section(self) -> str: @@ -68,6 +75,6 @@ def get_run_section(self) -> str: else: run_section = '' - self.log.debug(f'{run_section = }') + self.log.debug('run_section = %s', run_section) return run_section diff --git a/dev_tools/docker_dev/gdev/cmd/gen/web/cfg.py b/dev_tools/docker_dev/gdev/cmd/gen/web/cfg.py index a20a475f1625..e1209c874bd1 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/web/cfg.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/web/cfg.py @@ -18,4 +18,3 @@ class GenWebCfg(GenAbcCfg): Note that this class is empty, but kept to maintain a subcommand structure that is equivalent to the other subcommands, such as build. """ - pass diff --git a/dev_tools/docker_dev/gdev/cmd/gen/web/dockerfile.py b/dev_tools/docker_dev/gdev/cmd/gen/web/dockerfile.py index cce866b0ec45..f045d6d2887b 100644 --- a/dev_tools/docker_dev/gdev/cmd/gen/web/dockerfile.py +++ b/dev_tools/docker_dev/gdev/cmd/gen/web/dockerfile.py @@ -9,9 +9,8 @@ Module to generate the WEB section of the dockerfile. """ from gdev.third_party.atools import memoize +from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile from .cfg import GenWebCfg -from .._abc.dockerfile import GenAbcDockerfile - class GenWebDockerfile(GenAbcDockerfile): """ @@ -32,7 +31,7 @@ def get_from_section(self) -> str: """ from_section = f'FROM web_base AS {self.get_name()}' - self.log.debug(f'{from_section = }') + self.log.debug('from_section = %s', from_section) return from_section @@ -43,14 +42,15 @@ def get_run_section(self) -> str: """ if section_lines := self.cfg.get_section_lines(): + formatted_section_lines = ' \\\n '.join(section_lines) run_statement = ( - f'RUN wget ' - + ' \\\n '.join(section_lines) + 'RUN wget ' + + formatted_section_lines + ' \\\n && apt-get remove --autoremove -y wget' ) else: run_statement = '' - self.log.debug(f'{run_statement = }') + self.log.debug('run_statement = %s', run_statement) return run_statement diff --git a/dev_tools/docker_dev/gdev/custom/gaia_path.py b/dev_tools/docker_dev/gdev/custom/gaia_path.py new file mode 100644 index 000000000000..071c72828a33 --- /dev/null +++ b/dev_tools/docker_dev/gdev/custom/gaia_path.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +############################################# +# Copyright (c) Gaia Platform LLC +# All rights reserved. +############################################# + +""" +Module to provide various pathing utilities to make things simpler. +""" + +from __future__ import annotations + +from logging import getLogger +from pathlib import PosixPath +from subprocess import check_output +from textwrap import indent +from typing import Union + + +log = getLogger(__name__) + + +class GaiaPath(PosixPath): + """ + Class to provide various pathing utilities to make things simpler. + """ + _repo: GaiaPath = None + + def context(self) -> GaiaPath: + """ + Determine the basis on which to generated paths. + """ + if self.is_absolute(): + return self.relative_to(self.repo()) + return self + + def image_build(self) -> GaiaPath: + """ + Generate a path to the build directory. + """ + return GaiaPath.build(self.context()) + + def image_source(self) -> GaiaPath: + """ + Generate a path to the source directory. + """ + return GaiaPath.source(self.context()) + + @classmethod + def mixin(cls) -> GaiaPath: + """ + Determine the path to where the mixins are stored. + """ + return cls.repo() / 'dev_tools' / 'gdev' / 'mixin' + + @classmethod + def repo(cls) -> GaiaPath: + """ + Determine the path to root of the repository. + """ + if cls._repo is None: + repo = GaiaPath( + check_output( + 'git rev-parse --show-toplevel'.split(), + cwd=f'{GaiaPath(__file__).parent}' + ).decode().strip() + ) + + log.debug('repo = %s', repo) + + cls._repo = repo + + return cls._repo + + @classmethod + def build(cls, path: Union[GaiaPath, str] = '/') -> GaiaPath: + """ + Compute a path along the "/build" tree in the container. + """ + return GaiaPath('/build') / path + + @classmethod + def source(cls, path: Union[GaiaPath, str] = '/') -> GaiaPath: + """ + Compute a path along the "/source" tree in the container. + """ + return GaiaPath('/source') / path + + def write_text(self, data: str, encoding=None, errors=None) -> None: + """ + Write the specified text to the given file. + """ + log.debug('Writing to %s, text: \n%s', self, indent(data, " ")) + self.parent.mkdir(parents=True, exist_ok=True) + super().write_text(data, encoding=encoding, errors=errors) diff --git a/dev_tools/docker_dev/gdev/custom/pathlib.py b/dev_tools/docker_dev/gdev/custom/pathlib.py deleted file mode 100644 index 4d6911668862..000000000000 --- a/dev_tools/docker_dev/gdev/custom/pathlib.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 - -############################################# -# Copyright (c) Gaia Platform LLC -# All rights reserved. -############################################# - -from __future__ import annotations - -from logging import getLogger -from pathlib import PosixPath -from subprocess import check_output -from textwrap import indent -from typing import Union - - -log = getLogger(__name__) - - -class Path(PosixPath): - _repo: Path = None - - @classmethod - def build(cls, path: Union[Path, str] = '/') -> Path: - return Path('/build') / path - - def context(self) -> Path: - if self.is_absolute(): - return self.relative_to(self.repo()) - else: - return self - - def image_build(self) -> Path: - return Path.build(self.context()) - - def image_source(self) -> Path: - return Path.source(self.context()) - - @classmethod - def mixin(cls) -> Path: - return cls.repo() / 'dev_tools' / 'gdev' / 'mixin' - - @classmethod - def repo(cls) -> Path: - if cls._repo is None: - repo = Path( - check_output( - f'git rev-parse --show-toplevel'.split(), - cwd=f'{Path(__file__).parent}' - ).decode().strip() - ) - - log.debug(f'{repo = }') - - cls._repo = repo - - return cls._repo - - @classmethod - def source(cls, path: Union[Path, str] = '/') -> Path: - return Path('/source') / path - - def write_bytes(self, data: bytes) -> None: - log.debug(f'Write to {self}, bytes:\n{indent(data.decode(), " ")}') - self.parent.mkdir(parents=True, exist_ok=True) - super().write_bytes(data) - - def write_text(self, data: str, encoding=None, errors=None) -> None: - log.debug(f'Writing to {self}, text: \n{indent(data, " ")}') - self.parent.mkdir(parents=True, exist_ok=True) - super().write_text(data, encoding=encoding, errors=errors) diff --git a/dev_tools/docker_dev/gdev/dependency.py b/dev_tools/docker_dev/gdev/dependency.py index 528535e744cb..0cd518b45c81 100644 --- a/dev_tools/docker_dev/gdev/dependency.py +++ b/dev_tools/docker_dev/gdev/dependency.py @@ -15,22 +15,20 @@ from argparse import ArgumentParser, REMAINDER from dataclasses import dataclass from importlib import import_module -from importlib.util import find_spec -from inspect import getdoc, isabstract, iscoroutinefunction +from inspect import isabstract import logging -from pkgutil import iter_modules import platform import sys -from typing import FrozenSet, Sequence, Set, Tuple +from typing import Sequence +import argparse -from gdev.custom.pathlib import Path +from gdev.custom.gaia_path import GaiaPath from gdev.options import Options from gdev.mount import Mount from gdev.third_party.atools import memoize, memoize_db from gdev.third_party.argcomplete import autocomplete, FilesCompleter from gdev.parser_structure import ParserStructure from gdev.host import Host -import argparse @dataclass(frozen=True) class Dependency: @@ -38,16 +36,30 @@ class Dependency: Bob.Dependency """ - # The gdev CLI uses docstrings in the help output. This is what they'll see if a subclass does - # not provide a docstring. + # The gdev CLI uses docstrings in the help output. This is what they'll see if a + # subclass does not provide a docstring. options: Options - # These two classes are only present to handle the Abort exception, and early exit in some cases. + __LOG_LEVELS = [ + "CRITICAL", + "ERROR", + "WARNING", + "INFO", + "DEBUG"] + + + # These two classes are only present to handle the Abort exception, + # and early exit in some cases. class Exception(Exception): - pass + """ + Useless exception to be refactored out. + """ class Abort(Exception): + """ + Exception to know we aborted. + """ def __str__(self) -> str: return f'Abort: {super().__str__()}' @@ -78,7 +90,7 @@ def log(self) -> logging.Logger: if handler.level > logging.DEBUG: formatter = logging.Formatter(f'({self.options.target}) %(message)s') else: - formatter = logging.Formatter(f'%(levelname)s:%(name)s %(message)s') + formatter = logging.Formatter('%(levelname)s:%(name)s %(message)s') handler.setFormatter(formatter) log.addHandler(handler) return log @@ -141,7 +153,7 @@ def add_flags(parser: ArgumentParser) -> None: parser.add_argument( '--log-level', default=log_level_default, - choices=[name for _, name in sorted(logging._levelToName.items())], + choices=Dependency.__LOG_LEVELS, help=f'Log level. Default: "{log_level_default}"' ) @@ -160,12 +172,13 @@ def add_flags(parser: ArgumentParser) -> None: nargs='*', choices=sorted([ directory.name - for directory in Path.mixin().iterdir() + for directory in GaiaPath.mixin().iterdir() if directory.is_dir() ]), help=( f'Image mixins to use when creating a container. Mixins provide dev tools and' - f' configuration from targets in the "{Path.mixin().relative_to(Path.repo())}"' + f' configuration from targets in ' + f'the "{GaiaPath.mixin().relative_to(GaiaPath.repo())}"' f' directory. Default: "{mixins_default}"' ) ) @@ -210,14 +223,14 @@ def add_flags(parser: ArgumentParser) -> None: '--registry', default=registry_default, help=( - f'Registry to push images and query cached build stages.' + 'Registry to push images and query cached build stages.' f' Default: {registry_default}' ) ) parser.add_argument( 'args', nargs=REMAINDER, - help=f'Args to be forwarded on to docker run, if applicable.' + help='Args to be forwarded on to docker run, if applicable.' ) def inner(parser: ArgumentParser, parser_structure: ParserStructure) -> ArgumentParser: @@ -256,9 +269,9 @@ def run(self) -> None: return if hasattr(self, 'main'): - self.log.debug(f'Starting {type(self).__name__}.main') + self.log.debug('Starting %s.main', str(type(self).__name__)) self.main() - self.log.debug(f'Finished {type(self).__name__}.main') + self.log.debug('Finished %s.main', str(type(self).__name__)) # if self.options.log_level == 'DEBUG': # getters = [] @@ -282,7 +295,6 @@ def of_args(args: Sequence[str]) -> Dependency: if not parsed_args: Dependency.get_parser_structure.memoize.remove() parser.parse_args([*args, '--help']) - import sys sys.exit(1) if 'dry_dock' in parsed_args: @@ -301,7 +313,7 @@ def of_args(args: Sequence[str]) -> Dependency: mounts = [] for mount in parsed_args['mounts']: host_path, container_path = mount.split(':', 1) - host_path, container_path = Path(host_path), Path(container_path) + host_path, container_path = GaiaPath(host_path), GaiaPath(container_path) if not host_path.is_absolute(): host_path = host_path.absolute() if not container_path.is_absolute(): @@ -312,7 +324,7 @@ def of_args(args: Sequence[str]) -> Dependency: command_class = parsed_args.pop('command_class') command_module = parsed_args.pop('command_module') - options = Options(target=f'{Path.cwd().relative_to(Path.repo())}', **parsed_args) + options = Options(target=f'{GaiaPath.cwd().relative_to(GaiaPath.repo())}', **parsed_args) dependency = getattr(import_module(command_module), command_class)(options) diff --git a/dev_tools/docker_dev/gdev/host.py b/dev_tools/docker_dev/gdev/host.py index bb0ae29a2526..f3d496b556c6 100644 --- a/dev_tools/docker_dev/gdev/host.py +++ b/dev_tools/docker_dev/gdev/host.py @@ -46,35 +46,32 @@ def is_drydock_enabled(): @staticmethod @memoize - def __execute_sync(command: str, err_ok: bool, capture_output: bool) -> Optional[Sequence[str]]: + def __execute_sync(command: str, err_ok: bool, + capture_output: bool)-> Optional[Sequence[str]]: - log.debug(f'execute_sync {err_ok = } {capture_output = } {command = }') - child_process = subprocess.Popen( + log.debug('execute_sync err_ok = %s capture_output = %s command = %s', + err_ok, capture_output, command) + with subprocess.Popen( command.replace(" ", " ").split(" "), stdout=subprocess.PIPE if capture_output else None, - stderr=subprocess.PIPE if capture_output else None) - - process_return_code = child_process.wait() - log.debug(f'execute: return_code= {process_return_code}, command= {command}') - - if process_return_code == 0 or err_ok: - if child_process.stdout is None: - return tuple() - else: - process_stdout_output = "" - for line in io.TextIOWrapper(child_process.stdout, encoding="utf-8"): - process_stdout_output += line + stderr=subprocess.PIPE if capture_output else None) as child_process: + + process_return_code = child_process.wait() + log.debug('execute: return_code= %s, command= %s', process_return_code, command) + + if process_return_code == 0 or err_ok: + if child_process.stdout is None: + return tuple() + process_stdout_output = \ + "".join(io.TextIOWrapper(child_process.stdout, encoding="utf-8")) return tuple(process_stdout_output.strip().splitlines()) - else: if child_process.stdout is not None: - process_stdout_output = "" - for line in io.TextIOWrapper(child_process.stdout, encoding="utf-8"): - process_stdout_output += line + process_stdout_output = \ + "".join(io.TextIOWrapper(child_process.stdout, encoding="utf-8")) print(process_stdout_output, file=sys.stdout) if child_process.stderr is not None: - process_stderr_output = "" - for line in io.TextIOWrapper(child_process.stderr, encoding="utf-8"): - process_stderr_output += line + process_stderr_output = \ + "".join(io.TextIOWrapper(child_process.stderr, encoding="utf-8")) print(process_stderr_output, file=sys.stderr) sys.exit(process_return_code) diff --git a/dev_tools/docker_dev/gdev/main.py b/dev_tools/docker_dev/gdev/main.py index 1dc0942acdcd..fe7eb0e4acb6 100644 --- a/dev_tools/docker_dev/gdev/main.py +++ b/dev_tools/docker_dev/gdev/main.py @@ -10,30 +10,32 @@ """ import sys -from time import sleep +import logging +from gdev.dependency import Dependency +# pylint: disable=too-few-public-methods class DockerDev(): """ Class to provide a single entry point from the operating system. """ - def main(self): + @staticmethod + def main(): """ Main entry point from the operating system. """ - from gdev.dependency import Dependency dependency = Dependency.of_sys_argv() - import logging logging.basicConfig(level=dependency.options.log_level) try: dependency.cli_entrypoint() - except dependency.Exception as e: - print(f'\n{e}', file=sys.stderr) + except dependency.Exception as this_exception: + print(f'\n{this_exception}', file=sys.stderr) finally: logging.shutdown() return 0 +# pylint: enable=too-few-public-methods if __name__ == "__main__": DockerDev().main() diff --git a/dev_tools/docker_dev/gdev/mount.py b/dev_tools/docker_dev/gdev/mount.py index 5af62bb93fe7..7c3312bde282 100644 --- a/dev_tools/docker_dev/gdev/mount.py +++ b/dev_tools/docker_dev/gdev/mount.py @@ -10,12 +10,12 @@ """ from dataclasses import dataclass -from gdev.custom.pathlib import Path +from gdev.custom.gaia_path import GaiaPath @dataclass(frozen=True) class Mount: """ Class to represent a mount point between the docker container and the host system. """ - container_path: Path - host_path: Path + container_path: GaiaPath + host_path: GaiaPath diff --git a/dev_tools/docker_dev/gdev/options.py b/dev_tools/docker_dev/gdev/options.py index 0ff78dadfaf7..6697c503ad17 100644 --- a/dev_tools/docker_dev/gdev/options.py +++ b/dev_tools/docker_dev/gdev/options.py @@ -12,6 +12,7 @@ from typing import FrozenSet from gdev.mount import Mount +# pylint: disable=too-many-instance-attributes @dataclass(frozen=True) class Options: """ @@ -28,3 +29,4 @@ class Options: ports: FrozenSet[str] registry: str target: str +# pylint: enable=too-many-instance-attributes diff --git a/dev_tools/docker_dev/gdev/parser_structure.py b/dev_tools/docker_dev/gdev/parser_structure.py index b95c4135c9bf..30046e1b2c34 100644 --- a/dev_tools/docker_dev/gdev/parser_structure.py +++ b/dev_tools/docker_dev/gdev/parser_structure.py @@ -12,14 +12,9 @@ from typing import FrozenSet, Set, Tuple from importlib import import_module from importlib.util import find_spec -from inspect import getdoc, isabstract, iscoroutinefunction -from argparse import ArgumentParser, REMAINDER +from inspect import getdoc from dataclasses import dataclass -from importlib import import_module -from importlib.util import find_spec -from inspect import getdoc, isabstract, iscoroutinefunction from pkgutil import iter_modules -from typing import FrozenSet, Sequence, Set, Tuple @@ -28,7 +23,8 @@ class ParserStructure: """ Class to provide a description of the structure to be constructed. - Note that the `gdev.cmd` is a `path` within the package hierarchy, not calling out to a gdev.cmd script. + Note that the `gdev.cmd` is a `path` within the package hierarchy, + not calling out to a gdev.cmd script. """ command_parts: Tuple[str, ...] diff --git a/dev_tools/docker_dev/gdev/third_party/argcomplete.py b/dev_tools/docker_dev/gdev/third_party/argcomplete.py index f007389a989d..e3d0aad3c351 100644 --- a/dev_tools/docker_dev/gdev/third_party/argcomplete.py +++ b/dev_tools/docker_dev/gdev/third_party/argcomplete.py @@ -5,6 +5,11 @@ # All rights reserved. ############################################# +""" +Module to install the autocomplete package if it is not already installed. +""" + +# pylint: disable=import-self, unused-import try: from argcomplete import autocomplete from argcomplete.completers import FilesCompleter @@ -15,3 +20,4 @@ from argcomplete import autocomplete from argcomplete.completers import FilesCompleter +# pylint: enable=import-self, unused-import diff --git a/dev_tools/docker_dev/gdev/third_party/atools.py b/dev_tools/docker_dev/gdev/third_party/atools.py index 8155bb0f00cf..29c59e37aa7c 100644 --- a/dev_tools/docker_dev/gdev/third_party/atools.py +++ b/dev_tools/docker_dev/gdev/third_party/atools.py @@ -5,8 +5,13 @@ # All rights reserved. ############################################# +""" +Module to install the atools package if it is not already installed. +""" + from pathlib import Path as _Path +# pylint: disable=import-self, unused-import try: from atools import memoize except ImportError: @@ -15,6 +20,6 @@ _check_call('python3 -m pip install atools'.split()) from atools import memoize - +# pylint: enable=import-self, unused-import memoize_db = memoize(db_path=_Path.home() / '.memoize.gdev') diff --git a/dev_tools/docker_dev/test/gdev_execute.py b/dev_tools/docker_dev/test/gdev_execute.py index e599dcb8c4f1..d04ccfd70d30 100644 --- a/dev_tools/docker_dev/test/gdev_execute.py +++ b/dev_tools/docker_dev/test/gdev_execute.py @@ -5,40 +5,52 @@ # All rights reserved. ############################################# -from test.pytest_execute import InProcessExecution, InProcessResult -from gdev.__main__ import main -from gdev.main import DockerDev -import os -import io -import subprocess - """ -This code copied from: https://github.com/jackdewinter/pyscan/blob/test/test_scenarios.py with small changes. +Module to contain executors with which to execute the application within tests. + +This code copied from: https://github.com/jackdewinter/pyscan/blob/test/test_scenarios.py +with small changes. Any changes made to this code above the base code are copyright by Gaia Platform LLC. """ +import os +import io +import subprocess +from test.pytest_execute import InProcessExecution, InProcessResult +from gdev.__main__ import main +from gdev.main import DockerDev + +# pylint: disable=too-few-public-methods class SubprocessExecutor(): + """ + Class to use a subprocess to execute the tests. + """ def __init__(self, script_path=None): - self.__script_path = script_path if script_path else os.path.realpath(os.path.join(determine_repository_base_directory(), "dev_tools", "docker_dev", "gdev.sh")) + self.__script_path = script_path \ + if script_path else \ + os.path.realpath(os.path.join(determine_repository_base_directory(), \ + "dev_tools", "docker_dev", "gdev.sh")) def invoke_main(self, arguments=None, cwd=None): + """ + Invoke the main function with the proper parameters. + """ arguments = arguments.copy() arguments.insert(0, self.__script_path) cwd = cwd if cwd else determine_repository_production_directory() - new_process = subprocess.Popen(arguments, cwd=cwd,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - process_return_code = new_process.wait() + with subprocess.Popen(arguments, cwd=cwd,stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as new_process: - process_stdout_output = "" - for line in io.TextIOWrapper(new_process.stdout, encoding="utf-8"): - process_stdout_output += line + process_return_code = new_process.wait() - process_stderr_output = "" - for line in io.TextIOWrapper(new_process.stderr, encoding="utf-8"): - process_stderr_output += line + process_stdout_output = "".join(io.TextIOWrapper(new_process.stdout, encoding="utf-8")) + process_stderr_output = "".join(io.TextIOWrapper(new_process.stderr, encoding="utf-8")) - return InProcessResult(process_return_code, io.StringIO(process_stdout_output), io.StringIO(process_stderr_output)) + return InProcessResult(process_return_code, io.StringIO(process_stdout_output), \ + io.StringIO(process_stderr_output)) +# pylint: enable=too-few-public-methods class MainlineExecutor(InProcessExecution): """ @@ -52,39 +64,53 @@ def __init__(self, use_module=True, use_main=False): self.__entry_point = "__main__.py" if use_module else "main.py" def execute_main(self): + """ + Invoke the proper instance of the main function. + """ if self.__use_main: main() else: DockerDev().main() def get_main_name(self): + """ + Get the name of the entrypoint for "main". + """ return self.__entry_point def determine_repository_base_directory(): + """ + Determine the location of the repository's base directory. + """ script_directory = os.path.dirname(os.path.realpath(__file__)) base_directory = os.path.realpath(os.path.join(script_directory, "..", "..", "..")) return base_directory def determine_repository_production_directory(): - production_directory = os.path.realpath(os.path.join(determine_repository_base_directory(), "production")) + """ + Determine the location of the repository's production directory. + """ + production_directory = os.path.realpath( + os.path.join(determine_repository_base_directory(), "production")) return production_directory def determine_old_script_behavior(gdev_arguments): - original_gdev_script_path = os.path.realpath(os.path.join(determine_repository_base_directory(), "dev_tools", "gdev", "gdev.sh")) - executor = SubprocessExecutor(original_gdev_script_path) + """ + Determine the behavior of the old gdev script. + """ + original_gdev_script_path = os.path.realpath( + os.path.join(determine_repository_base_directory(), "dev_tools", "gdev", "gdev.sh")) + #executor = SubprocessExecutor(original_gdev_script_path) arguments_to_use = [original_gdev_script_path] arguments_to_use.extend(gdev_arguments) - new_process = subprocess.Popen(arguments_to_use, cwd=determine_repository_production_directory(),stdout=subprocess.PIPE,stderr=subprocess.PIPE) - process_return_code = new_process.wait() - - process_stdout_output = "" - for line in io.TextIOWrapper(new_process.stdout, encoding="utf-8"): - process_stdout_output += line + with subprocess.Popen(arguments_to_use, \ + cwd=determine_repository_production_directory(),stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as new_process: - process_stderr_output = "" - for line in io.TextIOWrapper(new_process.stderr, encoding="utf-8"): - process_stderr_output += line + process_return_code = new_process.wait() - return process_return_code, process_stdout_output, process_stderr_output + process_stdout_output = "".join(io.TextIOWrapper(new_process.stdout, encoding="utf-8")) + process_stderr_output = "".join(io.TextIOWrapper(new_process.stderr, encoding="utf-8")) + return process_return_code, process_stdout_output, process_stderr_output diff --git a/dev_tools/docker_dev/test/pytest_execute.py b/dev_tools/docker_dev/test/pytest_execute.py index 284d91e8f5c4..f4939aabd4df 100644 --- a/dev_tools/docker_dev/test/pytest_execute.py +++ b/dev_tools/docker_dev/test/pytest_execute.py @@ -64,7 +64,8 @@ def compare_versus_expected( if not was_found: assert ( False - ), f"Block\n---\n{next_text_block}\n---\nwas not found in\n---\n{actual_stream.getvalue()}" + ), f"Block\n---\n{next_text_block}\n---\nwas not found in" + \ + f"\n---\n{actual_stream.getvalue()}" elif actual_stream.getvalue().strip() != expected_text.strip(): diff = difflib.ndiff( expected_text.splitlines(), actual_stream.getvalue().splitlines() @@ -162,7 +163,8 @@ def assert_results( assert ( self.__return_code == error_code - ), f"Actual error code ({self.__return_code}) and expected error code ({error_code}) differ." + ), f"Actual error code ({self.__return_code}) and " + \ + f"expected error code ({error_code}) differ." finally: self.__std_out.close() @@ -226,7 +228,8 @@ def assert_resultant_file(cls, file_path, expected_contents): diff_values = "\n".join(list(diff)) assert ( False - ), f"Actual and expected contents of '{file_path}' are not equal:\n---\n{diff_values}\n---\n" + ), f"Actual and expected contents of '{file_path}' are not equal:" + \ + f"\n---\n{diff_values}\n---\n" # pylint: disable=too-few-public-methods diff --git a/dev_tools/docker_dev/test/test_scenarios.py b/dev_tools/docker_dev/test/test_scenarios.py index 583c632bbc48..44972cb01abc 100644 --- a/dev_tools/docker_dev/test/test_scenarios.py +++ b/dev_tools/docker_dev/test/test_scenarios.py @@ -5,10 +5,19 @@ # All rights reserved. ############################################# -from test.gdev_execute import MainlineExecutor, determine_old_script_behavior, determine_repository_production_directory, SubprocessExecutor, determine_repository_base_directory +""" +Module to provide high level scenario tests for the Docker_Dev project. +""" + +from test.gdev_execute import determine_old_script_behavior, \ + determine_repository_production_directory, SubprocessExecutor, \ + determine_repository_base_directory from gdev.host import Host def get_executor(): + """ + Get the executor to use for invoking the Gdev application for testing. + """ return SubprocessExecutor() def test_show_help_x(): @@ -19,10 +28,14 @@ def test_show_help_x(): # Arrange executor = get_executor() suppplied_arguments = ["--help"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) + expected_output = expected_output.replace(\ + "\nGaiaPlatform build and development environment tool.\n", "") # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -37,13 +50,16 @@ def test_show_help_build(): # Arrange executor = get_executor() suppplied_arguments = ["build", "--help"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert - expected_output = expected_output.replace("Dependency(options: 'Options')", "Bob.Dependency") + expected_output = expected_output.replace("Dependency(options: 'Options')", \ + "Class to provide for the `build` subcommand entry point.") execute_results.assert_results( expected_output, expected_error, expected_return_code ) @@ -56,13 +72,16 @@ def test_show_help_cfg(): # Arrange executor = get_executor() suppplied_arguments = ["cfg", "--help"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert - expected_output = expected_output.replace("Parse gdev.cfg for build rules.", "Parse the target gdev.cfg for build rules.") + expected_output = expected_output.replace("Parse gdev.cfg for build rules.", \ + "Class to provide for the `cfg` subcommand entry point.") execute_results.assert_results( expected_output, expected_error, expected_return_code ) @@ -74,13 +93,16 @@ def test_show_help_dockerfile(): # Arrange executor = get_executor() suppplied_arguments = ["dockerfile", "--help"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert - expected_output = expected_output.replace("Dependency(options: 'Options')", "Class to encapsulate the 'cfg' subcommand.") + expected_output = expected_output.replace("Dependency(options: 'Options')", \ + "Class to provide for the `dockerfile` subcommand entry point.") execute_results.assert_results( expected_output, expected_error, expected_return_code ) @@ -92,10 +114,15 @@ def test_show_help_gen(): # Arrange executor = get_executor() suppplied_arguments = ["gen", "--help"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) + expected_output = expected_output.replace("\nInternal component commands for top-level " + \ + "gdev commands. These should rarely\n" + \ + "be needed.\n", "") # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -110,13 +137,16 @@ def test_show_help_push(): # Arrange executor = get_executor() suppplied_arguments = ["push", "--help"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert - expected_output = expected_output.replace("Dependency(options: 'Options')", "Bob.Dependency") + expected_output = expected_output.replace("Dependency(options: 'Options')", \ + "Class to provide for the `push` subcommand entry point.") execute_results.assert_results( expected_output, expected_error, expected_return_code ) @@ -129,13 +159,16 @@ def test_show_help_run(): # Arrange executor = get_executor() suppplied_arguments = ["run", "--help"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert - expected_output = expected_output.replace("Dependency(options: 'Options')", "Bob.Dependency") + expected_output = expected_output.replace("Dependency(options: 'Options')", \ + "Class to provide for the `run` subcommand entry point.") execute_results.assert_results( expected_output, expected_error, expected_return_code ) @@ -160,7 +193,8 @@ def __find_docker_build_line(expected_output): print(f"build_line:{build_line}") return build_line -def __find_and_remove(line_to_look_in, part_to_look_for, look_at_end=False, search_for_end_whitespace=False, replace_with=None): +def __find_and_remove(line_to_look_in, part_to_look_for, look_at_end=False, \ + search_for_end_whitespace=False, replace_with=None): if look_at_end: assert line_to_look_in.endswith(part_to_look_for) @@ -181,15 +215,18 @@ def __construct_base_build_line(is_using_mixins=False): current_hash = Host.execute_and_get_line_sync('git rev-parse HEAD') run_type = 'custom' if is_using_mixins else 'run' - return f"-f {determine_repository_base_directory()}/.gdev/production/{run_type}.dockerfile.gdev " + \ + return f"-f {determine_repository_base_directory()}/.gdev/production" + \ + f"/{run_type}.dockerfile.gdev " + \ f"-t production__{run_type}:latest --label GitHash=\"{current_hash}\" " + \ "--build-arg BUILDKIT_INLINE_CACHE=1 --platform linux/amd64 --shm-size 1gb " + \ f"--ssh default {determine_repository_base_directory()}" def __construct_base_run_command_line(is_using_mixins=False): run_type = 'custom' if is_using_mixins else 'run' - return f"--rm --init --entrypoint /bin/bash --hostname production__{run_type} --platform linux/amd64 " + \ - f"--privileged --volume {determine_repository_base_directory()}:/source production__{run_type}:latest" + return f"--rm --init --entrypoint /bin/bash --hostname production__{run_type} " + \ + "--platform linux/amd64 " + \ + f"--privileged --volume {determine_repository_base_directory()}:" + \ + f"/source production__{run_type}:latest" def test_show_cfg(): @@ -200,17 +237,21 @@ def test_show_cfg(): # Arrange executor = get_executor() suppplied_arguments = ["cfg"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( expected_output, expected_error, expected_return_code ) - assert '# enable by setting "Debug"' in expected_output, "Original output missing config hint." - assert '# enable by setting "GaiaLLVMTests"' in expected_output, "Original output missing config hint." + assert '# enable by setting "Debug"' in expected_output, \ + "Original output missing config hint." + assert '# enable by setting "GaiaLLVMTests"' in expected_output, \ + "Original output missing config hint." def test_show_cfg_with_cfg_enable_debug(): """ @@ -221,17 +262,21 @@ def test_show_cfg_with_cfg_enable_debug(): # Arrange executor = get_executor() suppplied_arguments = ["cfg", "--cfg-enable", "Debug"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( expected_output, expected_error, expected_return_code ) - assert '# enable by setting "Debug"' not in expected_output, "Original output contains config hint." - assert '# enable by setting "GaiaLLVMTests"' in expected_output, "Original output missing config hint." + assert '# enable by setting "Debug"' not in expected_output, \ + "Original output contains config hint." + assert '# enable by setting "GaiaLLVMTests"' in expected_output, \ + "Original output missing config hint." def test_show_cfg_with_cfg_enable_debug_and_llvm(): """ @@ -242,17 +287,21 @@ def test_show_cfg_with_cfg_enable_debug_and_llvm(): # Arrange executor = get_executor() suppplied_arguments = ["cfg", "--cfg-enable", "Debug", "GaiaLLVMTests"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( expected_output, expected_error, expected_return_code ) - assert '# enable by setting "Debug"' not in expected_output, "Original output contains config hint." - assert '# enable by setting "GaiaLLVMTests"' not in expected_output, "Original output missing config hint." + assert '# enable by setting "Debug"' not in expected_output, \ + "Original output contains config hint." + assert '# enable by setting "GaiaLLVMTests"' not in expected_output, \ + "Original output missing config hint." def test_generate_dockerfile(): """ @@ -262,17 +311,21 @@ def test_generate_dockerfile(): # Arrange executor = get_executor() suppplied_arguments = ["dockerfile"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( expected_output, expected_error, expected_return_code ) - assert "-DCMAKE_BUILD_TYPE=Debug" not in expected_output, "Original output contains untriggered line." - assert "-DBUILD_GAIA_LLVM_TESTS=ON" not in expected_output, "Original output contains untriggered line." + assert "-DCMAKE_BUILD_TYPE=Debug" not in expected_output, \ + "Original output contains untriggered line." + assert "-DBUILD_GAIA_LLVM_TESTS=ON" not in expected_output, \ + "Original output contains untriggered line." def test_generate_dockerfile_debug(): """ @@ -286,17 +339,21 @@ def test_generate_dockerfile_debug(): # Arrange executor = get_executor() suppplied_arguments = ["dockerfile", "--cfg-enable", "Debug"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( expected_output, expected_error, expected_return_code ) - assert "-DCMAKE_BUILD_TYPE=Debug" in expected_output, "Original output does not contain triggered line." - assert "-DBUILD_GAIA_LLVM_TESTS=ON" not in expected_output, "Original output contains untriggered line." + assert "-DCMAKE_BUILD_TYPE=Debug" in expected_output, \ + "Original output does not contain triggered line." + assert "-DBUILD_GAIA_LLVM_TESTS=ON" not in expected_output, \ + "Original output contains untriggered line." def test_generate_dockerfile_debug_and_llvm(): """ @@ -310,17 +367,21 @@ def test_generate_dockerfile_debug_and_llvm(): # Arrange executor = get_executor() suppplied_arguments = ["dockerfile", "--cfg-enable", "Debug", "GaiaLLVMTests"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( expected_output, expected_error, expected_return_code ) - assert "-DCMAKE_BUILD_TYPE=Debug" in expected_output, "Original output does not contain triggered line." - assert "-DBUILD_GAIA_LLVM_TESTS=ON" in expected_output, "Original output does not contain triggered line." + assert "-DCMAKE_BUILD_TYPE=Debug" in expected_output, \ + "Original output does not contain triggered line." + assert "-DBUILD_GAIA_LLVM_TESTS=ON" in expected_output, \ + "Original output does not contain triggered line." def test_generate_dockerfile_debug_and_new_base_image(): """ @@ -334,10 +395,12 @@ def test_generate_dockerfile_debug_and_new_base_image(): # Arrange executor = get_executor() suppplied_arguments = ["dockerfile", "--cfg-enable", "Debug", "--base-image", "frogger"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -356,10 +419,12 @@ def test_generate_dockerfile_mixins_sshd(): # Arrange executor = get_executor() suppplied_arguments = ["dockerfile", "--mixins", "sshd"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -377,10 +442,12 @@ def test_generate_dockerfile_mixins_sshd_and_nano(): # Arrange executor = get_executor() suppplied_arguments = ["dockerfile", "--mixins", "sshd", "nano"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -400,10 +467,12 @@ def test_generate_docker_build(): # Arrange executor = get_executor() suppplied_arguments = ["build", "--dry-dock"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -422,10 +491,12 @@ def test_generate_docker_build_with_platform(): # Arrange executor = get_executor() suppplied_arguments = ["build", "--dry-dock", "--platform", "arm64"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -433,7 +504,8 @@ def test_generate_docker_build_with_platform(): ) build_line = __find_docker_build_line(expected_output) - build_line = __find_and_remove(build_line, " --platform linux/arm64",replace_with=" --platform linux/amd64") + build_line = __find_and_remove(build_line, " --platform linux/arm64",\ + replace_with=" --platform linux/amd64") assert build_line == __construct_base_build_line() def test_generate_docker_build_with_registry(): @@ -445,10 +517,12 @@ def test_generate_docker_build_with_registry(): # Arrange executor = get_executor() suppplied_arguments = ["build", "--dry-dock", "--registry", "localhost"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -456,7 +530,8 @@ def test_generate_docker_build_with_registry(): ) build_line = __find_docker_build_line(expected_output) - build_line = __find_and_remove(build_line, " --cache-from localhost/production__",search_for_end_whitespace=True,replace_with=" ") + build_line = __find_and_remove(build_line, " --cache-from localhost/production__",\ + search_for_end_whitespace=True,replace_with=" ") assert build_line == __construct_base_build_line() def test_generate_docker_build_with_mixins_sudo(): @@ -468,10 +543,12 @@ def test_generate_docker_build_with_mixins_sudo(): # Arrange executor = get_executor() suppplied_arguments = ["build", "--dry-dock", "--mixins", "sudo"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -491,10 +568,12 @@ def test_generate_docker_build_with_mixins_sudo_and_nano(): # Arrange executor = get_executor() suppplied_arguments = ["build", "--dry-dock", "--mixins", "sudo", "nano"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -515,10 +594,12 @@ def test_generate_docker_run(): # Arrange executor = get_executor() suppplied_arguments = ["run", "--dry-dock"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -541,10 +622,12 @@ def test_generate_docker_run_mixin_clion(): # Arrange executor = get_executor() suppplied_arguments = ["run", "--dry-dock", "--mixins", "clion"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -556,7 +639,8 @@ def test_generate_docker_run_mixin_clion(): assert build_line == __construct_base_build_line(is_using_mixins=True) run_line = __find_docker_run_line(expected_output) - run_line = __find_and_remove(run_line, " -p 22:22 --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --user 1000:1000") + run_line = __find_and_remove(run_line, \ + " -p 22:22 --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --user 1000:1000") assert run_line == __construct_base_run_command_line(is_using_mixins=True) def test_generate_docker_run_mixin_nano(): @@ -568,10 +652,12 @@ def test_generate_docker_run_mixin_nano(): # Arrange executor = get_executor() suppplied_arguments = ["run", "--dry-dock", "--mixins", "nano"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -595,10 +681,12 @@ def test_generate_docker_run_mounts(): # Arrange executor = get_executor() suppplied_arguments = ["run", "--dry-dock", "--mount", "/root:/host"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -610,7 +698,8 @@ def test_generate_docker_run_mounts(): run_line = __find_docker_run_line(expected_output) run_line = __find_and_remove(run_line, \ - '--mount type=volume,dst=/host,volume-driver=local,volume-opt=type=none,volume-opt=o=bind,volume-opt=device=/root ') + '--mount type=volume,dst=/host,volume-driver=local,volume-opt=type=none,' + \ + 'volume-opt=o=bind,volume-opt=device=/root ') assert run_line == __construct_base_run_command_line() @@ -624,10 +713,12 @@ def test_generate_docker_run_platform(): # Arrange executor = get_executor() suppplied_arguments = ["run", "--dry-dock", "--platform", "arm64"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -635,11 +726,13 @@ def test_generate_docker_run_platform(): ) build_line = __find_docker_build_line(expected_output) - build_line = __find_and_remove(build_line, " --platform linux/arm64", replace_with=" --platform linux/amd64") + build_line = __find_and_remove(build_line, " --platform linux/arm64", \ + replace_with=" --platform linux/amd64") assert build_line == __construct_base_build_line() run_line = __find_docker_run_line(expected_output) - run_line = __find_and_remove(run_line, " --platform linux/arm64", replace_with=" --platform linux/amd64") + run_line = __find_and_remove(run_line, " --platform linux/arm64", \ + replace_with=" --platform linux/amd64") assert run_line == __construct_base_run_command_line() @@ -652,10 +745,12 @@ def test_generate_docker_run_ports(): # Arrange executor = get_executor() suppplied_arguments = ["run", "--dry-dock", "--ports", "1234"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -680,10 +775,12 @@ def test_generate_docker_run_registry(): # Arrange executor = get_executor() suppplied_arguments = ["run", "--dry-dock", "--registry", "localhost"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -691,7 +788,8 @@ def test_generate_docker_run_registry(): ) build_line = __find_docker_build_line(expected_output) - build_line = __find_and_remove(build_line, "--cache-from localhost/production__", search_for_end_whitespace=True) + build_line = __find_and_remove(build_line, "--cache-from localhost/production__", \ + search_for_end_whitespace=True) assert build_line == __construct_base_build_line() run_line = __find_docker_run_line(expected_output) @@ -706,10 +804,12 @@ def test_generate_docker_run_force(): # Arrange executor = get_executor() suppplied_arguments = ["run", "--dry-dock", "--force"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( @@ -731,10 +831,12 @@ def test_generate_docker_run_args(): # Arrange executor = get_executor() suppplied_arguments = ["run", "--dry-dock", "not-an-argument"] - expected_return_code, expected_output, expected_error = determine_old_script_behavior(suppplied_arguments) + expected_return_code, expected_output, expected_error = \ + determine_old_script_behavior(suppplied_arguments) # Act - execute_results = executor.invoke_main(arguments=suppplied_arguments, cwd=determine_repository_production_directory()) + execute_results = executor.invoke_main(arguments=suppplied_arguments, \ + cwd=determine_repository_production_directory()) # Assert execute_results.assert_results( diff --git a/dev_tools/gdev/gdev/dependency.py b/dev_tools/gdev/gdev/dependency.py index a0ccd60e9f51..7d6366517c32 100644 --- a/dev_tools/gdev/gdev/dependency.py +++ b/dev_tools/gdev/gdev/dependency.py @@ -75,6 +75,13 @@ class Dependency: options: Options + __LOG_LEVELS = [ + "CRITICAL", + "ERROR", + "WARNING", + "INFO", + "DEBUG"] + class Exception(Exception): pass @@ -150,7 +157,7 @@ def add_flags(parser: ArgumentParser) -> None: parser.add_argument( '--log-level', default=log_level_default, - choices=[name for _, name in sorted(logging._levelToName.items())], + choices=[name for name in Dependency.__LOG_LEVELS], help=f'Log level. Default: "{log_level_default}"' ) parser.add_argument(