Skip to content

Commit

Permalink
Gaiaplat 1838 : Applying Black formatter to Docker_Dev (#1367)
Browse files Browse the repository at this point in the history
* GAIAPLAT-1838 - Applying Black formatter to docker_dev project.

https://gaiaplatform.atlassian.net/browse/GAIAPLAT-1838
  • Loading branch information
JackAtGaia authored Feb 23, 2022
1 parent 533a8df commit c6d2b90
Show file tree
Hide file tree
Showing 68 changed files with 1,235 additions and 812 deletions.
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -171,13 +171,13 @@ repos:
rev: 21.7b0
hooks:
- id: black
exclude: ^(dev_tools/gdev/|dev_tools/docker_dev/|production/tools/tests/gaiat/lit.cfg.py)
exclude: ^(dev_tools/gdev/|production/tools/tests/gaiat/lit.cfg.py)

- repo: https://github.com/PyCQA/flake8
rev: 4.0.1
hooks:
- id: flake8
exclude: ^(dev_tools/gdev/|dev_tools/docker_dev/|production/tools/tests/gaiat/lit.cfg.py)
exclude: ^(dev_tools/gdev/|production/tools/tests/gaiat/lit.cfg.py)
args:
- --ignore=E203,E501,W503

Expand Down
4 changes: 3 additions & 1 deletion dev_tools/docker_dev/gdev/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@
import sys
from gdev.main import DockerDev


def main() -> int:
"""
Module main entry point into the application.
"""
DockerDev().main()
return 0

if __name__ == '__main__':

if __name__ == "__main__":
sys.exit(main())
3 changes: 2 additions & 1 deletion dev_tools/docker_dev/gdev/cmd/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@
"""
Module to provide for the `cfg` subcommand entry point.
"""

from gdev.dependency import Dependency
from gdev.third_party.atools import memoize
from .gen.run.build import GenRunBuild
from gdev.cmd.gen.run.build import GenRunBuild


class Build(Dependency):
Expand Down
10 changes: 4 additions & 6 deletions dev_tools/docker_dev/gdev/cmd/cfg.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@
"""
Module to provide for the `cfg` subcommand entry point.
"""

from gdev.third_party.atools import memoize
from .gen.run.cfg import GenRunCfg
from gdev.cmd.gen.run.cfg import GenRunCfg


class Cfg(GenRunCfg):
Expand All @@ -23,10 +24,7 @@ def cli_entrypoint(self) -> None:
Execution entrypoint for this module.
"""
print(
'\n'.join(
self.get_lines(
cfg_enables=self.options.cfg_enables,
path=self.path
)
"\n".join(
self.get_lines(cfg_enables=self.options.cfg_enables, path=self.path)
)
)
3 changes: 2 additions & 1 deletion dev_tools/docker_dev/gdev/cmd/dockerfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@
"""
Module to provide for the `dockerfile` subcommand entry point.
"""

from gdev.dependency import Dependency
from gdev.third_party.atools import memoize
from .gen.run.dockerfile import GenRunDockerfile
from gdev.cmd.gen.run.dockerfile import GenRunDockerfile


class Dockerfile(Dependency):
Expand Down
83 changes: 42 additions & 41 deletions dev_tools/docker_dev/gdev/cmd/gen/_abc/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@

# We require buildkit support for inline caching of multi-stage dockerfiles. It's also way faster
# and the terminal output is relatively sane.
os.environ['DOCKER_BUILDKIT'] = '1'
os.environ['DOCKER_CLI_EXPERIMENTAL'] = 'enabled'
os.environ["DOCKER_BUILDKIT"] = "1"
os.environ["DOCKER_CLI_EXPERIMENTAL"] = "enabled"


class GenAbcBuild(Dependency, ABC):
Expand Down Expand Up @@ -53,12 +53,14 @@ def _get_actual_label_value(self, name: str) -> str:
"""
Request that docker provide information about the label value for the current image.
"""
if (line := Host.execute_and_get_line_sync(
f'docker image inspect'
if (
line := Host.execute_and_get_line_sync(
f"docker image inspect"
f' --format="{{{{.Config.Labels.{name}}}}}"'
f' {self.get_tag()}'
)) == '"<no value>"':
value = ''
f" {self.get_tag()}"
)
) == '"<no value>"':
value = ""
else:
value = line.strip('"')

Expand All @@ -70,7 +72,7 @@ def _get_actual_label_value_by_name(self) -> Mapping[str, str]:
Get the hash of an image with the actual label values that are called
for by the configuration.
"""
return {'GitHash': self._get_actual_label_value(name='GitHash')}
return {"GitHash": self._get_actual_label_value(name="GitHash")}

@memoize
def get_actual_label_value_by_name(self) -> Mapping[str, str]:
Expand All @@ -80,13 +82,14 @@ def get_actual_label_value_by_name(self) -> Mapping[str, str]:
"""

actual_label_value_by_name = self._get_actual_label_value_by_name()
self.log.debug('actual_label_value_by_name = %s', actual_label_value_by_name)
self.log.debug("actual_label_value_by_name = %s", actual_label_value_by_name)
return actual_label_value_by_name

@memoize
def __get_base_build_names(self) -> Iterable[str]:

seen_dockerfiles = set()

def inner(dockerfile: GenAbcDockerfile) -> Iterable[str]:
build_names = []
if dockerfile not in seen_dockerfiles:
Expand All @@ -99,7 +102,7 @@ def inner(dockerfile: GenAbcDockerfile) -> Iterable[str]:

base_build_names = tuple(inner(self.dockerfile))

self.log.debug('base_build_names = %s', base_build_names)
self.log.debug("base_build_names = %s", base_build_names)

return base_build_names

Expand All @@ -110,13 +113,13 @@ def get_sha(self) -> str:
"""

if lines := Host.execute_and_get_lines_sync(
f'docker image ls -q --no-trunc {self.get_tag()}'
f"docker image ls -q --no-trunc {self.get_tag()}"
):
sha = next(iter(lines))
else:
sha = ''
sha = ""

self.log.debug('sha = %s', sha)
self.log.debug("sha = %s", sha)

return sha

Expand All @@ -126,9 +129,9 @@ def get_tag(self) -> str:
Construct a tag from the name of the dockerfile.
"""

tag = f'{self.dockerfile.get_name()}:latest'
tag = f"{self.dockerfile.get_name()}:latest"

self.log.debug('tag = %s', tag)
self.log.debug("tag = %s", tag)

return tag

Expand All @@ -137,17 +140,17 @@ def __get_wanted_git_hash(self) -> str:
"""
Request that GIT provides information about the SHA of the HEAD node.
"""
wanted_git_hash = Host.execute_and_get_line_sync('git rev-parse HEAD')
wanted_git_hash = Host.execute_and_get_line_sync("git rev-parse HEAD")

self.log.debug('wanted_git_hash = %s', wanted_git_hash)
self.log.debug("wanted_git_hash = %s", wanted_git_hash)

return wanted_git_hash

def _get_wanted_label_value_by_name(self) -> Mapping[str, str]:
"""
Get the hash of the image with the label values that are called for by the configuration.
"""
return {'GitHash': self.__get_wanted_git_hash()}
return {"GitHash": self.__get_wanted_git_hash()}

@memoize
def get_wanted_label_value_by_name(self) -> Mapping[str, str]:
Expand All @@ -156,7 +159,7 @@ def get_wanted_label_value_by_name(self) -> Mapping[str, str]:
"""
wanted_label_value_by_name = self._get_wanted_label_value_by_name()

self.log.debug('wanted_label_value_by_name = %s', wanted_label_value_by_name)
self.log.debug("wanted_label_value_by_name = %s", wanted_label_value_by_name)

return wanted_label_value_by_name

Expand All @@ -169,39 +172,35 @@ def main(self) -> None:

cached_images = ""
if self.options.registry:
cached_images = ','.join(
[f'{self.options.registry}/{base_build_name}:latest'
for base_build_name in self.__get_base_build_names()])
cached_images = ",".join(
[
f"{self.options.registry}/{base_build_name}:latest"
for base_build_name in self.__get_base_build_names()
]
)
cached_images = f"--cache-from {cached_images}"

self.log.info('Creating image "%s"', self.get_tag())
Host.execute_sync(
f'docker buildx build'
f' -f {self.dockerfile.path}'
f' -t {self.get_tag()}'

f'''{''.join([
f"docker buildx build"
f" -f {self.dockerfile.path}"
f" -t {self.get_tag()}"
f"""{''.join([
f' --label {name}="{value}"'
for name, value
in (self.get_wanted_label_value_by_name()).items()
])}'''

])}"""
# Keep metadata about layers so that they can be used as a cache source.
f' --build-arg BUILDKIT_INLINE_CACHE=1'

f' --platform linux/{self.options.platform}'

f" --build-arg BUILDKIT_INLINE_CACHE=1"
f" --platform linux/{self.options.platform}"
# Required to run production.
f' --shm-size 1gb'

f" --shm-size 1gb"
# Allow cloning repos with ssh.
f' --ssh default'

f' {cached_images}'

f' {GaiaPath.repo()}'
f" --ssh default"
f" {cached_images}"
f" {GaiaPath.repo()}"
)
Host.execute_sync('docker image prune -f')
Host.execute_sync("docker image prune -f")

# pylint: disable=import-outside-toplevel
#
Expand All @@ -216,7 +215,9 @@ def cli_entrypoint(self) -> None:
build = self
else:
from gdev.cmd.gen._custom.build import GenCustomBuild

build = GenCustomBuild(options=self.options, base_build=self)

build.run()

# pylint: enable=import-outside-toplevel
Loading

0 comments on commit c6d2b90

Please sign in to comment.