From 50e0b928be3dd453c030c66f25b80d1434472367 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 21 Nov 2023 16:03:14 +0100 Subject: [PATCH] Make passing build args explicit in ci/prod builds (#35768) When building hte image, breeze converts some simple parameters passed as breeze command (with autocompletion and explanation) into much longer and more complex set of build args that are passed to `docker build` command. The way how passing hte args worked so far is that it was pretty implicit: * **kwargs were used to ingest `click` flags * parameters found as empty/None were filtered out from these * Build*Params dataclass was created out of such kwargs dict * argumenst from dataclass (with some customization) were converted to --build-arg (CAPITALIZED_PROPERTY_NAME) This had a lot of implicitness and it was not easy to understand whether the parameters passed were correct and how they passed through this chain. This change makes all the build arg much more explicit - without kwargs and dictionary. Each CI/PROD build param has now a method where it explicitly converts arguments into build-args - including specifying which of those are optional (where you can actually filter out Empty and None values) and which are required (where an actual value is expected). This PR also cleans up the click flags sequence and their presence as well as the output of help command (they were grouped with more related parameters) --- .../commands/ci_image_commands.py | 119 ++++++++--- .../commands/ci_image_commands_config.py | 29 ++- .../commands/production_image_commands.py | 160 +++++++++++---- .../production_image_commands_config.py | 44 ++-- .../airflow_breeze/params/build_ci_params.py | 113 +++++------ .../params/build_prod_params.py | 115 +++++------ .../params/common_build_params.py | 84 ++++---- .../airflow_breeze/utils/common_options.py | 12 +- .../utils/docker_command_utils.py | 42 +--- images/breeze/output_ci-image_build.svg | 160 ++++++++------- images/breeze/output_ci-image_build.txt | 2 +- images/breeze/output_prod-image_build.svg | 190 ++++++++++-------- images/breeze/output_prod-image_build.txt | 2 +- ...ase-management_prepare-airflow-package.txt | 2 +- ...e-management_prepare-provider-packages.txt | 2 +- 15 files changed, 619 insertions(+), 457 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 560f3852ecb0f..f33fdad2ef491 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -22,6 +22,7 @@ import subprocess import sys import time +from copy import deepcopy from functools import partial from pathlib import Path from typing import TYPE_CHECKING, Any, Callable @@ -32,10 +33,10 @@ from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( + option_additional_airflow_extras, option_additional_dev_apt_command, option_additional_dev_apt_deps, option_additional_dev_apt_env, - option_additional_extras, option_additional_pip_install_flags, option_additional_python_deps, option_airflow_constraints_location, @@ -53,7 +54,6 @@ option_docker_cache, option_dry_run, option_eager_upgrade_additional_requirements, - option_force_build, option_github_repository, option_github_token, option_image_name, @@ -77,7 +77,7 @@ option_upgrade_to_newer_dependencies, option_verbose, option_verify, - option_version_suffix_for_pypi, + option_version_suffix_for_pypi_ci, option_wait_for_image, ) from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm @@ -99,7 +99,6 @@ from airflow_breeze.utils.registry import login_to_github_docker_registry from airflow_breeze.utils.run_tests import verify_an_image from airflow_breeze.utils.run_utils import ( - filter_out_none, fix_group_permissions, instruct_build_image, is_repo_rebased, @@ -224,12 +223,6 @@ def get_exitcode(status: int) -> int: @ci_image.command(name="build") @option_python @option_debian_version -@option_run_in_parallel -@option_parallelism -@option_skip_cleanup -@option_debug_resources -@option_include_success_outputs -@option_python_versions @option_upgrade_to_newer_dependencies @option_upgrade_on_failure @option_platform_multiple @@ -239,18 +232,16 @@ def get_exitcode(status: int) -> int: @option_prepare_buildx_cache @option_push @option_install_providers_from_sources -@option_additional_extras +@option_additional_airflow_extras @option_additional_dev_apt_deps @option_additional_python_deps @option_additional_dev_apt_command @option_additional_dev_apt_env @option_builder @option_build_progress -@option_build_timeout_minutes @option_commit_sha @option_dev_apt_command @option_dev_apt_deps -@option_force_build @option_python_image @option_eager_upgrade_additional_requirements @option_airflow_constraints_location @@ -259,19 +250,58 @@ def get_exitcode(status: int) -> int: @option_tag_as_latest @option_additional_pip_install_flags @option_github_repository -@option_version_suffix_for_pypi +@option_version_suffix_for_pypi_ci +@option_build_timeout_minutes +@option_run_in_parallel +@option_parallelism +@option_skip_cleanup +@option_debug_resources +@option_include_success_outputs +@option_python_versions @option_verbose @option_dry_run @option_answer def build( + # Build options + python: str, + debian_version: str, + upgrade_to_newer_dependencies: bool, + upgrade_on_failure: bool, + platform: str | None, + github_token: str | None, + docker_cache: str, + image_tag: str, + prepare_buildx_cache: bool, + push: bool, + install_providers_from_sources: bool, + additional_airflow_extras: str | None, + additional_dev_apt_deps: str | None, + additional_python_deps: str | None, + additional_dev_apt_command: str | None, + additional_dev_apt_env: str | None, + builder: str, + build_progress: str, + commit_sha: str | None, + dev_apt_command: str | None, + dev_apt_deps: str | None, + eager_upgrade_additional_requirements: str | None, + airflow_constraints_location: str | None, + airflow_constraints_mode: str, + airflow_constraints_reference: str, + tag_as_latest: bool, + additional_pip_install_flags: str | None, + github_repository: str, + python_image: str | None, + version_suffix_for_pypi: str, + # Parallel building run_in_parallel: bool, parallelism: int, skip_cleanup: bool, debug_resources: bool, include_success_outputs, python_versions: str, + # Other options build_timeout_minutes: int | None, - **kwargs: dict[str, Any], ): """Build CI image. Include building multiple images for all python versions.""" @@ -306,16 +336,51 @@ def run_build(ci_image_params: BuildCiParams) -> None: perform_environment_checks() check_remote_ghcr_io_commands() - parameters_passed = filter_out_none(**kwargs) - parameters_passed["force_build"] = True fix_group_permissions() + base_build_params = BuildCiParams( + force_build=True, + python=python, + debian_version=debian_version, + upgrade_to_newer_dependencies=upgrade_to_newer_dependencies, + upgrade_on_failure=upgrade_on_failure, + github_token=github_token, + docker_cache=docker_cache, + image_tag=image_tag, + prepare_buildx_cache=prepare_buildx_cache, + push=push, + install_providers_from_sources=install_providers_from_sources, + additional_airflow_extras=additional_airflow_extras, + additional_python_deps=additional_python_deps, + additional_dev_apt_command=additional_dev_apt_command, + additional_dev_apt_env=additional_dev_apt_env, + builder=builder, + build_progress=build_progress, + commit_sha=commit_sha, + dev_apt_command=dev_apt_command, + dev_apt_deps=dev_apt_deps, + eager_upgrade_additional_requirements=eager_upgrade_additional_requirements, + airflow_constraints_location=airflow_constraints_location, + airflow_constraints_mode=airflow_constraints_mode, + airflow_constraints_reference=airflow_constraints_reference, + tag_as_latest=tag_as_latest, + additional_pip_install_flags=additional_pip_install_flags, + github_repository=github_repository, + python_image=python_image, + version_suffix_for_pypi=version_suffix_for_pypi, + ) + if platform: + base_build_params.platform = platform + if additional_dev_apt_deps: + # For CI image we only set additional_dev_apt_deps when we explicitly pass it + base_build_params.additional_dev_apt_deps = additional_dev_apt_deps + if run_in_parallel: python_version_list = get_python_version_list(python_versions) params_list: list[BuildCiParams] = [] for python in python_version_list: - params = BuildCiParams(**parameters_passed) - params.python = python - params_list.append(params) + build_params = deepcopy(base_build_params) + build_params.python = python + params_list.append(build_params) prepare_for_building_ci_image(params=params_list[0]) run_build_in_parallel( image_params_list=params_list, @@ -326,9 +391,8 @@ def run_build(ci_image_params: BuildCiParams) -> None: debug_resources=debug_resources, ) else: - params = BuildCiParams(**parameters_passed) - prepare_for_building_ci_image(params=params) - run_build(ci_image_params=params) + prepare_for_building_ci_image(params=base_build_params) + run_build(ci_image_params=base_build_params) @ci_image.command(name="pull") @@ -550,13 +614,6 @@ def run_build_ci_image( :param ci_image_params: CI image parameters :param output: output redirection """ - if not ci_image_params.version_suffix_for_pypi: - # We need that to handle the >= 2.7.0 limit we have for openlineage provider at least until - # Airflow 2.7.0 release is out, in order to avoid conflicting dependencies while building the image - # We are setting version_suffix_for_pypi to dev0 for CI builds where cache is prepared, so in - # order to have the cache used effectively, we should also locally force the version_suffix_for_pypi - # to dev0. We might evan leave it as default value in the future (to be decided after 2.7.0 release) - ci_image_params.version_suffix_for_pypi = "dev0" if ( ci_image_params.is_multi_platform() and not ci_image_params.push @@ -645,8 +702,6 @@ def rebuild_or_pull_ci_image_if_needed(command_params: ShellParams | BuildCiPara Rebuilds CI image if needed and user confirms it. :param command_params: parameters of the command to execute - - """ build_ci_image_check_cache = Path( BUILD_CACHE_DIR, command_params.airflow_branch, f".built_{command_params.python}" diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py index 87e6ab6e915cd..e128aa035b37d 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py @@ -35,7 +35,7 @@ "--image-tag", "--tag-as-latest", "--docker-cache", - "--force-build", + "--version-suffix-for-pypi", "--build-progress", ], }, @@ -51,24 +51,33 @@ ], }, { - "name": "Advanced options (for power users)", + "name": "Advanced build options (for power users)", "options": [ "--debian-version", + "--python-image", + "--commit-sha", + "--additional-pip-install-flags", "--install-providers-from-sources", + ], + }, + { + "name": "Selecting constraint location (for power users)", + "options": [ "--airflow-constraints-location", "--airflow-constraints-mode", "--airflow-constraints-reference", - "--python-image", - "--additional-python-deps", + ], + }, + { + "name": "Choosing dependencies and extras (for power users)", + "options": [ "--additional-airflow-extras", - "--additional-pip-install-flags", - "--additional-dev-apt-deps", - "--additional-dev-apt-env", - "--additional-dev-apt-command", + "--additional-python-deps", "--dev-apt-deps", + "--additional-dev-apt-deps", "--dev-apt-command", - "--version-suffix-for-pypi", - "--commit-sha", + "--additional-dev-apt-command", + "--additional-dev-apt-env", ], }, { diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 9f132aa3e7fac..ca1780738db33 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -18,7 +18,7 @@ import os import sys -from typing import Any +from copy import deepcopy import click @@ -27,10 +27,10 @@ from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( + option_additional_airflow_extras, option_additional_dev_apt_command, option_additional_dev_apt_deps, option_additional_dev_apt_env, - option_additional_extras, option_additional_pip_install_flags, option_additional_python_deps, option_additional_runtime_apt_command, @@ -39,6 +39,7 @@ option_airflow_constraints_location, option_airflow_constraints_mode_prod, option_airflow_constraints_reference_build, + option_answer, option_build_progress, option_builder, option_commit_sha, @@ -90,7 +91,7 @@ from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.registry import login_to_github_docker_registry from airflow_breeze.utils.run_tests import verify_an_image -from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command +from airflow_breeze.utils.run_utils import fix_group_permissions, run_command from airflow_breeze.utils.shared_options import get_dry_run, get_verbose @@ -151,26 +152,38 @@ def prod_image(): @prod_image.command(name="build") @option_python @option_debian_version -@option_run_in_parallel -@option_parallelism -@option_skip_cleanup -@option_debug_resources -@option_include_success_outputs -@option_python_versions @option_platform_multiple @option_github_token @option_docker_cache @option_image_tag_for_building @option_prepare_buildx_cache @option_push +@option_install_providers_from_sources +@click.option("-V", "--install-airflow-version", help="Install version of Airflow from PyPI.") +@option_additional_airflow_extras +@option_additional_dev_apt_deps +@option_additional_runtime_apt_deps +@option_additional_python_deps +@option_additional_dev_apt_command +@option_additional_dev_apt_env +@option_additional_runtime_apt_env +@option_additional_runtime_apt_command +@option_builder +@option_build_progress +@option_commit_sha +@option_dev_apt_command +@option_dev_apt_deps +@option_runtime_apt_command +@option_runtime_apt_deps @option_airflow_constraints_location @option_airflow_constraints_mode_prod @click.option( "--installation-method", help="Install Airflow from: sources or PyPI.", type=BetterChoice(ALLOWED_INSTALLATION_METHODS), + default=ALLOWED_INSTALLATION_METHODS[0], + show_default=True, ) -@option_install_providers_from_sources @click.option( "--install-packages-from-context", help="Install wheels from local docker-context-files when building image. " @@ -208,37 +221,72 @@ def prod_image(): help="Install Airflow using GitHub tag or branch.", ) @option_airflow_constraints_reference_build -@click.option("-V", "--install-airflow-version", help="Install version of Airflow from PyPI.") -@option_additional_extras -@option_additional_dev_apt_deps -@option_additional_runtime_apt_deps -@option_additional_python_deps -@option_additional_dev_apt_command -@option_additional_dev_apt_env -@option_additional_runtime_apt_env -@option_additional_runtime_apt_command -@option_builder -@option_build_progress -@option_dev_apt_command -@option_dev_apt_deps -@option_python_image -@option_runtime_apt_command -@option_runtime_apt_deps @option_tag_as_latest @option_additional_pip_install_flags @option_github_repository +@option_python_image @option_version_suffix_for_pypi -@option_commit_sha +@option_run_in_parallel +@option_parallelism +@option_skip_cleanup +@option_debug_resources +@option_include_success_outputs +@option_python_versions @option_verbose @option_dry_run +@option_answer def build( + # build options + python: str, + debian_version: str, + platform: str | None, + github_token: str | None, + docker_cache: str, + image_tag: str, + prepare_buildx_cache: bool, + push: bool, + install_providers_from_sources: bool, + install_airflow_version: str | None, + additional_airflow_extras: str | None, + additional_dev_apt_deps: str | None, + additional_runtime_apt_deps: str | None, + additional_python_deps: str | None, + additional_dev_apt_command: str | None, + additional_dev_apt_env: str | None, + additional_runtime_apt_command: str | None, + additional_runtime_apt_env: str | None, + builder: str, + build_progress: str, + commit_sha: str | None, + dev_apt_command: str | None, + dev_apt_deps: str | None, + runtime_apt_command: str | None, + runtime_apt_deps: str | None, + airflow_constraints_location: str | None, + airflow_constraints_mode: str, + installation_method: str, + install_packages_from_context: bool, + use_constraints_for_context_packages: bool, + cleanup_context: bool, + airflow_extras: str, + disable_mysql_client_installation: bool, + disable_mssql_client_installation: bool, + disable_postgres_client_installation: bool, + disable_airflow_repo_cache: bool, + install_airflow_reference: str | None, + airflow_constraints_reference: str | None, + tag_as_latest: bool, + additional_pip_install_flags: str | None, + github_repository: str, + python_image: str | None, + version_suffix_for_pypi: str, + # Parallel building run_in_parallel: bool, parallelism: int, skip_cleanup: bool, debug_resources: bool, - include_success_outputs: bool, + include_success_outputs, python_versions: str, - **kwargs: dict[str, Any], ): """ Build Production image. Include building multiple images for all or selected Python versions sequentially. @@ -252,14 +300,59 @@ def run_build(prod_image_params: BuildProdParams) -> None: perform_environment_checks() check_remote_ghcr_io_commands() - parameters_passed = filter_out_none(**kwargs) + base_build_params = BuildProdParams( + python=python, + debian_version=debian_version, + github_token=github_token, + docker_cache=docker_cache, + image_tag=image_tag, + prepare_buildx_cache=prepare_buildx_cache, + push=push, + install_providers_from_sources=install_providers_from_sources, + install_airflow_version=install_airflow_version, + additional_airflow_extras=additional_airflow_extras, + additional_dev_apt_deps=additional_dev_apt_deps, + additional_runtime_apt_deps=additional_runtime_apt_deps, + additional_python_deps=additional_python_deps, + additional_dev_apt_command=additional_dev_apt_command, + additional_dev_apt_env=additional_dev_apt_env, + additional_runtime_apt_command=additional_runtime_apt_command, + additional_runtime_apt_env=additional_runtime_apt_env, + builder=builder, + build_progress=build_progress, + commit_sha=commit_sha, + dev_apt_command=dev_apt_command, + dev_apt_deps=dev_apt_deps, + runtime_apt_command=runtime_apt_command, + runtime_apt_deps=runtime_apt_deps, + airflow_constraints_location=airflow_constraints_location, + airflow_constraints_mode=airflow_constraints_mode, + installation_method=installation_method, + install_packages_from_context=install_packages_from_context, + use_constraints_for_context_packages=use_constraints_for_context_packages, + cleanup_context=cleanup_context, + airflow_extras=airflow_extras, + disable_mysql_client_installation=disable_mysql_client_installation, + disable_mssql_client_installation=disable_mssql_client_installation, + disable_postgres_client_installation=disable_postgres_client_installation, + disable_airflow_repo_cache=disable_airflow_repo_cache, + install_airflow_reference=install_airflow_reference, + airflow_constraints_reference=airflow_constraints_reference, + tag_as_latest=tag_as_latest, + additional_pip_install_flags=additional_pip_install_flags, + github_repository=github_repository, + python_image=python_image, + version_suffix_for_pypi=version_suffix_for_pypi, + ) + if platform: + base_build_params.platform = platform fix_group_permissions() if run_in_parallel: python_version_list = get_python_version_list(python_versions) params_list: list[BuildProdParams] = [] for python in python_version_list: - params = BuildProdParams(**parameters_passed) + params = deepcopy(base_build_params) params.python = python params_list.append(params) prepare_for_building_prod_image(prod_image_params=params_list[0]) @@ -272,9 +365,8 @@ def run_build(prod_image_params: BuildProdParams) -> None: include_success_outputs=include_success_outputs, ) else: - params = BuildProdParams(**parameters_passed) - prepare_for_building_prod_image(prod_image_params=params) - run_build(prod_image_params=params) + prepare_for_building_prod_image(prod_image_params=base_build_params) + run_build(prod_image_params=base_build_params) @prod_image.command(name="pull") diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py index 713bd0bfa42c5..52291268f86e8 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py @@ -34,6 +34,7 @@ "--image-tag", "--tag-as-latest", "--docker-cache", + "--version-suffix-for-pypi", "--build-progress", ], }, @@ -49,44 +50,53 @@ ], }, { - "name": "Options for customizing images", + "name": "Advanced build options (for power users)", "options": [ + "--debian-version", + "--python-image", + "--commit-sha", + "--additional-pip-install-flags", "--install-providers-from-sources", - "--airflow-extras", + ], + }, + { + "name": "Selecting constraint location (for power users)", + "options": [ "--airflow-constraints-location", "--airflow-constraints-mode", "--airflow-constraints-reference", - "--python-image", + ], + }, + { + "name": "Choosing dependencies and extras (for power users)", + "options": [ + "--airflow-extras", "--additional-airflow-extras", - "--additional-pip-install-flags", "--additional-python-deps", - "--additional-runtime-apt-deps", - "--additional-runtime-apt-env", - "--additional-runtime-apt-command", + "--dev-apt-deps", "--additional-dev-apt-deps", - "--additional-dev-apt-env", + "--dev-apt-command", "--additional-dev-apt-command", + "--additional-dev-apt-env", "--runtime-apt-deps", + "--additional-runtime-apt-deps", "--runtime-apt-command", - "--dev-apt-deps", - "--dev-apt-command", - "--version-suffix-for-pypi", - "--commit-sha", + "--additional-runtime-apt-command", + "--additional-runtime-apt-env", ], }, { "name": "Advanced customization options (for specific customization needs)", "options": [ - "--debian-version", + "--installation-method", + "--install-airflow-reference", "--install-packages-from-context", - "--use-constraints-for-context-packages", "--cleanup-context", + "--use-constraints-for-context-packages", + "--disable-airflow-repo-cache", "--disable-mysql-client-installation", "--disable-mssql-client-installation", "--disable-postgres-client-installation", - "--disable-airflow-repo-cache", - "--install-airflow-reference", - "--installation-method", ], }, { diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index a713c280b54d9..b29ebacff0ae3 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import random from dataclasses import dataclass from pathlib import Path @@ -38,7 +39,7 @@ class BuildCiParams(CommonBuildParams): force_build: bool = False upgrade_to_newer_dependencies: bool = False upgrade_on_failure: bool = False - eager_upgrade_additional_requirements: str = "" + eager_upgrade_additional_requirements: str | None = None skip_provider_dependencies_check: bool = False @property @@ -49,72 +50,52 @@ def airflow_version(self): def image_type(self) -> str: return "CI" - @property - def extra_docker_build_flags(self) -> list[str]: - extra_ci_flags = [] - extra_ci_flags.extend( - ["--build-arg", f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}"] - ) - if self.airflow_constraints_location: - extra_ci_flags.extend( - ["--build-arg", f"AIRFLOW_CONSTRAINTS_LOCATION={self.airflow_constraints_location}"] - ) - if self.upgrade_to_newer_dependencies: - eager_upgrade_arg = self.eager_upgrade_additional_requirements.strip().replace("\n", "") - if eager_upgrade_arg: - extra_ci_flags.extend( - [ - "--build-arg", - f"EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS={eager_upgrade_arg}", - ] - ) - return super().extra_docker_build_flags + extra_ci_flags - @property def md5sum_cache_dir(self) -> Path: return Path(BUILD_CACHE_DIR, self.airflow_branch, self.python, "CI") - @property - def required_image_args(self) -> list[str]: - return [ - "airflow_branch", - "airflow_constraints_mode", - "airflow_constraints_reference", - "airflow_extras", - "airflow_image_date_created", - "airflow_image_repository", - "airflow_pre_cached_pip_packages", - "airflow_version", - "build_id", - "constraints_github_repository", - "python_base_image", - "upgrade_to_newer_dependencies", - ] - - @property - def optional_image_args(self) -> list[str]: - return [ - "additional_airflow_extras", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_pip_install_flags", - "additional_python_deps", - "additional_runtime_apt_command", - "additional_runtime_apt_deps", - "additional_runtime_apt_env", - "dev_apt_command", - "dev_apt_deps", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_airflow_extras", - "additional_pip_install_flags", - "additional_python_deps", - "version_suffix_for_pypi", - "commit_sha", - "build_progress", - ] - - def __post_init__(self): - pass + def prepare_arguments_for_docker_build_command(self) -> list[str]: + self.build_arg_values: list[str] = [] + # Required build args + self._req_arg("AIRFLOW_BRANCH", self.airflow_branch) + self._req_arg("AIRFLOW_CONSTRAINTS_MODE", self.airflow_constraints_mode) + self._req_arg("AIRFLOW_CONSTRAINTS_REFERENCE", self.airflow_constraints_reference) + self._req_arg("AIRFLOW_EXTRAS", self.airflow_extras) + self._req_arg("AIRFLOW_IMAGE_DATE_CREATED", self.airflow_image_date_created) + self._req_arg("AIRFLOW_IMAGE_REPOSITORY", self.airflow_image_repository) + self._req_arg("AIRFLOW_PRE_CACHED_PIP_PACKAGES", self.airflow_pre_cached_pip_packages) + self._req_arg("AIRFLOW_VERSION", self.airflow_version) + self._req_arg("BUILD_ID", self.build_id) + self._req_arg("CONSTRAINTS_GITHUB_REPOSITORY", self.constraints_github_repository) + self._req_arg("PYTHON_BASE_IMAGE", self.python_base_image) + if self.upgrade_to_newer_dependencies: + self._opt_arg("UPGRADE_TO_NEWER_DEPENDENCIES", f"{random.randrange(2**32):x}") + if self.eager_upgrade_additional_requirements: + # in case eager upgrade additional requirements have EOL, connect them together + self._opt_arg( + "EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS", + self.eager_upgrade_additional_requirements.replace("\n", ""), + ) + # optional build args + self._opt_arg("AIRFLOW_CONSTRAINTS_LOCATION", self.airflow_constraints_location) + self._opt_arg("ADDITIONAL_AIRFLOW_EXTRAS", self.additional_airflow_extras) + self._opt_arg("ADDITIONAL_DEV_APT_COMMAND", self.additional_dev_apt_command) + self._opt_arg("ADDITIONAL_DEV_APT_DEPS", self.additional_dev_apt_deps) + self._opt_arg("ADDITIONAL_DEV_APT_ENV", self.additional_dev_apt_env) + self._opt_arg("ADDITIONAL_PIP_INSTALL_FLAGS", self.additional_pip_install_flags) + self._opt_arg("ADDITIONAL_PYTHON_DEPS", self.additional_python_deps) + self._opt_arg("DEV_APT_COMMAND", self.dev_apt_command) + self._opt_arg("DEV_APT_DEPS", self.dev_apt_deps) + self._opt_arg("ADDITIONAL_DEV_APT_COMMAND", self.additional_dev_apt_command) + self._opt_arg("ADDITIONAL_DEV_APT_DEPS", self.additional_dev_apt_deps) + self._opt_arg("ADDITIONAL_DEV_APT_ENV", self.additional_dev_apt_env) + self._opt_arg("ADDITIONAL_AIRFLOW_EXTRAS", self.additional_airflow_extras) + self._opt_arg("ADDITIONAL_PIP_INSTALL_FLAGS", self.additional_pip_install_flags) + self._opt_arg("ADDITIONAL_PYTHON_DEPS", self.additional_python_deps) + self._opt_arg("VERSION_SUFFIX_FOR_PYPI", self.version_suffix_for_pypi) + self._opt_arg("COMMIT_SHA", self.commit_sha) + self._opt_arg("BUILD_PROGRESS", self.build_progress) + # Convert to build args + build_args = self._to_build_args() + # Add cache directive + return build_args diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index 585c7dc0c9690..ed9a6b674106f 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -38,9 +38,9 @@ class BuildProdParams(CommonBuildParams): PROD build parameters. Those parameters are used to determine command issued to build PROD image. """ - additional_runtime_apt_command: str = "" - additional_runtime_apt_deps: str = "" - additional_runtime_apt_env: str = "" + additional_runtime_apt_command: str | None = None + additional_runtime_apt_deps: str | None = None + additional_runtime_apt_env: str | None = None airflow_constraints_mode: str = "constraints" airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH cleanup_context: bool = False @@ -49,13 +49,13 @@ class BuildProdParams(CommonBuildParams): disable_mssql_client_installation: bool = False disable_mysql_client_installation: bool = False disable_postgres_client_installation: bool = False - install_airflow_reference: str = "" - install_airflow_version: str = "" + install_airflow_reference: str | None = None + install_airflow_version: str | None = None install_packages_from_context: bool = False use_constraints_for_context_packages: bool = False installation_method: str = "." - runtime_apt_command: str = "" - runtime_apt_deps: str = "" + runtime_apt_command: str | None = None + runtime_apt_deps: str | None = None @property def airflow_version(self) -> str: @@ -110,24 +110,20 @@ def args_for_remote_install(self) -> list: self.airflow_branch_for_pypi_preloading = AIRFLOW_BRANCH return build_args - @property - def extra_docker_build_flags(self) -> list[str]: + def _extra_prod_docker_build_flags(self) -> list[str]: extra_build_flags = [] if self.install_airflow_reference: - AIRFLOW_INSTALLATION_METHOD = ( - "https://github.com/apache/airflow/archive/" - + self.install_airflow_reference - + ".tar.gz#egg=apache-airflow" - ) extra_build_flags.extend( [ "--build-arg", - AIRFLOW_INSTALLATION_METHOD, + "https://github.com/apache/airflow/archive/" + + self.install_airflow_reference + + ".tar.gz#egg=apache-airflow", ] ) extra_build_flags.extend(self.args_for_remote_install) elif self.install_airflow_version: - if not re.match(r"^[0-9\.]+((a|b|rc|alpha|beta|pre)[0-9]+)?$", self.install_airflow_version): + if not re.match(r"^[0-9.]+((a|b|rc|alpha|beta|pre)[0-9]+)?$", self.install_airflow_version): get_console().print( f"\n[error]ERROR: Bad value for install-airflow-version:{self.install_airflow_version}" ) @@ -175,7 +171,7 @@ def extra_docker_build_flags(self) -> list[str]: f"io.artifacthub.package.logo-url={logo_url}", ] ) - return super().extra_docker_build_flags + extra_build_flags + return extra_build_flags @property def airflow_pre_cached_pip_packages(self) -> str: @@ -201,46 +197,45 @@ def docker_context_files(self) -> str: def airflow_image_kubernetes(self) -> str: return f"{self.airflow_image_name}-kubernetes" - @property - def required_image_args(self) -> list[str]: - return [ - "airflow_branch", - "airflow_constraints_mode", - "airflow_extras", - "airflow_image_date_created", - "airflow_image_readme_url", - "airflow_image_repository", - "airflow_pre_cached_pip_packages", - "airflow_version", - "build_id", - "constraints_github_repository", - "docker_context_files", - "install_mssql_client", - "install_mysql_client", - "install_packages_from_context", - "install_postgres_client", - "install_providers_from_sources", - "python_base_image", - ] - - @property - def optional_image_args(self) -> list[str]: - return [ - "additional_airflow_extras", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_pip_install_flags", - "additional_python_deps", - "additional_runtime_apt_command", - "additional_runtime_apt_deps", - "additional_runtime_apt_env", - "dev_apt_command", - "dev_apt_deps", - "runtime_apt_command", - "runtime_apt_deps", - "version_suffix_for_pypi", - "commit_sha", - "build_progress", - "use_constraints_for_context_packages", - ] + def prepare_arguments_for_docker_build_command(self) -> list[str]: + self.build_arg_values: list[str] = [] + # Required build args + self._req_arg("AIRFLOW_BRANCH", self.airflow_branch) + self._req_arg("AIRFLOW_CONSTRAINTS_MODE", self.airflow_constraints_mode) + self._req_arg("AIRFLOW_EXTRAS", self.airflow_extras) + self._req_arg("AIRFLOW_IMAGE_DATE_CREATED", self.airflow_image_date_created) + self._req_arg("AIRFLOW_IMAGE_README_URL", self.airflow_image_readme_url) + self._req_arg("AIRFLOW_IMAGE_REPOSITORY", self.airflow_image_repository) + self._req_arg("AIRFLOW_PRE_CACHED_PIP_PACKAGES", self.airflow_pre_cached_pip_packages) + self._req_arg("AIRFLOW_VERSION", self.airflow_version) + self._req_arg("BUILD_ID", self.build_id) + self._req_arg("CONSTRAINTS_GITHUB_REPOSITORY", self.constraints_github_repository) + self._req_arg("DOCKER_CONTEXT_FILES", self.docker_context_files) + self._req_arg("INSTALL_MSSQL_CLIENT", self.install_mssql_client) + self._req_arg("INSTALL_MYSQL_CLIENT", self.install_mysql_client) + self._req_arg("INSTALL_PACKAGES_FROM_CONTEXT", self.install_packages_from_context) + self._req_arg("INSTALL_POSTGRES_CLIENT", self.install_postgres_client) + self._req_arg("INSTALL_PROVIDERS_FROM_SOURCES", self.install_providers_from_sources) + self._req_arg("PYTHON_BASE_IMAGE", self.python_base_image) + # optional build args + self._opt_arg("AIRFLOW_CONSTRAINTS_LOCATION", self.airflow_constraints_location) + self._opt_arg("ADDITIONAL_AIRFLOW_EXTRAS", self.additional_airflow_extras) + self._opt_arg("ADDITIONAL_DEV_APT_COMMAND", self.additional_dev_apt_command) + self._opt_arg("ADDITIONAL_DEV_APT_DEPS", self.additional_dev_apt_deps) + self._opt_arg("ADDITIONAL_DEV_APT_ENV", self.additional_dev_apt_env) + self._opt_arg("ADDITIONAL_PIP_INSTALL_FLAGS", self.additional_pip_install_flags) + self._opt_arg("ADDITIONAL_PYTHON_DEPS", self.additional_python_deps) + self._opt_arg("ADDITIONAL_RUNTIME_APT_COMMAND", self.additional_runtime_apt_command) + self._opt_arg("ADDITIONAL_RUNTIME_APT_DEPS", self.additional_runtime_apt_deps) + self._opt_arg("ADDITIONAL_RUNTIME_APT_ENV", self.additional_runtime_apt_env) + self._opt_arg("DEV_APT_COMMAND", self.dev_apt_command) + self._opt_arg("DEV_APT_DEPS", self.dev_apt_deps) + self._opt_arg("RUNTIME_APT_COMMAND", self.runtime_apt_command) + self._opt_arg("RUNTIME_APT_DEPS", self.runtime_apt_deps) + self._opt_arg("VERSION_SUFFIX_FOR_PYPI", self.version_suffix_for_pypi) + self._opt_arg("COMMIT_SHA", self.commit_sha) + self._opt_arg("BUILD_PROGRESS", self.build_progress) + self._opt_arg("USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES", self.use_constraints_for_context_packages) + build_args = self._to_build_args() + build_args.extend(self._extra_prod_docker_build_flags()) + return build_args diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index 782d01bca44b1..29e748f8c8171 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -18,8 +18,9 @@ import os import sys -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime +from typing import Any from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import ( @@ -37,24 +38,23 @@ class CommonBuildParams: Common build parameters. Those parameters are common parameters for CI And PROD build. """ - additional_airflow_extras: str = "" - additional_dev_apt_command: str = "" - additional_dev_apt_deps: str = "" - additional_dev_apt_env: str = "" - additional_python_deps: str = "" - additional_pip_install_flags: str = "" + additional_airflow_extras: str | None = None + additional_dev_apt_command: str | None = None + additional_dev_apt_deps: str | None = None + additional_dev_apt_env: str | None = None + additional_python_deps: str | None = None + additional_pip_install_flags: str | None = None airflow_branch: str = os.environ.get("DEFAULT_BRANCH", AIRFLOW_BRANCH) default_constraints_branch: str = os.environ.get( "DEFAULT_CONSTRAINTS_BRANCH", DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH ) - airflow_constraints_location: str = "" - build_id: int = 0 + airflow_constraints_location: str | None = None builder: str = "autodetect" build_progress: str = ALLOWED_BUILD_PROGRESS[0] constraints_github_repository: str = APACHE_AIRFLOW_GITHUB_REPOSITORY - commit_sha: str = "" - dev_apt_command: str = "" - dev_apt_deps: str = "" + commit_sha: str | None = None + dev_apt_command: str | None = None + dev_apt_deps: str | None = None docker_cache: str = "registry" github_actions: str = os.environ.get("GITHUB_ACTIONS", "false") github_repository: str = APACHE_AIRFLOW_GITHUB_REPOSITORY @@ -68,14 +68,19 @@ class CommonBuildParams: python: str = "3.8" tag_as_latest: bool = False dry_run: bool = False - version_suffix_for_pypi: str = "" + version_suffix_for_pypi: str | None = None verbose: bool = False debian_version: str = "bookworm" + build_arg_values: list[str] = field(default_factory=list) @property def airflow_version(self): raise NotImplementedError() + @property + def build_id(self) -> str: + return os.environ.get("CI_BUILD_ID", "0") + @property def image_type(self) -> str: raise NotImplementedError() @@ -99,23 +104,14 @@ def airflow_image_name(self): return image @property - def extra_docker_build_flags(self) -> list[str]: - extra_flass = [] - if self.build_progress: - extra_flass.append(f"--progress={self.build_progress}") - return extra_flass - - @property - def docker_cache_directive(self) -> list[str]: - docker_cache_directive = [] + def common_docker_build_flags(self) -> list[str]: + extra_flags = [] + extra_flags.append(f"--progress={self.build_progress}") if self.docker_cache == "registry": - for platform in self.platforms: - docker_cache_directive.append(f"--cache-from={self.get_cache(platform)}") + extra_flags.append(f"--cache-from={self.get_cache(self.platform)}") elif self.docker_cache == "disabled": - docker_cache_directive.append("--no-cache") - else: - docker_cache_directive = [] - return docker_cache_directive + extra_flags.append("--no-cache") + return extra_flags @property def python_base_image(self): @@ -169,13 +165,31 @@ def preparing_latest_image(self) -> bool: def platforms(self) -> list[str]: return self.platform.split(",") - @property - def required_image_args(self) -> list[str]: - raise NotImplementedError() + def _build_arg(self, name: str, value: Any, optional: bool): + if value is None or "": + if optional: + return + else: + raise ValueError(f"Value for {name} cannot be empty or None") + if value is True: + str_value = "true" + elif value is False: + str_value = "false" + else: + str_value = str(value) if value is not None else "" + self.build_arg_values.append(f"{name}={str_value}") - @property - def optional_image_args(self) -> list[str]: + def _req_arg(self, name: str, value: Any): + self._build_arg(name, value, False) + + def _opt_arg(self, name: str, value: Any): + self._build_arg(name, value, True) + + def prepare_arguments_for_docker_build_command(self) -> list[str]: raise NotImplementedError() - def __post_init__(self): - pass + def _to_build_args(self): + build_args = [] + for arg in self.build_arg_values: + build_args.extend(["--build-arg", arg]) + return build_args diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 2a001f2757720..249faaafd1513 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -293,7 +293,7 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, help="When set, attempt to run upgrade to newer dependencies when regular build fails.", envvar="UPGRADE_ON_FAILURE", ) -option_additional_extras = click.option( +option_additional_airflow_extras = click.option( "--additional-airflow-extras", help="Additional extra package while installing Airflow in the image.", envvar="ADDITIONAL_AIRFLOW_EXTRAS", @@ -413,7 +413,13 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, option_version_suffix_for_pypi = click.option( "--version-suffix-for-pypi", help="Version suffix used for PyPI packages (alpha, beta, rc1, etc.).", - default="", + envvar="VERSION_SUFFIX_FOR_PYPI", +) +option_version_suffix_for_pypi_ci = click.option( + "--version-suffix-for-pypi", + help="Version suffix used for PyPI packages (alpha, beta, rc1, etc.).", + default="dev0", + show_default=True, envvar="VERSION_SUFFIX_FOR_PYPI", ) option_package_format = click.option( @@ -478,7 +484,6 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, option_airflow_constraints_location = click.option( "--airflow-constraints-location", type=str, - default="", help="If specified, it is used instead of calculating reference to the constraint file. " "It could be full remote URL to the location file, or local file placed in `docker-context-files` " "(in this case it has to start with /opt/airflow/docker-context-files).", @@ -615,7 +620,6 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, ) option_commit_sha = click.option( "--commit-sha", - default=None, show_default=True, envvar="COMMIT_SHA", help="Commit SHA that is used to build the images.", diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 36c976b2fed4a..78a8d69754ba0 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -388,35 +388,6 @@ def get_env_variable_value(arg_name: str, params: CommonBuildParams | ShellParam return value -def prepare_arguments_for_docker_build_command(image_params: CommonBuildParams) -> list[str]: - """ - Constructs docker compose command arguments list based on parameters passed. Maps arguments to - argument values. - - It maps: - * all the truthy/falsy values are converted to "true" / "false" respectively - * if upgrade_to_newer_dependencies is set to True, it is replaced by a random string to account - for the need of always triggering upgrade for docker build. - - :param image_params: parameters of the image - :return: list of `--build-arg` commands to use for the parameters passed - """ - - args_command = [] - for required_arg in image_params.required_image_args: - args_command.append("--build-arg") - args_command.append( - required_arg.upper() + "=" + get_env_variable_value(arg_name=required_arg, params=image_params) - ) - for optional_arg in image_params.optional_image_args: - param_value = get_env_variable_value(optional_arg, params=image_params) - if param_value: - args_command.append("--build-arg") - args_command.append(optional_arg.upper() + "=" + param_value) - args_command.extend(image_params.docker_cache_directive) - return args_command - - def prepare_docker_build_cache_command( image_params: CommonBuildParams, ) -> list[str]: @@ -427,16 +398,14 @@ def prepare_docker_build_cache_command( :return: Command to run as list of string """ - arguments = prepare_arguments_for_docker_build_command(image_params) - build_flags = image_params.extra_docker_build_flags final_command = [] final_command.extend(["docker"]) final_command.extend( ["buildx", "build", "--builder", get_and_use_docker_context(image_params.builder), "--progress=auto"] ) - final_command.extend(build_flags) + final_command.extend(image_params.common_docker_build_flags) final_command.extend(["--pull"]) - final_command.extend(arguments) + final_command.extend(image_params.prepare_arguments_for_docker_build_command()) final_command.extend(["--target", "main", "."]) final_command.extend( ["-f", "Dockerfile" if isinstance(image_params, BuildProdParams) else "Dockerfile.ci"] @@ -470,7 +439,6 @@ def prepare_base_build_command(image_params: CommonBuildParams) -> list[str]: "build", "--builder", get_and_use_docker_context(image_params.builder), - "--progress=auto", "--push" if image_params.push else "--load", ] ) @@ -488,17 +456,15 @@ def prepare_docker_build_command( :return: Command to run as list of string """ - arguments = prepare_arguments_for_docker_build_command(image_params) build_command = prepare_base_build_command( image_params=image_params, ) - build_flags = image_params.extra_docker_build_flags final_command = [] final_command.extend(["docker"]) final_command.extend(build_command) - final_command.extend(build_flags) + final_command.extend(image_params.common_docker_build_flags) final_command.extend(["--pull"]) - final_command.extend(arguments) + final_command.extend(image_params.prepare_arguments_for_docker_build_command()) final_command.extend(["-t", image_params.airflow_image_name_with_tag, "--target", "main", "."]) final_command.extend( ["-f", "Dockerfile" if isinstance(image_params, BuildProdParams) else "Dockerfile.ci"] diff --git a/images/breeze/output_ci-image_build.svg b/images/breeze/output_ci-image_build.svg index 2a7147c2fd763..57fb8ac402e76 100644 --- a/images/breeze/output_ci-image_build.svg +++ b/images/breeze/output_ci-image_build.svg @@ -1,4 +1,4 @@ - +