From 2d9bb46cd60583c56603141fe555e7e065950a6a Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 26 Feb 2024 13:10:31 +0100 Subject: [PATCH] Use `uv` as packaging tool used in CI builds (#37692) The `uv` tool released in Feb 2024 by ruff creators provides a way faster drop-in replacement to `pip` and we are using it now in our CI, when it can bring significant speed improvements (and soon possibly more features). This PR replaces `pip install` and `pip uninstall` with equivalent `uv pip install` and `uv pip uninstall` commands, controlled by a single `AIRFLOW_USE_UV` ARG. In CI images it is set to "true" so CI images are prepared using UV, but PROD images (which are also used during CI tests) are built using pip. This way we can get both - stability and compliance for user-facing `pip` installation and speed and new features coming from `uv`. --- .github/workflows/ci.yml | 2 +- Dockerfile | 169 +++++++++++------ Dockerfile.ci | 175 +++++++++++++----- dev/breeze/doc/ci/02_images.md | 3 + .../doc/images/output_ci-image_build.svg | 96 +++++----- .../doc/images/output_ci-image_build.txt | 2 +- .../doc/images/output_prod-image_build.svg | 18 +- .../doc/images/output_prod-image_build.txt | 2 +- .../commands/ci_image_commands.py | 11 ++ .../commands/ci_image_commands_config.py | 1 + .../production_image_commands_config.py | 4 +- .../commands/release_management_commands.py | 2 + .../airflow_breeze/params/build_ci_params.py | 2 + .../params/build_prod_params.py | 2 +- docs/docker-stack/build-arg-ref.rst | 4 + scripts/docker/common.sh | 71 ++++++- scripts/docker/entrypoint_ci.sh | 32 +++- .../docker/install_additional_dependencies.sh | 12 +- scripts/docker/install_airflow.sh | 22 +-- ...ll_airflow_dependencies_from_branch_tip.sh | 12 +- .../install_from_docker_context_files.sh | 23 +-- scripts/docker/install_mssql.sh | 1 - scripts/docker/install_pip_version.sh | 7 +- scripts/docker/install_pipx_tools.sh | 3 +- scripts/in_container/_in_container_utils.sh | 37 ++++ tests/cli/commands/test_webserver_command.py | 7 +- .../providers/weaviate/hooks/test_weaviate.py | 9 +- .../weaviate/operators/test_weaviate.py | 4 +- 28 files changed, 506 insertions(+), 227 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc730277ce64d..409dfaf44d0e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,7 +175,7 @@ jobs: # We only push CI cache as PROD cache usually does not gain as much from fresh cache because # it uses prepared airflow and provider packages that invalidate the cache anyway most of the time push-early-buildx-cache-to-github-registry: - timeout-minutes: 50 + timeout-minutes: 110 name: "Push Early Image Cache" runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} needs: diff --git a/Dockerfile b/Dockerfile index 4ac935925fd6a..f5efda7abe996 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,8 @@ ARG AIRFLOW_VERSION="2.8.1" ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm" ARG AIRFLOW_PIP_VERSION=24.0 +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="false" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md" @@ -336,7 +338,6 @@ set -euo pipefail common::get_colors declare -a packages -: "${AIRFLOW_PIP_VERSION:?Should be set}" : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" @@ -416,14 +417,13 @@ COPY <<"EOF" /install_pip_version.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location -common::install_pip_version +common::install_packaging_tool EOF # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -436,7 +436,6 @@ COPY <<"EOF" /install_airflow_dependencies_from_branch_tip.sh : "${AIRFLOW_BRANCH:?Should be set}" : "${INSTALL_MYSQL_CLIENT:?Should be true or false}" : "${INSTALL_POSTGRES_CLIENT:?Should be true or false}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" function install_airflow_dependencies_from_branch_tip() { echo @@ -452,26 +451,27 @@ function install_airflow_dependencies_from_branch_tip() { # dependencies that we can cache and reuse when installing airflow using constraints and latest # pyproject.toml in the next step (when we install regular airflow). set -x - pip install --root-user-action ignore \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" - common::install_pip_version + common::install_packaging_tool # Uninstall airflow and providers to keep only the dependencies. In the future when # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this # flag and skip the remove step. - pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL_CMD} freeze | grep apache-airflow-providers | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}" echo - pip uninstall --yes apache-airflow || true + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow || true } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow_dependencies_from_branch_tip EOF @@ -481,6 +481,10 @@ COPY <<"EOF" /common.sh #!/usr/bin/env bash set -euo pipefail +: "${AIRFLOW_PIP_VERSION:?Should be set}" +: "${AIRFLOW_UV_VERSION:?Should be set}" +: "${AIRFLOW_USE_UV:?Should be set}" + function common::get_colors() { COLOR_BLUE=$'\e[34m' COLOR_GREEN=$'\e[32m' @@ -494,6 +498,40 @@ function common::get_colors() { export COLOR_YELLOW } +function common::get_packaging_tool() { + ## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY CORRESPONDING FUNCTION IN + ## `scripts/in_container/_in_container_utils.sh` + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="uv" + export PACKAGING_TOOL_CMD="uv pip" + export EXTRA_INSTALL_FLAGS="" + export EXTRA_UNINSTALL_FLAGS="" + export RESOLUTION_HIGHEST_FLAG="--resolution highest" + export RESOLUTION_LOWEST_DIRECT_FLAG="--resolution lowest-direct" + # We need to lie about VIRTUAL_ENV to make uv works + # Until https://github.com/astral-sh/uv/issues/1396 is fixed + # In case we are running user installation, we need to set VIRTUAL_ENV to user's home + .local + if [[ ${PIP_USER=} == "true" ]]; then + VIRTUAL_ENV="${HOME}/.local" + else + VIRTUAL_ENV=$(python -c "import sys; print(sys.prefix)") + fi + export VIRTUAL_ENV + else + echo + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="pip" + export PACKAGING_TOOL_CMD="pip" + export EXTRA_INSTALL_FLAGS="--root-user-action ignore" + export EXTRA_UNINSTALL_FLAGS="--yes" + export RESOLUTION_HIGHEST_FLAG="--upgrade-strategy eager" + export RESOLUTION_LOWEST_DIRECT_FLAG="--upgrade --upgrade-strategy only-if-needed" + fi +} function common::get_airflow_version_specification() { if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=} @@ -529,20 +567,41 @@ function common::get_constraints_location() { fi } -function common::show_pip_version_and_location() { +function common::show_packaging_tool_version_and_location() { echo "PATH=${PATH}" - echo "pip on path: $(which pip)" - echo "Using pip: $(pip --version)" + if [[ ${PACKAGING_TOOL} == "pip" ]]; then + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo "pip on path: $(which pip)" + echo "Using pip: $(pip --version)" + else + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo "uv on path: $(which uv)" + echo "Using uv: $(uv --version)" + fi } -function common::install_pip_version() { +function common::install_packaging_tool() { echo echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then - pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" else - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + fi + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" + else + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" + fi fi mkdir -p "${HOME}/.local/bin" } @@ -601,8 +660,6 @@ COPY <<"EOF" /install_from_docker_context_files.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - function install_airflow_and_providers_from_docker_context_files(){ if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} @@ -619,7 +676,7 @@ function install_airflow_and_providers_from_docker_context_files(){ fi # shellcheck disable=SC2206 - local pip_flags=( + local packaging_flags=( # Don't quote this -- if it is empty we don't want it to create an # empty array element --find-links="file:///docker-context-files" @@ -669,7 +726,7 @@ function install_airflow_and_providers_from_docker_context_files(){ echo # force reinstall all airflow + provider packages with constraints found in set -x - pip install "${pip_flags[@]}" --root-user-action ignore --upgrade \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" --upgrade \ ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} set +x @@ -678,7 +735,7 @@ function install_airflow_and_providers_from_docker_context_files(){ echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}" echo set -x - pip install "${pip_flags[@]}" --root-user-action ignore \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} @@ -689,17 +746,16 @@ function install_airflow_and_providers_from_docker_context_files(){ echo "${COLOR_BLUE}Installing docker-context-files packages without constraints${COLOR_RESET}" echo set -x - pip install "${pip_flags[@]}" --root-user-action ignore \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} set +x fi - common::install_pip_version + common::install_packaging_tool pip check } function install_all_other_packages_from_docker_context_files() { - echo echo "${COLOR_BLUE}Force re-installing all other package from local files without dependencies${COLOR_RESET}" echo @@ -709,22 +765,22 @@ function install_all_other_packages_from_docker_context_files() { grep -v apache_airflow | grep -v apache-airflow || true) if [[ -n "${reinstalling_other_packages}" ]]; then set -x - pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \ - --root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages} - common::install_pip_version + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + --force-reinstall --no-deps --no-index ${reinstalling_other_packages} + common::install_packaging_tool set +x fi } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow_and_providers_from_docker_context_files -common::show_pip_version_and_location install_all_other_packages_from_docker_context_files EOF @@ -734,8 +790,6 @@ COPY <<"EOF" /install_airflow.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - function install_airflow() { # Coherence check for editable installation mode. if [[ ${AIRFLOW_INSTALLATION_METHOD} != "." && \ @@ -760,22 +814,21 @@ function install_airflow() { echo "${COLOR_BLUE}Remove airflow and all provider packages installed before potentially${COLOR_RESET}" echo set -x - pip freeze | grep apache-airflow | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL_CMD} freeze | grep apache-airflow | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Installing all packages with eager upgrade with ${AIRFLOW_INSTALL_EDITABLE_FLAG} mode${COLOR_RESET}" echo set -x - pip install --root-user-action ignore \ - --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} set +x - common::install_pip_version + common::install_packaging_tool echo - echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" + echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}" echo pip check else @@ -783,17 +836,17 @@ function install_airflow() { echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - common::install_pip_version + common::install_packaging_tool # then upgrade if needed without using constraints to account for new limits in pyproject.toml - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_LOWEST_DIRECT_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -804,10 +857,11 @@ function install_airflow() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow EOF @@ -819,7 +873,6 @@ set -euo pipefail : "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}" : "${ADDITIONAL_PYTHON_DEPS:?Should be set}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -829,10 +882,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -843,10 +896,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "${RESOLUTION_LOWEST_DIRECT_FLAG}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -856,10 +909,11 @@ function install_additional_dependencies() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_additional_dependencies EOF @@ -1246,6 +1300,8 @@ ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb" ARG INSTALL_MSSQL_CLIENT="true" ARG INSTALL_POSTGRES_CLIENT="true" ARG AIRFLOW_PIP_VERSION +ARG AIRFLOW_UV_VERSION +ARG AIRFLOW_USE_UV ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} \ @@ -1327,6 +1383,8 @@ RUN if [[ -f /docker-context-files/pip.conf ]]; then \ ARG ADDITIONAL_PIP_INSTALL_FLAGS="" ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \ AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \ @@ -1342,7 +1400,6 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ PATH=${PATH}:${AIRFLOW_USER_HOME_DIR}/.local/bin \ - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \ AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \ @@ -1458,12 +1515,16 @@ LABEL org.apache.airflow.distro="debian" \ ARG PYTHON_BASE_IMAGE ARG AIRFLOW_PIP_VERSION +ARG AIRFLOW_UV_VERSION +ARG AIRFLOW_USE_UV ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ # Make sure noninteractive debian install is used and language variables set DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 LD_LIBRARY_PATH=/usr/local/lib \ - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} + AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} ARG RUNTIME_APT_DEPS="" ARG ADDITIONAL_RUNTIME_APT_DEPS="" @@ -1503,10 +1564,14 @@ ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \ AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \ AIRFLOW_HOME=${AIRFLOW_HOME} -# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON +# THE 7 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON # AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED ARG AIRFLOW_PIP_VERSION=24.0 -ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="false" +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} COPY --from=scripts common.sh /scripts/docker/ # Only copy mysql/mssql installation scripts for now - so that changing the other diff --git a/Dockerfile.ci b/Dockerfile.ci index f5044a2ef9b4c..8e927094e8a9e 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -296,7 +296,6 @@ set -euo pipefail common::get_colors declare -a packages -: "${AIRFLOW_PIP_VERSION:?Should be set}" : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" @@ -376,14 +375,13 @@ COPY <<"EOF" /install_pip_version.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location -common::install_pip_version +common::install_packaging_tool EOF # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -396,7 +394,6 @@ COPY <<"EOF" /install_airflow_dependencies_from_branch_tip.sh : "${AIRFLOW_BRANCH:?Should be set}" : "${INSTALL_MYSQL_CLIENT:?Should be true or false}" : "${INSTALL_POSTGRES_CLIENT:?Should be true or false}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" function install_airflow_dependencies_from_branch_tip() { echo @@ -412,26 +409,27 @@ function install_airflow_dependencies_from_branch_tip() { # dependencies that we can cache and reuse when installing airflow using constraints and latest # pyproject.toml in the next step (when we install regular airflow). set -x - pip install --root-user-action ignore \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" - common::install_pip_version + common::install_packaging_tool # Uninstall airflow and providers to keep only the dependencies. In the future when # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this # flag and skip the remove step. - pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL_CMD} freeze | grep apache-airflow-providers | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}" echo - pip uninstall --yes apache-airflow || true + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow || true } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow_dependencies_from_branch_tip EOF @@ -441,6 +439,10 @@ COPY <<"EOF" /common.sh #!/usr/bin/env bash set -euo pipefail +: "${AIRFLOW_PIP_VERSION:?Should be set}" +: "${AIRFLOW_UV_VERSION:?Should be set}" +: "${AIRFLOW_USE_UV:?Should be set}" + function common::get_colors() { COLOR_BLUE=$'\e[34m' COLOR_GREEN=$'\e[32m' @@ -454,6 +456,40 @@ function common::get_colors() { export COLOR_YELLOW } +function common::get_packaging_tool() { + ## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY CORRESPONDING FUNCTION IN + ## `scripts/in_container/_in_container_utils.sh` + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="uv" + export PACKAGING_TOOL_CMD="uv pip" + export EXTRA_INSTALL_FLAGS="" + export EXTRA_UNINSTALL_FLAGS="" + export RESOLUTION_HIGHEST_FLAG="--resolution highest" + export RESOLUTION_LOWEST_DIRECT_FLAG="--resolution lowest-direct" + # We need to lie about VIRTUAL_ENV to make uv works + # Until https://github.com/astral-sh/uv/issues/1396 is fixed + # In case we are running user installation, we need to set VIRTUAL_ENV to user's home + .local + if [[ ${PIP_USER=} == "true" ]]; then + VIRTUAL_ENV="${HOME}/.local" + else + VIRTUAL_ENV=$(python -c "import sys; print(sys.prefix)") + fi + export VIRTUAL_ENV + else + echo + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="pip" + export PACKAGING_TOOL_CMD="pip" + export EXTRA_INSTALL_FLAGS="--root-user-action ignore" + export EXTRA_UNINSTALL_FLAGS="--yes" + export RESOLUTION_HIGHEST_FLAG="--upgrade-strategy eager" + export RESOLUTION_LOWEST_DIRECT_FLAG="--upgrade --upgrade-strategy only-if-needed" + fi +} function common::get_airflow_version_specification() { if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=} @@ -489,20 +525,41 @@ function common::get_constraints_location() { fi } -function common::show_pip_version_and_location() { +function common::show_packaging_tool_version_and_location() { echo "PATH=${PATH}" - echo "pip on path: $(which pip)" - echo "Using pip: $(pip --version)" + if [[ ${PACKAGING_TOOL} == "pip" ]]; then + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo "pip on path: $(which pip)" + echo "Using pip: $(pip --version)" + else + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo "uv on path: $(which uv)" + echo "Using uv: $(uv --version)" + fi } -function common::install_pip_version() { +function common::install_packaging_tool() { echo echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then - pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" else - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + fi + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" + else + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" + fi fi mkdir -p "${HOME}/.local/bin" } @@ -548,7 +605,7 @@ function install_pipx_tools() { echo "${COLOR_BLUE}Installing pipx tools${COLOR_RESET}" echo # Make sure PIPX is installed in latest version - pip install --root-user-action ignore --upgrade "pipx>=1.2.1" + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pipx>=1.2.1" if [[ $(uname -m) != "aarch64" ]]; then # Do not install mssql-cli for ARM # Install all the tools we need available in command line but without impacting the current environment @@ -562,6 +619,7 @@ function install_pipx_tools() { } common::get_colors +common::get_packaging_tool install_pipx_tools EOF @@ -572,8 +630,6 @@ COPY <<"EOF" /install_airflow.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - function install_airflow() { # Coherence check for editable installation mode. if [[ ${AIRFLOW_INSTALLATION_METHOD} != "." && \ @@ -598,22 +654,21 @@ function install_airflow() { echo "${COLOR_BLUE}Remove airflow and all provider packages installed before potentially${COLOR_RESET}" echo set -x - pip freeze | grep apache-airflow | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL_CMD} freeze | grep apache-airflow | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Installing all packages with eager upgrade with ${AIRFLOW_INSTALL_EDITABLE_FLAG} mode${COLOR_RESET}" echo set -x - pip install --root-user-action ignore \ - --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} set +x - common::install_pip_version + common::install_packaging_tool echo - echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" + echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}" echo pip check else @@ -621,17 +676,17 @@ function install_airflow() { echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - common::install_pip_version + common::install_packaging_tool # then upgrade if needed without using constraints to account for new limits in pyproject.toml - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_LOWEST_DIRECT_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -642,10 +697,11 @@ function install_airflow() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow EOF @@ -657,7 +713,6 @@ set -euo pipefail : "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}" : "${ADDITIONAL_PYTHON_DEPS:?Should be set}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -667,10 +722,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -681,10 +736,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "${RESOLUTION_LOWEST_DIRECT_FLAG}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -694,10 +749,11 @@ function install_additional_dependencies() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_additional_dependencies EOF @@ -893,8 +949,12 @@ function check_boto_upgrade() { echo echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" echo - pip uninstall --root-user-action ignore aiobotocore s3fs -y || true - pip install --root-user-action ignore --upgrade boto3 botocore + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true + # We need to include oss2 as dependency as otherwise jmespath will be bumped and it will not pass + # the pip check test, Similarly gcloud-aio-auth limit is needed to be included as it bumps cryptography + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore "oss2>=2.14.0" "gcloud-aio-auth>=4.0.0,<5.0.0" pip check } @@ -903,25 +963,31 @@ function check_pydantic() { echo echo "${COLOR_YELLOW}Reinstalling airflow from local sources to account for pyproject.toml changes${COLOR_RESET}" echo - pip install --root-user-action ignore -e . + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} -e . echo echo "${COLOR_YELLOW}Remove pydantic and 3rd party libraries that depend on it${COLOR_RESET}" echo - pip uninstall --root-user-action ignore pydantic aws-sam-translator openai pyiceberg qdrant-client cfn-lint -y + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} pydantic aws-sam-translator openai \ + pyiceberg qdrant-client cfn-lint weaviate-client pip check elif [[ ${PYDANTIC=} == "v1" ]]; then echo echo "${COLOR_YELLOW}Reinstalling airflow from local sources to account for pyproject.toml changes${COLOR_RESET}" echo - pip install --root-user-action ignore -e . + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} -e . echo - echo "${COLOR_YELLOW}Uninstalling pyicberg which is not compatible with Pydantic 1${COLOR_RESET}" + echo "${COLOR_YELLOW}Uninstalling dependencies which are not compatible with Pydantic 1${COLOR_RESET}" echo - pip uninstall pyiceberg -y + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} pyiceberg waeviate-client echo echo "${COLOR_YELLOW}Downgrading Pydantic to < 2${COLOR_RESET}" echo - pip install --upgrade "pydantic<2.0.0" + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0" pip check else echo @@ -939,7 +1005,8 @@ function check_download_sqlalchemy() { echo echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}" echo - pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}" + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "sqlalchemy==${min_sqlalchemy_version}" pip check } @@ -951,7 +1018,8 @@ function check_download_pendulum() { echo echo "${COLOR_BLUE}Downgrading pendulum to minimum supported version: ${min_pendulum_version}${COLOR_RESET}" echo - pip install --root-user-action ignore "pendulum==${min_pendulum_version}" + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "pendulum==${min_pendulum_version}" pip check } @@ -1042,10 +1110,14 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ COPY --from=scripts install_os_dependencies.sh /scripts/docker/ RUN bash /scripts/docker/install_os_dependencies.sh dev -# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON +# THE 7 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON # AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED ARG AIRFLOW_PIP_VERSION=24.0 -ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="true" +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} COPY --from=scripts common.sh /scripts/docker/ # Only copy mysql/mssql installation scripts for now - so that changing the other @@ -1108,9 +1180,13 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" ARG AIRFLOW_CI_BUILD_EPOCH="10" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" ARG AIRFLOW_PIP_VERSION=24.0 +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="true" # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" +# By default UV install run without cache to make image smaller +ARG UV_NO_CACHE="true" # By default PIP has progress bar but you can disable it. ARG PIP_PROGRESS_BAR="on" # Optimizing installation of Cassandra driver (in case there are no prebuilt wheels which is the @@ -1138,6 +1214,8 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \ # In the CI image we always: # * install MySQL, MsSQL # * install airflow from current sources, not from PyPI package @@ -1153,6 +1231,7 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_VERSION_SPECIFICATION="" \ PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} \ PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ + UV_NO_CACHE=${UV_NO_CACHE} \ ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \ CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY} \ CASS_DRIVER_NO_CYTHON=${CASS_DRIVER_NO_CYTHON} diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 4c6fc2c016bb7..7807d94b1e242 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -428,6 +428,7 @@ can be used for CI images: | `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing this number will reinstall all apt dependencies | | `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional `pip` flags passed to the installation commands (except when reinstalling `pip` itself) | | `PIP_NO_CACHE_DIR` | `true` | if true, then no pip cache will be stored | +| `UV_NO_CACHE` | `true` | if true, then no uv cache will be stored | | `HOME` | `/root` | Home directory of the root user (CI image has root user as default) | | `AIRFLOW_HOME` | `/root/airflow` | Airflow's HOME (that's where logs and sqlite databases are stored) | | `AIRFLOW_SOURCES` | `/opt/airflow` | Mounted sources of Airflow | @@ -447,6 +448,8 @@ can be used for CI images: | `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | | `AIRFLOW_PIP_VERSION` | `24.0` | PIP version used. | +| `AIRFLOW_UV_VERSION` | `0.1.10` | UV version used. | +| `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | | `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | Here are some examples of how CI images can built manually. CI is always diff --git a/dev/breeze/doc/images/output_ci-image_build.svg b/dev/breeze/doc/images/output_ci-image_build.svg index 17ea15e9e999f..f617eb05e72aa 100644 --- a/dev/breeze/doc/images/output_ci-image_build.svg +++ b/dev/breeze/doc/images/output_ci-image_build.svg @@ -1,4 +1,4 @@ - +