From c89ab99aced32c854a7806f3a2e00f8eaeb0764a Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 11 Nov 2024 05:40:20 +0100 Subject: [PATCH] Recommend and use ``uv`` instead of ``pipx`` in a few remaining places (#43866) The ``pipx`` has still been used and recommended in a few places in our documentation and in a few scripts/github actions. We also installed pipx in the container image to be able to run mmsql-cli, but this is not needed any more as mssql is not supported as metadata backend. This PR updates the remaining parts. --- .github/workflows/basic-tests.yml | 5 +- .github/workflows/check-providers.yml | 8 +- .github/workflows/news-fragment.yml | 2 +- .github/workflows/task-sdk-tests.yml | 4 +- Dockerfile.ci | 34 +---- INSTALL | 142 ++++++------------ .../03_contributors_quick_start.rst | 51 ++++--- .../contributors_quick_start_gitpod.rst | 18 +-- dev/README_RELEASE_AIRFLOW.md | 42 +++--- dev/breeze/README.md | 18 +-- dev/breeze/doc/01_installation.rst | 61 ++++---- dev/breeze/doc/04_troubleshooting.rst | 19 ++- dev/breeze/doc/10_advanced_breeze_topics.rst | 13 +- .../adr/0016-use-uv-tool-to-install-breeze.md | 56 +++++++ dev/breeze/pyproject.toml | 1 - .../src/airflow_breeze/utils/path_utils.py | 8 +- .../src/airflow_breeze/utils/run_utils.py | 4 +- .../installation/installing-from-pypi.rst | 4 +- docs/docker-stack/changelog.rst | 3 + scripts/ci/install_breeze.sh | 10 +- scripts/ci/pre_commit/update_installers.py | 9 ++ scripts/docker/install_pipx_tools.sh | 43 ------ scripts/tools/setup_breeze | 30 ++-- 23 files changed, 268 insertions(+), 317 deletions(-) create mode 100644 dev/breeze/doc/adr/0016-use-uv-tool-to-install-breeze.md delete mode 100644 scripts/docker/install_pipx_tools.sh diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 7930ba148906f..d6b63510e8924 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -187,9 +187,8 @@ jobs: working-directory: ./airflow-client-python - name: Install hatch run: | - python -m pip install --upgrade pipx - pipx ensurepath - pipx install hatch + python -m pip install --upgrade uv + uv tool install hatch - name: Run tests run: hatch run run-coverage env: diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml index f5ff95b73c2f7..3faf19b61f532 100644 --- a/.github/workflows/check-providers.yml +++ b/.github/workflows/check-providers.yml @@ -102,8 +102,8 @@ jobs: run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0 - name: "Verify wheel packages with twine" run: | - pipx uninstall twine || true - pipx install twine && twine check dist/*.whl + uv tool uninstall twine || true + uv tool install twine && twine check dist/*.whl - name: "Test providers issue generation automatically" run: > breeze release-management generate-issue-content-providers @@ -170,8 +170,8 @@ jobs: --version-suffix-for-pypi dev0 --package-format sdist - name: "Verify sdist packages with twine" run: | - pipx uninstall twine || true - pipx install twine && twine check dist/*.tar.gz + uv tool uninstall twine || true + uv tool install twine && twine check dist/*.tar.gz - name: "Generate source constraints from CI image" shell: bash run: > diff --git a/.github/workflows/news-fragment.yml b/.github/workflows/news-fragment.yml index 6691bf65f9d11..31b0f51b1f25c 100644 --- a/.github/workflows/news-fragment.yml +++ b/.github/workflows/news-fragment.yml @@ -38,7 +38,7 @@ jobs: - name: Check news fragment run: > - pipx run towncrier check + uv tool run towncrier check --dir . --config newsfragments/config.toml --compare-with origin/${{ github.base_ref }} diff --git a/.github/workflows/task-sdk-tests.yml b/.github/workflows/task-sdk-tests.yml index d1d152648cb8b..acc9872e6ed96 100644 --- a/.github/workflows/task-sdk-tests.yml +++ b/.github/workflows/task-sdk-tests.yml @@ -77,8 +77,8 @@ jobs: breeze release-management prepare-task-sdk-package --package-format wheel - name: "Verify wheel packages with twine" run: | - pipx uninstall twine || true - pipx install twine && twine check dist/*.whl + uv tool uninstall twine || true + uv tool install twine && twine check dist/*.whl - name: > Run unit tests for Airflow Task SDK:Python ${{ matrix.python-version }} run: > diff --git a/Dockerfile.ci b/Dockerfile.ci index 81f5ac0399bc2..5adb2d6210ab3 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -634,35 +634,6 @@ function common::import_trusted_gpg() { } EOF -# The content below is automatically copied from scripts/docker/install_pipx_tools.sh -COPY <<"EOF" /install_pipx_tools.sh -#!/usr/bin/env bash -. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" - -function install_pipx_tools() { - echo - echo "${COLOR_BLUE}Installing pipx tools${COLOR_RESET}" - echo - # Make sure PIPX is installed in latest version - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pipx>=1.2.1" - if [[ $(uname -m) != "aarch64" ]]; then - # Do not install mssql-cli for ARM - # Install all the tools we need available in command line but without impacting the current environment - pipx install mssql-cli - - # Unfortunately mssql-cli installed by `pipx` does not work out of the box because it uses - # its own execution bash script which is not compliant with the auto-activation of - # pipx venvs - we need to manually patch Python executable in the script to fix it: ¯\_(ツ)_/¯ - sed "s/python /\/root\/\.local\/pipx\/venvs\/mssql-cli\/bin\/python /" -i /root/.local/bin/mssql-cli - fi -} - -common::get_colors -common::get_packaging_tool - -install_pipx_tools -EOF - # The content below is automatically copied from scripts/docker/install_airflow.sh COPY <<"EOF" /install_airflow.sh #!/usr/bin/env bash @@ -1381,13 +1352,10 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ # The PATH is needed for PIPX to find the tools installed ENV PATH="/root/.local/bin:${PATH}" -COPY --from=scripts install_pipx_tools.sh /scripts/docker/ - # Install useful command line tools in their own virtualenv so that they do not clash with # dependencies installed in Airflow also reinstall PIP and UV to make sure they are installed # in the version specified above -RUN bash /scripts/docker/install_packaging_tools.sh; \ - bash /scripts/docker/install_pipx_tools.sh +RUN bash /scripts/docker/install_packaging_tools.sh # Airflow sources change frequently but dependency configuration won't change that often # We copy pyproject.toml and other files needed to perform setup of dependencies diff --git a/INSTALL b/INSTALL index edf738f5e114d..2a83248ad68b6 100644 --- a/INSTALL +++ b/INSTALL @@ -47,7 +47,8 @@ you get all sources in one place. This is the most convenient way to develop Air Otherwise, you have to install Airflow and Providers separately from sources in the same environment, which is not as convenient. -## Creating virtualenv +Creating virtualenv +------------------- Airflow pulls in quite a lot of dependencies to connect to other services. You generally want to test or run Airflow from a virtualenv to ensure those dependencies are separated from your system-wide versions. Using system-installed Python installation is strongly discouraged as the versions of Python @@ -59,7 +60,8 @@ Once you have a suitable Python version installed, you can create a virtualenv a python3 -m venv PATH_TO_YOUR_VENV source PATH_TO_YOUR_VENV/bin/activate -## Installing Airflow locally +Installing Airflow locally +-------------------------- Installing Airflow locally can be done using pip - note that this will install "development" version of Airflow, where all providers are installed from local sources (if available), not from `pypi`. @@ -79,7 +81,6 @@ this one contains the minimum set of tools and dependencies needed to run unit t pip install -e ".[devel]" - You can also install optional packages that are needed to run certain tests. In case of local installation for example, you can install all prerequisites for Google provider, tests, and all Hadoop providers with this command: @@ -97,122 +98,78 @@ Additionally when you want to develop providers you need to install providers co pip install -e "./providers" -# Using Hatch to manage your Python, virtualenvs, and build packages - -Airflow uses [hatch](https://hatch.pypa.io/) as a build and development tool. It is one of the popular -build tools and environment managers for Python, maintained by the Python Packaging Authority. -It is an optional tool that is only really needed when you want to build packages from sources, but -it is also very convenient to manage your Python versions and virtualenvs. - -Airflow project contains some pre-defined virtualenv definitions in `pyproject.toml` that can be -easily used by Hatch to create your local venvs. This is not necessary for you to develop and test -Airflow, but it is a convenient way to manage your local Python versions and virtualenvs. - -Installing Hatch ----------------- - -You can install Hatch using various other ways (including Gui installers). - -Example using `pipx`: - - pipx install hatch - -We recommend using `pipx` as you can manage installed Python apps easily and later use it -to upgrade `hatch` easily as needed with: +Using ``uv`` to manage your Python, virtualenvs, and install airflow for development +==================================================================================== - pipx upgrade hatch +While you can manually install airflow locally from sources, Airflow committers recommend using +[uv](https://docs.astral.sh/uv/) as a build and development tool. It is a modern, +recently introduced popular packaging front-end tool and environment managers for Python. +It is an optional tool that is only really needed when you want to build packages from sources, you can use +many other packaging frontends (for example ``hatch``) but ``uv`` is very fast and convenient to manage +also your Python versions and virtualenvs. Also we use `` -## Using Hatch to manage your Python versions - -You can also use Hatch to install and manage airflow virtualenvs and development -environments. For example, you can install Python 3.10 with this command: - - hatch python install 3.10 - -or install all Python versions that are used in Airflow: - - hatch python install all - -## Using Hatch to manage your virtualenvs - -Airflow has some pre-defined virtualenvs that you can use to develop and test airflow. -You can see the list of available envs with: - - hatch env show - -This is what it shows currently: +Installing ``uv`` +----------------- -┏━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Name ┃ Type ┃ Description ┃ -┡━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ default │ virtual │ Default environment with Python 3.9 for maximum compatibility │ -├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤ -│ airflow-39 │ virtual │ Environment with Python 3.9. No devel installed. │ -├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤ -│ airflow-310 │ virtual │ Environment with Python 3.10. No devel installed. │ -├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤ -│ airflow-311 │ virtual │ Environment with Python 3.11. No devel installed │ -├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤ -│ airflow-312 │ virtual │ Environment with Python 3.12. No devel installed │ -└─────────────┴─────────┴───────────────────────────────────────────────────────────────┘ +You can install uv following [the instructions](https://docs.astral.sh/uv/getting-started/installation/ on -The default env (if you have not used one explicitly) is `default` and it is a Python 3.9 -virtualenv for maximum compatibility with `devel` extra installed - this devel extra contains the minimum set -of dependencies and tools that should be used during unit testing of core Airflow and running all `airflow` -CLI commands - without support for providers or databases. +Using ``uv`` to manage your virtualenvs +--------------------------------------- -The other environments are just bare-bones Python virtualenvs with Airflow core requirements only, -without any extras installed and without any tools. They are much faster to create than the default -environment, and you can manually install either appropriate extras or tools that you need for -testing or development. +You can create a virtualenv with ``uv`` using the following command: - hatch env create + uv venv create -You can create specific environments by using them in create command: +You can sync to latest versions of airflow dependencies using: - hatch env create airflow-310 + uv sync -You can install extras in the environment by running pip command: +And if you want to use some extras (for example because you want to develop providers) you can add +extras to the command: - hatch -e airflow-310 run -- pip install -e ".[devel,google]" + uv sync --extra devel -And you can enter the environment by running a shell of your choice (for example, zsh) where you -can run any commands +You can also synchronize all extras: - hatch -e airflow-310 shell + uv sync --all-extras -Once you are in the environment (indicated usually by an updated prompt), you can just install -the extra dependencies you need: +Building airflow packages with Hatch +==================================== - [~/airflow] [airflow-310] pip install -e ".[devel,google]" +While building packages will work with any compliant packaging front-end tool, for reproducibility, we +recommend using ``hatch``. It is a modern, fast, and convenient tool to build packages from sources managed +by the Python Packaging Authority. It is also used by Airflow to build packages in CI/CD as well as by +release managers to build locally packages for verification of reproducibility of the build. -You can exit the environment by just exiting the shell. +Installing ``hatch`` +-------------------- -You can also see where Hatch created the virtualenvs and use it in your IDE or activate it manually: +More information about hatch can be found in https://hatch.pypa.io/ - hatch env find airflow-310 +We recommend to install ``hatch`` using ```uv tool`` command which will make hatch available as a CLI +command globally: -You will get a path similar to the following: + uv tool install hatch - /Users/jarek/Library/Application Support/hatch/env/virtual/apache-airflow/TReRdyYt/apache-airflow +You can still install ``hatch`` using ``pipx`` if you prefer: -Then you will find `python` binary and `activate` script in the `bin` sub-folder of this directory, and -you can configure your IDE to use this python virtualenv if you want to use that environment in your IDE. + pipx install hatch -You can also set the default environment name by the HATCH_ENV environment variable. -You can clean the environment by running the following: +It's important to keep your hatch up to date. You can do this by running: - hatch env prune + uv tool upgrade hatch -More information about hatch can be found in https://hatch.pypa.io/1.9/environment/ -## Using Hatch to build your packages +Using Hatch to build your packages +---------------------------------- You can use Hatch to build installable packages from the Airflow sources. Such package will -include all metadata configured in `pyproject.toml` and will be installable with pip. +include all metadata configured in `pyproject.toml` and will be installable with ``pip`` and and any other +PEP-compliant packaging front-end. -The packages will have pre-installed dependencies for providers that are available when Airflow is installed from PyPI. Both `wheel` and `sdist` packages are built by default. +The packages will have pre-installed dependencies for providers that are available when Airflow is i +onstalled from PyPI. Both `wheel` and `sdist` packages are built by default. hatch build @@ -221,7 +178,8 @@ You can also build only `wheel` or `sdist` packages: hatch build -t wheel hatch build -t sdist -## Installing recommended version of dependencies +Installing recommended version of dependencies +============================================== Whatever virtualenv solution you use, when you want to make sure you are using the same version of dependencies as in main, you can install the recommended version of the dependencies by using diff --git a/contributing-docs/03_contributors_quick_start.rst b/contributing-docs/03_contributors_quick_start.rst index 7869f254a6397..e6279936e6854 100644 --- a/contributing-docs/03_contributors_quick_start.rst +++ b/contributing-docs/03_contributors_quick_start.rst @@ -222,18 +222,20 @@ see in CI in your local environment. Setting up Breeze ----------------- -1. Install ``pipx`` (>=1.2.1) - follow the instructions in `Install pipx `_ +1. Install ``uv`` or ``pipx``. We recommend to install ``uv`` as general purpose python development + environment - you can install it via https://docs.astral.sh/uv/getting-started/installation/ or you can + install ``pipx`` (>=1.2.1) - follow the instructions in `Install pipx `_ It is important to install version of pipx >= 1.2.1 to workaround ``packaging`` breaking change introduced in September 2023 -2. Run ``pipx install -e ./dev/breeze`` in your checked-out repository. Make sure to follow any instructions - printed by ``pipx`` during the installation - this is needed to make sure that ``breeze`` command is - available in your PATH +2. Run ``uv tool install -e ./dev/breeze`` (or ``pipx install -e ./dev/breeze`` in your checked-out + repository. Make sure to follow any instructions printed by during the installation - this is needed + to make sure that ``breeze`` command is available in your PATH .. warning:: - If you see below warning - it means that you hit `known issue `_ - with ``packaging`` version 23.2: + If you see below warning while running pipx - it means that you hit the + `known issue `_ with ``packaging`` version 23.2: ⚠️ Ignoring --editable install option. pipx disallows it for anything but a local path, to avoid having to create a new src/ directory. @@ -245,7 +247,6 @@ Setting up Breeze pip install "packaging==23.1" pipx install -e ./dev/breeze --force - 3. Initialize breeze autocomplete .. code-block:: bash @@ -454,9 +455,7 @@ To avoid burden on CI infrastructure and to save time, Pre-commit hooks can be r .. note:: - We have recently started to recommend ``uv`` for our local development. Currently (October 2024) ``uv`` - speeds up installation more than 10x comparing to ``pip``. While we still describe ``pip`` and ``pipx`` - below, we also show the ``uv`` alternatives. + We have recently started to recommend ``uv`` for our local development. .. note:: @@ -464,26 +463,28 @@ To avoid burden on CI infrastructure and to save time, Pre-commit hooks can be r started to use Python 3.9+ features in Airflow and accompanying scripts. -Installing pre-commit is best done with ``pipx``: +Installing pre-commit is best done with ``uv`` (recommended) or ``pipx``: + +This will install ``pre-commit`` with ``uv``, and it will change it to use ``uv`` to install its own +virtualenvs. .. code-block:: bash - pipx install pre-commit + uv tool install pre-commit --with pre-commit-uv -You can still add uv support for pre-commit if you use pipx using the commands: +or .. code-block:: bash pipx install pre-commit - pipx inject - pipx inject prepare_breeze_and_image -Also, if you already use ``uvx`` instead of ``pipx``, use this command: +You can add ````uv`` support for ``pre-commit`` even you install it with ``pipx`` using the commands +(then pre-commit will use ``uv`` to create virtualenvs for the hooks): .. code-block:: bash - uv tool install pre-commit --with pre-commit-uv --force-reinstall - + pipx install pre-commit + pipx inject pre-commit pre-commit-uv # optionally if you want to use uv to install virtualenvs 1. Installing required packages @@ -499,11 +500,18 @@ on macOS, install via brew install libxml2 -2. Installing required Python packages +2. Installing pre-commit (if you have not done it yet): + +.. code-block:: bash + + uv tool install pre-commit --with pre-commit-uv + +or .. code-block:: bash pipx install pre-commit + pipx install inject pre-commit pre-commit-uv 3. Go to your project directory @@ -512,7 +520,7 @@ on macOS, install via cd ~/Projects/airflow -1. Running pre-commit hooks +4. Running pre-commit hooks .. code-block:: bash @@ -547,7 +555,6 @@ on macOS, install via pre-commit run --files airflow/utils/decorators.py tests/utils/test_task_group.py - 6. Running specific hook for selected files .. code-block:: bash @@ -558,7 +565,6 @@ on macOS, install via Run ruff............................................................Passed - 7. Enabling Pre-commit check before push. It will run pre-commit automatically before committing and stops the commit .. code-block:: bash @@ -574,7 +580,6 @@ on macOS, install via cd ~/Projects/airflow pre-commit uninstall - - For more information on visit |08_static_code_checks.rst| .. |08_static_code_checks.rst| raw:: html diff --git a/contributing-docs/quick-start-ide/contributors_quick_start_gitpod.rst b/contributing-docs/quick-start-ide/contributors_quick_start_gitpod.rst index 7a2c13d31fc1d..f5a0738cdb609 100644 --- a/contributing-docs/quick-start-ide/contributors_quick_start_gitpod.rst +++ b/contributing-docs/quick-start-ide/contributors_quick_start_gitpod.rst @@ -56,23 +56,7 @@ Set up Breeze in Gitpod Gitpod default image have all the required packages installed. -1. Run ``pipx install -e ./dev/breeze`` to install Breeze - -.. warning:: - - If you see below warning - it means that you hit `known issue `_ - with ``packaging`` version 23.2: - ⚠️ Ignoring ``--editable install`` option. ``pipx`` disallows it for anything but a local path, - to avoid having to create a new ``src/`` directory. - - The workaround is to downgrade packaging to 23.1 and re-running the ``pipx install`` command. for example - by running ``pip install "packaging<23.2"``. - - .. code-block:: bash - - pip install "packaging==23.1" - pipx install -e ./dev/breeze --force - +1. Run ``uv tool install -e ./dev/breeze`` (or ``pipx install -e ./dev/breeze`` ) to install Breeze 2. Run ``breeze`` to enter breeze in Gitpod. Setting up database in Breeze diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 1f56def27646c..08a8994529e23 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -214,35 +214,39 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag - Set environment variables - ```shell script +```shell script - # You can avoid repeating this command for every release if you will set it in .zshrc - # see https://unix.stackexchange.com/questions/608842/zshrc-export-gpg-tty-tty-says-not-a-tty - export GPG_TTY=$(tty) +# You can avoid repeating this command for every release if you will set it in .zshrc +# see https://unix.stackexchange.com/questions/608842/zshrc-export-gpg-tty-tty-says-not-a-tty +export GPG_TTY=$(tty) - # Set Version - export VERSION=2.1.2rc3 - export VERSION_SUFFIX=rc3 - export VERSION_BRANCH=2-1 - export VERSION_WITHOUT_RC=${VERSION/rc?/} +# Set Version +export VERSION=2.1.2rc3 +export VERSION_SUFFIX=rc3 +export VERSION_BRANCH=2-1 +export VERSION_WITHOUT_RC=${VERSION/rc?/} - # Set AIRFLOW_REPO_ROOT to the path of your git repo - export AIRFLOW_REPO_ROOT=$(pwd) +# Set AIRFLOW_REPO_ROOT to the path of your git repo +export AIRFLOW_REPO_ROOT=$(pwd) - # Example after cloning - git clone https://github.com/apache/airflow.git airflow - cd airflow - export AIRFLOW_REPO_ROOT=$(pwd) - ``` +# Example after cloning +git clone https://github.com/apache/airflow.git airflow +cd airflow +export AIRFLOW_REPO_ROOT=$(pwd) +``` - Install `breeze` command: - ```shell script - pipx install -e ./dev/breeze - ``` +```shell script +uv tool install -e ./dev/breeze +``` +or (if you prefer to use pipx): +```shell script +pipx install -e ./dev/breeze +``` - For major/minor version release, run the following commands to create the 'test' and 'stable' branches. diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 70ad4076750ad..b2c8ccf3b2eb7 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -35,27 +35,19 @@ for Airflow Development. This package should never be installed in "production" mode. The `breeze` entrypoint will actually fail if you do so. It is supposed to be installed only in [editable/development mode](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#working-in-development-mode) -directly from Airflow sources using `pipx` - usually with `--force` flag to account for re-installation -that might often be needed if dependencies change during development. +directly from Airflow sources using `uv tool``or ``pipx` - usually with `--force` flag to account +for re-installation that might often be needed if dependencies change during development. ```shell -pipx install -e ./dev/breeze --force +uv tool install -e ./dev/breeze --force ``` -NOTE! If you see below warning - it means that you hit [known issue](https://github.com/pypa/pipx/issues/1092) -with `packaging` version 23.2 -⚠️ Ignoring --editable install option. pipx disallows it for anything but a local path, -to avoid having to create a new src/ directory. - -The workaround is to downgrade packaging to 23.1 and re-running the `pipx install` command, for example -by running `pip install "packaging<23.2"`. +or ```shell -pip install "packaging<23.2" pipx install -e ./dev/breeze --force ``` - You can read more about Breeze in the [documentation](https://github.com/apache/airflow/blob/main/dev/breeze/doc/README.rst) This README file contains automatically generated hash of the `pyproject.toml` files that were @@ -136,6 +128,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: 5d32e2c150de1cc22d522f94d86500c739f434439ae064e35791ac795932a1f21509c3d0fcf9f2351e7901a32601190bd4cb640799620a44d0ff6d9244aef57c +Package config hash: 1a6bdff24f910175038dbd62c1c18dd091958ee2ffbb55ac7d5c93cc43f8f9ad5176093c135ac72031574292397164402a2c17a7c4f7f5fdb3c02e3d576109bf --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/doc/01_installation.rst b/dev/breeze/doc/01_installation.rst index 052dc3faca9f9..9c053bf280b66 100644 --- a/dev/breeze/doc/01_installation.rst +++ b/dev/breeze/doc/01_installation.rst @@ -262,25 +262,18 @@ Set your working directory to the root of this cloned repository. cd airflow -Run this command to install Breeze (make sure to use ``-e`` flag): +Run this command to install Breeze (make sure to use ``-e`` flag) - you can choose ``uv`` (recommended) or +``pipx``: -.. code-block:: bash - - pipx install -e ./dev/breeze -.. warning:: +.. code-block:: bash - If you see below warning - it means that you hit `known issue `_ - with ``packaging`` version 23.2: - ⚠️ Ignoring --editable install option. pipx disallows it for anything but a local path, - to avoid having to create a new src/ directory. + uv tool install -e ./dev/breeze - The workaround is to downgrade packaging to 23.1 and re-running the ``pipx install`` command. - .. code-block:: bash +.. code-block:: bash - pip install "packaging<23.2" - pipx install -e ./dev/breeze --force + pipx install -e ./dev/breeze .. note:: Note for Windows users @@ -289,6 +282,12 @@ Run this command to install Breeze (make sure to use ``-e`` flag): If you are on Windows, you should use Windows way to point to the ``dev/breeze`` sub-folder of Airflow either as absolute or relative path. For example: + .. code-block:: bash + + uv tool install -e dev\breeze + + or + .. code-block:: bash pipx install -e dev\breeze @@ -329,7 +328,13 @@ that Breeze works on .. warning:: Upgrading from earlier Python version If you used Breeze with Python 3.8 and when running it, it will complain that it needs Python 3.9. In this - case you should force-reinstall Breeze with ``pipx``: + case you should force-reinstall Breeze with ``uv`` (or ``pipx``): + + .. code-block:: bash + + uv tool install --force -e ./dev/breeze + + or .. code-block:: bash @@ -341,30 +346,35 @@ that Breeze works on If you are on Windows, you should use Windows way to point to the ``dev/breeze`` sub-folder of Airflow either as absolute or relative path. For example: + .. code-block:: bash + + uv tool install --force -e dev\breeze + + or + .. code-block:: bash pipx install --force -e dev\breeze - .. note:: creating pipx virtual env ``apache-airflow-breeze`` with a specific python version - In ``pipx install -e ./dev/breeze`` or ``pipx install -e dev\breeze``, ``pipx`` uses default - system python version to create virtual env for breeze. - We can use a specific version by providing python executable in ``--python`` argument. For example: + .. note:: creating virtual env for ``apache-airflow-breeze`` with a specific python version + The ``uv tool install`` or ``pipx install`` use default system python version to create virtual + env for breeze. You can use a specific version by providing python version in ``uv`` or + python executable in ``pipx`` in ``--python``. If you have breeze installed already with another Python version you can reinstall breeze with reinstall command .. code-block:: bash - pipx reinstall --python /Users/airflow/.pyenv/versions/3.9.16/bin/python apache-airflow-breeze + uv tool install --python 3.9.16 ./dev/breeze --force - Or you can uninstall breeze and install it with a specific python version: + or .. code-block:: bash - pipx uninstall apache-airflow-breeze - pipx install -e ./dev/breeze --python /Users/airflow/.pyenv/versions/3.9.16/bin/python + pipx install -e ./dev/breeze --python /Users/airflow/.pyenv/versions/3.9.16/bin/python --force Running Breeze for the first time @@ -477,19 +487,18 @@ Automating breeze installation ------------------------------ Breeze on POSIX-compliant systems (Linux, MacOS) can be automatically installed by running the -``scripts/tools/setup_breeze`` bash script. This includes checking and installing ``pipx``, setting up +``scripts/tools/setup_breeze`` bash script. This includes checking and installing ``uv``, setting up ``breeze`` with it and setting up autocomplete. Uninstalling Breeze ------------------- -Since Breeze is installed with ``pipx``, with ``pipx list``, you can list the installed packages. -Once you have the name of ``breeze`` package you can proceed to uninstall it. +Since Breeze is installed with ``uv tool`` or ``pipx``, you need to use the appropriate tool to uninstall it. .. code-block:: bash - pipx list + uv tool uninstall apache-airflow-breeze This will also remove breeze from the folder: ``${HOME}.local/bin/`` diff --git a/dev/breeze/doc/04_troubleshooting.rst b/dev/breeze/doc/04_troubleshooting.rst index 1eddfb60ca1d3..5e44c4dbf791a 100644 --- a/dev/breeze/doc/04_troubleshooting.rst +++ b/dev/breeze/doc/04_troubleshooting.rst @@ -91,23 +91,32 @@ or It means that your pre-commit hook is installed with (already End-Of-Life) Python 3.8 and you should reinstall it and clean pre-commit cache. -This can be done (if you use ``pipx`` to install ``pre-commit``): +This can be done with ``uv tool`` to install ``pre-commit``) + +.. code-block:: bash + + uv tool uninstall pre-commit + uv tool install pre-commit --python 3.9 --force --with pre-commit-uv + pre-commit clean + pre-commit install + +You can also use ``pipx`` .. code-block:: bash pipx uninstall pre-commit pipx install pre-commit --python $(which python3.9) --force # This one allows pre-commit to use uv for venvs installed by pre-commit - pipx inject pre-commit pre-commit-uv + pipx inject pre-commit pre-commit-uv # optionally if you want to use uv to install virtualenvs pre-commit clean pre-commit install If you installed ``pre-commit`` differently, you should remove and reinstall -it (and clean cache) in the way you installed it. +it (and clean cache) following the way you installed it. -Bad Interpreter Error ---------------------- +Bad Interpreter Error with ``pipx`` +----------------------------------- If you are experiencing bad interpreter errors ``zsh: /Users/eladkal/.local/bin/breeze: bad interpreter: /Users/eladkal/.local/pipx/venvs/apache-airflow-breeze/bin/python: no such file or directory`` diff --git a/dev/breeze/doc/10_advanced_breeze_topics.rst b/dev/breeze/doc/10_advanced_breeze_topics.rst index a4f9384863009..9bbf113cb7e62 100644 --- a/dev/breeze/doc/10_advanced_breeze_topics.rst +++ b/dev/breeze/doc/10_advanced_breeze_topics.rst @@ -29,17 +29,10 @@ Debugging/developing Breeze Breeze can be quite easily debugged with PyCharm/VSCode or any other IDE - but it might be less discoverable if you never tested modules and if you do not know how to bypass version check of breeze. -For testing, you can create your own virtual environment, or use the one that ``pipx`` created for you if you -already installed breeze following the recommended ``pipx install -e ./dev/breeze`` command. +For testing, you can create your own virtual environment, or use the one that ``uv`` or ``pipx`` created +for you if you already installed breeze following the recommended installation. -For local virtualenv, you can use ``pyenv`` or any other virtualenv wrapper. For example with ``pyenv``, -you can use ``pyenv virtualenv 3.9.6 airflow-breeze`` to create virtualenv called ``airflow-breeze`` -with Python 3.9.6. Then you can use ``pyenv activate airflow-breeze`` to activate it and install breeze -in editable mode with ``pip install -e ./dev/breeze``. - -For ``pipx`` virtualenv, you can use the virtualenv that ``pipx`` created for you. You can find the name -where ``pipx`` keeps their venvs via ``pipx list`` command. Usually it is -``${HOME}/.local/pipx/venvs/apache-airflow-breeze`` where ``$HOME`` is your home directory. +Or you can change your directory to The venv can be used for running breeze tests and for debugging breeze. While running tests should be usually "out-of-the-box" for most IDEs, once you configure ``./dev/breeze`` project to use the venv, diff --git a/dev/breeze/doc/adr/0016-use-uv-tool-to-install-breeze.md b/dev/breeze/doc/adr/0016-use-uv-tool-to-install-breeze.md new file mode 100644 index 0000000000000..d425b6c40aa34 --- /dev/null +++ b/dev/breeze/doc/adr/0016-use-uv-tool-to-install-breeze.md @@ -0,0 +1,56 @@ + + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [10. Use uv tool to install breeze](#10-use-uv-tool-to-install-breeze) + - [Status](#status) + - [Context](#context) + - [Decision](#decision) + - [Consequences](#consequences) + + + +# 10. Use uv tool to install breeze + +Date: 2024-11-11 + +## Status + +Accepted + +Supersedes [10. Use pipx to install breeze](0010-use-pipx-to-install-breeze.md) + +## Context + +The ``uv`` tools is a new modern python development environment management tool +and we adopt it in ``Airflow`` as recommended way to manage airflow local virtualenv and development +setup. It's much faster to install dependencies with ``uv`` than with ``pip`` and it has many +more features - including managing python interpreters, workspaces, syncing virtualenv and more. + +## Decision + +While it is still possible to install breeze using ``pipx``, we are now recommending ``uv`` and specifically +``uv tool`` as the way to install breeze. Contributors should use ``uv tool`` to install breeze. + +## Consequences + +Those who used ``pipx``, should clean-up and reinstall their environment with ``uv``. diff --git a/dev/breeze/pyproject.toml b/dev/breeze/pyproject.toml index f5b4b9b98c4e9..04eae4706e7f2 100644 --- a/dev/breeze/pyproject.toml +++ b/dev/breeze/pyproject.toml @@ -73,7 +73,6 @@ dependencies = [ "jinja2>=3.1.0", "jsonschema>=4.19.1", "packaging>=23.2", - "pipx>=1.4.1", "pre-commit>=3.5.0", "pre-commit-uv>=4.1.3", "psutil>=5.9.6", diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index cf04ecc278715..9add64abc45ac 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -96,8 +96,12 @@ def get_package_setup_metadata_hash() -> str: from importlib_metadata import distribution # type: ignore[no-redef, assignment] prefix = "Package config hash: " - - for line in distribution("apache-airflow-breeze").metadata.as_string().splitlines(keepends=False): + metadata = distribution("apache-airflow-breeze").metadata + try: + description = metadata.json["description"] # type: ignore[attr-defined] + except AttributeError: + description = metadata.as_string() + for line in description.splitlines(keepends=False): if line.startswith(prefix): return line[len(prefix) :] return "NOT FOUND" diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index 8396c0016dea3..25df86203cd3e 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -249,8 +249,8 @@ def assert_pre_commit_installed(): ) get_console().print( "\n[warning]With pipx you can install it with:[/]\n\n" - " pipx inject\n" - " pipx inject pre-commit pre-commit-uv\n" + " pipx inject pre-commit pre-commit-uv # optionally if you want to use uv to " + "install virtualenvs\n" ) else: get_console().print( diff --git a/docs/apache-airflow/installation/installing-from-pypi.rst b/docs/apache-airflow/installation/installing-from-pypi.rst index fa34287a31560..4b2774de6bd93 100644 --- a/docs/apache-airflow/installation/installing-from-pypi.rst +++ b/docs/apache-airflow/installation/installing-from-pypi.rst @@ -63,8 +63,8 @@ Those are just examples, see further for more explanation why those are the best .. note:: Generally speaking, Python community established practice is to perform application installation in a - virtualenv created with ``virtualenv`` or ``venv`` tools. You can also use ``pipx`` to install Airflow® in a - application dedicated virtual environment created for you. There are also other tools that can be used + virtualenv created with ``virtualenv`` or ``venv`` tools. You can also use ``uv`` or ``pipx`` to install + Airflow in application dedicated virtual environment created for you. There are also other tools that can be used to manage your virtualenv installation and you are free to choose how you are managing the environments. Airflow has no limitation regarding to the tool of your choice when it comes to virtual environment. diff --git a/docs/docker-stack/changelog.rst b/docs/docker-stack/changelog.rst index 9b135b80b5535..0b1ebf5897d61 100644 --- a/docs/docker-stack/changelog.rst +++ b/docs/docker-stack/changelog.rst @@ -39,6 +39,9 @@ Airflow 3.0 * The ``virtualenv`` package is no longer installed in the reference container. Airflow 3 and standard provider relies on ``venv`` module from Python standard library. + * There is no ``pipx`` and ``mssql-cli`` installed in the image by default. We recommend to use + ``uv tool`` instead of ``pipx`` and ``mssql-cli`` is not used in the image by default as we do not + have mssql metadata support any more. Airflow 2.10 ~~~~~~~~~~~~ diff --git a/scripts/ci/install_breeze.sh b/scripts/ci/install_breeze.sh index aa5a3160060bf..bdae70b95f544 100755 --- a/scripts/ci/install_breeze.sh +++ b/scripts/ci/install_breeze.sh @@ -21,13 +21,15 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/../../" PYTHON_ARG="" +PIP_VERSION="24.3.1" +UV_VERSION="0.5.1" if [[ ${PYTHON_VERSION=} != "" ]]; then PYTHON_ARG="--python=$(which python"${PYTHON_VERSION}") " fi -python -m pip install --upgrade pip==24.3.1 -python -m pip install "pipx>=1.4.1" -python -m pipx uninstall apache-airflow-breeze >/dev/null 2>&1 || true +python -m pip install --upgrade "pip==${PIP_VERSION}" +python -m pip install "uv==${UV_VERSION}" +uv tool uninstall apache-airflow-breeze >/dev/null 2>&1 || true # shellcheck disable=SC2086 -python -m pipx install ${PYTHON_ARG} --force --editable ./dev/breeze/ +uv tool install ${PYTHON_ARG} --force --editable ./dev/breeze/ echo '/home/runner/.local/bin' >> "${GITHUB_PATH}" diff --git a/scripts/ci/pre_commit/update_installers.py b/scripts/ci/pre_commit/update_installers.py index f55a937df0cdf..28de198824064 100755 --- a/scripts/ci/pre_commit/update_installers.py +++ b/scripts/ci/pre_commit/update_installers.py @@ -32,6 +32,7 @@ AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile.ci", AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "ci" / "install_breeze.sh", AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "docker" / "common.sh", + AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "tools" / "setup_breeze", AIRFLOW_SOURCES_ROOT_PATH / "pyproject.toml", AIRFLOW_SOURCES_ROOT_PATH / "dev" / "breeze" / "src" / "airflow_breeze" / "global_constants.py", AIRFLOW_SOURCES_ROOT_PATH @@ -60,12 +61,14 @@ def get_latest_pypi_version(package_name: str) -> str: AIRFLOW_PIP_PATTERN = re.compile(r"(AIRFLOW_PIP_VERSION=)([0-9.]+)") AIRFLOW_PIP_QUOTED_PATTERN = re.compile(r"(AIRFLOW_PIP_VERSION = )(\"[0-9.]+\")") PIP_QUOTED_PATTERN = re.compile(r"(PIP_VERSION = )(\"[0-9.]+\")") +PIP_QUOTED_PATTERN_NO_SPACES = re.compile(r"(PIP_VERSION=)(\"[0-9.]+\")") AIRFLOW_PIP_DOC_PATTERN = re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)") AIRFLOW_PIP_UPGRADE_PATTERN = re.compile(r"(python -m pip install --upgrade pip==)([0-9.]+)") AIRFLOW_UV_PATTERN = re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)") AIRFLOW_UV_QUOTED_PATTERN = re.compile(r"(AIRFLOW_UV_VERSION = )(\"[0-9.]+\")") UV_QUOTED_PATTERN = re.compile(r"(UV_VERSION = )(\"[0-9.]+\")") +UV_QUOTED_PATTERN_NO_SPACES = re.compile(r"(UV_VERSION=)(\"[0-9.]+\")") AIRFLOW_UV_DOC_PATTERN = re.compile(r"(\| *`AIRFLOW_UV_VERSION` *\| *)(`[0-9.]+`)( *\|)") UV_GREATER_PATTERN = re.compile(r'"(uv>=)([0-9]+)"') @@ -115,6 +118,9 @@ def replacer(match): new_content = replace_group_2_while_keeping_total_length( PIP_QUOTED_PATTERN, f'"{pip_version}"', new_content ) + new_content = replace_group_2_while_keeping_total_length( + PIP_QUOTED_PATTERN_NO_SPACES, f'"{pip_version}"', new_content + ) if UPGRADE_UV: new_content = replace_group_2_while_keeping_total_length( AIRFLOW_UV_PATTERN, uv_version, new_content @@ -128,6 +134,9 @@ def replacer(match): new_content = replace_group_2_while_keeping_total_length( UV_QUOTED_PATTERN, f'"{uv_version}"', new_content ) + new_content = replace_group_2_while_keeping_total_length( + UV_QUOTED_PATTERN_NO_SPACES, f'"{uv_version}"', new_content + ) if new_content != file_content: file.write_text(new_content) console.print(f"[bright_blue]Updated {file}") diff --git a/scripts/docker/install_pipx_tools.sh b/scripts/docker/install_pipx_tools.sh deleted file mode 100644 index 534dce8a93497..0000000000000 --- a/scripts/docker/install_pipx_tools.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck shell=bash disable=SC2086 -# shellcheck source=scripts/docker/common.sh -. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" - -function install_pipx_tools() { - echo - echo "${COLOR_BLUE}Installing pipx tools${COLOR_RESET}" - echo - # Make sure PIPX is installed in latest version - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pipx>=1.2.1" - if [[ $(uname -m) != "aarch64" ]]; then - # Do not install mssql-cli for ARM - # Install all the tools we need available in command line but without impacting the current environment - pipx install mssql-cli - - # Unfortunately mssql-cli installed by `pipx` does not work out of the box because it uses - # its own execution bash script which is not compliant with the auto-activation of - # pipx venvs - we need to manually patch Python executable in the script to fix it: ¯\_(ツ)_/¯ - sed "s/python /\/root\/\.local\/pipx\/venvs\/mssql-cli\/bin\/python /" -i /root/.local/bin/mssql-cli - fi -} - -common::get_colors -common::get_packaging_tool - -install_pipx_tools diff --git a/scripts/tools/setup_breeze b/scripts/tools/setup_breeze index c740b6b4b9bb4..78d4db3a7d974 100755 --- a/scripts/tools/setup_breeze +++ b/scripts/tools/setup_breeze @@ -27,13 +27,14 @@ COLOR_YELLOW=$'\e[33m' COLOR_BLUE=$'\e[34m' COLOR_RESET=$'\e[0m' +UV_VERSION="0.5.1" + function manual_instructions() { echo echo "${COLOR_BLUE}Please run those commands manually (you might need to restart shell between them)${COLOR_RESET}" echo - echo " python -m pip install \"pipx>=1.2.1\"" - echo " pipx ensurepath" - echo " pipx install -e '${AIRFLOW_SOURCES}/dev/breeze/'" + echo " python -m pip install \"uv==${UV_VERSION}\"" + echo " uv tool install -e '${AIRFLOW_SOURCES}/dev/breeze/'" echo " breeze setup autocomplete --force" echo echo " After that, both pipx and breeze should be available on your path" @@ -43,20 +44,20 @@ function manual_instructions() { function check_breeze_installed() { set +e - command -v "${BREEZE_BINARY}" >/dev/null + command -v "${BREEZE_BINARY}" >/dev/null 2>/dev/null local breeze_on_path=$? - command -v "pipx" >/dev/null - local pipx_on_path=$? + command -v "uv" >/dev/null 2>/dev/null + local uv_on_path=$? set -e - if [[ ${breeze_on_path} != "0" || ${pipx_on_path} != "0" ]]; then + if [[ ${breeze_on_path} != "0" || ${uv_on_path} != "0" ]]; then echo - if [[ ${pipx_on_path} != 0 ]]; then - echo "${COLOR_RED}The 'pipx' is not on path. It should be installed and 'pipx' should be available on your PATH.${COLOR_RESET}" + if [[ ${uv_on_path} != 0 ]]; then + echo "${COLOR_RED}The 'uv' is not on path. It should be installed and 'uv' should be available on your PATH.${COLOR_RESET}" export TIMEOUT=0 - if "${MY_DIR}/confirm" "Installing pipx?"; then - python -m pip install "pipx>=1.2.1" --upgrade + if "${MY_DIR}/confirm" "Installing uv?"; then + python -m pip install "uv==${UV_VERSION}" --upgrade echo - echo "${COLOR_YELLOW}Please close and re-open the shell and retry. You might need to add 'pipx' to the PATH!${COLOR_RESET}" + echo "${COLOR_YELLOW}Please close and re-open the shell and retry. You might need to add 'uv' to the PATH!${COLOR_RESET}" echo exit else @@ -67,11 +68,10 @@ function check_breeze_installed() { echo "${COLOR_RED}The '${BREEZE_BINARY}' is not on path. Breeze should be installed and 'breeze' should be available on your PATH!${COLOR_RESET}" export TIMEOUT=0 echo - echo "${COLOR_YELLOW}Installing Breeze. This will install breeze via pipx and modify your local rc file for ${SHELL}${COLOR_RESET}" + echo "${COLOR_YELLOW}Installing Breeze. This will install breeze via uv and modify your ${SHELL} to run it${COLOR_RESET}" echo if "${MY_DIR}/confirm" "installing and modifying the startup scripts"; then - pipx ensurepath --force - pipx install -e "${AIRFLOW_SOURCES}/dev/breeze/" --force + uv tool install -e "${AIRFLOW_SOURCES}/dev/breeze/" --force ${BREEZE_BINARY} setup-autocomplete --force --answer yes echo echo "${COLOR_YELLOW}Please close and re-open the shell and retry. Then rerun your last command!${COLOR_RESET}"