Add feature to build "chicken-egg" packages from sources (apache#35890)

When we build on ci a pre-release version of provider, and we want to include packages that have >= CURRENTLY_RELEASED_VERSION we have to make sure that those packages are built from sources during building of PROD image. Otherwise they will not be installable on CURRENT_VERSION.dev0, CURRENT_VERSION.rc* etc. Until we "Actually" release a provider we should have a way to build such provider from sources. This is the CI version of it, once we have it working we can also apply it to the workflow that releases images to dockerhub.
astronomer · Dec 1, 2023 · cf052dc · cf052dc
1 parent eed6427
commit cf052dc
Show file tree

Hide file tree

Showing 19 changed files with 120 additions and 47 deletions.
diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml
@@ -22,6 +22,9 @@ inputs:
   build-provider-packages:
     description: 'Whether to build provider packages from sources'
     required: true
+  chicken-egg-providers:
+    description: 'List of chicken-egg provider packages to build from sources'
+    required: true
 runs:
   using: "composite"
   steps:
@@ -41,6 +44,15 @@ runs:
         --package-list-file ./airflow/providers/installed_providers.txt
         --package-format wheel --version-suffix-for-pypi dev0
       if: ${{ inputs.build-provider-packages == 'true' }}
+    - name: "Prepare chicken-eggs provider packages"
+      # In case of provider packages which use latest dev0 version of providers, we should prepare them
+      # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency
+      # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0
+      shell: bash
+      run: >
+        breeze release-management prepare-provider-packages
+        --package-format wheel --version-suffix-for-pypi dev0 ${{ inputs.chicken-egg-providers }}
+      if: ${{ inputs.build-provider-packages != 'true' && inputs.chicken-egg-providers != '' }}
     - name: "Prepare airflow package"
       shell: bash
       run: >

diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -74,6 +74,7 @@ jobs:
       is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
       is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
       is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
+      chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
       target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha ||
           github.event.pull_request.head.sha ||
           github.sha
@@ -293,6 +294,7 @@ jobs:
         uses: ./.github/actions/build-prod-images
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+          chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -121,6 +121,7 @@ jobs:
       is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
       is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
       is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
+      chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
       has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
       source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
       pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
@@ -474,6 +475,7 @@ jobs:
       RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
       PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      VERSION_SUFFIX_FOR_PYPI: "dev0"
     if: needs.build-info.outputs.ci-image-build == 'true'
     steps:
       - name: Cleanup repo
@@ -502,12 +504,23 @@ jobs:
         run: >
           breeze release-management generate-constraints --run-in-parallel
           --airflow-constraints-mode constraints-no-providers
+      - name: "Prepare chicken-eggs provider packages"
+        # In case of provider packages which use latest dev0 version of providers, we should prepare them
+        # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency
+        # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0
+        shell: bash
+        run: >
+          breeze release-management prepare-provider-packages
+          --package-format wheel --version-suffix-for-pypi dev0
+          ${{ needs.build-info.outputs.chicken-egg-providers }}
+        if: needs.build-info.outputs.chicken-egg-providers != ''
       - name: "PyPI constraints"
         shell: bash
         timeout-minutes: 25
         run: >
           breeze release-management generate-constraints --run-in-parallel
           --airflow-constraints-mode constraints
+          --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}"
       - name: "Dependency upgrade summary"
         shell: bash
         run: |
@@ -1608,6 +1621,7 @@ jobs:
         if: needs.build-info.outputs.in-workflow-build == 'true'
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+          chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
@@ -1647,6 +1661,7 @@ jobs:
         uses: ./.github/actions/build-prod-images
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+          chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}

diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml
@@ -753,7 +753,7 @@ additional-extras:
       - apache-airflow-providers-cncf-kubernetes>=7.2.0
   - name: s3fs
     dependencies:
-      - s3fs>=2023.9.2
+      - s3fs>=2023.10.0
   - name: python3-saml
     dependencies:
       - python3-saml>=1.16.0

diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml
@@ -87,7 +87,7 @@ dependencies:
   - gcloud-aio-auth>=4.0.0,<5.0.0
   - gcloud-aio-bigquery>=6.1.2
   - gcloud-aio-storage
-  - gcsfs>=2023.9.2
+  - gcsfs>=2023.10.0
   - google-ads>=22.1.0
   - google-api-core>=2.11.0
   - google-api-python-client>=1.6.0

diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml
@@ -70,7 +70,7 @@ versions:
 
 dependencies:
   - apache-airflow>=2.5.0
-  - adlfs>=2023.9.2
+  - adlfs>=2023.10.0
   - azure-batch>=8.0.0
   - azure-cosmos>=4.0.0
   - azure-mgmt-cosmosdb

diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -684,21 +684,30 @@ def run_generate_constraints_in_parallel(
 @option_image_tag_for_running
 @option_debug_release_management
 @option_airflow_constraints_mode_ci
+@click.option(
+    "--chicken-egg-providers",
+    default="",
+    help="List of chicken-egg provider packages - "
+    "those that have airflow_version >= current_version and should "
+    "be installed in CI from locally built packages with >= current_version.dev0 ",
+    envvar="CHICKEN_EGG_PROVIDERS",
+)
 @option_github_repository
 @option_verbose
 @option_dry_run
 @option_answer
 def generate_constraints(
+    airflow_constraints_mode: str,
+    debug: bool,
+    debug_resources: bool,
+    github_repository: str,
+    image_tag: str | None,
+    parallelism: int,
     python: str,
+    python_versions: str,
     run_in_parallel: bool,
-    parallelism: int,
     skip_cleanup: bool,
-    debug_resources: bool,
-    python_versions: str,
-    image_tag: str | None,
-    debug: bool,
-    airflow_constraints_mode: str,
-    github_repository: str,
+    chicken_egg_providers: str,
 ):
     perform_environment_checks()
     check_remote_ghcr_io_commands()
@@ -742,6 +751,7 @@ def generate_constraints(
                 python=python,
                 github_repository=github_repository,
                 airflow_constraints_mode=airflow_constraints_mode,
+                chicken_egg_providers=chicken_egg_providers,
             )
             for python in python_version_list
         ]
@@ -762,6 +772,7 @@ def generate_constraints(
             skip_image_upgrade_check=True,
             quiet=True,
             airflow_constraints_mode=airflow_constraints_mode,
+            chicken_egg_providers=chicken_egg_providers,
         )
         return_code, info = run_generate_constraints(
             shell_params=shell_params,

diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
@@ -134,6 +134,7 @@
                 "--image-tag",
                 "--python",
                 "--airflow-constraints-mode",
+                "--chicken-egg-providers",
                 "--debug",
                 "--github-repository",
             ],

diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -118,6 +118,7 @@ class ShellParams:
     builder: str = "autodetect"
     celery_broker: str = DEFAULT_CELERY_BROKER
     celery_flower: bool = False
+    chicken_egg_providers: str = ""
     collect_only: bool = False
     database_isolation: bool = False
     db_reset: bool = False
@@ -435,6 +436,7 @@ def env_variables_for_docker_commands(self) -> _Environ:
         _set_var(_env, "BREEZE", "true")
         _set_var(_env, "BREEZE_INIT_COMMAND", None, "")
         _set_var(_env, "CELERY_FLOWER", self.celery_flower)
+        _set_var(_env, "CHICKEN_EGG_PROVIDERS", self.chicken_egg_providers)
         _set_var(_env, "CI", None, "false")
         _set_var(_env, "CI_BUILD_ID", None, "0")
         _set_var(_env, "CI_EVENT_TYPE", None, "pull_request")

diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -56,10 +56,6 @@
 # TODO: use single source of truth for those
 # for now we need to keep them in sync with the ones in setup.py
 PREINSTALLED_PROVIDERS = [
-    #   Until we cut off the 2.8.0 branch and bump current airflow version to 2.9.0, we should
-    #   Keep common.io commented out in order ot be able to generate PyPI constraints because
-    #   The version from PyPI has requirement of apache-airflow>=2.8.0
-    #   "common.io",
     "common.sql",
     "ftp",
     "http",

diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -1019,3 +1019,8 @@ def mssql_parallelism(self) -> int:
     @cached_property
     def has_migrations(self) -> bool:
         return any([file.startswith("airflow/migrations/") for file in self._files])
+
+    @cached_property
+    def chicken_egg_providers(self) -> str:
+        """Space separated list of providers with chicken-egg problem and should be built from sources."""
+        return "common.io"
diff --git a/docs/apache-airflow-providers-google/index.rst b/docs/apache-airflow-providers-google/index.rst
@@ -116,7 +116,7 @@ PIP package                              Version required
 ``gcloud-aio-auth``                      ``>=4.0.0,<5.0.0``
 ``gcloud-aio-bigquery``                  ``>=6.1.2``
 ``gcloud-aio-storage``
-``gcsfs``                                ``>=2023.9.2``
+``gcsfs``                                ``>=2023.10.0``
 ``google-ads``                           ``>=22.1.0``
 ``google-api-core``                      ``>=2.11.0``
 ``google-api-python-client``             ``>=1.6.0``

diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst
@@ -106,7 +106,7 @@ The minimum Apache Airflow version supported by this provider package is ``2.5.0
 PIP package                       Version required
 ================================  ==================
 ``apache-airflow``                ``>=2.5.0``
-``adlfs``                         ``>=2023.9.2``
+``adlfs``                         ``>=2023.10.0``
 ``azure-batch``                   ``>=8.0.0``
 ``azure-cosmos``                  ``>=4.0.0``
 ``azure-mgmt-cosmosdb``

diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
@@ -415,7 +415,7 @@
       "gcloud-aio-auth>=4.0.0,<5.0.0",
       "gcloud-aio-bigquery>=6.1.2",
       "gcloud-aio-storage",
-      "gcsfs>=2023.9.2",
+      "gcsfs>=2023.10.0",
       "google-ads>=22.1.0",
       "google-api-core>=2.11.0",
       "google-api-python-client>=1.6.0",
@@ -557,7 +557,7 @@
   "microsoft.azure": {
     "deps": [
       "adal>=1.2.7",
-      "adlfs>=2023.9.2",
+      "adlfs>=2023.10.0",
       "apache-airflow>=2.5.0",
       "azure-batch>=8.0.0",
       "azure-cosmos>=4.0.0",