Add script to move providers to the new directory structure (apache#4…

…5945)

.github/boring-cyborg.yml

@@ -27,8 +27,8 @@ labelPRBasedOnFilePath:
     - providers/tests/alibaba/**/*
     - providers/tests/system/alibaba/**/*
 
-  provider:amazon-aws:
-    - providers/src/airflow/providers/amazon/aws/**/*
+  provider:amazon:
+    - providers/src/airflow/providers/amazon/**/*
     - providers/tests/amazon/aws/**/*
     - docs/apache-airflow-providers-amazon/**/*
     - providers/tests/system/amazon/aws/**/*

dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py

@@ -1183,18 +1183,22 @@ def _regenerate_pyproject_toml(context: dict[str, Any], provider_details: Provid
         if in_required_dependencies and line == "]":
             in_required_dependencies = False
             continue
-        if line == "[project.optional-dependencies]":
-            in_optional_dependencies = True
-            continue
-        if in_optional_dependencies and line == "":
-            in_optional_dependencies = False
-            continue
         if line == "[dependency-groups]":
             in_dependency_groups = True
             continue
         if in_dependency_groups and line == "":
             in_dependency_groups = False
             continue
+        if in_dependency_groups and line.startswith("["):
+            in_dependency_groups = False
+        if line == "[project.optional-dependencies]":
+            in_optional_dependencies = True
+            continue
+        if in_optional_dependencies and line == "":
+            in_optional_dependencies = False
+            continue
+        if in_optional_dependencies and line.startswith("["):
+            in_optional_dependencies = False
         if in_required_dependencies:
             required_dependencies.append(line)
         if in_optional_dependencies:
@@ -1209,7 +1213,9 @@ def _regenerate_pyproject_toml(context: dict[str, Any], provider_details: Provid
 
     # Add cross-provider dependencies to the optional dependencies if they are missing
     for module in PROVIDER_DEPENDENCIES.get(provider_details.provider_id)["cross-providers-deps"]:
-        if f'"{module}" = [' not in optional_dependencies:
+        if f'"{module}" = [' not in optional_dependencies and get_pip_package_name(module) not in "\n".join(
+            required_dependencies
+        ):
             optional_dependencies.append(f'"{module}" = [')
             optional_dependencies.append(f'    "{get_pip_package_name(module)}"')
             optional_dependencies.append("]")
@@ -1221,6 +1227,8 @@ def _regenerate_pyproject_toml(context: dict[str, Any], provider_details: Provid
         context=context,
         extension=".toml",
         autoescape=False,
+        lstrip_blocks=True,
+        trim_blocks=True,
         keep_trailing_newline=True,
     )
     get_pyproject_toml_path.write_text(get_pyproject_toml_content)

dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py

@@ -137,6 +137,8 @@ def _prepare_pyproject_toml_file(context: dict[str, Any], target_path: Path):
         context=context,
         extension=".toml",
         autoescape=False,
+        lstrip_blocks=True,
+        trim_blocks=True,
         keep_trailing_newline=True,
     )
     (target_path / "pyproject.toml").write_text(manifest_content)

dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2

@@ -69,25 +69,26 @@ classifiers = [
     "Topic :: System :: Monitoring",
 ]
 requires-python = "~=3.9"
+
 # The dependencies should be modified in place in the generated file
 # Any change in the dependencies is preserved when the file is regenerated
 dependencies = [
 {{ INSTALL_REQUIREMENTS }}
 ]
+{% if EXTRAS_REQUIREMENTS %}
 
-{%- if EXTRAS_REQUIREMENTS %}
 # The optional dependencies should be modified in place in the generated file
 # Any change in the dependencies is preserved when the file is regenerated
 [project.optional-dependencies]
 {{ EXTRAS_REQUIREMENTS }}
-{%- endif %}
+{% endif %}
+{% if DEPENDENCY_GROUPS %}
 
-{%- if DEPENDENCY_GROUPS %}
 # The dependency groups should be modified in place in the generated file
 # Any change in the dependencies is preserved when the file is regenerated
 [dependency-groups]
 {{ DEPENDENCY_GROUPS }}
-{%- endif %}
+{% endif %}
 
 [project.urls]
 "Documentation" = "https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME }}/{{RELEASE}}"
@@ -100,13 +101,13 @@ dependencies = [
 
 [project.entry-points."apache_airflow_provider"]
 provider_info = "airflow.providers.{{ PROVIDER_ID }}.get_provider_info:get_provider_info"
+{% if PLUGINS %}
 
-{%- if PLUGINS %}
 [project.entry-points."airflow.plugins"]
-{%- for plugin in PLUGINS %}
+{% for plugin in PLUGINS %}
 {{ plugin.name }} = "{{ plugin.package_name }}:{{ plugin.class_name }}"
-{%- endfor %}
-{%- endif %}
+{% endfor %}
+{% endif %}
 
 [tool.flit.module]
 name = "airflow.providers.{{ PROVIDER_ID }}"

dev/breeze/src/airflow_breeze/utils/packages.py

@@ -834,6 +834,8 @@ def render_template(
     context: dict[str, Any],
     extension: str,
     autoescape: bool = True,
+    lstrip_blocks: bool = False,
+    trim_blocks: bool = False,
     keep_trailing_newline: bool = False,
 ) -> str:
     """
@@ -842,6 +844,8 @@ def render_template(
     :param context: Jinja2 context
     :param extension: Target file extension
     :param autoescape: Whether to autoescape HTML
+    :param lstrip_blocks: Whether to strip leading blocks
+    :param trim_blocks: Whether to trim blocks
     :param keep_trailing_newline: Whether to keep the newline in rendered output
     :return: rendered template
     """
@@ -852,6 +856,8 @@ def render_template(
         loader=template_loader,
         undefined=jinja2.StrictUndefined,
         autoescape=autoescape,
+        lstrip_blocks=lstrip_blocks,
+        trim_blocks=trim_blocks,
         keep_trailing_newline=keep_trailing_newline,
     )
     template = template_env.get_template(f"{template_name}_TEMPLATE{extension}.jinja2")

dev/moving_providers/README.md

@@ -0,0 +1,87 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [Moving providers to new structure](#moving-providers-to-new-structure)
+  - [How to use the script](#how-to-use-the-script)
+  - [Options](#options)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+# Moving providers to new structure
+
+We are moving providers to a new structure, where each provider has a separate sub-project in
+"providers" sub-folder.
+
+This means that we need to migrate some 90+ providers to the new structure. This is a big task and while we
+could do it in one huge PR, it would be disruptive and likely take some time to review and fix some individual
+edge-cases - even if we have automated most of the work.
+
+This directory contains a script that contributors can use to move a provider (or a few providers to the
+new structure as a starting point for their PR. Most of the work is automated, but there will be likely
+some manual adjustments needed in more complex cases.
+
+## How to use the script
+
+The script follows https://peps.python.org/pep-0723/ and uses inlined dependencies - so it can be run as-is
+by modern tools without creating dedicated virtualenv - the virtualenv with dependencies is
+created on-the-fly by PEP 723 compatible tools. For example one can use uv to run it:
+
+```shell
+uv run dev/moving_providers/move_providers.py alibaba
+```
+
+## Options
+
+
+> [!NOTE]
+> You can see all the options by running the script with `--help` option:
+>
+> ```shell
+> uv run dev/moving_providers/move_providers.py --help
+> ```
+
+By default the script runs in `--dry-run` mode, which means it will not make any changes to the file system,
+but will print what it would do. To actually move the files, you need to pass `--no-dry-run` option and you
+will be asked to commit the code and create a PR:
+
+```shell
+uv run dev/moving_providers/move_providers.py alibaba --no-dry-run
+```
+
+You can specify multiple providers to move in one go:
+
+```shell
+uv run dev/moving_providers/move_providers.py alibaba amazon  microsoft.azure
+```
+
+You can specify `--verbose` option to see more details about what the script is doing:
+
+```shell
+uv run dev/moving_providers/move_providers.py alibaba --verbose
+```
+
+You can also specify `--quiet` option to see less output:
+
+```shell
+uv run dev/moving_providers/move_providers.py alibaba --quiet
+```