From 07dc0b6068415cb877cd98406d1acac70aba4c4b Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 24 Nov 2024 12:21:57 +0000 Subject: [PATCH 01/15] move openapi tests to breeze container --- .github/workflows/basic-tests.yml | 102 ------- .github/workflows/ci.yml | 17 ++ .github/workflows/openapi-tests.yml | 86 ++++++ Dockerfile.ci | 34 +++ clients/python/test_python_client.py | 116 ++++---- dev/breeze/doc/05_test_commands.rst | 23 ++ ...utput_setup_check-all-params-in-groups.svg | 4 +- ...utput_setup_check-all-params-in-groups.txt | 2 +- ...output_setup_regenerate-command-images.svg | 30 +- ...output_setup_regenerate-command-images.txt | 2 +- dev/breeze/doc/images/output_testing.svg | 12 +- dev/breeze/doc/images/output_testing.txt | 2 +- .../images/output_testing_openapi-tests.svg | 260 ++++++++++++++++++ .../images/output_testing_openapi-tests.txt | 1 + .../common_package_installation_options.py | 15 + .../commands/testing_commands.py | 88 ++++++ .../commands/testing_commands_config.py | 24 +- .../src/airflow_breeze/global_constants.py | 1 + .../src/airflow_breeze/params/shell_params.py | 8 + .../src/airflow_breeze/utils/run_tests.py | 21 +- scripts/docker/entrypoint_ci.sh | 34 +++ .../install_airflow_python_client.py | 91 ++++++ 22 files changed, 786 insertions(+), 187 deletions(-) create mode 100644 .github/workflows/openapi-tests.yml create mode 100644 dev/breeze/doc/images/output_testing_openapi-tests.svg create mode 100644 dev/breeze/doc/images/output_testing_openapi-tests.txt create mode 100644 scripts/in_container/install_airflow_python_client.py diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 14f0628e454bb..4276573b95fe2 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -153,108 +153,6 @@ jobs: env: FORCE_COLOR: 2 - test-openapi-client: - timeout-minutes: 10 - name: "Test OpenAPI client" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} - if: inputs.needs-api-codegen == 'true' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - fetch-depth: 2 - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - repository: "apache/airflow-client-python" - fetch-depth: 1 - persist-credentials: false - path: ./airflow-client-python - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: "Generate client with breeze" - run: > - breeze release-management prepare-python-client --package-format both - --version-suffix-for-pypi dev0 --python-client-repo ./airflow-client-python - - name: "Show diff" - run: git diff --color HEAD - working-directory: ./airflow-client-python - - name: Install hatch - run: | - python -m pip install --upgrade uv - uv tool install hatch - - name: Run tests - run: hatch run run-coverage - env: - HATCH_ENV: "test" - working-directory: ./clients/python - - name: "Install source version of required packages" - run: | - breeze release-management prepare-provider-packages \ - fab \ - standard \ - common.sql \ - sqlite \ - --package-format wheel \ - --skip-tag-check \ - --version-suffix-for-pypi dev0 - pip install . \ - dist/apache_airflow_providers_fab-*.whl \ - dist/apache_airflow_providers_standard-*.whl \ - dist/apache_airflow_providers_common_sql-*.whl \ - dist/apache_airflow_providers_sqlite-*.whl - breeze release-management prepare-task-sdk-package --package-format wheel - pip install ./dist/apache_airflow_task_sdk-*.whl - - name: "Install Python client" - run: pip install ./dist/apache_airflow_client-*.whl - - name: "Initialize Airflow DB and start webserver" - run: | - airflow db init - # Let scheduler runs a few loops and get all DAG files from example DAGs serialized to DB - airflow scheduler --num-runs 100 - airflow users create --username admin --password admin --firstname Admin --lastname Admin \ - --role Admin --email admin@example.org - killall python || true # just in case there is a webserver running in the background - nohup airflow webserver --port 8080 & - echo "Started webserver" - env: - AIRFLOW__API__AUTH_BACKENDS: >- - airflow.api.auth.backend.session,airflow.providers.fab.auth_manager.api.auth.backend.basic_auth - AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "True" - AIRFLOW__CORE__LOAD_EXAMPLES: "True" - AIRFLOW_HOME: "${{ github.workspace }}/airflow_home" - - name: "Waiting for the webserver to be available" - run: | - timeout 30 bash -c 'until nc -z $0 $1; do echo "sleeping"; sleep 1; done' localhost 8080 - sleep 5 - - name: "Run test python client" - run: python ./clients/python/test_python_client.py - env: - FORCE_COLOR: "standard" - - name: "Stop running webserver" - run: killall python || true # just in case there is a webserver running in the background - if: always() - - name: "Upload python client packages" - uses: actions/upload-artifact@v4 - with: - name: python-client-packages - path: ./dist/apache_airflow_client-* - retention-days: 7 - if-no-files-found: error - - name: "Upload logs from failed tests" - uses: actions/upload-artifact@v4 - if: failure() - with: - name: python-client-failed-logs - path: "${{ github.workspace }}/airflow_home/logs" - retention-days: 7 - # Those checks are run if no image needs to be built for checks. This is for simple changes that # Do not touch any of the python code or any of the important files that might require building # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 115d78a49913a..bd8a285a98e62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -366,6 +366,7 @@ jobs: needs.build-info.outputs.default-branch == 'main' && needs.build-info.outputs.latest-versions-only != 'true' + tests-postgres: name: "Postgres tests" uses: ./.github/workflows/run-unit-tests.yml @@ -506,6 +507,22 @@ jobs: include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + test-openapi: + name: "OpenAPI tests" + uses: ./.github/workflows/openapi-tests.yml + needs: [build-info, wait-for-ci-images] + permissions: + contents: read + packages: read + secrets: inherit + with: + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + image-tag: ${{ needs.build-info.outputs.image-tag }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + if: needs.build-info.outputs.needs-api-codegen == 'true' + tests-integration-system: name: Integration and System Tests needs: [build-info, wait-for-ci-images] diff --git a/.github/workflows/openapi-tests.yml b/.github/workflows/openapi-tests.yml new file mode 100644 index 0000000000000..2f8c72414cd8e --- /dev/null +++ b/.github/workflows/openapi-tests.yml @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: OpenAPI client +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + runs-on-as-json-public: + description: "The array of labels (in json form) determining public runners." + required: true + type: string + debug-resources: + description: "Whether to debug resources or not (true/false)" + required: true + type: string + run-coverage: + description: "Whether to run coverage or not (true/false)" + required: true + type: string + image-tag: + description: "Tag to set for the image" + required: true + type: string + default-python-version: + description: "Which version of python should be used by default" + required: true + type: string +jobs: + test-openapi-client: + timeout-minutes: 60 + name: "Test OpenAPI client" + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + env: + DEBUG_RESOURCES: "${{ inputs.debug-resources }}" + ENABLE_COVERAGE: "${{ inputs.run-coverage }}" + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + IMAGE_TAG: "${{ inputs.image-tag }}" + JOB_ID: "openapi-client-tests" + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + VERBOSE: "true" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + fetch-depth: 2 + persist-credentials: false + - name: "Cleanup docker" + run: ./scripts/ci/cleanup_docker.sh + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + repository: "apache/airflow-client-python" + fetch-depth: 1 + persist-credentials: false + path: ./airflow-client-python + - name: "Prepare breeze & CI image: ${{inputs.default-python-version}}:${{inputs.image-tag}}" + uses: ./.github/actions/prepare_breeze_and_image + - name: "Generate airflow python client" + run: > + breeze release-management prepare-python-client --package-format both + --version-suffix-for-pypi dev0 --python-client-repo ./airflow-client-python + - name: "Show diff" + run: git diff --color HEAD + working-directory: ./airflow-client-python + - name: "Open API Tests with python client" + run: breeze testing openapi-tests diff --git a/Dockerfile.ci b/Dockerfile.ci index b3686b782ad93..f1990ca085c85 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1155,12 +1155,46 @@ function check_force_lowest_dependencies() { set +x } +function check_airflow_python_client_installation() { + if [[ ${INSTALL_AIRFLOW_PYTHON_CLIENT=} != "true" ]]; then + return + fi + python "${IN_CONTAINER_DIR}/install_airflow_python_client.py" +} + +function start_airflow_minimal_webserver_with_examples(){ + if [[ ${START_AIRFLOW_MINIMAL_WEBSERVER_WITH_EXAMPLES=} != "true" ]]; then + return + fi + export AIRFLOW__CORE__LOAD_EXAMPLES=True + echo + echo "${COLOR_BLUE}Initializing database${COLOR_RESET}" + echo + airflow db migrate + echo + echo "${COLOR_BLUE}Database initialized${COLOR_RESET}" + echo + echo "${COLOR_BLUE}Parsing example dags${COLOR_RESET}" + echo + airflow scheduler --num-runs 100 + echo "Example dags parsing finished" + echo "Create admin user" + airflow users create -u admin -p admin -f Thor -l Administrator -r Admin -e admin@email.domain + echo "Admin user created" + echo + echo "${COLOR_BLUE}Starting airflow webserver${COLOR_RESET}" + echo + airflow webserver --port 8080 --daemon && sleep 60 +} + determine_airflow_to_use environment_initialization check_boto_upgrade check_downgrade_sqlalchemy check_downgrade_pendulum check_force_lowest_dependencies +check_airflow_python_client_installation +start_airflow_minimal_webserver_with_examples check_run_tests "${@}" exec /bin/bash "${@}" diff --git a/clients/python/test_python_client.py b/clients/python/test_python_client.py index 5d0accdc019ff..54adedba82ba4 100644 --- a/clients/python/test_python_client.py +++ b/clients/python/test_python_client.py @@ -63,68 +63,70 @@ # or AIRFLOW__CORE__LOAD_EXAMPLES environment variable set to True DAG_ID = "example_bash_operator" -# Enter a context with an instance of the API client -with airflow_client.client.ApiClient(configuration) as api_client: - errors = False - print("[blue]Getting DAG list") - dag_api_instance = dag_api.DAGApi(api_client) - try: - api_response = dag_api_instance.get_dags() - print(api_response) - except airflow_client.client.OpenApiException as e: - print(f"[red]Exception when calling DagAPI->get_dags: {e}\n") - errors = True - else: - print("[green]Getting DAG list successful") +# Enter a context with an instance of the API client +def test_python_client(): + with airflow_client.client.ApiClient(configuration) as api_client: + errors = False - print("[blue]Getting Tasks for a DAG") - try: - api_response = dag_api_instance.get_tasks(DAG_ID) - print(api_response) - except airflow_client.client.exceptions.OpenApiException as e: - print(f"[red]Exception when calling DagAPI->get_tasks: {e}\n") - errors = True - else: - print("[green]Getting Tasks successful") + print("[blue]Getting DAG list") + dag_api_instance = dag_api.DAGApi(api_client) + try: + api_response = dag_api_instance.get_dags() + print(api_response) + except airflow_client.client.OpenApiException as e: + print(f"[red]Exception when calling DagAPI->get_dags: {e}\n") + errors = True + else: + print("[green]Getting DAG list successful") - print("[blue]Triggering a DAG run") - dag_run_api_instance = dag_run_api.DAGRunApi(api_client) - try: - # Create a DAGRun object (no dag_id should be specified because it is read-only property of DAGRun) - # dag_run id is generated randomly to allow multiple executions of the script - dag_run = DAGRun( - dag_run_id="some_test_run_" + uuid.uuid4().hex, - ) - api_response = dag_run_api_instance.post_dag_run(DAG_ID, dag_run) - print(api_response) - except airflow_client.client.exceptions.OpenApiException as e: - print(f"[red]Exception when calling DAGRunAPI->post_dag_run: {e}\n") - errors = True - else: - print("[green]Posting DAG Run successful") + print("[blue]Getting Tasks for a DAG") + try: + api_response = dag_api_instance.get_tasks(DAG_ID) + print(api_response) + except airflow_client.client.exceptions.OpenApiException as e: + print(f"[red]Exception when calling DagAPI->get_tasks: {e}\n") + errors = True + else: + print("[green]Getting Tasks successful") - # Get current configuration. Note, this is disabled by default with most installation. - # You need to set `expose_config = True` in Airflow configuration in order to retrieve configuration. - conf_api_instance = config_api.ConfigApi(api_client) - try: - api_response = conf_api_instance.get_config() - print(api_response) - except airflow_client.client.OpenApiException as e: - if "FORBIDDEN" in str(e): - print( - "[yellow]You need to set `expose_config = True` in Airflow configuration" - " in order to retrieve configuration." + print("[blue]Triggering a DAG run") + dag_run_api_instance = dag_run_api.DAGRunApi(api_client) + try: + # Create a DAGRun object (no dag_id should be specified because it is read-only property of DAGRun) + # dag_run id is generated randomly to allow multiple executions of the script + dag_run = DAGRun( + dag_run_id="some_test_run_" + uuid.uuid4().hex, ) - print("[bright_blue]This is OK. Exposing config is disabled by default.") - else: + api_response = dag_run_api_instance.post_dag_run(DAG_ID, dag_run) + print(api_response) + except airflow_client.client.exceptions.OpenApiException as e: print(f"[red]Exception when calling DAGRunAPI->post_dag_run: {e}\n") errors = True - else: - print("[green]Config retrieved successfully") + else: + print("[green]Posting DAG Run successful") - if errors: - print("\n[red]There were errors while running the script - see above for details") - sys.exit(1) - else: - print("\n[green]Everything went well") + # Get current configuration. Note, this is disabled by default with most installation. + # You need to set `expose_config = True` in Airflow configuration in order to retrieve configuration. + conf_api_instance = config_api.ConfigApi(api_client) + try: + api_response = conf_api_instance.get_config() + print(api_response) + except airflow_client.client.OpenApiException as e: + if "FORBIDDEN" in str(e): + print( + "[yellow]You need to set `expose_config = True` in Airflow configuration" + " in order to retrieve configuration." + ) + print("[bright_blue]This is OK. Exposing config is disabled by default.") + else: + print(f"[red]Exception when calling DAGRunAPI->post_dag_run: {e}\n") + errors = True + else: + print("[green]Config retrieved successfully") + + if errors: + print("\n[red]There were errors while running the script - see above for details") + sys.exit(1) + else: + print("\n[green]Everything went well") diff --git a/dev/breeze/doc/05_test_commands.rst b/dev/breeze/doc/05_test_commands.rst index f5b3a78eef78f..07d5cc85bbd1a 100644 --- a/dev/breeze/doc/05_test_commands.rst +++ b/dev/breeze/doc/05_test_commands.rst @@ -218,6 +218,29 @@ Here is the detailed set of options for the ``breeze testing providers-integrati :alt: Breeze testing providers-integration-tests +Running OpenAPI Client tests +................................... + +To run OpenAPI Client tests, you need to have airflow python client packaged in dist folder. +To package the client, clone the airflow-python-client repository and run the following command: + +.. code-block:: bash + + breeze release-management prepare-python-client --package-format both + --version-suffix-for-pypi dev0 --python-client-repo ./airflow-client-python + +.. code-block:: bash + + breeze testing openapi-tests + +Here is the detailed set of options for the ``breeze testing openapi-tests`` command. + +.. image:: ./images/output_testing_openpapi-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_openapi-tests.svg + :width: 100% + :alt: Breeze testing openapi-tests + + Running system tests .................... diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg index 2281855ed930d..5d49a48a80408 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg @@ -202,8 +202,8 @@ │setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images | setup:self-upgrade │ │| setup:synchronize-local-mounts | setup:version | shell | start-airflow | static-checks | testing |   │ │testing:core-integration-tests | testing:core-tests | testing:docker-compose-tests | testing:helm-tests│ -│| testing:providers-integration-tests | testing:providers-tests | testing:system-tests |               │ -│testing:task-sdk-tests)                                                                                │ +│| testing:openapi-tests | testing:providers-integration-tests | testing:providers-tests |              │ +│testing:system-tests | testing:task-sdk-tests)                                                         │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--verbose-vPrint verbose information about performed steps.│ diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt index 8af1fd4596083..134bf0d88815d 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt @@ -1 +1 @@ -5b1f2b4c7ccd4fb99efaa8f48c721c34 +a96cde419d0d6dee102ce903e9c33772 diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg index 7a3bcdedd717e..b9206ffd4e58f 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg @@ -1,4 +1,4 @@ - + │setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images |                 │ │setup:self-upgrade | setup:synchronize-local-mounts | setup:version | shell | start-airflow |       │ │static-checks | testing | testing:core-integration-tests | testing:core-tests |                     │ -│testing:docker-compose-tests | testing:helm-tests | testing:providers-integration-tests |           │ -│testing:providers-tests | testing:system-tests | testing:task-sdk-tests)                            │ -│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ -│together with --command flag or --force.                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│testing:docker-compose-tests | testing:helm-tests | testing:openapi-tests |                         │ +│testing:providers-integration-tests | testing:providers-tests | testing:system-tests |              │ +│testing:task-sdk-tests)                                                                             │ +│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ +│together with --command flag or --force.                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt index 8cc4669293051..91cd9da729c0f 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt @@ -1 +1 @@ -cef764d3eaa21feef72d2f259627ade6 +8946afaa796b15e7239a8b073e672c32 diff --git a/dev/breeze/doc/images/output_testing.svg b/dev/breeze/doc/images/output_testing.svg index b3e5ddf681bdb..bcebf5814aa3d 100644 --- a/dev/breeze/doc/images/output_testing.svg +++ b/dev/breeze/doc/images/output_testing.svg @@ -1,4 +1,4 @@ - +