From 83a6cb4ff3fa08d22c85e6d50e9e2b8a903f8bb9 Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Wed, 21 Aug 2024 13:20:06 +0530 Subject: [PATCH] Airflow 2.10.0 has been released (#41528) * Airflow 2.10.0 has been released * Update the k8 versions * Fix conflicts * Fix Readme --- .github/ISSUE_TEMPLATE/airflow_bug_report.yml | 2 +- Dockerfile | 2 +- README.md | 16 +- RELEASE_NOTES.rst | 291 ++++++++++++++++++ airflow/reproducible_build.yaml | 4 +- .../installation/supported-versions.rst | 2 +- generated/PYPI_README.md | 14 +- newsfragments/37948.feature.rst | 1 - newsfragments/38891.significant.rst | 10 - newsfragments/39336.significant.rst | 7 - newsfragments/39823.bugfix.rst | 1 - newsfragments/40145.significant.rst | 5 - newsfragments/40379.improvement.rst | 1 - newsfragments/40701.feature.rst | 1 - newsfragments/40703.feature.rst | 1 - newsfragments/40874.significant.rst | 1 - newsfragments/41039.feature.rst | 1 - newsfragments/41116.feature.rst | 1 - scripts/ci/pre_commit/supported_versions.py | 2 +- 19 files changed, 312 insertions(+), 51 deletions(-) delete mode 100644 newsfragments/37948.feature.rst delete mode 100644 newsfragments/38891.significant.rst delete mode 100644 newsfragments/39336.significant.rst delete mode 100644 newsfragments/39823.bugfix.rst delete mode 100644 newsfragments/40145.significant.rst delete mode 100644 newsfragments/40379.improvement.rst delete mode 100644 newsfragments/40701.feature.rst delete mode 100644 newsfragments/40703.feature.rst delete mode 100644 newsfragments/40874.significant.rst delete mode 100644 newsfragments/41039.feature.rst delete mode 100644 newsfragments/41116.feature.rst diff --git a/.github/ISSUE_TEMPLATE/airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_bug_report.yml index 2a482ffc7fbda..0b9697edf00f7 100644 --- a/.github/ISSUE_TEMPLATE/airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_bug_report.yml @@ -25,7 +25,7 @@ body: the latest release or main to see if the issue is fixed before reporting it. multiple: false options: - - "2.10.0rc1" + - "2.10.0" - "2.9.3" - "main (development)" - "Other Airflow 2 version (please specify below)" diff --git a/Dockerfile b/Dockerfile index aff4094603553..c280e8c80b140 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ ARG AIRFLOW_UID="50000" ARG AIRFLOW_USER_HOME_DIR=/home/airflow # latest released version here -ARG AIRFLOW_VERSION="2.9.3" +ARG AIRFLOW_VERSION="2.10.0" ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm" diff --git a/README.md b/README.md index 4816a83531e8a..9556022ce5afb 100644 --- a/README.md +++ b/README.md @@ -97,13 +97,13 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.9.3) | +| | Main version (dev) | Stable version (2.10.0) | |------------|----------------------------|----------------------------| | Python | 3.8, 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12 | | Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | -| Kubernetes | 1.28, 1.29, 1.30, 1.31 | 1.26, 1.27, 1.28, 1.29 | +| Kubernetes | 1.28, 1.29, 1.30, 1.31 | 1.27, 1.28, 1.29, 1.30 | | PostgreSQL | 12, 13, 14, 15, 16 | 12, 13, 14, 15, 16 | -| MySQL | 8.0, 8.4, Innovation | 8.0, Innovation | +| MySQL | 8.0, 8.4, Innovation | 8.0, 8.4, Innovation | | SQLite | 3.15.0+ | 3.15.0+ | \* Experimental @@ -177,15 +177,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==2.9.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.8.txt" +pip install 'apache-airflow==2.10.0' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.8.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==2.8.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.8.txt" +pip install 'apache-airflow[postgres,google]==2.10.0' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.8.txt" ``` For information on installing provider packages, check @@ -290,7 +290,7 @@ Apache Airflow version life cycle: | Version | Current Patch/Minor | State | First Release | Limited Support | EOL/Terminated | |-----------|-----------------------|-----------|-----------------|-------------------|------------------| -| 2 | 2.9.3 | Supported | Dec 17, 2020 | TBD | TBD | +| 2 | 2.10.0 | Supported | Dec 17, 2020 | TBD | TBD | | 1.10 | 1.10.15 | EOL | Aug 27, 2018 | Dec 17, 2020 | June 17, 2021 | | 1.9 | 1.9.0 | EOL | Jan 03, 2018 | Aug 27, 2018 | Aug 27, 2018 | | 1.8 | 1.8.2 | EOL | Mar 19, 2017 | Jan 03, 2018 | Jan 03, 2018 | diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index da34dfc9bef07..a5e6d9974c50e 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -21,6 +21,297 @@ .. towncrier release notes start +Airflow 2.10.0 (2024-08-15) +--------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Datasets no longer trigger inactive DAGs (#38891) +""""""""""""""""""""""""""""""""""""""""""""""""" + +Previously, when a DAG is paused or removed, incoming dataset events would still +trigger it, and the DAG would run when it is unpaused or added back in a DAG +file. This has been changed; a DAG's dataset schedule can now only be satisfied +by events that occur when the DAG is active. While this is a breaking change, +the previous behavior is considered a bug. + +The behavior of time-based scheduling is unchanged, including the timetable part +of ``DatasetOrTimeSchedule``. + +``try_number`` is no longer incremented during task execution (#39336) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Previously, the try number (``try_number``) was incremented at the beginning of task execution on the worker. This was problematic for many reasons. +For one it meant that the try number was incremented when it was not supposed to, namely when resuming from reschedule or deferral. And it also resulted in +the try number being "wrong" when the task had not yet started. The workarounds for these two issues caused a lot of confusion. + +Now, instead, the try number for a task run is determined at the time the task is scheduled, and does not change in flight, and it is never decremented. +So after the task runs, the observed try number remains the same as it was when the task was running; only when there is a "new try" will the try number be incremented again. + +One consequence of this change is, if users were "manually" running tasks (e.g. by calling ``ti.run()`` directly, or command line ``airflow tasks run``), +try number will no longer be incremented. Airflow assumes that tasks are always run after being scheduled by the scheduler, so we do not regard this as a breaking change. + +``/logout`` endpoint in FAB Auth Manager is now CSRF protected (#40145) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The ``/logout`` endpoint's method in FAB Auth Manager has been changed from ``GET`` to ``POST`` in all existing +AuthViews (``AuthDBView``, ``AuthLDAPView``, ``AuthOAuthView``, ``AuthOIDView``, ``AuthRemoteUserView``), and +now includes CSRF protection to enhance security and prevent unauthorized logouts. + +OpenTelemetry Traces for Apache Airflow (#37948). +""""""""""""""""""""""""""""""""""""""""""""""""" +This new feature adds capability for Apache Airflow to emit 1) airflow system traces of scheduler, +triggerer, executor, processor 2) DAG run traces for deployed DAG runs in OpenTelemetry format. Previously, only metrics were supported which emitted metrics in OpenTelemetry. +This new feature will add richer data for users to use OpenTelemetry standard to emit and send their trace data to OTLP compatible endpoints. + +Decorator for Task Flow ``(@skip_if, @run_if)`` to make it simple to apply whether or not to skip a Task. (#41116) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +This feature adds a decorator to make it simple to skip a Task. + +Using Multiple Executors Concurrently (#40701) +"""""""""""""""""""""""""""""""""""""""""""""" +Previously known as hybrid executors, this new feature allows Airflow to use multiple executors concurrently. DAGs, or even individual tasks, can be configured +to use a specific executor that suits its needs best. A single DAG can contain tasks all using different executors. Please see the Airflow documentation for +more details. Note: This feature is still experimental. See `documentation on Executor `_ for a more detailed description. + +Scarf based telemetry: Does Airflow collect any telemetry data? (#39510) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +Airflow integrates Scarf to collect basic usage data during operation. Deployments can opt-out of data collection by setting the ``[usage_data_collection]enabled`` option to False, or the SCARF_ANALYTICS=false environment variable. +See `FAQ on this `_ for more information. + + +New Features +"""""""""""" +- AIP-61 Hybrid Execution (`AIP-61 `_) +- AIP-62 Getting Lineage from Hook Instrumentation (`AIP-62 `_) +- AIP-64 TaskInstance Try History (`AIP-64 `_) +- AIP-44 Internal API (`AIP-44 `_) +- Enable ending the task directly from the triggerer without going into the worker. (#40084) +- Extend dataset dependencies (#40868) +- Feature/add token authentication to internal api (#40899) +- Add DatasetAlias to support dynamic Dataset Event Emission and Dataset Creation (#40478) +- Add example DAGs for inlet_events (#39893) +- Implement ``accessors`` to read dataset events defined as inlet (#39367) +- Decorator for Task Flow, to make it simple to apply whether or not to skip a Task. (#41116) +- Add start execution from triggerer support to dynamic task mapping (#39912) +- Add try_number to log table (#40739) +- Added ds_format_locale method in macros which allows localizing datetime formatting using Babel (#40746) +- Add DatasetAlias to support dynamic Dataset Event Emission and Dataset Creation (#40478, #40723, #40809, #41264, #40830, #40693, #41302) +- Use sentinel to mark dag as removed on re-serialization (#39825) +- Add parameter for the last number of queries to the DB in DAG file processing stats (#40323) +- Add prototype version dark mode for Airflow UI (#39355) +- Add ability to mark some tasks as successful in ``dag test`` (#40010) +- Allow use of callable for template_fields (#37028) +- Filter running/failed and active/paused dags on the home page(#39701) +- Add metrics about task CPU and memory usage (#39650) +- UI changes for DAG Re-parsing feature (#39636) +- Add Scarf based telemetry (#39510, #41318) +- Add dag re-parsing request endpoint (#39138) +- Redirect to new DAGRun after trigger from Grid view (#39569) +- Display ``endDate`` in task instance tooltip. (#39547) +- Implement ``accessors`` to read dataset events defined as inlet (#39367, #39893) +- Add color to log lines in UI for error and warnings based on keywords (#39006) +- Add Rendered k8s pod spec tab to ti details view (#39141) +- Make audit log before/after filterable (#39120) +- Consolidate grid collapse actions to a single full screen toggle (#39070) +- Implement Metadata to emit runtime extra (#38650) +- Add executor field to the DB and parameter to the operators (#38474) +- Implement context accessor for DatasetEvent extra (#38481) +- Add dataset event info to dag graph (#41012) +- Add button to toggle datasets on/off in dag graph (#41200) +- Add ``run_if`` & ``skip_if`` decorators (#41116) +- Add dag_stats rest api endpoint (#41017) +- Add listeners for Dag import errors (#39739) +- Allowing DateTimeSensorAsync, FileSensor and TimeSensorAsync to start execution from trigger during dynamic task mapping (#41182) + + +Improvements +"""""""""""" +- Allow set Dag Run resource into Dag Level permission: extends Dag's access_control feature to allow Dag Run resource permissions. (#40703) +- Improve security and error handling for the internal API (#40999) +- Datasets UI Improvements (#40871) +- Change DAG Audit log tab to Event Log (#40967) +- Make standalone dag file processor works in DB isolation mode (#40916) +- Show only the source on the consumer DAG page and only triggered DAG run in the producer DAG page (#41300) +- Update metrics names to allow multiple executors to report metrics (#40778) +- Format DAG run count (#39684) +- Update styles for ``renderedjson`` component (#40964) +- Improve ATTRIBUTE_REMOVED sentinel to use class and more context (#40920) +- Make XCom display as react json (#40640) +- Replace usages of task context logger with the log table (#40867) +- Rollback for all retry exceptions (#40882) (#40883) +- Support rendering ObjectStoragePath value (#40638) +- Add try_number and map_index as params for log event endpoint (#40845) +- Rotate fernet key in batches to limit memory usage (#40786) +- Add gauge metric for 'last_num_of_db_queries' parameter (#40833) +- Set parallelism log messages to warning level for better visibility (#39298) +- Add error handling for encoding the dag runs (#40222) +- Use params instead of dag_run.conf in example DAG (#40759) +- Load Example Plugins with Example DAGs (#39999) +- Stop deferring TimeDeltaSensorAsync task when the target_dttm is in the past (#40719) +- Send important executor logs to task logs (#40468) +- Open external links in new tabs (#40635) +- Attempt to add ReactJSON view to rendered templates (#40639) +- Speeding up regex match time for custom warnings (#40513) +- Refactor DAG.dataset_triggers into the timetable class (#39321) +- add next_kwargs to StartTriggerArgs (#40376) +- Improve UI error handling (#40350) +- Remove double warning in CLI when config value is deprecated (#40319) +- Implement XComArg concat() (#40172) +- Added ``get_extra_dejson`` method with nested parameter which allows you to specify if you want the nested json as string to be also deserialized (#39811) +- Add executor field to the task instance API (#40034) +- Support checking for db path absoluteness on Windows (#40069) +- Introduce StartTriggerArgs and prevent start trigger initialization in scheduler (#39585) +- Add task documentation to details tab in grid view (#39899) +- Allow executors to be specified with only the class name of the Executor (#40131) +- Remove obsolete conditional logic related to try_number (#40104) +- Allow Task Group Ids to be passed as branches in BranchMixIn (#38883) +- Javascript connection form will apply CodeMirror to all textarea's dynamically (#39812) +- Determine needs_expansion at time of serialization (#39604) +- Add indexes on dag_id column in referencing tables to speed up deletion of dag records (#39638) +- Add task failed dependencies to details page (#38449) +- Remove webserver try_number adjustment (#39623) +- Implement slicing in lazy sequence (#39483) +- Unify lazy db sequence implementations (#39426) +- Add ``__getattr__`` to task decorator stub (#39425) +- Allow passing labels to FAB Views registered via Plugins (#39444) +- Simpler error message when trying to offline migrate with sqlite (#39441) +- Add soft_fail to TriggerDagRunOperator (#39173) +- Rename "dataset event" in context to use "outlet" (#39397) +- Resolve ``RemovedIn20Warning`` in ``airflow task`` command (#39244) +- Determine fail_stop on client side when db isolated (#39258) +- Refactor cloudpickle support in Python operators/decorators (#39270) +- Update trigger kwargs migration to specify existing_nullable (#39361) +- Allowing tasks to start execution directly from triggerer without going to worker (#38674) +- Better ``db migrate`` error messages (#39268) +- Add stacklevel into the ``suppress_and_warn`` warning (#39263) +- Support searching by dag_display_name (#39008) +- Allow sort by on all fields in MappedInstances.tsx (#38090) +- Expose count of scheduled tasks in metrics (#38899) +- Use ``declarative_base`` from ``sqlalchemy.orm`` instead of ``sqlalchemy.ext.declarative`` (#39134) +- Add example DAG to demonstrate emitting approaches (#38821) +- Give ``on_task_instance_failed`` access to the error that caused the failure (#38155) +- Simplify dataset serialization (#38694) +- Add heartbeat recovery message to jobs (#34457) +- Remove select_column option in TaskInstance.get_task_instance (#38571) +- Don't create session in get_dag if not reading dags from database (#38553) +- Add a migration script for encrypted trigger kwargs (#38358) +- Implement render_templates on TaskInstancePydantic (#38559) +- Handle optional session in _refresh_from_db (#38572) +- Make type annotation less confusing in task_command.py (#38561) +- Use fetch_dagrun directly to avoid session creation (#38557) +- Added ``output_processor`` parameter to ``BashProcessor`` (#40843) +- Improve serialization for Database Isolation Mode (#41239) +- Only orphan non-orphaned Datasets (#40806) +- Adjust gantt width based on task history dates (#41192) +- Enable scrolling on legend with high number of elements. (#41187) + +Bug Fixes +""""""""" +- Bugfix for get_parsing_context() when ran with LocalExecutor (#40738) +- Validating provider documentation urls before displaying in views (#40933) +- Move import to make PythonOperator working on Windows (#40424) +- Fix dataset_with_extra_from_classic_operator example DAG (#40747) +- Call listener on_task_instance_failed() after ti state is changed (#41053) +- Add ``never_fail`` in BaseSensor (#40915) +- Fix tasks API endpoint when DAG doesn't have ``start_date`` (#40878) +- Fix and adjust URL generation for UI grid and older runs (#40764) +- Rotate fernet key optimization (#40758) +- Fix class instance vs. class type in validate_database_executor_compatibility() call (#40626) +- Clean up dark mode (#40466) +- Validate expected types for args for DAG, BaseOperator and TaskGroup (#40269) +- Exponential Backoff Not Functioning in BaseSensorOperator Reschedule Mode (#39823) +- local task job: add timeout, to not kill on_task_instance_success listener prematurely (#39890) +- Move Post Execution Log Grouping behind Exception Print (#40146) +- Fix triggerer race condition in HA setting (#38666) +- Pass triggered or existing DAG Run logical date to DagStateTrigger (#39960) +- Passing ``external_task_group_id`` to ``WorkflowTrigger`` (#39617) +- ECS Executor: Set tasks to RUNNING state once active (#39212) +- Only heartbeat if necessary in backfill loop (#39399) +- Fix trigger kwarg encryption migration (#39246) +- Fix decryption of trigger kwargs when downgrading. (#38743) +- Fix wrong link in TriggeredDagRuns (#41166) +- Pass MapIndex to LogLink component for external log systems (#41125) +- Add NonCachingRotatingFileHandler for worker task (#41064) +- Add argument include_xcom in method resolve an optional value (#41062) +- Sanitizing file names in example_bash_decorator DAG (#40949) +- Show dataset aliases in dependency graphs (#41128) +- Render Dataset Conditions in DAG Graph view (#41137) +- Add task duration plot across dagruns (#40755) +- Add start execution from trigger support for existing core sensors (#41021) +- add example dag for dataset_alias (#41037) +- Add dataset alias unique constraint and remove wrong dataset alias removing logic (#41097) +- Set "has_outlet_datasets" to true if "dataset alias" exists (#41091) +- Make HookLineageCollector group datasets by (#41034) +- Enhance start_trigger_args serialization (#40993) +- Refactor ``BaseSensorOperator`` introduce ``skip_policy`` parameter (#40924) +- Fix viewing logs from triggerer when task is deferred (#41272) +- Refactor how triggered dag run url is replaced (#41259) +- Added support for additional sql alchemy session args (#41048) +- Allow empty list in TriggerDagRun failed_state (#41249) +- Clean up the exception handler when run_as_user is the airflow user (#41241) +- Collapse docs when click and folded (#41214) +- Update updated_at when saving to db as session.merge does not trigger on-update (#40782) +- Fix query count statistics when parsing DAF file (#41149) +- Method Resolution Order in operators without ``__init__`` (#41086) +- Ensure try_number incremented for empty operator (#40426) + +Miscellaneous +""""""""""""" +- Remove the Experimental flag from ``OTel`` Traces (#40874) +- Bump packaging version to 23.0 in order to fix issue with older otel (#40865) +- Simplify _auth_manager_is_authorized_map function (#40803) +- Use correct unknown executor exception in scheduler job (#40700) +- Add D1 ``pydocstyle`` rules to pyproject.toml (#40569) +- Enable enforcing ``pydocstyle`` rule D213 in ruff. (#40448, #40464) +- Update ``Dag.test()`` to run with an executor if desired (#40205) +- Update jest and babel minor versions (#40203) +- Refactor BashOperator and Bash decorator for consistency and simplicity (#39871) +- Add ``AirflowInternalRuntimeError`` for raise ``non catchable`` errors (#38778) +- ruff version bump 0.4.5 (#39849) +- Bump ``pytest`` to 8.0+ (#39450) +- Remove stale comment about TI index (#39470) +- Configure ``back_populates`` between ``DagScheduleDatasetReference.dag`` and ``DagModel.schedule_dataset_references`` (#39392) +- Remove deprecation warnings in endpoints.py (#39389) +- Fix SQLA deprecations in Airflow core (#39211) +- Use class-bound attribute directly in SA (#39198, #39195) +- Fix stacklevel for TaskContextLogger (#39142) +- Capture warnings during collect DAGs (#39109) +- Resolve ``B028`` (no-explicit-stacklevel) in core (#39123) +- Rename model ``ImportError`` to ``ParseImportError`` for avoid shadowing with builtin exception (#39116) +- Add option to support cloudpickle in PythonVenv/External Operator (#38531) +- Suppress ``SubDagOperator`` examples warnings (#39057) +- Add log for running callback (#38892) +- Use ``model_dump`` instead of ``dict`` for serialize Pydantic V2 model (#38933) +- Widen cheat sheet column to avoid wrapping commands (#38888) +- Update hatchling to latest version (1.22.5) (#38780) +- bump uv to 0.1.29 (#38758) +- Add missing serializations found during provider tests fixing (#41252) +- Bump ``ws`` from 7.5.5 to 7.5.10 in /airflow/www (#40288) +- Improve typing for allowed/failed_states in TriggerDagRunOperator (#39855) + +Doc Only Changes +"""""""""""""""" +- Add ``filesystems`` and ``dataset-uris`` to "how to create your own provider" page (#40801) +- Fix (TM) to (R) in Airflow repository (#40783) +- Set ``otel_on`` to True in example airflow.cfg (#40712) +- Add warning for _AIRFLOW_PATCH_GEVENT (#40677) +- Update multi-team diagram proposal after Airflow 3 discussions (#40671) +- Add stronger warning that MSSQL is not supported and no longer functional (#40565) +- Fix misleading mac menu structure in howto (#40440) +- Update k8s supported version in docs (#39878) +- Add compatibility note for Listeners (#39544) +- Update edge label image in documentation example with the new graph view (#38802) +- Update UI doc screenshots (#38680) +- Add section "Manipulating queued dataset events through REST API" (#41022) +- Add information about lack of security guarantees for docker compose (#41072) +- Add links to example dags in use params section (#41031) +- Change ``task_id`` from ``send_email`` to ``send_email_notification`` in ``taskflow.rst`` (#41060) +- Remove unnecessary nginx redirect rule from reverse proxy documentation (#38953) + + + Airflow 2.9.3 (2024-07-15) -------------------------- diff --git a/airflow/reproducible_build.yaml b/airflow/reproducible_build.yaml index 775aaa70ae9bf..24eb1ab9cb526 100644 --- a/airflow/reproducible_build.yaml +++ b/airflow/reproducible_build.yaml @@ -1,2 +1,2 @@ -release-notes-hash: 62779daa28762b157d814861dc91adcf -source-date-epoch: 1723560829 +release-notes-hash: 21b1c588582fcbd521c30e73b4b560b4 +source-date-epoch: 1724146775 diff --git a/docs/apache-airflow/installation/supported-versions.rst b/docs/apache-airflow/installation/supported-versions.rst index 5d9f9df700bc5..0fbeb683526f6 100644 --- a/docs/apache-airflow/installation/supported-versions.rst +++ b/docs/apache-airflow/installation/supported-versions.rst @@ -29,7 +29,7 @@ Apache Airflow® version life cycle: ========= ===================== ========= =============== ================= ================ Version Current Patch/Minor State First Release Limited Support EOL/Terminated ========= ===================== ========= =============== ================= ================ -2 2.9.3 Supported Dec 17, 2020 TBD TBD +2 2.10.0 Supported Dec 17, 2020 TBD TBD 1.10 1.10.15 EOL Aug 27, 2018 Dec 17, 2020 June 17, 2021 1.9 1.9.0 EOL Jan 03, 2018 Aug 27, 2018 Aug 27, 2018 1.8 1.8.2 EOL Mar 19, 2017 Jan 03, 2018 Jan 03, 2018 diff --git a/generated/PYPI_README.md b/generated/PYPI_README.md index 59ae93a64ed03..05d5bdedaf3b8 100644 --- a/generated/PYPI_README.md +++ b/generated/PYPI_README.md @@ -54,13 +54,13 @@ Use Airflow to author workflows as directed acyclic graphs (DAGs) of tasks. The Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.9.3) | +| | Main version (dev) | Stable version (2.10.0) | |------------|----------------------------|----------------------------| | Python | 3.8, 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12 | | Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | -| Kubernetes | 1.28, 1.29, 1.30, 1.31 | 1.26, 1.27, 1.28, 1.29 | +| Kubernetes | 1.28, 1.29, 1.30, 1.31 | 1.27, 1.28, 1.29, 1.30 | | PostgreSQL | 12, 13, 14, 15, 16 | 12, 13, 14, 15, 16 | -| MySQL | 8.0, 8.4, Innovation | 8.0, Innovation | +| MySQL | 8.0, 8.4, Innovation | 8.0, 8.4, Innovation | | SQLite | 3.15.0+ | 3.15.0+ | \* Experimental @@ -130,15 +130,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==2.9.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.8.txt" +pip install 'apache-airflow==2.10.0' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.8.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==2.8.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.8.txt" +pip install 'apache-airflow[postgres,google]==2.10.0' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.8.txt" ``` For information on installing provider packages, check diff --git a/newsfragments/37948.feature.rst b/newsfragments/37948.feature.rst deleted file mode 100644 index 440788c264523..0000000000000 --- a/newsfragments/37948.feature.rst +++ /dev/null @@ -1 +0,0 @@ -OpenTelemetry Traces for Apache Airflow. This new feature adds capability for Apache Airflow to emit 1) airflow system traces of scheduler, triggerer, executor, processor 2) DAG run traces for deployed DAG runs in OpenTelemetry format. Previously, only metrics were supported which emitted metrics in OpenTelemetry. This new feature will add richer data for users to use OpenTelemetry standard to emitt and send their trace data to OTLP compatible endpoints. diff --git a/newsfragments/38891.significant.rst b/newsfragments/38891.significant.rst deleted file mode 100644 index 82caa4cdfc60d..0000000000000 --- a/newsfragments/38891.significant.rst +++ /dev/null @@ -1,10 +0,0 @@ -Datasets no longer trigger inactive DAGs - -Previously, when a DAG is paused or removed, incoming dataset events would still -trigger it, and the DAG would run when it is unpaused or added back in a DAG -file. This has been changed; a DAG's dataset schedule can now only be satisfied -by events that occur when the DAG is active. While this is a breaking change, -the previous behavior is considered a bug. - -The behavior of time-based scheduling is unchanged, including the timetable part -of ``DatasetOrTimeSchedule``. diff --git a/newsfragments/39336.significant.rst b/newsfragments/39336.significant.rst deleted file mode 100644 index 750a1807881e4..0000000000000 --- a/newsfragments/39336.significant.rst +++ /dev/null @@ -1,7 +0,0 @@ -``try_number`` is no longer incremented during task execution - -Previously, the try number (``try_number``) was incremented at the beginning of task execution on the worker. This was problematic for many reasons. For one it meant that the try number was incremented when it was not supposed to, namely when resuming from reschedule or deferral. And it also resulted in the try number being "wrong" when the task had not yet started. The workarounds for these two issues caused a lot of confusion. - -Now, instead, the try number for a task run is determined at the time the task is scheduled, and does not change in flight, and it is never decremented. So after the task runs, the observed try number remains the same as it was when the task was running; only when there is a "new try" will the try number be incremented again. - -One consequence of this change is, if users were "manually" running tasks (e.g. by calling ``ti.run()`` directly, or command line ``airflow tasks run``), try number will no longer be incremented. Airflow assumes that tasks are always run after being scheduled by the scheduler, so we do not regard this as a breaking change. diff --git a/newsfragments/39823.bugfix.rst b/newsfragments/39823.bugfix.rst deleted file mode 100644 index 7a774258a4732..0000000000000 --- a/newsfragments/39823.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed ``BaseSensorOperator`` with exponential backoff and reschedule mode by estimating try number based on ``run_duration``; previously, sensors had a fixed reschedule interval. diff --git a/newsfragments/40145.significant.rst b/newsfragments/40145.significant.rst deleted file mode 100644 index beedfc7746d43..0000000000000 --- a/newsfragments/40145.significant.rst +++ /dev/null @@ -1,5 +0,0 @@ -``/logout`` endpoint in FAB Auth Manager is now CSRF protected - -The ``/logout`` endpoint's method in FAB Auth Manager has been changed from ``GET`` to ``POST`` in all existing -AuthViews (``AuthDBView``, ``AuthLDAPView``, ``AuthOAuthView``, ``AuthOIDView``, ``AuthRemoteUserView``), and -now includes CSRF protection to enhance security and prevent unauthorized logouts. diff --git a/newsfragments/40379.improvement.rst b/newsfragments/40379.improvement.rst deleted file mode 100644 index ecccde2065a1d..0000000000000 --- a/newsfragments/40379.improvement.rst +++ /dev/null @@ -1 +0,0 @@ -``chunk_size`` parameter is added to ``LocalFilesystemToGCSOperator``, enabling file uploads in multiple chunks of a specified size. diff --git a/newsfragments/40701.feature.rst b/newsfragments/40701.feature.rst deleted file mode 100644 index 48f928962862a..0000000000000 --- a/newsfragments/40701.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Using Multiple Executors Concurrently: Previously known as hybrid executors, this new feature allows Airflow to use multiple executors concurrently. DAGs, or even individual tasks, can be configured to use a specific executor that suits its needs best. A single DAG can contain tasks all using different executors. Please see the Airflow documentation for more details. Note: This feature is still experimental. diff --git a/newsfragments/40703.feature.rst b/newsfragments/40703.feature.rst deleted file mode 100644 index 4fd2fddf7e66a..0000000000000 --- a/newsfragments/40703.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Allow set Dag Run resource into Dag Level permission: extends Dag's access_control feature to allow Dag Run resource permissions. diff --git a/newsfragments/40874.significant.rst b/newsfragments/40874.significant.rst deleted file mode 100644 index 0677d131d557e..0000000000000 --- a/newsfragments/40874.significant.rst +++ /dev/null @@ -1 +0,0 @@ -Support for OpenTelemetry Traces is no longer "Experimental" diff --git a/newsfragments/41039.feature.rst b/newsfragments/41039.feature.rst deleted file mode 100644 index c696d25f874a8..0000000000000 --- a/newsfragments/41039.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Enable ``get_current_context()`` to work in virtual environments. The following ``Operators`` are affected: ``PythonVirtualenvOperator``, ``BranchPythonVirtualenvOperator``, ``ExternalPythonOperator``, ``BranchExternalPythonOperator`` diff --git a/newsfragments/41116.feature.rst b/newsfragments/41116.feature.rst deleted file mode 100644 index f5fa13d5295f4..0000000000000 --- a/newsfragments/41116.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Decorator for Task Flow, to make it simple to apply whether or not to skip a Task. diff --git a/scripts/ci/pre_commit/supported_versions.py b/scripts/ci/pre_commit/supported_versions.py index 5bd530a1ed3bc..0fb68e5afc4b6 100755 --- a/scripts/ci/pre_commit/supported_versions.py +++ b/scripts/ci/pre_commit/supported_versions.py @@ -27,7 +27,7 @@ HEADERS = ("Version", "Current Patch/Minor", "State", "First Release", "Limited Support", "EOL/Terminated") SUPPORTED_VERSIONS = ( - ("2", "2.9.3", "Supported", "Dec 17, 2020", "TBD", "TBD"), + ("2", "2.10.0", "Supported", "Dec 17, 2020", "TBD", "TBD"), ("1.10", "1.10.15", "EOL", "Aug 27, 2018", "Dec 17, 2020", "June 17, 2021"), ("1.9", "1.9.0", "EOL", "Jan 03, 2018", "Aug 27, 2018", "Aug 27, 2018"), ("1.8", "1.8.2", "EOL", "Mar 19, 2017", "Jan 03, 2018", "Jan 03, 2018"),