From 0fb130f8862498930cd3a06dbd6838d36fa255b4 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Mon, 19 Aug 2024 23:24:52 +0530 Subject: [PATCH] Show deprecation status of operators and sensors on the docs operator list page --- .../providers/amazon/aws/operators/batch.py | 3 ++ .../providers/amazon/aws/operators/emr.py | 5 +++ .../amazon/aws/operators/redshift_cluster.py | 15 +++++++ .../amazon/aws/operators/redshift_data.py | 5 +++ .../amazon/aws/operators/sagemaker.py | 15 +++++++ .../providers/amazon/aws/sensors/batch.py | 3 ++ .../providers/amazon/aws/sensors/emr.py | 9 ++++ .../amazon/aws/sensors/redshift_cluster.py | 5 +++ astronomer/providers/amazon/aws/sensors/s3.py | 20 +++++++-- .../providers/apache/livy/operators/livy.py | 3 ++ .../kubernetes/operators/kubernetes_pod.py | 5 +++ .../providers/core/sensors/external_task.py | 3 ++ .../providers/core/sensors/filesystem.py | 3 ++ .../databricks/operators/databricks.py | 10 +++++ .../providers/dbt/cloud/operators/dbt.py | 5 +++ astronomer/providers/dbt/cloud/sensors/dbt.py | 3 ++ .../google/cloud/operators/bigquery.py | 15 +++++++ .../google/cloud/operators/dataproc.py | 20 +++++++++ .../cloud/operators/kubernetes_engine.py | 5 +++ .../google/cloud/sensors/bigquery.py | 5 +++ .../providers/google/cloud/sensors/gcs.py | 20 +++++++++ astronomer/providers/http/sensors/http.py | 3 ++ .../microsoft/azure/operators/data_factory.py | 3 ++ .../microsoft/azure/sensors/data_factory.py | 3 ++ .../providers/microsoft/azure/sensors/wasb.py | 8 ++++ astronomer/providers/sftp/sensors/sftp.py | 3 ++ .../snowflake/operators/snowflake.py | 5 +++ docs/_ext/traverse_operators_sensors.py | 42 +++++++++++++++---- docs/providers/operators_and_sensors_list.rst | 6 +++ setup.cfg | 10 +++-- 30 files changed, 244 insertions(+), 16 deletions(-) diff --git a/astronomer/providers/amazon/aws/operators/batch.py b/astronomer/providers/amazon/aws/operators/batch.py index 1eda818ea..d7aff1ea0 100644 --- a/astronomer/providers/amazon/aws/operators/batch.py +++ b/astronomer/providers/amazon/aws/operators/batch.py @@ -23,6 +23,9 @@ class BatchOperatorAsync(BatchOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.amazon.aws.operators.batch import BatchOperator" + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( ( diff --git a/astronomer/providers/amazon/aws/operators/emr.py b/astronomer/providers/amazon/aws/operators/emr.py index c21bfe261..ea582a0cd 100644 --- a/astronomer/providers/amazon/aws/operators/emr.py +++ b/astronomer/providers/amazon/aws/operators/emr.py @@ -9,6 +9,11 @@ class EmrContainerOperatorAsync(EmrContainerOperator): Please use :class: `~airflow.providers.amazon.aws.operators.emr.EmrContainerOperator`. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.amazon.aws.operators.emr import EmrContainerOperator" + ) + def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] warnings.warn( ( diff --git a/astronomer/providers/amazon/aws/operators/redshift_cluster.py b/astronomer/providers/amazon/aws/operators/redshift_cluster.py index 4e107d83e..aba1dd139 100644 --- a/astronomer/providers/amazon/aws/operators/redshift_cluster.py +++ b/astronomer/providers/amazon/aws/operators/redshift_cluster.py @@ -13,6 +13,11 @@ class RedshiftDeleteClusterOperatorAsync(RedshiftDeleteClusterOperator): Please use :class: `~airflow.providers.amazon.aws.operators.redshift_cluster.RedshiftDeleteClusterOperator`. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.amazon.aws.operators.redshift_cluster import RedshiftDeleteClusterOperator" + ) + def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] warnings.warn( ( @@ -32,6 +37,11 @@ class RedshiftResumeClusterOperatorAsync(RedshiftResumeClusterOperator): Please use :class: `~airflow.providers.amazon.aws.operators.redshift_cluster.RedshiftResumeClusterOperator`. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.amazon.aws.operators.redshift_cluster import RedshiftResumeClusterOperator" + ) + def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] warnings.warn( ( @@ -51,6 +61,11 @@ class RedshiftPauseClusterOperatorAsync(RedshiftPauseClusterOperator): Please use :class: `~airflow.providers.amazon.aws.operators.redshift_cluster.RedshiftPauseClusterOperator`. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.amazon.aws.operators.redshift_cluster import RedshiftPauseClusterOperator" + ) + def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] warnings.warn( ( diff --git a/astronomer/providers/amazon/aws/operators/redshift_data.py b/astronomer/providers/amazon/aws/operators/redshift_data.py index 9b6ebcb5d..5599806c7 100644 --- a/astronomer/providers/amazon/aws/operators/redshift_data.py +++ b/astronomer/providers/amazon/aws/operators/redshift_data.py @@ -11,6 +11,11 @@ class RedshiftDataOperatorAsync(RedshiftDataOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.amazon.aws.operators.redshift_data import RedshiftDataOperator" + ) + def __init__( self, **kwargs: Any, diff --git a/astronomer/providers/amazon/aws/operators/sagemaker.py b/astronomer/providers/amazon/aws/operators/sagemaker.py index d388a3244..830e9c760 100644 --- a/astronomer/providers/amazon/aws/operators/sagemaker.py +++ b/astronomer/providers/amazon/aws/operators/sagemaker.py @@ -24,6 +24,11 @@ class SageMakerProcessingOperatorAsync(SageMakerProcessingOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.amazon.aws.operators.sagemaker import SageMakerProcessingOperator" + ) + def __init__(self, **kwargs: Any) -> None: warnings.warn( ( @@ -44,6 +49,11 @@ class SageMakerTransformOperatorAsync(SageMakerTransformOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.amazon.aws.operators.sagemaker import SageMakerTransformOperator" + ) + def __init__(self, **kwargs: Any) -> None: warnings.warn( ( @@ -64,6 +74,11 @@ class SageMakerTrainingOperatorAsync(SageMakerTrainingOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.amazon.aws.operators.sagemaker import SageMakerTrainingOperator" + ) + def __init__(self, **kwargs: Any) -> None: warnings.warn( ( diff --git a/astronomer/providers/amazon/aws/sensors/batch.py b/astronomer/providers/amazon/aws/sensors/batch.py index 7c49018ae..de2bf6931 100644 --- a/astronomer/providers/amazon/aws/sensors/batch.py +++ b/astronomer/providers/amazon/aws/sensors/batch.py @@ -9,6 +9,9 @@ class BatchSensorAsync(BatchSensor): Please use :class: `~airflow.providers.amazon.aws.sensors.batch.BatchSensor`. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.amazon.aws.sensors.batch import BatchSensor" + def __init__(self, *args, **kwargs) -> None: # type: ignore[no-untyped-def] warnings.warn( ( diff --git a/astronomer/providers/amazon/aws/sensors/emr.py b/astronomer/providers/amazon/aws/sensors/emr.py index e5e9bbde9..fca4d6aea 100644 --- a/astronomer/providers/amazon/aws/sensors/emr.py +++ b/astronomer/providers/amazon/aws/sensors/emr.py @@ -15,6 +15,9 @@ class EmrContainerSensorAsync(EmrContainerSensor): Please use :class: `~airflow.providers.amazon.aws.sensors.emr.EmrContainerSensor`. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.amazon.aws.sensors.emr import EmrContainerSensor" + def __init__(self, *args, **kwargs) -> None: # type: ignore[no-untyped-def] poll_interval = kwargs.pop("poll_interval") if poll_interval: @@ -44,6 +47,9 @@ class EmrStepSensorAsync(EmrStepSensor): Please use :class: `~airflow.providers.amazon.aws.sensors.emr.EmrStepSensor`. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.amazon.aws.sensors.emr import EmrStepSensor" + def __init__(self, *args, **kwargs) -> None: # type: ignore[no-untyped-def] warnings.warn( ( @@ -63,6 +69,9 @@ class EmrJobFlowSensorAsync(EmrJobFlowSensor): Please use :class: `~airflow.providers.amazon.aws.sensors.emr.EmrJobFlowSensor`. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.amazon.aws.sensors.emr import EmrJobFlowSensor" + def __init__(self, *args, **kwargs) -> None: # type: ignore[no-untyped-def] warnings.warn( ( diff --git a/astronomer/providers/amazon/aws/sensors/redshift_cluster.py b/astronomer/providers/amazon/aws/sensors/redshift_cluster.py index 6f919bf22..3a568da2d 100644 --- a/astronomer/providers/amazon/aws/sensors/redshift_cluster.py +++ b/astronomer/providers/amazon/aws/sensors/redshift_cluster.py @@ -11,6 +11,11 @@ class RedshiftClusterSensorAsync(RedshiftClusterSensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.amazon.aws.sensors.redshift_cluster import RedshiftClusterSensor" + ) + def __init__( self, **kwargs: Any, diff --git a/astronomer/providers/amazon/aws/sensors/s3.py b/astronomer/providers/amazon/aws/sensors/s3.py index ab0858c7e..f2c4924cc 100644 --- a/astronomer/providers/amazon/aws/sensors/s3.py +++ b/astronomer/providers/amazon/aws/sensors/s3.py @@ -14,6 +14,9 @@ class S3KeySensorAsync(S3KeySensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor" + def __init__(self, **kwargs: Any) -> None: warnings.warn( ( @@ -30,9 +33,12 @@ def __init__(self, **kwargs: Any) -> None: class S3KeySizeSensorAsync(S3KeySensorAsync): """ This class is deprecated. - Please use :class: `~astronomer.providers.amazon.aws.sensor.s3.S3KeySensorAsync`. + Please use :class: `~airflow.providers.amazon.aws.sensors.s3.S3KeySensor`. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor" + def __init__( self, *, @@ -42,7 +48,7 @@ def __init__( warnings.warn( """ S3KeySizeSensorAsync is deprecated. - Please use `astronomer.providers.amazon.aws.sensor.s3.S3KeySensorAsync`. + Please use `airflow.providers.amazon.aws.sensors.s3.S3KeySensor`. """, DeprecationWarning, stacklevel=2, @@ -57,6 +63,9 @@ class S3KeysUnchangedSensorAsync(S3KeysUnchangedSensor): Please use :class: `~airflow.providers.amazon.aws.sensors.s3.S3KeysUnchangedSensor`. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.amazon.aws.sensors.s3 import S3KeysUnchangedSensor" + def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] warnings.warn( ( @@ -73,9 +82,12 @@ def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] class S3PrefixSensorAsync(BaseSensorOperator): """ This class is deprecated. - Please use :class: `~astronomer.providers.amazon.aws.sensor.s3.S3KeySensorAsync`. + Please use :class: `~airflow.providers.amazon.aws.sensors.s3.S3KeySensor`. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor" + def __init__( self, *, @@ -89,7 +101,7 @@ def __init__( warnings.warn( """ S3PrefixSensor is deprecated. - Please use `astronomer.providers.amazon.aws.sensor.s3.S3KeySensorAsync`. + Please use `airflow.providers.amazon.aws.sensors.s3.S3KeySensor`. """, DeprecationWarning, stacklevel=2, diff --git a/astronomer/providers/apache/livy/operators/livy.py b/astronomer/providers/apache/livy/operators/livy.py index a2b8a0dad..cd83eb60e 100644 --- a/astronomer/providers/apache/livy/operators/livy.py +++ b/astronomer/providers/apache/livy/operators/livy.py @@ -15,6 +15,9 @@ class LivyOperatorAsync(LivyOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.apache.livy.operators.livy import LivyOperator" + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( "This class is deprecated. " diff --git a/astronomer/providers/cncf/kubernetes/operators/kubernetes_pod.py b/astronomer/providers/cncf/kubernetes/operators/kubernetes_pod.py index 603e99f05..76d682d82 100644 --- a/astronomer/providers/cncf/kubernetes/operators/kubernetes_pod.py +++ b/astronomer/providers/cncf/kubernetes/operators/kubernetes_pod.py @@ -21,6 +21,11 @@ class KubernetesPodOperatorAsync(KubernetesPodOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator" + ) + def __init__(self, **kwargs: Any): warnings.warn( ( diff --git a/astronomer/providers/core/sensors/external_task.py b/astronomer/providers/core/sensors/external_task.py index b1b277e8b..df9dad411 100644 --- a/astronomer/providers/core/sensors/external_task.py +++ b/astronomer/providers/core/sensors/external_task.py @@ -21,6 +21,9 @@ class ExternalTaskSensorAsync(ExternalTaskSensor): # noqa: D101 + is_deprecated = True + post_deprecation_replacement = "from airflow.sensors.external_task import ExternalTaskSensor" + def __init__( self, poke_interval: float = 5.0, diff --git a/astronomer/providers/core/sensors/filesystem.py b/astronomer/providers/core/sensors/filesystem.py index 53258d5a5..74b2c47f0 100644 --- a/astronomer/providers/core/sensors/filesystem.py +++ b/astronomer/providers/core/sensors/filesystem.py @@ -26,6 +26,9 @@ class FileSensorAsync(FileSensor): ``**`` in glob filepath parameter. Defaults to ``False``. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.sensors.filesystem import FileSensor" + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( ( diff --git a/astronomer/providers/databricks/operators/databricks.py b/astronomer/providers/databricks/operators/databricks.py index e5bb334fa..9cb913287 100644 --- a/astronomer/providers/databricks/operators/databricks.py +++ b/astronomer/providers/databricks/operators/databricks.py @@ -16,6 +16,11 @@ class DatabricksSubmitRunOperatorAsync(DatabricksSubmitRunOperator): `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.databricks.operators.databricks import DatabricksSubmitRunOperator" + ) + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( "This class is deprecated." @@ -32,6 +37,11 @@ class DatabricksRunNowOperatorAsync(DatabricksRunNowOperator): `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.databricks.operators.databricks import DatabricksRunNowOperator" + ) + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( "This class is deprecated." diff --git a/astronomer/providers/dbt/cloud/operators/dbt.py b/astronomer/providers/dbt/cloud/operators/dbt.py index 6a8f48c4b..0f3fd5c60 100644 --- a/astronomer/providers/dbt/cloud/operators/dbt.py +++ b/astronomer/providers/dbt/cloud/operators/dbt.py @@ -17,6 +17,11 @@ class DbtCloudRunJobOperatorAsync(DbtCloudRunJobOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.dbt.cloud.operators.dbt import DbtCloudRunJobOperator" + ) + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( ( diff --git a/astronomer/providers/dbt/cloud/sensors/dbt.py b/astronomer/providers/dbt/cloud/sensors/dbt.py index e6b1683e8..9ea6420ae 100644 --- a/astronomer/providers/dbt/cloud/sensors/dbt.py +++ b/astronomer/providers/dbt/cloud/sensors/dbt.py @@ -13,6 +13,9 @@ class DbtCloudJobRunSensorAsync(DbtCloudJobRunSensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.dbt.cloud.sensors import DbtCloudJobRunSensor" + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( ( diff --git a/astronomer/providers/google/cloud/operators/bigquery.py b/astronomer/providers/google/cloud/operators/bigquery.py index 3d07efc2e..540f62326 100644 --- a/astronomer/providers/google/cloud/operators/bigquery.py +++ b/astronomer/providers/google/cloud/operators/bigquery.py @@ -23,6 +23,9 @@ class BigQueryInsertJobOperatorAsync(BigQueryInsertJobOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from google.cloud.operators.bigquery import BigQueryInsertJobOperator" + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( ( @@ -45,6 +48,9 @@ class BigQueryCheckOperatorAsync(BigQueryCheckOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from google.cloud.operators.bigquery import BigQueryCheckOperator" + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( ( @@ -65,6 +71,9 @@ class BigQueryGetDataOperatorAsync(BigQueryGetDataOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from google.cloud.operators.bigquery import BigQueryGetDataOperator" + def __init__(self, *args: Any, use_legacy_sql: bool = False, **kwargs: Any) -> None: warnings.warn( ( @@ -86,6 +95,9 @@ class BigQueryIntervalCheckOperatorAsync(BigQueryIntervalCheckOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from google.cloud.operators.bigquery import BigQueryIntervalCheckOperator" + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( ( @@ -106,6 +118,9 @@ class BigQueryValueCheckOperatorAsync(BigQueryValueCheckOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from google.cloud.operators.bigquery import BigQueryValueCheckOperator" + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( ( diff --git a/astronomer/providers/google/cloud/operators/dataproc.py b/astronomer/providers/google/cloud/operators/dataproc.py index dff462020..d698af355 100644 --- a/astronomer/providers/google/cloud/operators/dataproc.py +++ b/astronomer/providers/google/cloud/operators/dataproc.py @@ -20,6 +20,11 @@ class DataprocCreateClusterOperatorAsync(DataprocCreateClusterOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.google.cloud.operators.dataproc import DataprocCreateClusterOperator" + ) + def __init__( self, *, @@ -46,6 +51,11 @@ class DataprocDeleteClusterOperatorAsync(DataprocDeleteClusterOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.google.cloud.operators.dataproc import DataprocDeleteClusterOperator" + ) + def __init__( self, *, @@ -74,6 +84,11 @@ class DataprocSubmitJobOperatorAsync(DataprocSubmitJobOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.google.cloud.operators.dataproc import DataprocSubmitJobOperator" + ) + def __init__(self, *args: Any, **kwargs: Any): warnings.warn( ( @@ -94,6 +109,11 @@ class DataprocUpdateClusterOperatorAsync(DataprocUpdateClusterOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.google.cloud.operators.dataproc import DataprocUpdateClusterOperator" + ) + def __init__( self, *, diff --git a/astronomer/providers/google/cloud/operators/kubernetes_engine.py b/astronomer/providers/google/cloud/operators/kubernetes_engine.py index 5a70b4bcd..103ad6e26 100644 --- a/astronomer/providers/google/cloud/operators/kubernetes_engine.py +++ b/astronomer/providers/google/cloud/operators/kubernetes_engine.py @@ -15,6 +15,11 @@ class GKEStartPodOperatorAsync(GKEStartPodOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.google.cloud.operators.kubernetes_engine import GKEStartPodOperator" + ) + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( ( diff --git a/astronomer/providers/google/cloud/sensors/bigquery.py b/astronomer/providers/google/cloud/sensors/bigquery.py index 030857b15..fbb47c89b 100644 --- a/astronomer/providers/google/cloud/sensors/bigquery.py +++ b/astronomer/providers/google/cloud/sensors/bigquery.py @@ -15,6 +15,11 @@ class BigQueryTableExistenceSensorAsync(BigQueryTableExistenceSensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.google.cloud.sensors.bigquery import BigQueryTableExistenceSensor" + ) + def __init__( self, gcp_conn_id: str = "google_cloud_default", diff --git a/astronomer/providers/google/cloud/sensors/gcs.py b/astronomer/providers/google/cloud/sensors/gcs.py index 0465e2de7..03512c7ca 100644 --- a/astronomer/providers/google/cloud/sensors/gcs.py +++ b/astronomer/providers/google/cloud/sensors/gcs.py @@ -20,6 +20,11 @@ class GCSObjectExistenceSensorAsync(GCSObjectExistenceSensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.google.cloud.sensors.gcs import GCSObjectExistenceSensor" + ) + def __init__( self, *args: Any, @@ -55,6 +60,11 @@ class GCSObjectsWithPrefixExistenceSensorAsync(GCSObjectsWithPrefixExistenceSens and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.google.cloud.sensors.gcs import GCSObjectsWithPrefixExistenceSensor" + ) + def __init__( self, *args: Any, @@ -90,6 +100,11 @@ class GCSUploadSessionCompleteSensorAsync(GCSUploadSessionCompleteSensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.google.cloud.sensors.gcs import GCSUploadSessionCompleteSensor" + ) + def __init__( self, *args: Any, @@ -126,6 +141,11 @@ class GCSObjectUpdateSensorAsync(GCSObjectUpdateSensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.google.cloud.sensors.gcs import GCSObjectUpdateSensor" + ) + def __init__( self, *args: Any, diff --git a/astronomer/providers/http/sensors/http.py b/astronomer/providers/http/sensors/http.py index 73f7869a5..a029718b4 100644 --- a/astronomer/providers/http/sensors/http.py +++ b/astronomer/providers/http/sensors/http.py @@ -11,6 +11,9 @@ class HttpSensorAsync(HttpSensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.http.sensors.http import HttpSensor" + def __init__(self, *args: Any, **kwargs: Any) -> None: warnings.warn( ( diff --git a/astronomer/providers/microsoft/azure/operators/data_factory.py b/astronomer/providers/microsoft/azure/operators/data_factory.py index 811948e6c..459c085c4 100644 --- a/astronomer/providers/microsoft/azure/operators/data_factory.py +++ b/astronomer/providers/microsoft/azure/operators/data_factory.py @@ -12,6 +12,9 @@ class AzureDataFactoryRunPipelineOperatorAsync(AzureDataFactoryRunPipelineOperat and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.microsoft.azure.operators.data_factory import AzureDataFactoryRunPipelineOperator" + def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] warnings.warn( ( diff --git a/astronomer/providers/microsoft/azure/sensors/data_factory.py b/astronomer/providers/microsoft/azure/sensors/data_factory.py index ac9413ac6..6748fca87 100644 --- a/astronomer/providers/microsoft/azure/sensors/data_factory.py +++ b/astronomer/providers/microsoft/azure/sensors/data_factory.py @@ -11,6 +11,9 @@ class AzureDataFactoryPipelineRunStatusSensorAsync(AzureDataFactoryPipelineRunSt instead and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.microsoft.azure.sensors.data_factory import AzureDataFactoryPipelineRunStatusSensor" + def __init__( self, *args: Any, diff --git a/astronomer/providers/microsoft/azure/sensors/wasb.py b/astronomer/providers/microsoft/azure/sensors/wasb.py index b663b953e..cbe0fc9f4 100644 --- a/astronomer/providers/microsoft/azure/sensors/wasb.py +++ b/astronomer/providers/microsoft/azure/sensors/wasb.py @@ -14,6 +14,9 @@ class WasbBlobSensorAsync(WasbBlobSensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.microsoft.azure.sensors.wasb import WasbBlobSensor" + def __init__( self, *args: Any, @@ -48,6 +51,11 @@ class WasbPrefixSensorAsync(WasbPrefixSensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.microsoft.azure.sensors.wasb import WasbPrefixSensor" + ) + def __init__( self, *args: Any, diff --git a/astronomer/providers/sftp/sensors/sftp.py b/astronomer/providers/sftp/sensors/sftp.py index b70ed3cdb..adf1ae837 100644 --- a/astronomer/providers/sftp/sensors/sftp.py +++ b/astronomer/providers/sftp/sensors/sftp.py @@ -11,6 +11,9 @@ class SFTPSensorAsync(SFTPSensor): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = "from airflow.providers.sftp.sensors.sftp import SFTPSensor" + def __init__( self, *args: Any, diff --git a/astronomer/providers/snowflake/operators/snowflake.py b/astronomer/providers/snowflake/operators/snowflake.py index 35854c346..cf4cd1aef 100644 --- a/astronomer/providers/snowflake/operators/snowflake.py +++ b/astronomer/providers/snowflake/operators/snowflake.py @@ -216,6 +216,11 @@ class SnowflakeSqlApiOperatorAsync(SnowflakeSqlApiOperator): and set `deferrable` param to `True` instead. """ + is_deprecated = True + post_deprecation_replacement = ( + "from airflow.providers.snowflake.operators.snowflake import SnowflakeSqlApiOperator" + ) + def __init__(self, *args: Any, **kwargs: Any): warnings.warn( ( diff --git a/docs/_ext/traverse_operators_sensors.py b/docs/_ext/traverse_operators_sensors.py index 13891b1de..bb98912c0 100644 --- a/docs/_ext/traverse_operators_sensors.py +++ b/docs/_ext/traverse_operators_sensors.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import ast +import importlib import os from pathlib import Path -from typing import List, Tuple from docutils import nodes from sphinx.util.docutils import SphinxDirective @@ -20,9 +22,9 @@ def collect_elements( directory_path: str, - files: List[str], + files: list[str], element_type: str, - elements_list: List[Tuple[str, str]], + elements_list: list[tuple[str, str, str, str]], current_module_path: str, ): """ @@ -47,7 +49,13 @@ def collect_elements( if isinstance(element, ast.ClassDef): element_name = element.name if element_type in element_name and ASYNC_SUBSTRING in element_name: - elements_list.append((element_name, module_import_path)) + module = importlib.import_module(module_import_path) + cls = getattr(module, element_name) + is_deprecated = "Yes" if getattr(cls, "is_deprecated", False) else "No" + post_deprecation_replacement = str(getattr(cls, "post_deprecation_replacement", "")) + elements_list.append( + (element_name, is_deprecated, module_import_path, post_deprecation_replacement) + ) def search_providers(): @@ -79,7 +87,13 @@ def run(self): """Generates raw html to list the operators and sensors parsed using `ast`.""" operators, sensors = search_providers() operators_html = ( - "

Operators

" "" "" "" "" + "

Operators

" + "
#Operator nameImport path
" + "" + "" + "" + "" + "" ) for index, operator in enumerate(operators, start=1): class_def_link = operator[1].replace(".", "/") + "/index.html#" + operator[1] + "." + operator[0] @@ -87,7 +101,9 @@ def run(self): f"" f"" f"" - f"" + f"" + f"" + f"" f"" ) # The below script generates the URL for the class definition by extracting the selected doc version from @@ -101,14 +117,24 @@ def run(self): ) operators_html += "
#Operator nameIs deprecatedImport pathPost deprecation replacement
{index}{operator[0]}
from {operator[1]} import {operator[0]}
{operator[1]}
from {operator[2]} import {operator[0]}
{operator[3]}

" - sensors_html = "

Sensors

" "" "" "" "" + sensors_html = ( + "

Sensors

" + "
#Sensor nameImport path
" + "" + "" + "" + "" + "" + ) for index, sensor in enumerate(sensors, start=1): class_def_link = sensor[1].replace(".", "/") + "/index.html#" + sensor[1] + "." + sensor[0] sensors_html += ( f"" f"" f"" - f"" + f"" + f"" + f"" f"" ) sensors_html += ( diff --git a/docs/providers/operators_and_sensors_list.rst b/docs/providers/operators_and_sensors_list.rst index 9e0ef2258..3b755d8e8 100644 --- a/docs/providers/operators_and_sensors_list.rst +++ b/docs/providers/operators_and_sensors_list.rst @@ -1,6 +1,12 @@ Available Operators and Sensors ------------------------------- +Since ``astronomer-providers>=1.19.0``, most of the operators and sensors are now deprecated and their instantiations +are proxied to their upstream Apache Airflow providers' deferrable counterparts. +Please check the deprecation status in the ``Is deprecated`` column in the tables below. If the status is ``Yes``, +then you're suggested to use the replacement suggestion provided in the ``Post deprecation replacement`` column and +pass the ``deferrable=True`` param to the operator/sensor instantiation in your DAG. + .. traverse_operators_sensors:: :hidden: :maxdepth: 2 diff --git a/setup.cfg b/setup.cfg index c38912a8d..95bec947b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -78,10 +78,6 @@ snowflake = openlineage = openlineage-airflow>=0.12.0 -docs = - sphinx - sphinx-autoapi - sphinx-copybutton tests = aioresponses asynctest @@ -139,6 +135,12 @@ all = paramiko snowflake-sqlalchemy>=1.4.4 # Temporary solution for https://github.com/astronomer/astronomer-providers/issues/958, we should pin apache-airflow-providers-snowflake version after it pins this package to great than or equal to 1.4.4. +docs = + sphinx + sphinx-autoapi + sphinx-copybutton + .[all] + [options.packages.find] include = astronomer.*
#Sensor nameImport pathIs deprecatedReplacement import path
{index}{sensor[0]}
from {sensor[1]} import {sensor[0]}
{sensor[1]}
from {sensor[2]} import {sensor[0]}
{sensor[3]}