From c1faaf3745dd631d4491202ed245cf8190f35697 Mon Sep 17 00:00:00 2001 From: vincbeck <97131062+vincbeck@users.noreply.github.com> Date: Mon, 7 Mar 2022 17:27:32 -0500 Subject: [PATCH] Add sample dag and doc for RedshiftToS3Operator (#22060) --- .../example_dags/example_redshift_to_s3.py | 43 ++++++++++++++ .../amazon/aws/transfers/redshift_to_s3.py | 4 ++ airflow/providers/amazon/provider.yaml | 1 + .../operators/transfer/redshift_to_s3.rst | 56 +++++++++++++++++++ .../operators/transfer/s3_to_redshift.rst | 16 +++--- 5 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 airflow/providers/amazon/aws/example_dags/example_redshift_to_s3.py create mode 100644 docs/apache-airflow-providers-amazon/operators/transfer/redshift_to_s3.rst diff --git a/airflow/providers/amazon/aws/example_dags/example_redshift_to_s3.py b/airflow/providers/amazon/aws/example_dags/example_redshift_to_s3.py new file mode 100644 index 0000000000000..8116e02dc165c --- /dev/null +++ b/airflow/providers/amazon/aws/example_dags/example_redshift_to_s3.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datetime import datetime +from os import getenv + +from airflow import DAG +from airflow.providers.amazon.aws.transfers.redshift_to_s3 import RedshiftToS3Operator + +S3_BUCKET_NAME = getenv("S3_BUCKET_NAME", "s3_bucket_name") +S3_KEY = getenv("S3_KEY", "s3_key") +REDSHIFT_TABLE = getenv("REDSHIFT_TABLE", "redshift_table") + +with DAG( + dag_id="example_redshift_to_s3", + start_date=datetime(2021, 1, 1), + schedule_interval=None, + catchup=False, + tags=['example'], +) as dag: + # [START howto_transfer_redshift_to_s3] + task_transfer_redshift_to_s3 = RedshiftToS3Operator( + task_id='transfer_redshift_to_s3', + s3_bucket=S3_BUCKET_NAME, + s3_key=S3_KEY, + schema='PUBLIC', + table=REDSHIFT_TABLE, + ) + # [END howto_transfer_redshift_to_s3] diff --git a/airflow/providers/amazon/aws/transfers/redshift_to_s3.py b/airflow/providers/amazon/aws/transfers/redshift_to_s3.py index de14a1c0c86a4..22068be4131af 100644 --- a/airflow/providers/amazon/aws/transfers/redshift_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/redshift_to_s3.py @@ -31,6 +31,10 @@ class RedshiftToS3Operator(BaseOperator): """ Executes an UNLOAD command to s3 as a CSV with headers + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:RedshiftToS3Operator` + :param s3_bucket: reference to a specific S3 bucket :param s3_key: reference to a specific S3 key. If ``table_as_file_name`` is set to False, this param must include the desired file name diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 472c5a413dc87..2324b82c3035f 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -472,6 +472,7 @@ transfers: python-module: airflow.providers.amazon.aws.transfers.mysql_to_s3 - source-integration-name: Amazon Redshift target-integration-name: Amazon Simple Storage Service (S3) + how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/redshift_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.redshift_to_s3 - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: Amazon Redshift diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/redshift_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/redshift_to_s3.rst new file mode 100644 index 0000000000000..08f9df628bac2 --- /dev/null +++ b/docs/apache-airflow-providers-amazon/operators/transfer/redshift_to_s3.rst @@ -0,0 +1,56 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. _howto/operator:RedshiftToS3Operator: + +Amazon Redshift to S3 Transfer Operator +======================================= + +Use the RedshiftToS3Operator transfer to copy the data from an Amazon Redshift table into an Amazon Simple Storage +Service (S3) file. + +Prerequisite Tasks +^^^^^^^^^^^^^^^^^^ + +.. include:: ../_partials/prerequisite_tasks.rst + +Amazon Redshift To Amazon S3 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This operator loads data from an Amazon Redshift table to an existing Amazon S3 bucket. + +To get more information about this operator visit: +:class:`~airflow.providers.amazon.aws.transfers.redshift_to_s3.RedshiftToS3Operator` + +Example usage: + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_redshift_to_s3.py + :language: python + :dedent: 4 + :start-after: [START howto_transfer_redshift_to_s3] + :end-before: [END howto_transfer_redshift_to_s3] + +You can find more information to the ``UNLOAD`` command used +`here `__. + +Reference +^^^^^^^^^ + +* `AWS UNLOAD from Amazon Redshift Documentation `__ +* `AWS boto3 Library Documentation for Amazon Redshift `__ +* `AWS boto3 Library Documentation for Amazon S3 `__ diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_redshift.rst b/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_redshift.rst index efd50bb46ddfc..05db5a9fe7bc9 100644 --- a/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_redshift.rst +++ b/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_redshift.rst @@ -18,22 +18,23 @@ .. _howto/operator:S3ToRedshiftOperator: -Amazon Redshift to S3 Transfer Operator +Amazon S3 to Redshift Transfer Operator ======================================= -Use the S3ToRedshiftOperator transfer to copy the data from a Amazon S3 file into an Amazon Redshift table. +Use the S3ToRedshiftOperator transfer to copy the data from an Amazon Simple Storage Service (S3) file into an +Amazon Redshift table. Prerequisite Tasks ^^^^^^^^^^^^^^^^^^ .. include:: ../_partials/prerequisite_tasks.rst -S3 To Redshift -^^^^^^^^^^^^^^ +Amazon S3 To Amazon Redshift +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -This operator loads data from S3 to an existing Amazon Redshift table. +This operator loads data from Amazon S3 to an existing Amazon Redshift table. -To get more information about operator visit: +To get more information about this operator visit: :class:`~airflow.providers.amazon.aws.transfers.s3_to_redshift.S3ToRedshiftOperator` Example usage: @@ -51,4 +52,5 @@ Reference ^^^^^^^^^ * `AWS COPY from Amazon S3 Documentation `__ -* `AWS boto3 Library Documentation for S3 `__ +* `AWS boto3 Library Documentation for Amazon S3 `__ +* `AWS boto3 Library Documentation for Amazon Redshift `__