Extract secret from gcs

stellar · Dec 16, 2024 · 4c24f70 · 4c24f70
1 parent 06884ed
commit 4c24f70
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 1 deletion.
diff --git a/dags/external_data_dag.py b/dags/external_data_dag.py
@@ -22,6 +22,7 @@
     get_airflow_metadata,
 )
 from stellar_etl_airflow.default import get_default_dag_args, init_sentry
+from stellar_etl_airflow.utils import retrieve_secret
 
 init_sentry()
 
@@ -65,7 +66,10 @@
         "{{ subtract_data_interval(dag, data_interval_end).isoformat() }}",
     ],
     use_gcs=True,
-    env_vars={"RETOOL_API_KEY": "{{ var.value.retool_api_key }}"},
+    env_vars={
+        "RETOOL_API_KEY": "{{ var.value.retool_api_key }}",
+        "AIRFLOW_SECRET": retrieve_secret("retool-api-key-test"),
+    },
 )
 
 

diff --git a/dags/stellar_etl_airflow/utils.py b/dags/stellar_etl_airflow/utils.py
@@ -4,6 +4,7 @@
 
 from airflow.configuration import conf
 from airflow.models import Variable
+from airflow.providers.google.cloud.hooks.secret_manager import SecretManagerHook
 from airflow.utils.state import TaskInstanceState
 
 base_log_folder = conf.get("logging", "base_log_folder")
@@ -100,3 +101,14 @@ def skip_retry_dbt_errors(context) -> None:
         return
     else:
         return
+
+
+def retrieve_secret(secret_name: str):
+    """
+    Extracts secret from GCS secrets
+    """
+
+    hook = SecretManagerHook()
+
+    secret_value = hook.get_secret(secret_name)
+    return secret_value