From a8f3c0f8852255fc56808b6ae887f3b5f185c291 Mon Sep 17 00:00:00 2001 From: Cayod Date: Mon, 26 Feb 2024 10:49:33 -0300 Subject: [PATCH] Update elementary task --- dags/stellar_etl_airflow/build_dbt_task.py | 24 +------------------ .../build_elementary_task.py | 2 +- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/dags/stellar_etl_airflow/build_dbt_task.py b/dags/stellar_etl_airflow/build_dbt_task.py index 885e8a84..84036597 100644 --- a/dags/stellar_etl_airflow/build_dbt_task.py +++ b/dags/stellar_etl_airflow/build_dbt_task.py @@ -2,7 +2,6 @@ from datetime import timedelta from airflow.configuration import conf -from airflow.kubernetes.secret import Secret from airflow.models import Variable from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import ( KubernetesPodOperator, @@ -66,22 +65,7 @@ def dbt_task( operator="", command_type="build", resource_cfg="default", - alert=True, ): - secret_env = Secret( - deploy_type="env", - deploy_target="SLACK_TOKEN", - secret="slack-token-elementary", - key="token", - ) - elementary_alert = [ - "edr", - "monitor", - "--slack-token", - "$SLACK_TOKEN", - "--slack-channel-name", - "{{ var.value.slack_elementary_channel }}", - ] namespace = conf.get("kubernetes", "NAMESPACE") if namespace == "default": config_file_location = Variable.get("kube_config_location") @@ -100,7 +84,7 @@ def dbt_task( dbt_image = "{{ var.value.dbt_image_name }}" - args = ["dbt", command_type, f"--{flag}"] + args = [command_type, f"--{flag}"] models = [] if tag: @@ -118,10 +102,6 @@ def dbt_task( if Variable.get("dbt_full_refresh_models", deserialize_json=True).get(task_name): args.append("--full-refresh") - if alert: - args.append("&&") - args.extend(elementary_alert) - logging.info(f"sh commands to run in pod: {args}") return KubernetesPodOperator( @@ -145,9 +125,7 @@ def dbt_task( "EXECUTION_DATE": "{{ ds }}", }, image=dbt_image, - cmds=["sh", "-c"], arguments=args, - secrets=[secret_env], dag=dag, do_xcom_push=True, is_delete_operator_pod=True, diff --git a/dags/stellar_etl_airflow/build_elementary_task.py b/dags/stellar_etl_airflow/build_elementary_task.py index ce206fc8..00e26b1f 100644 --- a/dags/stellar_etl_airflow/build_elementary_task.py +++ b/dags/stellar_etl_airflow/build_elementary_task.py @@ -47,7 +47,7 @@ def elementary_task( ) affinity = Variable.get("affinity", deserialize_json=True).get(resource_cfg) - dbt_image = "{{ var.value.dbt_image_name }}" + dbt_image = "us-central1-docker.pkg.dev/test-hubble-319619/stellar-dbt-elementary/stellar-dbt:latest" logging.info(f"sh commands to run in pod: {args}")