Skip to content

Commit

Permalink
Add dag for recency tests
Browse files Browse the repository at this point in the history
  • Loading branch information
amishas157 committed Sep 20, 2024
1 parent ff7275f commit 9a62165
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 0 deletions.
40 changes: 40 additions & 0 deletions dags/dbt_recency_tests_dag.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
from datetime import datetime

from airflow import DAG
from airflow.operators.empty import EmptyOperator
from kubernetes.client import models as k8s
from stellar_etl_airflow.build_dbt_task import dbt_task
from stellar_etl_airflow.build_elementary_slack_alert_task import elementary_task
from stellar_etl_airflow.default import (
alert_sla_miss,
get_default_dag_args,
init_sentry,
)

init_sentry()

with DAG(
"dbt_recency_tests",
default_args=get_default_dag_args(),
start_date=datetime(2024, 6, 25, 0, 0),
description="This DAG runs non-model dbt tests half-hourly cadence",
schedule="*/30 * * * *", # Runs every 30 minutes
user_defined_filters={
"container_resources": lambda s: k8s.V1ResourceRequirements(requests=s),
},
max_active_runs=1,
catchup=False,
tags=["dbt-data-quality"],
# sla_miss_callback=alert_sla_miss,
) as dag:

# DBT tests to run
recency_tests = dbt_task(
dag,
command_type="test",
tag="recency",
resource_cfg="dbt",
run_recency_test="true",
)

recency_tests
1 change: 1 addition & 0 deletions dags/stellar_etl_airflow/build_dbt_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ def dbt_task(
excluded=None,
resource_cfg="default",
run_singular_test="false",
run_recency_test="false"
):
namespace = conf.get("kubernetes", "NAMESPACE")
if namespace == "default":
Expand Down

0 comments on commit 9a62165

Please sign in to comment.