Skip to content

Commit

Permalink
Use exclude package name on dbt tasks
Browse files Browse the repository at this point in the history
  • Loading branch information
sydneynotthecity committed Nov 14, 2024
1 parent 8cdd5ec commit 825d896
Show file tree
Hide file tree
Showing 4 changed files with 15 additions and 16 deletions.
2 changes: 1 addition & 1 deletion airflow_variables_dev.json
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@
"partnership_assets__account_holders_activity_fact": false,
"partnership_assets__asset_activity_fact": false
},
"dbt_image_name": "stellar/stellar-dbt:420c216df",
"dbt_image_name": "stellar/stellar-dbt-dev:4b8a2ecc4",
"dbt_internal_source_db": "test-hubble-319619",
"dbt_internal_source_schema": "test_crypto_stellar_internal",
"dbt_job_execution_timeout_seconds": 300,
Expand Down
4 changes: 1 addition & 3 deletions dags/dbt_enriched_base_tables_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,7 @@
wait_on_state_table = build_cross_deps(dag, "wait_on_state_table", "state_table_export")

# DBT models to run
enriched_history_operations_task = dbt_task(
dag, tag="enriched_history_operations", operator="+", excluded="singular_test"
)
enriched_history_operations_task = dbt_task(dag, tag="enriched_history_operations", operator="+")
current_state_task = dbt_task(dag, tag="current_state", operator="+")

# DAG task graph
Expand Down
23 changes: 12 additions & 11 deletions dags/dbt_stellar_marts_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,23 +33,24 @@
)

# DBT models to run
ohlc_task = dbt_task(dag, tag="ohlc", operator="+")
ohlc_task = dbt_task(dag, tag="ohlc", operator="+", excluded="stellar_dbt_public")
liquidity_pool_trade_volume_task = dbt_task(
dag, tag="liquidity_pool_trade_volume", operator="+"
dag, tag="liquidity_pool_trade_volume", operator="+", excluded="stellar_dbt_public"
)

liquidity_providers_task = dbt_task(dag, tag="liquidity_providers", operator="+")
liquidity_pools_values_task = dbt_task(dag, tag="liquidity_pools_value", operator="+")
liquidity_providers_task = dbt_task(dag, tag="liquidity_providers", excluded="stellar_dbt_public")
liquidity_pools_values_task = dbt_task(dag, tag="liquidity_pools_value", operator="+", excluded="stellar_dbt_public")
liquidity_pools_value_history_task = dbt_task(
dag, tag="liquidity_pools_value_history", operator="+"
dag, tag="liquidity_pools_value_history", operator="+", excluded="stellar_dbt_public"
)
trade_agg_task = dbt_task(dag, tag="trade_agg", operator="+")
fee_stats_agg_task = dbt_task(dag, tag="fee_stats", operator="+")
asset_stats_agg_task = dbt_task(dag, tag="asset_stats", operator="+")
network_stats_agg_task = dbt_task(dag, tag="network_stats", operator="+")
partnership_assets_task = dbt_task(dag, tag="partnership_assets", operator="+")
fee_stats_agg_task = dbt_task(dag, tag="fee_stats")
asset_stats_agg_task = dbt_task(dag, tag="asset_stats", operator="+", excluded="stellar_dbt_public")
network_stats_agg_task = dbt_task(dag, tag="network_stats", excluded="stellar_dbt_public")
partnership_assets_task = dbt_task(dag, tag="partnership_assets", operator="+", excluded="stellar_dbt_public")
history_assets = dbt_task(dag, tag="history_assets", operator="+")
soroban = dbt_task(dag, tag="soroban", operator="+")
# Disable soroban tables because they're broken
# soroban = dbt_task(dag, tag="soroban", operator="+")
# Disable snapshot state tables because they're broken
# snapshot_state = dbt_task(dag, tag="snapshot_state")
# Disable releveant_asset_trades due to bugs in SCD tables
Expand All @@ -67,6 +68,6 @@
wait_on_dbt_enriched_base_tables >> network_stats_agg_task
wait_on_dbt_enriched_base_tables >> partnership_assets_task
wait_on_dbt_enriched_base_tables >> history_assets
wait_on_dbt_enriched_base_tables >> soroban
# wait_on_dbt_enriched_base_tables >> soroban
# wait_on_dbt_enriched_base_tables >> snapshot_state
# wait_on_dbt_enriched_base_tables >> relevant_asset_trades
2 changes: 1 addition & 1 deletion dags/dbt_stellar_marts_mgi_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@

# DBT models to run

mgi_task = dbt_task(dag, tag="mgi", operator="+")
mgi_task = dbt_task(dag, tag="mgi", operator="+", excluded="stellar_dbt_public")

wait_on_dbt_enriched_base_tables >> mgi_task
wait_on_partner_pipeline_dag >> mgi_task

0 comments on commit 825d896

Please sign in to comment.