From 4e86886f898285aeb5a2161ac121c0dd28c5e601 Mon Sep 17 00:00:00 2001 From: chowbao Date: Fri, 26 Jul 2024 13:12:04 -0400 Subject: [PATCH] Add json field to history_operations (#448) * Add json field to history_operations * Disable relevant assets * lint * Update stellar-etl image --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- dags/dbt_stellar_marts_dag.py | 5 +++-- schemas/history_operations_schema.json | 5 +++++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 5522b27d..38698607 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -154,7 +154,7 @@ }, "gcs_exported_data_bucket_name": "us-central1-test-hubble-43c3e190-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:741ee9b", + "image_name": "stellar/stellar-etl:37b7123", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "k8s_namespace": "hubble-composer", diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index ad25ccd4..7adf8508 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -155,7 +155,7 @@ }, "gcs_exported_data_bucket_name": "us-central1-hubble-14c4ca64-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:741ee9b", + "image_name": "stellar/stellar-etl:37b7123", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "k8s_namespace": "hubble-composer", diff --git a/dags/dbt_stellar_marts_dag.py b/dags/dbt_stellar_marts_dag.py index 47920366..1fb201af 100644 --- a/dags/dbt_stellar_marts_dag.py +++ b/dags/dbt_stellar_marts_dag.py @@ -52,7 +52,8 @@ history_assets = dbt_task(dag, tag="history_assets") soroban = dbt_task(dag, tag="soroban") snapshot_state = dbt_task(dag, tag="snapshot_state") -relevant_asset_trades = dbt_task(dag, tag="relevant_asset_trades") +# Disable releveant_asset_trades due to bugs in SCD tables +# relevant_asset_trades = dbt_task(dag, tag="relevant_asset_trades") # DAG task graph wait_on_dbt_enriched_base_tables >> ohlc_task >> liquidity_pool_trade_volume_task @@ -71,4 +72,4 @@ wait_on_dbt_enriched_base_tables >> history_assets wait_on_dbt_enriched_base_tables >> soroban wait_on_dbt_enriched_base_tables >> snapshot_state -wait_on_dbt_enriched_base_tables >> relevant_asset_trades +# wait_on_dbt_enriched_base_tables >> relevant_asset_trades diff --git a/schemas/history_operations_schema.json b/schemas/history_operations_schema.json index 755b85bf..09ff1ee6 100644 --- a/schemas/history_operations_schema.json +++ b/schemas/history_operations_schema.json @@ -1176,5 +1176,10 @@ "mode": "NULLABLE", "name": "ledger_sequence", "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "details_json", + "type": "JSON" } ]