From a091f34fea1f1463e3f0fecc799283bfa8126fb9 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Nov 2023 10:33:54 -0400 Subject: [PATCH 1/8] Add closed_at to schemas --- airflow_variables_dev.json | 2 +- schemas/account_signers_schema.json | 5 +++++ schemas/accounts_schema.json | 5 +++++ schemas/claimable_balances_schema.json | 5 +++++ schemas/config_settings_schema.json | 5 +++++ schemas/contract_code_schema.json | 5 +++++ schemas/contract_data_schema.json | 5 +++++ schemas/expiration_schema.json | 5 +++++ schemas/liquidity_pools_schema.json | 5 +++++ schemas/offers_schema.json | 5 +++++ schemas/trust_lines_schema.json | 5 +++++ 11 files changed, 51 insertions(+), 1 deletion(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 8f550768..60df5e27 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -108,7 +108,7 @@ "dbt_token_uri": "https://oauth2.googleapis.com/token", "gcs_exported_data_bucket_name": "us-central1-hubble-1pt5-dev-7db0e004-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:0ea7a30", + "image_name": "chowbao/stellar-etl:eb3fdcf", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", diff --git a/schemas/account_signers_schema.json b/schemas/account_signers_schema.json index 6a722791..a8be4e3f 100644 --- a/schemas/account_signers_schema.json +++ b/schemas/account_signers_schema.json @@ -48,5 +48,10 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" } ] diff --git a/schemas/accounts_schema.json b/schemas/accounts_schema.json index 4b36b15f..419eef3d 100644 --- a/schemas/accounts_schema.json +++ b/schemas/accounts_schema.json @@ -118,5 +118,10 @@ "mode": "NULLABLE", "name": "sequence_time", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" } ] diff --git a/schemas/claimable_balances_schema.json b/schemas/claimable_balances_schema.json index 25811818..ea0b9846 100644 --- a/schemas/claimable_balances_schema.json +++ b/schemas/claimable_balances_schema.json @@ -458,5 +458,10 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" } ] diff --git a/schemas/config_settings_schema.json b/schemas/config_settings_schema.json index cb3eb2e9..3a9dbf7f 100644 --- a/schemas/config_settings_schema.json +++ b/schemas/config_settings_schema.json @@ -277,5 +277,10 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" } ] diff --git a/schemas/contract_code_schema.json b/schemas/contract_code_schema.json index 7c141cc9..b25e1af0 100644 --- a/schemas/contract_code_schema.json +++ b/schemas/contract_code_schema.json @@ -38,5 +38,10 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" } ] diff --git a/schemas/contract_data_schema.json b/schemas/contract_data_schema.json index 351e0f9c..9d3e617b 100644 --- a/schemas/contract_data_schema.json +++ b/schemas/contract_data_schema.json @@ -68,5 +68,10 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" } ] diff --git a/schemas/expiration_schema.json b/schemas/expiration_schema.json index e1389a97..54d8c76a 100644 --- a/schemas/expiration_schema.json +++ b/schemas/expiration_schema.json @@ -38,5 +38,10 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" } ] diff --git a/schemas/liquidity_pools_schema.json b/schemas/liquidity_pools_schema.json index e8be2062..51af0628 100644 --- a/schemas/liquidity_pools_schema.json +++ b/schemas/liquidity_pools_schema.json @@ -103,5 +103,10 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" } ] diff --git a/schemas/offers_schema.json b/schemas/offers_schema.json index 6a298ec9..a507f13b 100644 --- a/schemas/offers_schema.json +++ b/schemas/offers_schema.json @@ -108,5 +108,10 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" } ] diff --git a/schemas/trust_lines_schema.json b/schemas/trust_lines_schema.json index 40b73aee..16c168fd 100644 --- a/schemas/trust_lines_schema.json +++ b/schemas/trust_lines_schema.json @@ -93,5 +93,10 @@ "mode": "NULLABLE", "name": "sponsor", "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" } ] From 8409b395f2f708922033ad6d846d5d2761b92e34 Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Thu, 2 Nov 2023 13:15:16 -0500 Subject: [PATCH 2/8] Update requirements-ci.txt --- requirements-ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-ci.txt b/requirements-ci.txt index eba53579..e2b0c909 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -1,5 +1,5 @@ alembic==1.10.4 -apache-airflow==2.3.3 +apache-airflow==2.3.4 apache-airflow-providers-cncf-kubernetes==4.4.0 apache-airflow-providers-common-sql==1.4.0 apache-airflow-providers-ftp==3.3.1 From a98da537b44a50357e6ced1c4493c414276905cc Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Thu, 2 Nov 2023 14:37:13 -0500 Subject: [PATCH 3/8] Update requirements-ci.txt --- requirements-ci.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-ci.txt b/requirements-ci.txt index e2b0c909..c4dd7129 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -12,3 +12,4 @@ freezegun==1.2.0 jsonlines==3.0 pytest==7.2.2 sentry-sdk==1.19.0 +werkzeug==2.3.0 From 4effb27bb72b3a427b77c098ad371a56ec3d5523 Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Thu, 2 Nov 2023 14:46:29 -0500 Subject: [PATCH 4/8] Update requirements-ci.txt --- requirements-ci.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-ci.txt b/requirements-ci.txt index c4dd7129..48cbcc2e 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -8,8 +8,9 @@ apache-airflow-providers-http==4.3.0 apache-airflow-providers-imap==3.1.1 apache-airflow-providers-sqlite==3.3.2 docker==3.7.3 +flask==2.2.5 freezegun==1.2.0 jsonlines==3.0 pytest==7.2.2 sentry-sdk==1.19.0 -werkzeug==2.3.0 +werkzeug==2.2.3 From feddeccfa18186f55186c9948a3f2fb101f4cd20 Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Thu, 2 Nov 2023 14:51:56 -0500 Subject: [PATCH 5/8] Update requirements-ci.txt --- requirements-ci.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements-ci.txt b/requirements-ci.txt index 48cbcc2e..441dddf0 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -1,5 +1,5 @@ alembic==1.10.4 -apache-airflow==2.3.4 +apache-airflow==2.3.3 apache-airflow-providers-cncf-kubernetes==4.4.0 apache-airflow-providers-common-sql==1.4.0 apache-airflow-providers-ftp==3.3.1 @@ -7,8 +7,10 @@ apache-airflow-providers-google==8.6.0 apache-airflow-providers-http==4.3.0 apache-airflow-providers-imap==3.1.1 apache-airflow-providers-sqlite==3.3.2 +connexion==2.14.2 docker==3.7.3 flask==2.2.5 +flask-appbuilder==4.1.2 freezegun==1.2.0 jsonlines==3.0 pytest==7.2.2 From 72f0dbb09978e102505c056ef70cf83ce898df0c Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Nov 2023 16:00:50 -0400 Subject: [PATCH 6/8] Update stellar etl image --- airflow_variables_dev.json | 2 +- dags/marts_tables_dag.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 60df5e27..f7831279 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -108,7 +108,7 @@ "dbt_token_uri": "https://oauth2.googleapis.com/token", "gcs_exported_data_bucket_name": "us-central1-hubble-1pt5-dev-7db0e004-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "chowbao/stellar-etl:eb3fdcf", + "image_name": "chowbao/stellar-etl:df85b2d", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", diff --git a/dags/marts_tables_dag.py b/dags/marts_tables_dag.py index f73fbe91..1d99d2a4 100644 --- a/dags/marts_tables_dag.py +++ b/dags/marts_tables_dag.py @@ -20,7 +20,7 @@ ) wait_on_partnership_assets_dag = build_cross_deps( - dag, "wait_on_partnership_assets_pipeline", "partnership_assets", time_delta=10 + dag, "wait_on_partnership_assets_pipeline", "partnership_assets", time_delta=30 ) # tasks for staging tables for marts From 4fdcad1819cb1a6726657f8b62b0a2442faee607 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Nov 2023 16:23:49 -0400 Subject: [PATCH 7/8] Add ledger sequence to schemas --- schemas/account_signers_schema.json | 5 +++++ schemas/accounts_schema.json | 5 +++++ schemas/claimable_balances_schema.json | 5 +++++ schemas/config_settings_schema.json | 5 +++++ schemas/contract_code_schema.json | 5 +++++ schemas/contract_data_schema.json | 5 +++++ schemas/expiration_schema.json | 5 +++++ schemas/liquidity_pools_schema.json | 5 +++++ schemas/offers_schema.json | 5 +++++ schemas/trust_lines_schema.json | 5 +++++ 10 files changed, 50 insertions(+) diff --git a/schemas/account_signers_schema.json b/schemas/account_signers_schema.json index a8be4e3f..fd92d8a6 100644 --- a/schemas/account_signers_schema.json +++ b/schemas/account_signers_schema.json @@ -53,5 +53,10 @@ "mode": "NULLABLE", "name": "closed_at", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/accounts_schema.json b/schemas/accounts_schema.json index 419eef3d..cbcb7c43 100644 --- a/schemas/accounts_schema.json +++ b/schemas/accounts_schema.json @@ -123,5 +123,10 @@ "mode": "NULLABLE", "name": "closed_at", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/claimable_balances_schema.json b/schemas/claimable_balances_schema.json index ea0b9846..b06056a5 100644 --- a/schemas/claimable_balances_schema.json +++ b/schemas/claimable_balances_schema.json @@ -463,5 +463,10 @@ "mode": "NULLABLE", "name": "closed_at", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/config_settings_schema.json b/schemas/config_settings_schema.json index 3a9dbf7f..9900c870 100644 --- a/schemas/config_settings_schema.json +++ b/schemas/config_settings_schema.json @@ -282,5 +282,10 @@ "mode": "NULLABLE", "name": "closed_at", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/contract_code_schema.json b/schemas/contract_code_schema.json index b25e1af0..dc39cdfb 100644 --- a/schemas/contract_code_schema.json +++ b/schemas/contract_code_schema.json @@ -43,5 +43,10 @@ "mode": "NULLABLE", "name": "closed_at", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/contract_data_schema.json b/schemas/contract_data_schema.json index 9d3e617b..65f65d38 100644 --- a/schemas/contract_data_schema.json +++ b/schemas/contract_data_schema.json @@ -73,5 +73,10 @@ "mode": "NULLABLE", "name": "closed_at", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/expiration_schema.json b/schemas/expiration_schema.json index 54d8c76a..e2a5bf5d 100644 --- a/schemas/expiration_schema.json +++ b/schemas/expiration_schema.json @@ -43,5 +43,10 @@ "mode": "NULLABLE", "name": "closed_at", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/liquidity_pools_schema.json b/schemas/liquidity_pools_schema.json index 51af0628..6861eccb 100644 --- a/schemas/liquidity_pools_schema.json +++ b/schemas/liquidity_pools_schema.json @@ -108,5 +108,10 @@ "mode": "NULLABLE", "name": "closed_at", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/offers_schema.json b/schemas/offers_schema.json index a507f13b..e5104ca0 100644 --- a/schemas/offers_schema.json +++ b/schemas/offers_schema.json @@ -113,5 +113,10 @@ "mode": "NULLABLE", "name": "closed_at", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/trust_lines_schema.json b/schemas/trust_lines_schema.json index 16c168fd..9b4e734e 100644 --- a/schemas/trust_lines_schema.json +++ b/schemas/trust_lines_schema.json @@ -98,5 +98,10 @@ "mode": "NULLABLE", "name": "closed_at", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] From c3938822589aea0fc65846d597e14032dfbb58ad Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 2 Nov 2023 16:58:14 -0400 Subject: [PATCH 8/8] Update stellar-etl image --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index f7831279..d2a0f1d7 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -108,7 +108,7 @@ "dbt_token_uri": "https://oauth2.googleapis.com/token", "gcs_exported_data_bucket_name": "us-central1-hubble-1pt5-dev-7db0e004-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "chowbao/stellar-etl:df85b2d", + "image_name": "stellar/stellar-etl:08fd8de", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 8b17bc63..dbd3159b 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -130,7 +130,7 @@ "dbt_token_uri": "https://oauth2.googleapis.com/token", "gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:0ea7a30", + "image_name": "stellar/stellar-etl:08fd8de", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "",