diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 8f550768..d2a0f1d7 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -108,7 +108,7 @@ "dbt_token_uri": "https://oauth2.googleapis.com/token", "gcs_exported_data_bucket_name": "us-central1-hubble-1pt5-dev-7db0e004-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:0ea7a30", + "image_name": "stellar/stellar-etl:08fd8de", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 8b17bc63..dbd3159b 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -130,7 +130,7 @@ "dbt_token_uri": "https://oauth2.googleapis.com/token", "gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:0ea7a30", + "image_name": "stellar/stellar-etl:08fd8de", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", diff --git a/dags/marts_tables_dag.py b/dags/marts_tables_dag.py index f73fbe91..1d99d2a4 100644 --- a/dags/marts_tables_dag.py +++ b/dags/marts_tables_dag.py @@ -20,7 +20,7 @@ ) wait_on_partnership_assets_dag = build_cross_deps( - dag, "wait_on_partnership_assets_pipeline", "partnership_assets", time_delta=10 + dag, "wait_on_partnership_assets_pipeline", "partnership_assets", time_delta=30 ) # tasks for staging tables for marts diff --git a/requirements-ci.txt b/requirements-ci.txt index eba53579..441dddf0 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -7,8 +7,12 @@ apache-airflow-providers-google==8.6.0 apache-airflow-providers-http==4.3.0 apache-airflow-providers-imap==3.1.1 apache-airflow-providers-sqlite==3.3.2 +connexion==2.14.2 docker==3.7.3 +flask==2.2.5 +flask-appbuilder==4.1.2 freezegun==1.2.0 jsonlines==3.0 pytest==7.2.2 sentry-sdk==1.19.0 +werkzeug==2.2.3 diff --git a/schemas/account_signers_schema.json b/schemas/account_signers_schema.json index 6a722791..fd92d8a6 100644 --- a/schemas/account_signers_schema.json +++ b/schemas/account_signers_schema.json @@ -48,5 +48,15 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/accounts_schema.json b/schemas/accounts_schema.json index 4b36b15f..cbcb7c43 100644 --- a/schemas/accounts_schema.json +++ b/schemas/accounts_schema.json @@ -118,5 +118,15 @@ "mode": "NULLABLE", "name": "sequence_time", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/claimable_balances_schema.json b/schemas/claimable_balances_schema.json index 25811818..b06056a5 100644 --- a/schemas/claimable_balances_schema.json +++ b/schemas/claimable_balances_schema.json @@ -458,5 +458,15 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/config_settings_schema.json b/schemas/config_settings_schema.json index cb3eb2e9..9900c870 100644 --- a/schemas/config_settings_schema.json +++ b/schemas/config_settings_schema.json @@ -277,5 +277,15 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/contract_code_schema.json b/schemas/contract_code_schema.json index 7c141cc9..dc39cdfb 100644 --- a/schemas/contract_code_schema.json +++ b/schemas/contract_code_schema.json @@ -38,5 +38,15 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/contract_data_schema.json b/schemas/contract_data_schema.json index 351e0f9c..65f65d38 100644 --- a/schemas/contract_data_schema.json +++ b/schemas/contract_data_schema.json @@ -68,5 +68,15 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/expiration_schema.json b/schemas/expiration_schema.json index e1389a97..e2a5bf5d 100644 --- a/schemas/expiration_schema.json +++ b/schemas/expiration_schema.json @@ -38,5 +38,15 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/liquidity_pools_schema.json b/schemas/liquidity_pools_schema.json index e8be2062..6861eccb 100644 --- a/schemas/liquidity_pools_schema.json +++ b/schemas/liquidity_pools_schema.json @@ -103,5 +103,15 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/offers_schema.json b/schemas/offers_schema.json index 6a298ec9..e5104ca0 100644 --- a/schemas/offers_schema.json +++ b/schemas/offers_schema.json @@ -108,5 +108,15 @@ "mode": "NULLABLE", "name": "batch_insert_ts", "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ] diff --git a/schemas/trust_lines_schema.json b/schemas/trust_lines_schema.json index 40b73aee..9b4e734e 100644 --- a/schemas/trust_lines_schema.json +++ b/schemas/trust_lines_schema.json @@ -93,5 +93,15 @@ "mode": "NULLABLE", "name": "sponsor", "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" } ]