From e96355718df910df139da9cd838d3d06781f1617 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 30 Jul 2024 10:48:28 -0500 Subject: [PATCH 1/5] Add schema for signatures (#466) * Add schema for signatures * update image to test * use repeated to denote array * Update etl image --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- schemas/history_transactions_schema.json | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 0f615b3b..82650c6e 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -154,7 +154,7 @@ }, "gcs_exported_data_bucket_name": "us-central1-test-hubble-43c3e190-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:37b7123", + "image_name": "amishastellar/stellar-etl:ffe275c", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "k8s_namespace": "hubble-composer", diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 6cdf49f2..8a1f0628 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -155,7 +155,7 @@ }, "gcs_exported_data_bucket_name": "us-central1-hubble-14c4ca64-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:37b7123", + "image_name": "stellar/stellar-etl:ffe275c", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "k8s_namespace": "hubble-composer", diff --git a/schemas/history_transactions_schema.json b/schemas/history_transactions_schema.json index 656c08f4..4e0c6518 100644 --- a/schemas/history_transactions_schema.json +++ b/schemas/history_transactions_schema.json @@ -208,5 +208,10 @@ "mode": "NULLABLE", "name": "rent_fee_charged", "type": "INTEGER" + }, + { + "mode": "REPEATED", + "name": "signatures", + "type": "STRING" } ] From 29be552f327480f52783c6fef004e4ed3c5f25ca Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 30 Jul 2024 13:09:50 -0500 Subject: [PATCH 2/5] Add account of signer accounts when exporting transactions (#468) * Add account of signer accounts when exporting transactions * update images --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- schemas/history_transactions_schema.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 82650c6e..a97867d2 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -154,7 +154,7 @@ }, "gcs_exported_data_bucket_name": "us-central1-test-hubble-43c3e190-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "amishastellar/stellar-etl:ffe275c", + "image_name": "amishastellar/stellar-etl:63b5833", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "k8s_namespace": "hubble-composer", diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 8a1f0628..4033d0e2 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -155,7 +155,7 @@ }, "gcs_exported_data_bucket_name": "us-central1-hubble-14c4ca64-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:ffe275c", + "image_name": "stellar/stellar-etl:63b5833", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "k8s_namespace": "hubble-composer", diff --git a/schemas/history_transactions_schema.json b/schemas/history_transactions_schema.json index 4e0c6518..809523f2 100644 --- a/schemas/history_transactions_schema.json +++ b/schemas/history_transactions_schema.json @@ -211,7 +211,7 @@ }, { "mode": "REPEATED", - "name": "signatures", + "name": "tx_signers", "type": "STRING" } ] From 3def885a387c0947e11ba3d64c11f98ef619fa90 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Wed, 31 Jul 2024 16:33:23 -0500 Subject: [PATCH 3/5] Fix image name for dev (#469) --- airflow_variables_dev.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index a97867d2..3e347df4 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -154,7 +154,7 @@ }, "gcs_exported_data_bucket_name": "us-central1-test-hubble-43c3e190-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "amishastellar/stellar-etl:63b5833", + "image_name": "stellar/stellar-etl:63b5833", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "k8s_namespace": "hubble-composer", From 14067647e538b61c2a599a5f1cbe27f0e3993b9f Mon Sep 17 00:00:00 2001 From: harsha-stellar-data Date: Thu, 1 Aug 2024 17:51:19 -0400 Subject: [PATCH 4/5] Updated dbt image name in Test (changes related to Hubble-420) (#470) * Updated dbt image name in Test (changes related to Hubble-420) * Updated dbt image in Prod variables json * - Updated the dev image name to point to Hubbe 420 PR - Reverted Prod variable changes (will update once testing is successful) * Updated dbt_image_name to point to the latest build --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 3e347df4..f38e67cf 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -122,7 +122,7 @@ "partnership_assets__account_holders_activity_fact": false, "partnership_assets__asset_activity_fact": false }, - "dbt_image_name": "stellar/stellar-dbt:37e6d34", + "dbt_image_name": "stellar/stellar-dbt:5398e95", "dbt_internal_source_db": "test-hubble-319619", "dbt_internal_source_schema": "test_crypto_stellar_internal", "dbt_job_execution_timeout_seconds": 300, diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 4033d0e2..6a3311ae 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -123,7 +123,7 @@ "partnership_assets__asset_activity_fact": false, "trade_agg": false }, - "dbt_image_name": "stellar/stellar-dbt:37e6d34", + "dbt_image_name": "stellar/stellar-dbt:5398e95", "dbt_internal_source_db": "hubble-261722", "dbt_internal_source_schema": "crypto_stellar_internal_2", "dbt_job_execution_timeout_seconds": 1800, From a5ebd26aebf3126364724f0cd060982ad1046d3c Mon Sep 17 00:00:00 2001 From: harsha-stellar-data Date: Mon, 5 Aug 2024 12:49:09 -0400 Subject: [PATCH 5/5] Hubble 420 dbt image name change (#471) * Updated dbt image name in Test (changes related to Hubble-420) * Updated dbt image in Prod variables json * - Updated the dev image name to point to Hubbe 420 PR - Reverted Prod variable changes (will update once testing is successful) * Updated dbt_image_name to point to the latest build * - Updated the dbt_image name in Dev and Prod airflow variables --- airflow_variables_dev.json | 2 +- airflow_variables_prod.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index f38e67cf..f8b55627 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -122,7 +122,7 @@ "partnership_assets__account_holders_activity_fact": false, "partnership_assets__asset_activity_fact": false }, - "dbt_image_name": "stellar/stellar-dbt:5398e95", + "dbt_image_name": "stellar/stellar-dbt:d2f5024", "dbt_internal_source_db": "test-hubble-319619", "dbt_internal_source_schema": "test_crypto_stellar_internal", "dbt_job_execution_timeout_seconds": 300, diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 6a3311ae..fa9bf53e 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -123,7 +123,7 @@ "partnership_assets__asset_activity_fact": false, "trade_agg": false }, - "dbt_image_name": "stellar/stellar-dbt:5398e95", + "dbt_image_name": "stellar/stellar-dbt:d2f5024", "dbt_internal_source_db": "hubble-261722", "dbt_internal_source_schema": "crypto_stellar_internal_2", "dbt_job_execution_timeout_seconds": 1800,