From 9fa65e897a2c51caeb9790b8f890b23d4e07c0ae Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Wed, 17 Apr 2024 16:30:13 -0500 Subject: [PATCH 01/11] Update etl and dbt images --- airflow_variables_dev.json | 4 ++-- schemas/history_operations_schema.json | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 8d9bde81..31f97227 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -121,7 +121,7 @@ "partnership_assets__account_holders_activity_fact": false, "partnership_assets__asset_activity_fact": false }, - "dbt_image_name": "stellar/stellar-dbt:b621192", + "dbt_image_name": "stellar/stellar-dbt:8ae3438", "dbt_job_execution_timeout_seconds": 300, "dbt_job_retries": 1, "dbt_mart_dataset": "test_sdf_marts", @@ -131,7 +131,7 @@ "dbt_threads": 12, "gcs_exported_data_bucket_name": "us-central1-hubble-1pt5-dev-7db0e004-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:75c9a9c", + "image_name": "stellar/stellar-etl:395f1f8", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", diff --git a/schemas/history_operations_schema.json b/schemas/history_operations_schema.json index 3ebe8982..7726aadb 100644 --- a/schemas/history_operations_schema.json +++ b/schemas/history_operations_schema.json @@ -1074,6 +1074,11 @@ "mode": "NULLABLE", "name": "contract_code_hash", "type": "STRING" + }, + { + "mode": "REPEATED", + "name": "key_hash", + "type": "STRING" } ], "mode": "NULLABLE", From 4cde3270d58f68c58618cc062b6c61e506ffe766 Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Wed, 17 Apr 2024 16:38:54 -0500 Subject: [PATCH 02/11] Repoint CI to composer 2 --- airflow_variables_dev.json | 73 ++------------------------------------ 1 file changed, 2 insertions(+), 71 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 31f97227..ffc9bdd3 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -1,74 +1,5 @@ { - "affinity": { - "default": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "cloud.google.com/gke-nodepool", - "operator": "In", - "values": ["default-pool"] - } - ] - } - ] - } - } - }, - "cc": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "cloud.google.com/gke-nodepool", - "operator": "In", - "values": ["pool-2"] - } - ] - } - ] - } - } - }, - "wocc": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "cloud.google.com/gke-nodepool", - "operator": "In", - "values": ["default-pool"] - } - ] - } - ] - } - } - }, - "state": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "cloud.google.com/gke-nodepool", - "operator": "In", - "values": ["default-pool"] - } - ] - } - ] - } - } - } - }, + "affinity": {}, "api_key_path": "/home/airflow/gcs/data/apiKey.json", "bq_dataset": "test_crypto_stellar_internal", "bq_project": "test-hubble-319619", @@ -129,7 +60,7 @@ "dbt_project": "test-hubble-319619", "dbt_target": "test", "dbt_threads": 12, - "gcs_exported_data_bucket_name": "us-central1-hubble-1pt5-dev-7db0e004-bucket", + "gcs_exported_data_bucket_name": "us-central1-test-hubble-2-5f1f2dbf-bucket", "gcs_exported_object_prefix": "dag-exported", "image_name": "stellar/stellar-etl:395f1f8", "image_output_path": "/etl/exported_data/", From 72e1cad96050d1f4e0187845fe2c974784fab671 Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Wed, 17 Apr 2024 16:42:32 -0500 Subject: [PATCH 03/11] Add missing ci changes --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc1df335..a5dff094 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,8 +59,8 @@ jobs: - id: "get-credentials" uses: "google-github-actions/get-gke-credentials@v2" with: - cluster_name: "us-central1-hubble-1pt5-dev-7db0e004-gke" - location: "us-central1-c" + cluster_name: "us-central1-test-hubble-2-5f1f2dbf-gke" + location: "us-central1" - name: Pytest run: pytest dags/ @@ -98,20 +98,20 @@ jobs: run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET env: GOOGLE_CLOUD_PROJECT: test-hubble-319619 - BUCKET: us-central1-hubble-1pt5-dev-7db0e004-bucket + BUCKET: us-central1-test-hubble-2-5f1f2dbf-bucket - name: Update Airflow variables uses: actions-hub/gcloud@master env: PROJECT_ID: test-hubble-319619 APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" - COMPOSER_ENVIRONMENT: hubble-1pt5-dev + COMPOSER_ENVIRONMENT: test-hubble-2 LOCATION: us-central1 with: args: > components install kubectl && gcloud composer environments run $COMPOSER_ENVIRONMENT --location $LOCATION variables import - -- gcsfuse/variables.json + -- gcsfuse/actual_mounth_path/variables.json promote-to-prod: runs-on: ubuntu-latest From abf330f49036b7901f950f6a5b1fca998270e29b Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Wed, 17 Apr 2024 16:48:31 -0500 Subject: [PATCH 04/11] Fix dir name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5dff094..60ace9ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,7 @@ jobs: args: > components install kubectl && gcloud composer environments run $COMPOSER_ENVIRONMENT --location $LOCATION variables import - -- gcsfuse/actual_mounth_path/variables.json + -- gcsfuse/actual_mount_path/variables.json promote-to-prod: runs-on: ubuntu-latest From 2547c0429efe11791532bcb1c51dc1bece0cf9b1 Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Wed, 17 Apr 2024 17:19:32 -0500 Subject: [PATCH 05/11] Change key_hash to ledger_key_hash --- schemas/history_operations_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/history_operations_schema.json b/schemas/history_operations_schema.json index 7726aadb..c5cdd219 100644 --- a/schemas/history_operations_schema.json +++ b/schemas/history_operations_schema.json @@ -1077,7 +1077,7 @@ }, { "mode": "REPEATED", - "name": "key_hash", + "name": "ledger_key_hash", "type": "STRING" } ], From b0d165aeccf8726d7586f4129a9daa8ceb5e09bf Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Wed, 17 Apr 2024 20:39:40 -0500 Subject: [PATCH 06/11] revert bad build_export_task command --- dags/stellar_etl_airflow/build_export_task.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dags/stellar_etl_airflow/build_export_task.py b/dags/stellar_etl_airflow/build_export_task.py index ded28975..1d15e099 100644 --- a/dags/stellar_etl_airflow/build_export_task.py +++ b/dags/stellar_etl_airflow/build_export_task.py @@ -198,8 +198,7 @@ def build_export_task( else: arguments = f""" {etl_cmd_string} 2>> stderr.out && echo "{{\\"output\\": \\"{output_file}\\", - \\"failed_transforms\\": `grep failed_transforms stderr.out | cut -d\\",\\" -f2 | cut -d\\":\\" -f2`\\", - \\"succcessful_transforms\\": `grep successful_transforms stderr.out | cut -d\\",\\" -f2 | cut -d\\":\\" -f2`}}" >> /airflow/xcom/return.json + \\"failed_transforms\\": `grep failed_transforms stderr.out | cut -d\\",\\" -f2 | cut -d\\":\\" -f2`}}" >> /airflow/xcom/return.json """ return KubernetesPodOperator( service_account_name=Variable.get("k8s_service_account"), From 4bee5ea0ca34658ee4fdf7ba3ccc9abc032ae75b Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Wed, 17 Apr 2024 21:07:33 -0500 Subject: [PATCH 07/11] readd liquidity pools --- dags/dbt_sdf_marts_dag.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dags/dbt_sdf_marts_dag.py b/dags/dbt_sdf_marts_dag.py index 558fdf52..47995d84 100644 --- a/dags/dbt_sdf_marts_dag.py +++ b/dags/dbt_sdf_marts_dag.py @@ -38,8 +38,8 @@ mgi_task = dbt_task(dag, tag="mgi") liquidity_providers_task = dbt_task(dag, tag="liquidity_providers") -# liquidity_pools_values_task = dbt_task(dag, tag="liquidity_pools_value") -# liquidity_pools_value_history_task = dbt_task(dag, tag="liquidity_pools_value_history") +liquidity_pools_values_task = dbt_task(dag, tag="liquidity_pools_value") +liquidity_pools_value_history_task = dbt_task(dag, tag="liquidity_pools_value_history") trade_agg_task = dbt_task(dag, tag="trade_agg") fee_stats_agg_task = dbt_task(dag, tag="fee_stats") asset_stats_agg_task = dbt_task(dag, tag="asset_stats") @@ -57,8 +57,8 @@ wait_on_partner_pipeline_dag >> mgi_task wait_on_dbt_enriched_base_tables >> liquidity_providers_task -# wait_on_dbt_enriched_base_tables >> liquidity_pools_values_task -# wait_on_dbt_enriched_base_tables >> liquidity_pools_value_history_task +wait_on_dbt_enriched_base_tables >> liquidity_pools_values_task +wait_on_dbt_enriched_base_tables >> liquidity_pools_value_history_task wait_on_dbt_enriched_base_tables >> trade_agg_task wait_on_dbt_enriched_base_tables >> fee_stats_agg_task wait_on_dbt_enriched_base_tables >> asset_stats_agg_task @@ -69,8 +69,8 @@ mgi_task >> elementary liquidity_providers_task >> elementary -# liquidity_pools_values_task >> elementary -# liquidity_pools_value_history_task >> elementary +liquidity_pools_values_task >> elementary +liquidity_pools_value_history_task >> elementary trade_agg_task >> elementary fee_stats_agg_task >> elementary asset_stats_agg_task >> elementary From b17ac166080529811eb9aca503751fe20e363007 Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Wed, 17 Apr 2024 22:34:11 -0500 Subject: [PATCH 08/11] add snapshot state task --- dags/dbt_sdf_marts_dag.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dags/dbt_sdf_marts_dag.py b/dags/dbt_sdf_marts_dag.py index 47995d84..8c29900f 100644 --- a/dags/dbt_sdf_marts_dag.py +++ b/dags/dbt_sdf_marts_dag.py @@ -47,6 +47,7 @@ partnership_assets_task = dbt_task(dag, tag="partnership_assets") history_assets = dbt_task(dag, tag="history_assets") soroban = dbt_task(dag, tag="soroban") +snapshot_state = dbt_task(dag, tag="snapshot_state") elementary = elementary_task(dag, "dbt_sdf_marts") @@ -66,6 +67,7 @@ wait_on_dbt_enriched_base_tables >> partnership_assets_task wait_on_dbt_enriched_base_tables >> history_assets wait_on_dbt_enriched_base_tables >> soroban +wait_on_dbt_enriched_base_tables >> snapshot_state mgi_task >> elementary liquidity_providers_task >> elementary @@ -79,3 +81,4 @@ history_assets >> elementary soroban >> elementary liquidity_pool_trade_volume_task >> elementary +snapshot_state >> elementary From 4eaf7648e1d68a1e2823b19993f8a3103a82bba7 Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Thu, 18 Apr 2024 08:35:03 -0500 Subject: [PATCH 09/11] Update prod config and ci --- .github/workflows/release.yml | 10 ++-- airflow_variables_prod.json | 94 ++--------------------------------- 2 files changed, 9 insertions(+), 95 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1ba61839..f089d9db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,8 +39,8 @@ jobs: - id: "get-credentials" uses: "google-github-actions/get-gke-credentials@v2" with: - cluster_name: "us-central1-hubble-1pt5-dev-7db0e004-gke" - location: "us-central1-c" + cluster_name: "us-central1-hubble-14c4ca64-gke" + location: "us-central1" - name: Pytest run: pytest dags/ @@ -75,17 +75,17 @@ jobs: run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET --env prod env: GOOGLE_CLOUD_PROJECT: hubble-261722 - BUCKET: us-central1-hubble-2-d948d67b-bucket + BUCKET: us-central1-hubble-14c4ca64-bucket - name: Update Airflow variables uses: actions-hub/gcloud@master env: PROJECT_ID: hubble-261722 APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" - COMPOSER_ENVIRONMENT: hubble-2 + COMPOSER_ENVIRONMENT: hubble LOCATION: us-central1 with: args: > components install kubectl && gcloud composer environments run $COMPOSER_ENVIRONMENT --location $LOCATION variables import - -- gcsfuse/variables.json + -- gcsfuse/actual_mount_path/variables.json diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 105a02c5..43fe0c2f 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -1,91 +1,5 @@ { - "affinity": { - "default": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "cloud.google.com/gke-nodepool", - "operator": "In", - "values": ["default-pool"] - } - ] - } - ] - } - } - }, - "backfill": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "cloud.google.com/gke-nodepool", - "operator": "In", - "values": ["hubble-etl"] - } - ] - } - ] - } - } - }, - "cc": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "cloud.google.com/gke-nodepool", - "operator": "In", - "values": ["hubble-etl"] - } - ] - } - ] - } - } - }, - "wocc": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "cloud.google.com/gke-nodepool", - "operator": "In", - "values": ["default-pool"] - } - ] - } - ] - } - } - }, - "state": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "cloud.google.com/gke-nodepool", - "operator": "In", - "values": ["state-table"] - } - ] - } - ] - } - } - } - }, + "affinity": {}, "api_key_path": "/home/airflow/gcs/data/apiKey.json", "bq_dataset": "crypto_stellar_internal_2", "bq_project": "hubble-261722", @@ -143,7 +57,7 @@ "partnership_assets__asset_activity_fact": false, "trade_agg": false }, - "dbt_image_name": "stellar/stellar-dbt:b621192", + "dbt_image_name": "stellar/stellar-dbt:8ae3438", "dbt_job_execution_timeout_seconds": 1800, "dbt_job_retries": 1, "dbt_mart_dataset": "sdf_marts", @@ -151,9 +65,9 @@ "dbt_project": "hubble-261722", "dbt_target": "prod", "dbt_threads": 12, - "gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket", + "gcs_exported_data_bucket_name": "us-central1-hubble-14c4ca64-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:75c9a9c", + "image_name": "stellar/stellar-etl:395f1f8", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", From 38e25f0dee8cb1fdde90e35710893cf36c1fa3ab Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Thu, 18 Apr 2024 08:40:17 -0500 Subject: [PATCH 10/11] Revert mainnet ci changes --- .github/workflows/release.yml | 10 ++-- airflow_variables_prod.json | 90 ++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f089d9db..1ba61839 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,8 +39,8 @@ jobs: - id: "get-credentials" uses: "google-github-actions/get-gke-credentials@v2" with: - cluster_name: "us-central1-hubble-14c4ca64-gke" - location: "us-central1" + cluster_name: "us-central1-hubble-1pt5-dev-7db0e004-gke" + location: "us-central1-c" - name: Pytest run: pytest dags/ @@ -75,17 +75,17 @@ jobs: run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET --env prod env: GOOGLE_CLOUD_PROJECT: hubble-261722 - BUCKET: us-central1-hubble-14c4ca64-bucket + BUCKET: us-central1-hubble-2-d948d67b-bucket - name: Update Airflow variables uses: actions-hub/gcloud@master env: PROJECT_ID: hubble-261722 APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}" - COMPOSER_ENVIRONMENT: hubble + COMPOSER_ENVIRONMENT: hubble-2 LOCATION: us-central1 with: args: > components install kubectl && gcloud composer environments run $COMPOSER_ENVIRONMENT --location $LOCATION variables import - -- gcsfuse/actual_mount_path/variables.json + -- gcsfuse/variables.json diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index 43fe0c2f..7af9edc7 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -1,5 +1,91 @@ { - "affinity": {}, + "affinity": { + "default": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "cloud.google.com/gke-nodepool", + "operator": "In", + "values": ["default-pool"] + } + ] + } + ] + } + } + }, + "backfill": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "cloud.google.com/gke-nodepool", + "operator": "In", + "values": ["hubble-etl"] + } + ] + } + ] + } + } + }, + "cc": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "cloud.google.com/gke-nodepool", + "operator": "In", + "values": ["hubble-etl"] + } + ] + } + ] + } + } + }, + "wocc": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "cloud.google.com/gke-nodepool", + "operator": "In", + "values": ["default-pool"] + } + ] + } + ] + } + } + }, + "state": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "cloud.google.com/gke-nodepool", + "operator": "In", + "values": ["state-table"] + } + ] + } + ] + } + } + } + }, "api_key_path": "/home/airflow/gcs/data/apiKey.json", "bq_dataset": "crypto_stellar_internal_2", "bq_project": "hubble-261722", @@ -65,7 +151,7 @@ "dbt_project": "hubble-261722", "dbt_target": "prod", "dbt_threads": 12, - "gcs_exported_data_bucket_name": "us-central1-hubble-14c4ca64-bucket", + "gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket", "gcs_exported_object_prefix": "dag-exported", "image_name": "stellar/stellar-etl:395f1f8", "image_output_path": "/etl/exported_data/", From 355f27a3dd164be93d5e59108d198fc69e22fb65 Mon Sep 17 00:00:00 2001 From: sydneynotthecity Date: Thu, 18 Apr 2024 09:53:58 -0500 Subject: [PATCH 11/11] Update test gke creds --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1ba61839..8a4b2070 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,8 +39,8 @@ jobs: - id: "get-credentials" uses: "google-github-actions/get-gke-credentials@v2" with: - cluster_name: "us-central1-hubble-1pt5-dev-7db0e004-gke" - location: "us-central1-c" + cluster_name: "us-central1-test-hubble-2-5f1f2dbf-gke" + location: "us-central1" - name: Pytest run: pytest dags/