From bac100ea394cb52b39a6ce6be467ee96eed795a4 Mon Sep 17 00:00:00 2001 From: laysabit Date: Thu, 14 Mar 2024 19:02:53 -0300 Subject: [PATCH] test #5 --- .github/workflows/diff-quality.yml | 3 +-- .sqlfluffignore | 6 ++++++ models/marts/enriched_history_operations.sql | 10 +++++----- models/marts/fee_stats_agg.sql | 4 ++-- .../account_signers_current.sql | 8 ++++---- .../ledger_current_state/accounts_current.sql | 14 +++++++------- .../liquidity_pools_current.sql | 8 ++++---- .../marts/ledger_current_state/offers_current.sql | 8 ++++---- .../ledger_current_state/trust_lines_current.sql | 8 ++++---- models/marts/ledger_current_state/ttl_current.sql | 8 ++++---- models/staging/stg_account_signers.sql | 2 +- models/staging/stg_accounts.sql | 2 +- models/staging/stg_claimable_balances.sql | 2 +- models/staging/stg_contract_data.sql | 2 +- models/staging/stg_history_effects.sql | 2 +- models/staging/stg_history_ledgers.sql | 2 +- models/staging/stg_history_operations.sql | 2 +- models/staging/stg_history_trades.sql | 2 +- models/staging/stg_history_transactions.sql | 4 ++-- models/staging/stg_liquidity_pools.sql | 2 +- models/staging/stg_offers.sql | 2 +- models/staging/stg_trust_lines.sql | 2 +- models/staging/stg_ttl.sql | 2 +- 23 files changed, 55 insertions(+), 50 deletions(-) create mode 100644 .sqlfluffignore diff --git a/.github/workflows/diff-quality.yml b/.github/workflows/diff-quality.yml index fa25a48..42d0fa5 100644 --- a/.github/workflows/diff-quality.yml +++ b/.github/workflows/diff-quality.yml @@ -41,5 +41,4 @@ jobs: - name: Checkout source branch and diff quality run: | git checkout ${{ github.ref_name }} - dbt docs generate - # diff-quality --violations=sqlfluff \ No newline at end of file + diff-quality --violations=sqlfluff --compare-branch=origin/master \ No newline at end of file diff --git a/.sqlfluffignore b/.sqlfluffignore new file mode 100644 index 0000000..9f74fc5 --- /dev/null +++ b/.sqlfluffignore @@ -0,0 +1,6 @@ +test/ +macros/ +snapshots/ +env/ +dbt_packages/ +target/ \ No newline at end of file diff --git a/models/marts/enriched_history_operations.sql b/models/marts/enriched_history_operations.sql index 7445022..dc3d0cd 100644 --- a/models/marts/enriched_history_operations.sql +++ b/models/marts/enriched_history_operations.sql @@ -32,10 +32,10 @@ with , batch_run_date , batch_insert_ts from {{ ref('stg_history_ledgers') }} - where + where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) - {% if is_incremental() %} + {% if is_incremental() %} and cast(batch_run_date as date) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) and date(closed_at) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) {% endif %} @@ -76,7 +76,7 @@ with where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) - {% if is_incremental() %} + {% if is_incremental() %} and cast(batch_run_date as date) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) and date(closed_at) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) {% endif %} @@ -195,7 +195,7 @@ with , asset_balance_changes , parameters , parameters_decoded - , `function` + , function , address , soroban_operation_type , extend_to @@ -205,7 +205,7 @@ with where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) - {% if is_incremental() %} + {% if is_incremental() %} and cast(batch_run_date as date) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) and date(closed_at) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) {% endif %} diff --git a/models/marts/fee_stats_agg.sql b/models/marts/fee_stats_agg.sql index 1bd2a74..75ce89f 100644 --- a/models/marts/fee_stats_agg.sql +++ b/models/marts/fee_stats_agg.sql @@ -84,7 +84,7 @@ with where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) - {% if is_incremental() %} + {% if is_incremental() %} and cast(batch_run_date as date) >= date('{{ dbt_airflow_macros.ds() }}') -- batch run is the min bound of a batch and date(closed_at) >= date('{{ dbt_airflow_macros.ds() }}') {% endif %} @@ -104,7 +104,7 @@ with where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) - {% if is_incremental() %} + {% if is_incremental() %} and cast(batch_run_date as date) >= date('{{ dbt_airflow_macros.ds() }}') -- batch run is the min bound of a batch and date(closed_at) >= date('{{ dbt_airflow_macros.ds() }}') {% endif %} diff --git a/models/marts/ledger_current_state/account_signers_current.sql b/models/marts/ledger_current_state/account_signers_current.sql index c82852a..08b9e23 100644 --- a/models/marts/ledger_current_state/account_signers_current.sql +++ b/models/marts/ledger_current_state/account_signers_current.sql @@ -28,10 +28,10 @@ with , s.batch_run_date , s.batch_insert_ts , row_number() - over ( - partition by s.account_id, s.signer - order by s.last_modified_ledger desc, s.ledger_entry_change desc - ) as row_nr + over ( + partition by s.account_id, s.signer + order by s.last_modified_ledger desc, s.ledger_entry_change desc + ) as row_nr from {{ ref('stg_account_signers') }} as s join {{ ref('stg_history_ledgers') }} as l on s.last_modified_ledger = l.sequence diff --git a/models/marts/ledger_current_state/accounts_current.sql b/models/marts/ledger_current_state/accounts_current.sql index cd8409a..35fea3f 100644 --- a/models/marts/ledger_current_state/accounts_current.sql +++ b/models/marts/ledger_current_state/accounts_current.sql @@ -39,13 +39,13 @@ with , a.batch_run_date , a.batch_insert_ts , row_number() - over ( - partition by a.account_id - order by - a.last_modified_ledger desc - , a.ledger_entry_change desc - ) - as row_nr + over ( + partition by a.account_id + order by + a.last_modified_ledger desc + , a.ledger_entry_change desc + ) + as row_nr from {{ ref('stg_accounts') }} as a join {{ ref('stg_history_ledgers') }} as l on a.last_modified_ledger = l.sequence diff --git a/models/marts/ledger_current_state/liquidity_pools_current.sql b/models/marts/ledger_current_state/liquidity_pools_current.sql index 7d8526b..7ebb4c3 100644 --- a/models/marts/ledger_current_state/liquidity_pools_current.sql +++ b/models/marts/ledger_current_state/liquidity_pools_current.sql @@ -38,10 +38,10 @@ with , lp.batch_run_date , lp.batch_insert_ts , row_number() - over ( - partition by lp.liquidity_pool_id - order by lp.last_modified_ledger desc, lp.ledger_entry_change desc - ) as row_nr + over ( + partition by lp.liquidity_pool_id + order by lp.last_modified_ledger desc, lp.ledger_entry_change desc + ) as row_nr from {{ ref('stg_liquidity_pools') }} as lp join {{ ref('stg_history_ledgers') }} as l on lp.last_modified_ledger = l.sequence diff --git a/models/marts/ledger_current_state/offers_current.sql b/models/marts/ledger_current_state/offers_current.sql index baf398c..8b406dc 100644 --- a/models/marts/ledger_current_state/offers_current.sql +++ b/models/marts/ledger_current_state/offers_current.sql @@ -32,10 +32,10 @@ with , o.batch_run_date , o.batch_insert_ts , row_number() - over ( - partition by o.offer_id - order by o.last_modified_ledger desc, o.ledger_entry_change desc - ) as row_nr + over ( + partition by o.offer_id + order by o.last_modified_ledger desc, o.ledger_entry_change desc + ) as row_nr from {{ ref('stg_offers') }} as o join {{ ref('stg_history_ledgers') }} as l on o.last_modified_ledger = l.sequence diff --git a/models/marts/ledger_current_state/trust_lines_current.sql b/models/marts/ledger_current_state/trust_lines_current.sql index b4962d2..35b32de 100644 --- a/models/marts/ledger_current_state/trust_lines_current.sql +++ b/models/marts/ledger_current_state/trust_lines_current.sql @@ -35,10 +35,10 @@ with , tl.batch_run_date , tl.batch_insert_ts , row_number() - over ( - partition by tl.account_id, tl.asset_code, tl.asset_issuer, tl.liquidity_pool_id - order by tl.last_modified_ledger desc, tl.ledger_entry_change desc - ) as row_nr + over ( + partition by tl.account_id, tl.asset_code, tl.asset_issuer, tl.liquidity_pool_id + order by tl.last_modified_ledger desc, tl.ledger_entry_change desc + ) as row_nr from {{ ref('stg_trust_lines') }} as tl join {{ ref('stg_history_ledgers') }} as l on tl.last_modified_ledger = l.sequence diff --git a/models/marts/ledger_current_state/ttl_current.sql b/models/marts/ledger_current_state/ttl_current.sql index ebe297a..50d410a 100644 --- a/models/marts/ledger_current_state/ttl_current.sql +++ b/models/marts/ledger_current_state/ttl_current.sql @@ -23,10 +23,10 @@ with , ttl.batch_run_date , ttl.batch_insert_ts , row_number() - over ( - partition by ttl.key_hash - order by ttl.last_modified_ledger desc, ledger_entry_change - ) as rn + over ( + partition by ttl.key_hash + order by ttl.last_modified_ledger desc, ledger_entry_change + ) as rn from {{ ref('stg_ttl') }} as ttl {% if is_incremental() %} -- limit the number of partitions fetched incrementally diff --git a/models/staging/stg_account_signers.sql b/models/staging/stg_account_signers.sql index 78549f0..67350cf 100644 --- a/models/staging/stg_account_signers.sql +++ b/models/staging/stg_account_signers.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'account_signers')}} + from {{ source('crypto_stellar', 'account_signers') }} ) , account_signers as ( diff --git a/models/staging/stg_accounts.sql b/models/staging/stg_accounts.sql index ee3bfff..81b35b2 100644 --- a/models/staging/stg_accounts.sql +++ b/models/staging/stg_accounts.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'accounts')}} + from {{ source('crypto_stellar', 'accounts') }} ) , accounts as ( diff --git a/models/staging/stg_claimable_balances.sql b/models/staging/stg_claimable_balances.sql index efd3141..12565ca 100644 --- a/models/staging/stg_claimable_balances.sql +++ b/models/staging/stg_claimable_balances.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'claimable_balances')}} + from {{ source('crypto_stellar', 'claimable_balances') }} ) , claimable_balance as ( diff --git a/models/staging/stg_contract_data.sql b/models/staging/stg_contract_data.sql index 94642cd..12af69d 100644 --- a/models/staging/stg_contract_data.sql +++ b/models/staging/stg_contract_data.sql @@ -32,4 +32,4 @@ with ) select * -from raw_table +from contract_data diff --git a/models/staging/stg_history_effects.sql b/models/staging/stg_history_effects.sql index 3c861bd..4d9d615 100644 --- a/models/staging/stg_history_effects.sql +++ b/models/staging/stg_history_effects.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'history_effects')}} + from {{ source('crypto_stellar', 'history_effects') }} ) , history_effects as ( diff --git a/models/staging/stg_history_ledgers.sql b/models/staging/stg_history_ledgers.sql index 72659ce..4d338ab 100644 --- a/models/staging/stg_history_ledgers.sql +++ b/models/staging/stg_history_ledgers.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'history_ledgers')}} + from {{ source('crypto_stellar', 'history_ledgers') }} ) , history_ledgers as ( diff --git a/models/staging/stg_history_operations.sql b/models/staging/stg_history_operations.sql index a0397c6..6a4736e 100644 --- a/models/staging/stg_history_operations.sql +++ b/models/staging/stg_history_operations.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'history_operations')}} + from {{ source('crypto_stellar', 'history_operations') }} ) , history_operations as ( diff --git a/models/staging/stg_history_trades.sql b/models/staging/stg_history_trades.sql index 48bc089..eae90ca 100644 --- a/models/staging/stg_history_trades.sql +++ b/models/staging/stg_history_trades.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'history_trades')}} + from {{ source('crypto_stellar', 'history_trades') }} ) , history_trades as ( diff --git a/models/staging/stg_history_transactions.sql b/models/staging/stg_history_transactions.sql index 546523d..71a475f 100644 --- a/models/staging/stg_history_transactions.sql +++ b/models/staging/stg_history_transactions.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'history_transactions')}} + from {{ source('crypto_stellar', 'history_transactions') }} ) , history_transactions as ( @@ -45,7 +45,7 @@ with , closed_at , batch_id , batch_run_date - , batch_insert_ts + , batch_insert_ts from raw_table ) diff --git a/models/staging/stg_liquidity_pools.sql b/models/staging/stg_liquidity_pools.sql index 30ac477..11810c2 100644 --- a/models/staging/stg_liquidity_pools.sql +++ b/models/staging/stg_liquidity_pools.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'liquidity_pools')}} + from {{ source('crypto_stellar', 'liquidity_pools') }} ) , liquidity_pool as ( diff --git a/models/staging/stg_offers.sql b/models/staging/stg_offers.sql index 098e871..4bd2702 100644 --- a/models/staging/stg_offers.sql +++ b/models/staging/stg_offers.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'offers')}} + from {{ source('crypto_stellar', 'offers') }} ) , offers as ( diff --git a/models/staging/stg_trust_lines.sql b/models/staging/stg_trust_lines.sql index e9d1e46..2ca8556 100644 --- a/models/staging/stg_trust_lines.sql +++ b/models/staging/stg_trust_lines.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'trust_lines')}} + from {{ source('crypto_stellar', 'trust_lines') }} ) , trust_lines as ( diff --git a/models/staging/stg_ttl.sql b/models/staging/stg_ttl.sql index 5ef6dcb..4076476 100644 --- a/models/staging/stg_ttl.sql +++ b/models/staging/stg_ttl.sql @@ -6,7 +6,7 @@ with raw_table as ( select * - from {{ source('crypto_stellar', 'ttl')}} + from {{ source('crypto_stellar', 'ttl') }} ) , ttl as (