diff --git a/dbt_subprojects/daily_spellbook/macros/project/lifi/lifi_extract_bridge_data_macro.sql b/dbt_subprojects/daily_spellbook/macros/project/lifi/lifi_extract_bridge_data_macro.sql new file mode 100644 index 00000000000..b1e3162564d --- /dev/null +++ b/dbt_subprojects/daily_spellbook/macros/project/lifi/lifi_extract_bridge_data_macro.sql @@ -0,0 +1,30 @@ +{% macro lifi_extract_bridge_data(blockchain) %} + +{% set bridge_data_fields = [ + 'transactionId', + 'bridge', + 'integrator', + 'referrer', + 'sendingAssetId', + 'receiver', + 'minAmount', + 'destinationChainId' +] %} + +select + contract_address, + evt_tx_hash, + evt_index, + evt_block_time, + evt_block_number, + {% for field in bridge_data_fields %} + json_extract_scalar(bridgeData, '$.{{ field }}') as {{ field }}, + {% endfor %} + '{{ blockchain }}' as source_chain, + {{ dbt_utils.generate_surrogate_key(['evt_tx_hash', 'evt_index']) }} as transfer_id +from {{ source('lifi_' ~ blockchain, 'LiFiDiamond_v2_evt_LiFiTransferStarted') }} +{% if is_incremental() %} +where {{ incremental_predicate('evt_block_time') }} +{% endif %} + +{% endmacro %} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/LiFiDiamond_v2_evt_LiFiTransferStarted.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/LiFiDiamond_v2_evt_LiFiTransferStarted.sql index 7eead433b77..1f1975f80b5 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/lifi/LiFiDiamond_v2_evt_LiFiTransferStarted.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/LiFiDiamond_v2_evt_LiFiTransferStarted.sql @@ -1,8 +1,8 @@ {{ config( schema = 'lifi', - alias = 'transfers', - materialized = 'incremental' + alias = 'LiFiDiamond_v2_evt_LiFiTransferStarted', + materialized = 'view', ) }} @@ -33,11 +33,9 @@ with chain_transfers as ( minAmount, destinationChainId, source_chain, - sender + tx_from, + transfer_id from {{ ref('lifi_' ~ chain ~ '_transfers') }} - {% if is_incremental() %} - where evt_block_time >= date_trunc('day', now() - interval '7' day) - {% endif %} {% if not loop.last %} union all @@ -60,5 +58,6 @@ select minAmount, destinationChainId, source_chain, - sender + tx_from, + transfer_id from chain_transfers \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/arbitrum/lifi_arbitrum_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/arbitrum/lifi_arbitrum_transfers.sql index 661b8ef25b3..ef4bb0d5396 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/lifi/arbitrum/lifi_arbitrum_transfers.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/arbitrum/lifi_arbitrum_transfers.sql @@ -1,38 +1,22 @@ {{ config( schema = 'lifi_arbitrum', alias = 'transfers', - materialized = 'view' + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] ) }} with source_data as ( - select - contract_address, - evt_tx_hash, - evt_index, - evt_block_time, - evt_block_number, - json_extract_scalar(bridgeData, '$.transactionId') as transactionId, - json_extract_scalar(bridgeData, '$.bridge') as bridge, - json_extract_scalar(bridgeData, '$.integrator') as integrator, - json_extract_scalar(bridgeData, '$.referrer') as referrer, - json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId, - json_extract_scalar(bridgeData, '$.receiver') as receiver, - json_extract_scalar(bridgeData, '$.minAmount') as minAmount, - json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId, - 'arbitrum' as source_chain - from {{ source('lifi_arbitrum', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }} -), -transactions as ( - select - "from" as sender, - hash as tx_hash - from {{ source('arbitrum', 'transactions') }} + {{ lifi_extract_bridge_data('arbitrum') }} ) -select - s.*, - t.sender -from source_data s -inner join transactions t - on s.evt_tx_hash = t.tx_hash +{{ + add_tx_columns( + model_cte = 'source_data' + , blockchain = 'arbitrum' + , columns = ['from'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/avalanche/lifi_avalanche_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/avalanche/lifi_avalanche_transfers.sql index 2d4e11862c6..fab3fb48b98 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/lifi/avalanche/lifi_avalanche_transfers.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/avalanche/lifi_avalanche_transfers.sql @@ -1,38 +1,22 @@ {{ config( schema = 'lifi_avalanche', alias = 'transfers', - materialized = 'view' + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] ) }} with source_data as ( - select - contract_address, - evt_tx_hash, - evt_index, - evt_block_time, - evt_block_number, - json_extract_scalar(bridgeData, '$.transactionId') as transactionId, - json_extract_scalar(bridgeData, '$.bridge') as bridge, - json_extract_scalar(bridgeData, '$.integrator') as integrator, - json_extract_scalar(bridgeData, '$.referrer') as referrer, - json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId, - json_extract_scalar(bridgeData, '$.receiver') as receiver, - json_extract_scalar(bridgeData, '$.minAmount') as minAmount, - json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId, - 'avalanche' as source_chain - from {{ source('lifi_avalanche_c', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }} -), -transactions as ( - select - "from" as sender, - hash as tx_hash - from {{ source('avalanche_c', 'transactions') }} + {{ lifi_extract_bridge_data('avalanche') }} ) -select - s.*, - t.sender -from source_data s -inner join transactions t - on s.evt_tx_hash = t.tx_hash +{{ + add_tx_columns( + model_cte = 'source_data' + , blockchain = 'avalanche' + , columns = ['from'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/bnb/lifi_bnb_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/bnb/lifi_bnb_transfers.sql index af8ca6eb0b8..c45a58eb6ba 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/lifi/bnb/lifi_bnb_transfers.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/bnb/lifi_bnb_transfers.sql @@ -1,38 +1,22 @@ {{ config( schema = 'lifi_bnb', alias = 'transfers', - materialized = 'view' + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] ) }} with source_data as ( - select - contract_address, - evt_tx_hash, - evt_index, - evt_block_time, - evt_block_number, - json_extract_scalar(bridgeData, '$.transactionId') as transactionId, - json_extract_scalar(bridgeData, '$.bridge') as bridge, - json_extract_scalar(bridgeData, '$.integrator') as integrator, - json_extract_scalar(bridgeData, '$.referrer') as referrer, - json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId, - json_extract_scalar(bridgeData, '$.receiver') as receiver, - json_extract_scalar(bridgeData, '$.minAmount') as minAmount, - json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId, - 'bnb' as source_chain - from {{ source('lifi_bnb', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }} -), -transactions as ( - select - "from" as sender, - hash as tx_hash - from {{ source('bnb', 'transactions') }} + {{ lifi_extract_bridge_data('bnb') }} ) -select - s.*, - t.sender -from source_data s -inner join transactions t - on s.evt_tx_hash = t.tx_hash +{{ + add_tx_columns( + model_cte = 'source_data' + , blockchain = 'bnb' + , columns = ['from'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/ethereum/lifi_ethereum_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/ethereum/lifi_ethereum_transfers.sql index a1c36d2da7d..9b866e5b69b 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/lifi/ethereum/lifi_ethereum_transfers.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/ethereum/lifi_ethereum_transfers.sql @@ -1,38 +1,22 @@ {{ config( schema = 'lifi_ethereum', alias = 'transfers', - materialized = 'view' + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] ) }} with source_data as ( - select - contract_address, - evt_tx_hash, - evt_index, - evt_block_time, - evt_block_number, - json_extract_scalar(bridgeData, '$.transactionId') as transactionId, - json_extract_scalar(bridgeData, '$.bridge') as bridge, - json_extract_scalar(bridgeData, '$.integrator') as integrator, - json_extract_scalar(bridgeData, '$.referrer') as referrer, - json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId, - json_extract_scalar(bridgeData, '$.receiver') as receiver, - json_extract_scalar(bridgeData, '$.minAmount') as minAmount, - json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId, - 'ethereum' as source_chain - from {{ source('lifi_ethereum', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }} -), -transactions as ( - select - "from" as sender, - hash as tx_hash - from {{ source('ethereum', 'transactions') }} + {{ lifi_extract_bridge_data('ethereum') }} ) -select - s.*, - t.sender -from source_data s -inner join transactions t - on s.evt_tx_hash = t.tx_hash +{{ + add_tx_columns( + model_cte = 'source_data' + , blockchain = 'ethereum' + , columns = ['from'] + ) +}} \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/fantom/lifi_fantom_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/fantom/lifi_fantom_transfers.sql index 31fe5c038f2..816200bffa5 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/lifi/fantom/lifi_fantom_transfers.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/fantom/lifi_fantom_transfers.sql @@ -1,38 +1,22 @@ {{ config( schema = 'lifi_fantom', alias = 'transfers', - materialized = 'view' + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] ) }} with source_data as ( - select - contract_address, - evt_tx_hash, - evt_index, - evt_block_time, - evt_block_number, - json_extract_scalar(bridgeData, '$.transactionId') as transactionId, - json_extract_scalar(bridgeData, '$.bridge') as bridge, - json_extract_scalar(bridgeData, '$.integrator') as integrator, - json_extract_scalar(bridgeData, '$.referrer') as referrer, - json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId, - json_extract_scalar(bridgeData, '$.receiver') as receiver, - json_extract_scalar(bridgeData, '$.minAmount') as minAmount, - json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId, - 'fantom' as source_chain - from {{ source('lifi_fantom', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }} -), -transactions as ( - select - "from" as sender, - hash as tx_hash - from {{ source('fantom', 'transactions') }} + {{ lifi_extract_bridge_data('fantom') }} ) -select - s.*, - t.sender -from source_data s -inner join transactions t - on s.evt_tx_hash = t.tx_hash +{{ + add_tx_columns( + model_cte = 'source_data' + , blockchain = 'fantom' + , columns = ['from'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/gnosis/lifi_gnosis_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/gnosis/lifi_gnosis_transfers.sql index 4b1748fe7bb..f1baab9df8e 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/lifi/gnosis/lifi_gnosis_transfers.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/gnosis/lifi_gnosis_transfers.sql @@ -1,38 +1,22 @@ {{ config( schema = 'lifi_gnosis', alias = 'transfers', - materialized = 'view' + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] ) }} with source_data as ( - select - contract_address, - evt_tx_hash, - evt_index, - evt_block_time, - evt_block_number, - json_extract_scalar(bridgeData, '$.transactionId') as transactionId, - json_extract_scalar(bridgeData, '$.bridge') as bridge, - json_extract_scalar(bridgeData, '$.integrator') as integrator, - json_extract_scalar(bridgeData, '$.referrer') as referrer, - json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId, - json_extract_scalar(bridgeData, '$.receiver') as receiver, - json_extract_scalar(bridgeData, '$.minAmount') as minAmount, - json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId, - 'gnosis' as source_chain - from {{ source('lifi_gnosis', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }} -), -transactions as ( - select - "from" as sender, - hash as tx_hash - from {{ source('gnosis', 'transactions') }} + {{ lifi_extract_bridge_data('gnosis') }} ) -select - s.*, - t.sender -from source_data s -inner join transactions t - on s.evt_tx_hash = t.tx_hash +{{ + add_tx_columns( + model_cte = 'source_data' + , blockchain = 'gnosis' + , columns = ['from'] + ) +}} diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/schema.yml b/dbt_subprojects/daily_spellbook/models/_projects/lifi/schema.yml index 8718028c92f..8fac0f5c181 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/lifi/schema.yml +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/schema.yml @@ -5,35 +5,40 @@ models: description: "Ethereum LiFi transfer events" columns: &common_columns - name: contract_address - description: "Contract address of the LiFi Diamond" + description: "Contract address of LiFi Diamond" - name: evt_tx_hash description: "Transaction hash" - name: evt_index - description: "Event index in the transaction" + description: "Event index" - name: evt_block_time - description: "Block timestamp of the event" + description: "Timestamp of the block" - name: evt_block_number - description: "Block number of the event" + description: "Block number" - name: transactionId - description: "Unique identifier for the transfer transaction" + description: "Unique transaction ID from LiFi" - name: bridge - description: "Bridge protocol used" + description: "Bridge used for the transfer" - name: integrator - description: "Integration partner" + description: "Integrator of the transfer" - name: referrer - description: "Referrer address" + description: "Referrer of the transfer" - name: sendingAssetId - description: "Source token address" + description: "Asset being transferred" - name: receiver - description: "Recipient address" + description: "Address receiving the transfer" - name: minAmount description: "Minimum amount to be received" - name: destinationChainId description: "Target chain ID for the transfer" - name: source_chain description: "Source blockchain" - - name: sender + - name: tx_from description: "Address that initiated the transaction" + - name: transfer_id + description: "Unique identifier for the transfer" + tests: + - unique + - not_null - name: lifi_arbitrum_transfers description: "Arbitrum LiFi transfer events" @@ -44,7 +49,7 @@ models: columns: *common_columns - name: lifi_bnb_transfers - description: "BNB Chain LiFi transfer events" + description: "BNB LiFi transfer events" columns: *common_columns - name: lifi_fantom_transfers @@ -52,19 +57,13 @@ models: columns: *common_columns - name: lifi_gnosis_transfers - description: "Gnosis Chain LiFi transfer events" + description: "Gnosis LiFi transfer events" columns: *common_columns - name: lifi_zksync_transfers description: "zkSync LiFi transfer events" columns: *common_columns - - name: LiFiDiamond_v2_evt_LiFiTransferStarted + - name: lifi_LiFiDiamond_v2_evt_LiFiTransferStarted description: "Combined LiFi transfer events across all supported chains" - columns: *common_columns - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - evt_tx_hash - - evt_index - - source_chain + columns: *common_columns \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/zksync/lifi_zksync_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/zksync/lifi_zksync_transfers.sql index ed2514140ee..f2dae1c9813 100644 --- a/dbt_subprojects/daily_spellbook/models/_projects/lifi/zksync/lifi_zksync_transfers.sql +++ b/dbt_subprojects/daily_spellbook/models/_projects/lifi/zksync/lifi_zksync_transfers.sql @@ -1,38 +1,22 @@ {{ config( schema = 'lifi_zksync', alias = 'transfers', - materialized = 'view' + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['transfer_id'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')] ) }} with source_data as ( - select - contract_address, - evt_tx_hash, - evt_index, - evt_block_time, - evt_block_number, - json_extract_scalar(bridgeData, '$.transactionId') as transactionId, - json_extract_scalar(bridgeData, '$.bridge') as bridge, - json_extract_scalar(bridgeData, '$.integrator') as integrator, - json_extract_scalar(bridgeData, '$.referrer') as referrer, - json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId, - json_extract_scalar(bridgeData, '$.receiver') as receiver, - json_extract_scalar(bridgeData, '$.minAmount') as minAmount, - json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId, - 'zksync' as source_chain - from {{ source('lifi_zksync', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }} -), -transactions as ( - select - "from" as sender, - hash as tx_hash - from {{ source('zksync', 'transactions') }} + {{ lifi_extract_bridge_data('zksync') }} ) -select - s.*, - t.sender -from source_data s -inner join transactions t - on s.evt_tx_hash = t.tx_hash +{{ + add_tx_columns( + model_cte = 'source_data' + , blockchain = 'zksync' + , columns = ['from'] + ) +}}