1. create macro lifi_extract_bridge_data_macro.sql

and apply it in the models 2. update the models' config 3. flip materialization strategy
duneanalytics · Nov 26, 2024 · 8a5a222 · 8a5a222
1 parent 72af65d
commit 8a5a222
Show file tree

Hide file tree

Showing 10 changed files with 147 additions and 231 deletions.
diff --git a/dbt_subprojects/daily_spellbook/macros/project/lifi/lifi_extract_bridge_data_macro.sql b/dbt_subprojects/daily_spellbook/macros/project/lifi/lifi_extract_bridge_data_macro.sql
@@ -0,0 +1,30 @@
+{% macro lifi_extract_bridge_data(blockchain) %}
+
+{% set bridge_data_fields = [
+    'transactionId',
+    'bridge',
+    'integrator',
+    'referrer',
+    'sendingAssetId',
+    'receiver',
+    'minAmount',
+    'destinationChainId'
+] %}
+
+select
+    contract_address,
+    evt_tx_hash,
+    evt_index,
+    evt_block_time,
+    evt_block_number,
+    {% for field in bridge_data_fields %}
+    json_extract_scalar(bridgeData, '$.{{ field }}') as {{ field }},
+    {% endfor %}
+    '{{ blockchain }}' as source_chain,
+    {{ dbt_utils.generate_surrogate_key(['evt_tx_hash', 'evt_index']) }} as transfer_id
+from {{ source('lifi_' ~ blockchain, 'LiFiDiamond_v2_evt_LiFiTransferStarted') }}
+{% if is_incremental() %}
+where {{ incremental_predicate('evt_block_time') }}
+{% endif %}
+
+{% endmacro %}
diff --git a/...projects/daily_spellbook/models/_projects/lifi/LiFiDiamond_v2_evt_LiFiTransferStarted.sql b/...projects/daily_spellbook/models/_projects/lifi/LiFiDiamond_v2_evt_LiFiTransferStarted.sql
@@ -1,8 +1,8 @@
 {{
     config(
         schema = 'lifi',
-        alias = 'transfers',
-        materialized = 'incremental'
+        alias = 'LiFiDiamond_v2_evt_LiFiTransferStarted',
+        materialized = 'view',
     )
 }}
 
@@ -33,11 +33,9 @@ with chain_transfers as (
         minAmount,
         destinationChainId,
         source_chain,
-        sender
+        tx_from,
+        transfer_id
     from {{ ref('lifi_' ~ chain ~ '_transfers') }}
-    {% if is_incremental() %}
-    where evt_block_time >= date_trunc('day', now() - interval '7' day)
-    {% endif %}
 
     {% if not loop.last %}
     union all
@@ -60,5 +58,6 @@ select
     minAmount,
     destinationChainId,
     source_chain,
-    sender
+    tx_from,
+    transfer_id
 from chain_transfers
diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/arbitrum/lifi_arbitrum_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/arbitrum/lifi_arbitrum_transfers.sql
@@ -1,38 +1,22 @@
 {{ config(
     schema = 'lifi_arbitrum',
     alias = 'transfers',
-    materialized = 'view'
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['transfer_id'],
+    incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')]
     )
 }}
 
 with source_data as (
-    select
-        contract_address,
-        evt_tx_hash,
-        evt_index,
-        evt_block_time,
-        evt_block_number,
-        json_extract_scalar(bridgeData, '$.transactionId') as transactionId,
-        json_extract_scalar(bridgeData, '$.bridge') as bridge,
-        json_extract_scalar(bridgeData, '$.integrator') as integrator,
-        json_extract_scalar(bridgeData, '$.referrer') as referrer,
-        json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId,
-        json_extract_scalar(bridgeData, '$.receiver') as receiver,
-        json_extract_scalar(bridgeData, '$.minAmount') as minAmount,
-        json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId,
-        'arbitrum' as source_chain
-    from {{ source('lifi_arbitrum', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }}
-),
-transactions as (
-    select 
-        "from" as sender,
-        hash as tx_hash
-    from {{ source('arbitrum', 'transactions') }}
+    {{ lifi_extract_bridge_data('arbitrum') }}
 )
 
-select 
-    s.*,
-    t.sender
-from source_data s
-inner join transactions t
-    on s.evt_tx_hash = t.tx_hash
+{{
+    add_tx_columns(
+        model_cte = 'source_data'
+        , blockchain = 'arbitrum'
+        , columns = ['from']
+    )
+}}
diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/avalanche/lifi_avalanche_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/avalanche/lifi_avalanche_transfers.sql
@@ -1,38 +1,22 @@
 {{ config(
     schema = 'lifi_avalanche',
     alias = 'transfers',
-    materialized = 'view'
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['transfer_id'],
+    incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')]
     )
 }}
 
 with source_data as (
-    select
-        contract_address,
-        evt_tx_hash,
-        evt_index,
-        evt_block_time,
-        evt_block_number,
-        json_extract_scalar(bridgeData, '$.transactionId') as transactionId,
-        json_extract_scalar(bridgeData, '$.bridge') as bridge,
-        json_extract_scalar(bridgeData, '$.integrator') as integrator,
-        json_extract_scalar(bridgeData, '$.referrer') as referrer,
-        json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId,
-        json_extract_scalar(bridgeData, '$.receiver') as receiver,
-        json_extract_scalar(bridgeData, '$.minAmount') as minAmount,
-        json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId,
-        'avalanche' as source_chain
-    from {{ source('lifi_avalanche_c', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }}
-),
-transactions as (
-    select 
-        "from" as sender,
-        hash as tx_hash
-    from {{ source('avalanche_c', 'transactions') }}
+    {{ lifi_extract_bridge_data('avalanche') }}
 )
 
-select 
-    s.*,
-    t.sender
-from source_data s
-inner join transactions t
-    on s.evt_tx_hash = t.tx_hash
+{{
+    add_tx_columns(
+        model_cte = 'source_data'
+        , blockchain = 'avalanche'
+        , columns = ['from']
+    )
+}}
diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/bnb/lifi_bnb_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/bnb/lifi_bnb_transfers.sql
@@ -1,38 +1,22 @@
 {{ config(
     schema = 'lifi_bnb',
     alias = 'transfers',
-    materialized = 'view'
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['transfer_id'],
+    incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')]
     )
 }}
 
 with source_data as (
-    select
-        contract_address,
-        evt_tx_hash,
-        evt_index,
-        evt_block_time,
-        evt_block_number,
-        json_extract_scalar(bridgeData, '$.transactionId') as transactionId,
-        json_extract_scalar(bridgeData, '$.bridge') as bridge,
-        json_extract_scalar(bridgeData, '$.integrator') as integrator,
-        json_extract_scalar(bridgeData, '$.referrer') as referrer,
-        json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId,
-        json_extract_scalar(bridgeData, '$.receiver') as receiver,
-        json_extract_scalar(bridgeData, '$.minAmount') as minAmount,
-        json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId,
-        'bnb' as source_chain
-    from {{ source('lifi_bnb', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }}
-),
-transactions as (
-    select 
-        "from" as sender,
-        hash as tx_hash
-    from {{ source('bnb', 'transactions') }}
+    {{ lifi_extract_bridge_data('bnb') }}
 )
 
-select 
-    s.*,
-    t.sender
-from source_data s
-inner join transactions t
-    on s.evt_tx_hash = t.tx_hash
+{{
+    add_tx_columns(
+        model_cte = 'source_data'
+        , blockchain = 'bnb'
+        , columns = ['from']
+    )
+}}
diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/ethereum/lifi_ethereum_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/ethereum/lifi_ethereum_transfers.sql
@@ -1,38 +1,22 @@
 {{ config(
     schema = 'lifi_ethereum',
     alias = 'transfers',
-    materialized = 'view'
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['transfer_id'],
+    incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')]
     )
 }}
 
 with source_data as (
-    select
-        contract_address,
-        evt_tx_hash,
-        evt_index,
-        evt_block_time,
-        evt_block_number,
-        json_extract_scalar(bridgeData, '$.transactionId') as transactionId,
-        json_extract_scalar(bridgeData, '$.bridge') as bridge,
-        json_extract_scalar(bridgeData, '$.integrator') as integrator,
-        json_extract_scalar(bridgeData, '$.referrer') as referrer,
-        json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId,
-        json_extract_scalar(bridgeData, '$.receiver') as receiver,
-        json_extract_scalar(bridgeData, '$.minAmount') as minAmount,
-        json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId,
-        'ethereum' as source_chain
-    from {{ source('lifi_ethereum', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }}
-),
-transactions as (
-    select 
-        "from" as sender,
-        hash as tx_hash
-    from {{ source('ethereum', 'transactions') }}
+    {{ lifi_extract_bridge_data('ethereum') }}
 )
 
-select 
-    s.*,
-    t.sender
-from source_data s
-inner join transactions t
-    on s.evt_tx_hash = t.tx_hash
+{{
+    add_tx_columns(
+        model_cte = 'source_data'
+        , blockchain = 'ethereum'
+        , columns = ['from']
+    )
+}}
diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/fantom/lifi_fantom_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/fantom/lifi_fantom_transfers.sql
@@ -1,38 +1,22 @@
 {{ config(
     schema = 'lifi_fantom',
     alias = 'transfers',
-    materialized = 'view'
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['transfer_id'],
+    incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')]
     )
 }}
 
 with source_data as (
-    select
-        contract_address,
-        evt_tx_hash,
-        evt_index,
-        evt_block_time,
-        evt_block_number,
-        json_extract_scalar(bridgeData, '$.transactionId') as transactionId,
-        json_extract_scalar(bridgeData, '$.bridge') as bridge,
-        json_extract_scalar(bridgeData, '$.integrator') as integrator,
-        json_extract_scalar(bridgeData, '$.referrer') as referrer,
-        json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId,
-        json_extract_scalar(bridgeData, '$.receiver') as receiver,
-        json_extract_scalar(bridgeData, '$.minAmount') as minAmount,
-        json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId,
-        'fantom' as source_chain
-    from {{ source('lifi_fantom', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }}
-),
-transactions as (
-    select 
-        "from" as sender,
-        hash as tx_hash
-    from {{ source('fantom', 'transactions') }}
+    {{ lifi_extract_bridge_data('fantom') }}
 )
 
-select 
-    s.*,
-    t.sender
-from source_data s
-inner join transactions t
-    on s.evt_tx_hash = t.tx_hash
+{{
+    add_tx_columns(
+        model_cte = 'source_data'
+        , blockchain = 'fantom'
+        , columns = ['from']
+    )
+}}
diff --git a/dbt_subprojects/daily_spellbook/models/_projects/lifi/gnosis/lifi_gnosis_transfers.sql b/dbt_subprojects/daily_spellbook/models/_projects/lifi/gnosis/lifi_gnosis_transfers.sql
@@ -1,38 +1,22 @@
 {{ config(
     schema = 'lifi_gnosis',
     alias = 'transfers',
-    materialized = 'view'
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['transfer_id'],
+    incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')]
     )
 }}
 
 with source_data as (
-    select
-        contract_address,
-        evt_tx_hash,
-        evt_index,
-        evt_block_time,
-        evt_block_number,
-        json_extract_scalar(bridgeData, '$.transactionId') as transactionId,
-        json_extract_scalar(bridgeData, '$.bridge') as bridge,
-        json_extract_scalar(bridgeData, '$.integrator') as integrator,
-        json_extract_scalar(bridgeData, '$.referrer') as referrer,
-        json_extract_scalar(bridgeData, '$.sendingAssetId') as sendingAssetId,
-        json_extract_scalar(bridgeData, '$.receiver') as receiver,
-        json_extract_scalar(bridgeData, '$.minAmount') as minAmount,
-        json_extract_scalar(bridgeData, '$.destinationChainId') as destinationChainId,
-        'gnosis' as source_chain
-    from {{ source('lifi_gnosis', 'LiFiDiamond_v2_evt_LiFiTransferStarted') }}
-),
-transactions as (
-    select 
-        "from" as sender,
-        hash as tx_hash
-    from {{ source('gnosis', 'transactions') }}
+    {{ lifi_extract_bridge_data('gnosis') }}
 )
 
-select 
-    s.*,
-    t.sender
-from source_data s
-inner join transactions t
-    on s.evt_tx_hash = t.tx_hash
+{{
+    add_tx_columns(
+        model_cte = 'source_data'
+        , blockchain = 'gnosis'
+        , columns = ['from']
+    )
+}}