Dune index: cleanup models

dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_fees_evm.sql

@@ -0,0 +1,18 @@
+{% macro metrics_fees_evm(blockchain) %}
+
+select
+    blockchain
+    , block_date
+    , sum(tx_fee_usd) as gas_fees_usd
+from
+    {{ source('gas', 'fees') }}
+where blockchain = '{{blockchain}}'
+{% if is_incremental() %}
+and
+    {{ incremental_predicate('block_date') }}
+{% endif %}
+group by
+    blockchain
+    ,block_date
+
+{% endmacro %}

dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_transactions_evm.sql

@@ -0,0 +1,19 @@
+{% macro metrics_transactions_evm(blockchain) %}
+
+select
+        blockchain
+        , block_date
+        , approx_distinct(tx_hash) as tx_count --max 2% error, which is fine
+from
+    {{ source('tokens', 'transfers') }}
+where
+    blockchain = '{{ blockchain }}'
+    and amount_usd >=1
+    {% if is_incremental() %}
+    and {{ incremental_predicate('block_date') }}
+    {% endif %}
+group by
+    blockchain
+    , block_date
+
+{% endmacro %}

dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_transfers_daily.sql → dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_transfers_evm.sql

@@ -1,13 +1,4 @@
-{{ config(
-        schema = 'metrics'
-        , alias = 'net_transfers_daily'
-        , materialized = 'incremental'
-        , file_format = 'delta'
-        , incremental_strategy = 'merge'
-        , unique_key = ['blockchain', 'block_date']
-        , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')]
-        )
-}}
+{% macro metrics_transfers_evm(blockchain) %}
 
 with raw_transfers as (
     select
@@ -19,7 +10,7 @@ with raw_transfers as (
     from
         {{ source('tokens', 'transfers') }}
     where
-        1 = 1
+        blockchain = '{{blockchain}}'
         {% if is_incremental() %}
         and {{ incremental_predicate('block_date') }}
         {% endif %}
@@ -40,7 +31,7 @@ with raw_transfers as (
     from
         {{ source('tokens', 'transfers') }}
     where
-        1 = 1
+        blockchain = '{{blockchain}}'
         {% if is_incremental() %}
         and {{ incremental_predicate('block_date') }}
         {% endif %}
@@ -60,6 +51,7 @@ with raw_transfers as (
     inner join
         {{ source('labels', 'owner_details') }} as od
         on oa.owner_key = od.owner_key
+    where oa.blockchain = '{{blockchain}}'
 ), transfers_amount as (
     select
         t.blockchain
@@ -108,3 +100,5 @@ where
 group by
     blockchain
     , block_date
+
+{% endmacro %}

dbt_subprojects/daily_spellbook/models/_metrics/dune_index/_schema.yml

@@ -4,7 +4,7 @@ models:
   - name: metrics_dune_index_daily
     meta:
       sector: metrics
-      contributors: jeff-dude
+      contributors: jeff-dude, 0xRob
     config:
       tags: ['metrics', 'dune', 'index', 'daily']
     description: "Combine transactions, transfers and fees index values to get a daily dune index value"
@@ -13,10 +13,11 @@ models:
           combination_of_columns:
             - blockchain
             - block_date
+
   - name: metrics_dune_index_stats
     meta:
       sector: metrics
       contributors: jeff-dude
     config:
-      tags: ['metrics', 'dune', 'index', 'daily']
-    description: "View containing various time aggregations of the dune index"
+      tags: [ 'metrics', 'dune', 'index', 'daily' ]
+    description: "View containing various time aggregations of the dune index"

dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_daily.sql

@@ -1,21 +1,71 @@
 {{ config(
         schema = 'metrics'
         , alias = 'dune_index_daily'
-        , materialized = 'view'
+        , materialized = 'incremental'
+        , file_format = 'delta'
+        , incremental_strategy = 'merge'
+        , unique_key = ['blockchain', 'block_date']
+        , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')]
         )
 }}
 
+{% set baseline_date = '2018-01-01' %}
+{% set start_date = '2015-08-21' %}
+
+with
+fees as (
+    select
+        blockchain
+        , block_date
+        , gas_fees_usd
+        , (gas_fees_usd / (select sum(gas_fees_usd) from {{ ref('metrics_gas_fees_daily') }} where block_date = date '{{ baseline_date }}')) * 10 as fees_index
+    from
+        {{ ref('metrics_gas_fees_daily') }}
+    where
+        block_date >= date '{{ start_date }}'
+        {% if is_incremental() %}
+        and {{ incremental_predicate('block_date') }}
+        {% endif %}
+),
+transactions as (
+    select
+        blockchain
+        , block_date
+        , tx_count
+        , (tx_count / cast((select sum(tx_count) from {{ ref('metrics_transactions_daily') }} where block_date = date '{{ baseline_date }}') as double)) * 10 as tx_index
+    from
+        {{ ref('metrics_transactions_daily') }}
+    where
+        block_date >= date '{{ start_date }}'
+        {% if is_incremental() %}
+        and {{ incremental_predicate('block_date') }}
+        {% endif %}
+)
+,transfers as (
+    select
+        blockchain
+        , block_date
+        , net_transfer_amount_usd
+        , (net_transfer_amount_usd / cast((select sum(net_transfer_amount_usd) from {{ ref('metrics_transfers_daily') }} where block_date = date '{{ baseline_date }}') as double)) * 10 as transfers_index
+    from
+        {{ ref('metrics_transfers_daily') }}
+    where
+        block_date >= date '{{ start_date }}'
+        {% if is_incremental() %}
+        and {{ incremental_predicate('block_date') }}
+        {% endif %}
+)
+
 select
     blockchain
     , block_date
-    , coalesce(f.fees_index,0) as fees_index
-    , coalesce(tr.transfers_index,0) as transfers_index
-    , coalesce(tx.tx_index,0) as tx_index
-    , coalesce(f.fees_index,0)*0.45 + coalesce(tr.transfers_index,0)*0.45 + coalesce(tx.tx_index,0)*0.10 as dune_index
-from {{ ref('metrics_fees_index_daily') }} as f
-left join
-    {{ ref('metrics_transfers_index_daily') }} as tr
-    using (blockchain, block_date)
-left join
-    {{ ref('metrics_transactions_index_daily') }} as tx
-     using (blockchain, block_date)
+    , 0.45 * coalesce(fees_index,0) + 0.45 * coalesce(transfers_index,0) + 0.10 * coalesce(tx_index,0) as dune_index
+    , coalesce(fees_index,0) as fees_index
+    , coalesce(transfers_index,0) as transfers_index
+    , coalesce(tx_index,0) as tx_index
+    , gas_fees_usd
+    , tx_count
+    , net_transfer_amount_usd
+from fees
+left join transfers using (blockchain, block_date)
+left join transactions using (blockchain, block_date)

dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_stats.sql

@@ -157,4 +157,4 @@ from
 inner join weekly_stats as w
     on d.blockchain = w.blockchain
 inner join monthly_stats as m
-    on d.blockchain = m.blockchain
+    on d.blockchain = m.blockchain

dbt_subprojects/daily_spellbook/models/_metrics/fees/_schema.yml

@@ -8,27 +8,11 @@ models:
     config:
       tags: ['metrics', 'fees', 'gas', 'daily']
     description: "Sum of total fees spent per day across all chains available in gas.fees and gas_solana.fees tables"
-    data_tests:
-      - dbt_utils.unique_combination_of_columns:
-          combination_of_columns:
-            - blockchain
-            - block_date
+
   - name: metrics_gas_fees_stats
     meta:
       sector: metrics
       contributors: jeff-dude
     config:
       tags: ['metrics', 'fees', 'gas']
-    description: "View of gas fees per blockchain aggregated to various levels. The goal is to output one row per chain with stats availble for use in counter visuals."
-  - name: metrics_fees_index_daily
-    meta:
-      sector: metrics
-      contributors: jeff-dude
-    config:
-      tags: ['metrics', 'fees', 'gas', 'index', 'daily']
-    description: "Each day, per chain, compare the adoption of activity based on total gas fees relative to the baseline expectation"
-    data_tests:
-      - dbt_utils.unique_combination_of_columns:
-          combination_of_columns:
-            - blockchain
-            - block_date
+    description: "View of gas fees per blockchain aggregated to various levels. The goal is to output one row per chain."

dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/bitcoin/_schema.yml

@@ -0,0 +1,15 @@
+version: 2
+
+models:
+  - name: metrics_bitcoin_gas_fees_daily
+    meta:
+      sector: metrics
+      contributors: jeff-dude
+    config:
+      tags: ['metrics', 'fees', 'gas', 'daily']
+    description: "Sum of total fees spent per day"
+    data_tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - blockchain
+            - block_date

dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/bitcoin/metrics_bitcoin_gas_fees_daily.sql

@@ -0,0 +1,46 @@
+{{ config(
+        schema = 'metrics_bitcoin'
+        , alias = 'gas_fees_daily'
+        , materialized = 'incremental'
+        , file_format = 'delta'
+        , incremental_strategy = 'merge'
+        , unique_key = ['blockchain', 'block_date']
+        , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')]
+        )
+}}
+
+with prices as (
+        select
+            day
+            , price
+        from
+            {{ source('prices', 'usd_daily') }}
+        where
+            symbol = 'BTC'
+            and blockchain is null
+            {% if is_incremental() %}
+            and {{ incremental_predicate('day') }}
+            {% endif %}
+)
+, bitcoin_fees as (
+        select
+            date as block_date
+            , sum(total_fees) as daily_fee
+        from
+            {{ source('bitcoin', 'blocks') }}
+        where
+            date < cast(date_trunc('day', now()) as date) --exclude current day to match prices.usd_daily
+            {% if is_incremental() %}
+            and {{ incremental_predicate('date') }}
+            {% endif %}
+        group by
+            date
+)
+select
+    'bitcoin' as blockchain
+    , block_date
+    , (daily_fee * price) as gas_fees_usd
+from
+    bitcoin_fees
+inner join prices
+    on block_date = day