Allow override of incremental look-back period

models/dbt_queries.sql

@@ -51,7 +51,7 @@ select
 from {{ ref('query_history_enriched') }}
 where dbt_metadata is not null
     {% if is_incremental() %}
-        -- Conservatively re-process the last 3 days to account for late arriving rates data
-        -- which changes the cost per query
-        and end_time > (select dateadd(day, -3, max(end_time)) from {{ this }})
+        -- Conservatively re-process the last 3 days to account for late arriving rates data which changes the cost per query. 
+        -- Allow an override from project variable
+        and end_time > (select dateadd(day, -{{ var('dbt_snowflake_monitoring_incremental_days', '3') }}, max(end_time)) from {{ this }})
     {% endif %}

models/dbt_queries.yml

@@ -2,7 +2,7 @@ version: 2
 
 models:
   - name: dbt_queries
-    description: Filtered version of query_history_enriched just for queries issued by dbt. Adds additional dbt-specific columns.
+    description: Filtered version of query_history_enriched just for queries issued by dbt. Adds additional dbt-specific columns. Incrementally pulls the last 3 days by default to account for late arriving rates data. This can be overriden by passing a value for dbt_snowflake_monitoring_incremental_days.
     columns:
       - name: dbt_snowflake_query_tags_version
         description: Version of the dbt-snowflake-query-tags package that generated the metadata

models/query_history_enriched.sql

@@ -24,19 +24,19 @@ query_history as (
     from {{ ref('stg_query_history') }}
 
     {% if is_incremental() %}
-        -- Conservatively re-process the last 3 days to account for late arriving rates data
+        -- Conservatively re-process the last 3 days to account for late arriving rates data. Allow an override from project variable
         -- which changes the cost per query
-        where end_time > (select dateadd(day, -3, max(end_time)) from {{ this }})
+        where end_time > (select dateadd(day, -{{ var('dbt_snowflake_monitoring_incremental_days', '3') }}, max(end_time)) from {{ this }})
     {% endif %}
 ),
 
 cost_per_query as (
     select *
     from {{ ref('cost_per_query') }}
     {% if is_incremental() %}
-        -- Conservatively re-process the last 3 days to account for late arriving rates data
+        -- Conservatively re-process the last 3 days to account for late arriving rates data. Allow an override from project variable
         -- which changes the cost per query
-        where end_time > (select dateadd(day, -3, max(end_time)) from {{ this }})
+        where end_time > (select dateadd(day, -{{ var('dbt_snowflake_monitoring_incremental_days', '3') }}, max(end_time)) from {{ this }})
     {% endif %}
 )