Merge branch 'main' into feature/calculate-sql-complexity

integration_tests/dbt_project.yml

@@ -67,4 +67,5 @@ vars:
   new_model_type_folder_name: 'my_new_models'
   new_model_type_prefixes: 'nwmdl_'
   insert_batch_size: 100
-  too_many_joins_threshold: 3
+  too_many_joins_threshold: 3
+  chained_views_threshold: 2

integration_tests/models/dbt_project_evaluator_schema_tests/core.yml

@@ -20,7 +20,9 @@ models:
       - name: on_schema_change
         description: this contains the on_schema_change setting for incremental models. This column was sometimes an empty string, so should be tested to detect regressions
         tests:
-          - is_not_empty_string
+          - not_null:
+              config:
+                where: resource_type = 'model'
       - name: model_type
         tests: 
           - dbt_utils.expression_is_true:

integration_tests/models/marts/_int_model_5.yml

@@ -0,0 +1,9 @@
+models: 
+  - name: int_model_5
+    latest_version: 2
+    versions: 
+     - v: 1
+       deprecation_date: 2024-01-01
+     - v: 2
+       deprecation_date: 2029-01-01
+     - v: 3

integration_tests/models/marts/fct_model_6.sql

@@ -5,86 +5,89 @@
 }}
 
 select 1 as id 
--- from {{ ref('stg_model_3') }}
 
--- union all 
--- select
---     3 as id
--- from my_db.my_schema.my_table
--- union all 
--- select
---     3 as id
--- from 'my_db'.'my_schema'.'my_table'
--- union all 
--- select
---     3 as id
--- from "my_db"."my_schema"."my_table"
--- union all 
--- select
---     3 as id
--- from `my_db`.`my_schema`.`my_table`
--- union all 
--- select
---     3 as id
--- from [my_db].[my_schema].[my_table]
+-- depends on {{ ref('stg_model_3') }} 
 
--- union all
--- select 
---     4 as id
--- from my_schema.raw_relation_5
--- union all
--- select 
---     4 as id
--- from 'my_schema'.'raw_relation_5' 
--- union all
--- select 
---     4 as id
--- from "my_schema"."raw_relation_5"
--- union all
--- select 
---     4 as id
--- from `my_schema`.`raw_relation_5`
--- union all
--- select 
---     4 as id
--- from [my_schema].[raw_relation_5] 
+{#
+  from {{ ref('stg_model_3') }}   
+  union all 
+  select
+      3 as id
+  from my_db.my_schema.my_table
+  union all 
+  select
+      3 as id
+  from 'my_db'.'my_schema'.'my_table'
+  union all 
+  select
+      3 as id
+  from "my_db"."my_schema"."my_table"
+  union all 
+  select
+      3 as id
+  from `my_db`.`my_schema`.`my_table`
+  union all 
+  select
+      3 as id
+  from [my_db].[my_schema].[my_table]   
+  union all
+  select 
+      4 as id
+  from my_schema.raw_relation_5
+  union all
+  select 
+      4 as id
+  from 'my_schema'.'raw_relation_5' 
+  union all
+  select 
+      4 as id
+  from "my_schema"."raw_relation_5"
+  union all
+  select 
+      4 as id
+  from `my_schema`.`raw_relation_5`
+  union all
+  select 
+      4 as id
+  from [my_schema].[raw_relation_5]    
+  union all
+  select 
+      4 as id
+  from `raw_relation_1` 
+  union all
+  select 
+      4 as id
+  from "raw_relation_2" 
+  union all
+  select 
+      4 as id
+  from [raw_relation_3]  
+  union all
+  select 
+      4 as id
+  from 'raw_relation_4'    
+  union all
+  select
+      4 as id
+  from {{ var("my_table_reference") }}
+  union all
+  select
+      4 as id
+  from {{ var('my_table_reference') }}   
+  union all
+
+  -- the following is SQL commented so it should not be found as a hard coded reference
+--   select
+--       6 as id
+--   from my_db.my_schema.my_table222
+
+  select
+      5 as id
+  from {{ var("my_table_reference", "table_d") }}
+  union all
+  select
+      5 as id
+  from {{ var('my_table_reference', 'table_d') }}
 
 
--- union all
--- select 
---     4 as id
--- from `raw_relation_1` 
--- union all
--- select 
---     4 as id
--- from "raw_relation_2" 
--- union all
--- select 
---     4 as id
--- from [raw_relation_3]  
--- union all
--- select 
---     4 as id
--- from 'raw_relation_4' 
-
--- union all
--- select
---     4 as id
--- from {{ var("my_table_reference") }}
--- union all
--- select
---     4 as id
--- from {{ var('my_table_reference') }}
-
-
--- union all
--- select
---     5 as id
--- from {{ var("my_table_reference", "table_d") }}
--- union all
--- select
---     5 as id
--- from {{ var('my_table_reference', 'table_d') }}
--- select
---     7 as id
--- from {{ var('my_table_reference', 'table_d') }}
+#}

integration_tests/models/marts/int_model_5_v1.sql

@@ -0,0 +1 @@
+select 1 as id

integration_tests/models/marts/int_model_5_v3.sql

@@ -0,0 +1 @@
+select 1 as id

integration_tests/models/staging/source_1/schema.yml

@@ -5,7 +5,9 @@ models:
     description: test
     columns:
       - name: id 
-        description: hocus pocus
+        description: >
+          hocus pocus
+          pocus hocus
         tests:
           - unique
   - name: stg_model_3

integration_tests/seeds/dag/test_fct_rejoining_of_upstream_concepts.csv

@@ -1,2 +1,2 @@
 parent,child,parent_and_child,is_loop_independent
-stg_model_1,int_model_5,int_model_4,true
+stg_model_1,int_model_5.v2,int_model_4,true

integration_tests/seeds/dag/test_fct_root_models.csv

@@ -1,2 +1,4 @@
 child
+int_model_5.v1
+int_model_5.v3
 model_8

integration_tests/seeds/docs/test_fct_documentation_coverage.csv

@@ -1,2 +1,2 @@
 total_models,documented_models,documentation_coverage_pct,staging_documentation_coverage_pct,intermediate_documentation_coverage_pct,marts_documentation_coverage_pct,other_documentation_coverage_pct
-16,4,25,20.00,0.00,0.00,75.00
+18,4,22.22,20.00,0.00,0.00,75.00

integration_tests/seeds/docs/test_fct_undocumented_models.csv

@@ -8,6 +8,8 @@ model_8,marts
 fct_model_9,marts
 dim_model_7,marts
 int_model_4,intermediate
-int_model_5,intermediate
+int_model_5.v1,intermediate
+int_model_5.v2,intermediate
+int_model_5.v3,intermediate
 report_3,other
 fct_model_10,marts

integration_tests/seeds/performance/test_fct_chained_views_dependencies.csv

@@ -1,17 +1,2 @@
 parent,child,distance
 stg_model_1,dim_model_7,3
-stg_model_1,int_model_5,2
-stg_model_2,dim_model_7,2
-int_model_4,dim_model_7,2
-stg_model_1,dim_model_7,2
-stg_model_4,dim_model_7,1
-stg_model_2,stg_model_4,1
-int_model_4,int_model_5,1
-int_model_5,dim_model_7,1
-stg_model_1,int_model_5,1
-stg_model_1,int_model_4,1
-fct_model_9,stg_model_5,1
-stg_model_3,fct_model_6,1
-stg_model_1,fct_model_10,1
-stg_model_2,fct_model_10,1
-stg_model_3,fct_model_10,1

integration_tests/seeds/structure/test_fct_model_directories.csv

@@ -4,4 +4,6 @@ stg_model_4,models/staging/stg_model_4.sql,models/staging/source_1/stg_model_4.s
 stg_model_3,models/staging/source_1/stg_model_3.sql,models/staging/source_2/stg_model_3.sql
 dim_model_7,models/marts/intermediate/dim_model_7.sql,models/.../marts/.../dim_model_7.sql
 int_model_4,models/marts/int_model_4.sql,models/.../intermediate/.../int_model_4.sql
-int_model_5,models/marts/int_model_5.sql,models/.../intermediate/.../int_model_5.sql
+int_model_5.v1,models/marts/int_model_5_v1.sql,models/.../intermediate/.../int_model_5_v1.sql
+int_model_5.v2,models/marts/int_model_5_v2.sql,models/.../intermediate/.../int_model_5_v2.sql
+int_model_5.v3,models/marts/int_model_5_v3.sql,models/.../intermediate/.../int_model_5_v3.sql

integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv

@@ -3,7 +3,9 @@ dim_model_7,FALSE,0
 fct_model_6,FALSE,0
 model_8,FALSE,0
 fct_model_9,FALSE,0
-int_model_5,FALSE,0
+int_model_5.v1,FALSE,0
+int_model_5.v2,FALSE,0
+int_model_5.v3,FALSE,0
 report_1,FALSE,0
 report_2,FALSE,0
 report_3,FALSE,0

integration_tests/seeds/tests/test_fct_test_coverage.csv

@@ -1,2 +1,2 @@
 total_models,total_tests,tested_models,test_coverage_pct,staging_test_coverage_pct,intermediate_test_coverage_pct,marts_test_coverage_pct,other_test_coverage_pct,test_to_model_ratio
-16,13,6,37.50,80.00,50.00,0.00,25.00,0.8125
+18,13,6,33.33,80.00,25.00,0.00,25.00,0.7222

integration_tests/seeds/tests/tests_seeds.yml

@@ -23,7 +23,7 @@ seeds:
             - total_models
             - total_tests
             - tested_models
-            - test_coverage_pct
+            - "{{ 'test_coverage_pct' if not target.name in ['duckdb', 'databricks', 'trino'] else 'tested_models' }}"
             - test_to_model_ratio
             - staging_test_coverage_pct
             - intermediate_test_coverage_pct

macros/find_all_hard_coded_references.sql

@@ -14,6 +14,11 @@
         {%- set model_raw_sql = '' -%}
         {%- endif -%}
 
+        {# we remove the comments that start with -- , or other characters configured #}
+        {%- set re = modules.re -%}
+        {%- set comment_chars_match = "(" ~ var('comment_chars') | join("|") ~ ").*" -%}
+        {%- set model_raw_sql_no_comments = re.sub(comment_chars_match, '', model_raw_sql) -%}
+
         {#-
             REGEX Explanations
 
@@ -234,7 +239,7 @@
 
         {%- for regex_name, regex_pattern in from_hard_coded_references.items() -%}
 
-            {%- set all_regex_matches = re.findall(regex_pattern, model_raw_sql) -%}
+            {%- set all_regex_matches = re.findall(regex_pattern, model_raw_sql_no_comments) -%}
 
                 {%- for match in all_regex_matches -%}
 

macros/unpack/get_node_values.sql

@@ -29,6 +29,8 @@
                 wrap_string_with_quotes(node.group),
                 wrap_string_with_quotes(node.access),
                 wrap_string_with_quotes(node.latest_version),
+                wrap_string_with_quotes(node.version),
+                wrap_string_with_quotes(node.deprecation_date),
                 "cast(" ~ contract | trim  ~ " as boolean)",
                 node.columns.values() | list | length,
                 node.columns.values() | list | selectattr('description') | list | length,

macros/unpack/type_large_string.sql

@@ -0,0 +1,11 @@
+{%- macro type_large_string() -%}
+    {{ return(adapter.dispatch('type_large_string', 'dbt_project_evaluator')()) }}
+{%- endmacro -%}
+
+{%- macro default__type_large_string() -%}
+    {{ return(dbt.type_string()) }}
+{%- endmacro -%}
+
+{%- macro redshift__type_large_string() -%}
+    varchar(5000)
+{%- endmacro -%}

macros/wrap_string_with_quotes.sql

@@ -1,7 +1,14 @@
 {% macro wrap_string_with_quotes(str) %}
-  {% if not str %}
+  {% if str is none %}
     {{ return('cast(NULL as ' ~ dbt.type_string() ~ ')') }}
   {% else %}
-    {{ return("'" ~ str ~ "'") }}
+    {{ dbt.string_literal(str) }}
   {% endif %}
 {% endmacro %}
+
+{#
+  To be removed when https://github.com/dbt-labs/dbt-bigquery/pull/1089 is merged
+#}
+{% macro bigquery__string_literal(value) -%}
+  '''{{ value }}'''
+{%- endmacro %}

models/marts/core/int_all_graph_resources.sql

@@ -34,6 +34,7 @@ unioned_with_calc as (
         *,
         case 
             when resource_type = 'source' then  {{ dbt.concat(['source_name',"'.'",'name']) }}
+            when coalesce(version, '') != '' then {{ dbt.concat(['name',"'.v'",'version']) }} 
             else name 
         end as resource_name,
         case
@@ -83,6 +84,8 @@ joined as (
         unioned_with_calc.access, 
         unioned_with_calc.access = 'public' as is_public, 
         unioned_with_calc.latest_version, 
+        unioned_with_calc.version, 
+        unioned_with_calc.deprecation_date, 
         unioned_with_calc.is_contract_enforced, 
         unioned_with_calc.total_defined_columns, 
         unioned_with_calc.total_described_columns, 

models/marts/dag/dag.yml

@@ -6,7 +6,7 @@ models:
     tests:
       - is_empty
   - name: fct_duplicate_sources
-    descritption: "This table shows each source database location with more than one node in your dbt project."
+    description: "This table shows each source database location with more than one node in your dbt project."
     tests:
       - is_empty
   - name: fct_marts_or_intermediate_dependent_on_source
@@ -24,7 +24,7 @@ models:
   - name: fct_rejoining_of_upstream_concepts
     description: >
       "All cases where one of the parent's direct children (child) is ALSO the direct child of ANOTHER one of the 
-      parent's direct childen (parent_and_child). Only includes cases where the model "in between" the parent and child 
+      parent's direct children (parent_and_child). Only includes cases where the model "in between" the parent and child 
       has NO other downstream dependencies.
     tests:
       - is_empty