From f7fb71c0b545440ce9848333a414b7a293c7fcfc Mon Sep 17 00:00:00 2001 From: Artem Karpenko Date: Thu, 11 Aug 2022 13:08:04 +0300 Subject: [PATCH 1/2] Handling the Postgres identifier length limits --- macros/postgres__create_constraints.sql | 47 ++++++++++++++++++++++--- packages.yml | 3 ++ 2 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 packages.yml diff --git a/macros/postgres__create_constraints.sql b/macros/postgres__create_constraints.sql index 11ff230..c7ee400 100644 --- a/macros/postgres__create_constraints.sql +++ b/macros/postgres__create_constraints.sql @@ -1,6 +1,18 @@ {# PostgreSQL specific implementation to create a primary key #} -{%- macro postgres__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false) -%} - {%- set constraint_name = (table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} +{%- macro postgres__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false) -%} + + {% set constraint_name_query %} + select {{ dbt_utils.hash( dbt_utils.string_literal( table_relation.identifier ~ "_" ~ column_names|join('_') ) ) }} + {%endset%} + + {% set results = run_query(constraint_name_query) %} + + {% if execute %} + {% set constraint_name = "PK_" ~ results.columns[0].values()[0] %} + {% else %} + {% set constraint_name = "" %} + {% endif %} + {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} {#- Check that the table does not already have this PK/UK -#} @@ -28,8 +40,21 @@ {# PostgreSQL specific implementation to create a unique key #} -{%- macro postgres__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false) -%} - {%- set constraint_name = (table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} +{%- macro postgres__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false) -%} + + {%- set constraint_name_query %} + select {{ dbt_utils.hash( dbt_utils.string_literal( table_relation.identifier ~ "_" ~ column_names|join('_') ) ) }} + {%endset%} + + {% set results = run_query(constraint_name_query) %} + + {% if execute %} + {% set constraint_name = "UK_" ~ results.columns[0].values()[0] %} + {% else %} + {% set constraint_name = "" %} + {% endif %} + + {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} {#- Check that the table does not already have this PK/UK -#} @@ -58,7 +83,19 @@ {# PostgreSQL specific implementation to create a foreign key #} {%- macro postgres__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns=true) -%} - {%- set constraint_name = (fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} + + {%- set constraint_name_query %} + select {{ dbt_utils.hash( dbt_utils.string_literal( fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ) ) }} + {%endset%} + + {% set results = run_query(constraint_name_query) %} + + {% if execute %} + {% set constraint_name = "FK_" ~ results.columns[0].values()[0] %} + {% else %} + {% set constraint_name = "" %} + {% endif %} + {%- set fk_columns_csv = dbt_constraints.get_quoted_column_csv(fk_column_names, quote_columns) -%} {%- set pk_columns_csv = dbt_constraints.get_quoted_column_csv(pk_column_names, quote_columns) -%} {#- Check that the PK table has a PK or UK -#} diff --git a/packages.yml b/packages.yml new file mode 100644 index 0000000..538aeec --- /dev/null +++ b/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: dbt-labs/dbt_utils + version: [">=0.8.0", "<0.9.0"] From 5c87682d7ef9fdcaf312148cc1fb78e6627a6e36 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Thu, 11 Aug 2022 10:28:09 -0400 Subject: [PATCH 2/2] Removed dbt-utils dependency; fixed Oracle --- .../models/fact_order_line_longcol.sql | 12 ++++ integration_tests/models/schema.yml | 32 ++++++++++ macros/oracle__create_constraints.sql | 35 +++++++++-- macros/postgres__create_constraints.sql | 62 ++++++++----------- 4 files changed, 101 insertions(+), 40 deletions(-) create mode 100644 integration_tests/models/fact_order_line_longcol.sql diff --git a/integration_tests/models/fact_order_line_longcol.sql b/integration_tests/models/fact_order_line_longcol.sql new file mode 100644 index 0000000..8238a22 --- /dev/null +++ b/integration_tests/models/fact_order_line_longcol.sql @@ -0,0 +1,12 @@ +/* + Test +dim_order_copy____________ +*/ +SELECT + l_orderkey as l_____________________orderkey, + l_linenumber as l___________________linenumber, + l_partkey as l______________________partkey, + l_suppkey l______________________suppkey, + integration_id as l_______________integration_id +FROM +{{ ref('fact_order_line') }} O diff --git a/integration_tests/models/schema.yml b/integration_tests/models/schema.yml index 4a4f7d5..2b2412f 100644 --- a/integration_tests/models/schema.yml +++ b/integration_tests/models/schema.yml @@ -223,3 +223,35 @@ models: - name: dim_part_supplier_missing_con description: "Table is missing constraints to test FK won't be generated to it" + + - name: fact_order_line_longcol + description: "Fact Order Lines with long column names" + columns: + - name: l_____________________orderkey + description: "FK to dim_orders and first key in PK" + tests: + - relationships: + to: ref('dim_orders') + field: o_orderkey + - name: l___________________linenumber + description: "Order Line Number and second key in PK" + - name: l_______________integration_id + description: "Concatenation of PK colums for the unique and not_null tests" + tests: + - unique + - not_null + tests: + # Demonstration that the primary_key test can accept multiple columns + - dbt_constraints.primary_key: + column_names: + - l_____________________orderkey + - l___________________linenumber + # Test multi-column FK + - dbt_constraints.foreign_key: + fk_column_names: + - l______________________partkey + - l______________________suppkey + pk_table_name: ref('dim_part_supplier') + pk_column_names: + - ps_partkey + - ps_suppkey diff --git a/macros/oracle__create_constraints.sql b/macros/oracle__create_constraints.sql index 03d305a..dd413ba 100644 --- a/macros/oracle__create_constraints.sql +++ b/macros/oracle__create_constraints.sql @@ -1,6 +1,15 @@ {# Oracle specific implementation to create a primary key #} {%- macro oracle__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false) -%} {%- set constraint_name = (table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} + + {%- if constraint_name|length > 30 %} + {%- set constraint_name_query %} + select 'PK_' || ora_hash( '{{ constraint_name }}' ) as "constraint_name" from dual + {%- endset -%} + {%- set results = run_query(constraint_name_query) -%} + {%- set constraint_name = results.columns[0].values()[0] -%} + {% endif %} + {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} {#- Check that the table does not already have this PK/UK -#} @@ -35,6 +44,15 @@ END; {# Oracle specific implementation to create a unique key #} {%- macro oracle__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false) -%} {%- set constraint_name = (table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} + + {%- if constraint_name|length > 30 %} + {%- set constraint_name_query %} + select 'UK_' || ora_hash( '{{ constraint_name }}' ) as "constraint_name" from dual + {%- endset -%} + {%- set results = run_query(constraint_name_query) -%} + {%- set constraint_name = results.columns[0].values()[0] -%} + {% endif %} + {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} {#- Check that the table does not already have this PK/UK -#} @@ -69,6 +87,15 @@ END; {# Oracle specific implementation to create a foreign key #} {%- macro oracle__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns=true) -%} {%- set constraint_name = (fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} + + {%- if constraint_name|length > 30 %} + {%- set constraint_name_query %} + select 'FK_' || ora_hash( '{{ constraint_name }}' ) as "constraint_name" from dual + {%- endset -%} + {%- set results = run_query(constraint_name_query) -%} + {%- set constraint_name = results.columns[0].values()[0] -%} + {% endif %} + {%- set fk_columns_csv = dbt_constraints.get_quoted_column_csv(fk_column_names, quote_columns) -%} {%- set pk_columns_csv = dbt_constraints.get_quoted_column_csv(pk_column_names, quote_columns) -%} {#- Check that the PK table has a PK or UK -#} @@ -118,8 +145,8 @@ from and cons.owner = cols.owner where cons.constraint_type in ( 'P', 'U' ) - and cons.owner = '{{table_relation.schema}}' - and cons.table_name = '{{table_relation.identifier}}' + and upper(cons.owner) = upper('{{table_relation.schema}}') + and upper(cons.table_name) = upper('{{table_relation.identifier}}') order by 1, 2 {%- endset -%} {%- do log("Lookup: " ~ lookup_query, info=false) -%} @@ -152,8 +179,8 @@ from and cons.owner = cols.owner where cons.constraint_type in ( 'R' ) - and cons.owner = '{{table_relation.schema}}' - and cons.table_name = '{{table_relation.identifier}}' + and upper(cons.owner) = upper('{{table_relation.schema}}') + and upper(cons.table_name) = upper('{{table_relation.identifier}}') order by 1, 2 {%- endset -%} {%- do log("Lookup: " ~ lookup_query, info=false) -%} diff --git a/macros/postgres__create_constraints.sql b/macros/postgres__create_constraints.sql index c7ee400..99e6444 100644 --- a/macros/postgres__create_constraints.sql +++ b/macros/postgres__create_constraints.sql @@ -1,16 +1,13 @@ {# PostgreSQL specific implementation to create a primary key #} -{%- macro postgres__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false) -%} - - {% set constraint_name_query %} - select {{ dbt_utils.hash( dbt_utils.string_literal( table_relation.identifier ~ "_" ~ column_names|join('_') ) ) }} - {%endset%} - - {% set results = run_query(constraint_name_query) %} - - {% if execute %} - {% set constraint_name = "PK_" ~ results.columns[0].values()[0] %} - {% else %} - {% set constraint_name = "" %} +{%- macro postgres__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false) -%} + {%- set constraint_name = (table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} + + {%- if constraint_name|length > 63 %} + {%- set constraint_name_query %} + select 'PK_' || md5( '{{ constraint_name }}' )::varchar as "constraint_name" + {%- endset -%} + {%- set results = run_query(constraint_name_query) -%} + {%- set constraint_name = results.columns[0].values()[0] -%} {% endif %} {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} @@ -40,20 +37,16 @@ {# PostgreSQL specific implementation to create a unique key #} -{%- macro postgres__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false) -%} - - {%- set constraint_name_query %} - select {{ dbt_utils.hash( dbt_utils.string_literal( table_relation.identifier ~ "_" ~ column_names|join('_') ) ) }} - {%endset%} - - {% set results = run_query(constraint_name_query) %} - - {% if execute %} - {% set constraint_name = "UK_" ~ results.columns[0].values()[0] %} - {% else %} - {% set constraint_name = "" %} - {% endif %} +{%- macro postgres__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false) -%} + {%- set constraint_name = (table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} + {%- if constraint_name|length > 63 %} + {%- set constraint_name_query %} + select 'UK_' || md5( '{{ constraint_name }}' )::varchar as "constraint_name" + {%- endset -%} + {%- set results = run_query(constraint_name_query) -%} + {%- set constraint_name = results.columns[0].values()[0] -%} + {% endif %} {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} @@ -83,17 +76,14 @@ {# PostgreSQL specific implementation to create a foreign key #} {%- macro postgres__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns=true) -%} - - {%- set constraint_name_query %} - select {{ dbt_utils.hash( dbt_utils.string_literal( fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ) ) }} - {%endset%} - - {% set results = run_query(constraint_name_query) %} - - {% if execute %} - {% set constraint_name = "FK_" ~ results.columns[0].values()[0] %} - {% else %} - {% set constraint_name = "" %} + {%- set constraint_name = (fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} + + {%- if constraint_name|length > 63 %} + {%- set constraint_name_query %} + select 'FK_' || md5( '{{ constraint_name }}' )::varchar as "constraint_name" + {%- endset -%} + {%- set results = run_query(constraint_name_query) -%} + {%- set constraint_name = results.columns[0].values()[0] -%} {% endif %} {%- set fk_columns_csv = dbt_constraints.get_quoted_column_csv(fk_column_names, quote_columns) -%}