From 45b4516650c2d7ea49b08c6be087efecc2705a41 Mon Sep 17 00:00:00 2001 From: sachinthakur96 Date: Thu, 9 Nov 2023 13:07:34 +0530 Subject: [PATCH 01/10] Adding --- tests/functional/adapter/ephemeral/test_ephemeral.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/functional/adapter/ephemeral/test_ephemeral.py b/tests/functional/adapter/ephemeral/test_ephemeral.py index db170b1..cd3a088 100644 --- a/tests/functional/adapter/ephemeral/test_ephemeral.py +++ b/tests/functional/adapter/ephemeral/test_ephemeral.py @@ -25,6 +25,4 @@ def test_ephemeral_multi_snowflake(self, project): assert len(results) == 3 check_relations_equal(project.adapter, ["seed", "dependent"]) check_relations_equal(project.adapter, ["seed", "double_dependent"]) - check_relations_equal(project.adapter, ["seed", "super_dependent"]) - - + check_relations_equal(project.adapter, ["seed", "super_dependent"]) \ No newline at end of file From b51d8986ddebdae4ff096ecd1012cdaff3336b4a Mon Sep 17 00:00:00 2001 From: sachinthakur96 Date: Thu, 9 Nov 2023 13:18:28 +0530 Subject: [PATCH 02/10] Adding --- tests/functional/adapter/test_clone.py | 124 ------------------------- 1 file changed, 124 deletions(-) diff --git a/tests/functional/adapter/test_clone.py b/tests/functional/adapter/test_clone.py index e8ae2c4..e69de29 100644 --- a/tests/functional/adapter/test_clone.py +++ b/tests/functional/adapter/test_clone.py @@ -1,124 +0,0 @@ -from copy import deepcopy -from collections import Counter -from dbt.tests.util import run_dbt -from dbt.tests.adapter.dbt_clone.test_dbt_clone import BaseClonePossible - -import pytest -import shutil -import os - - -class TestSnowflakeClonePossible(BaseClonePossible): - - def test_can_clone_true(self, project, unique_schema, other_schema): - project.create_test_schema(other_schema) - self.run_and_save_state(project.project_root, with_snapshot=True) - - clone_args = [ - "clone", - "--state", - "state", - "--target", - "otherschema", - ] - - results = run_dbt(clone_args) - assert len(results) == 4 - print(results) - - schema_relations = project.adapter.list_relations( - database=project.database, schema=other_schema - ) - types = [r.type for r in schema_relations] - count_types = Counter(types) - assert count_types == Counter({"table": 3, "view": 1}) - - # objects already exist, so this is a no-op - results = run_dbt(clone_args) - assert len(results) == 4 - assert all("no-op" in r.message.lower() for r in results) - - # recreate all objects - assert len(results) == 4 - - # select only models this time - results = run_dbt([*clone_args, "--resource-type", "model"]) - assert len(results) == 2 - assert all("no-op" in r.message.lower() for r in results) - - # @pytest.fixture(autouse=True) - # def clean_up(self, project): - # yield - # with project.adapter.connection_named("__test"): - # relation = project.adapter.Relation.create( - # database=project.database, schema=f"{project.test_schema}_SEEDS" - # ) - # project.adapter.drop_schema(relation) - - # relation = project.adapter.Relation.create( - # database=project.database, schema=project.test_schema - # ) - # project.adapter.drop_schema(relation) - - # pass - - -table_model_1_sql = """ - {{ config( - materialized='table', - transient=true, - ) }} - select 1 as fun - """ - - - - - -class TestSnowflakeCloneTrainsentTable: - @pytest.fixture(scope="class") - def models(self): - return { - "table_model.sql": table_model_1_sql, - } - - @pytest.fixture(scope="class") - def other_schema(self, unique_schema): - return unique_schema + "_other" - - @pytest.fixture(scope="class") - def profiles_config_update(self, dbt_profile_target, unique_schema, other_schema): - outputs = {"default": dbt_profile_target, "otherschema": deepcopy(dbt_profile_target)} - outputs["default"]["schema"] = unique_schema - outputs["otherschema"]["schema"] = other_schema - return {"test": {"outputs": outputs, "target": "default"}} - - def copy_state(self, project_root): - state_path = os.path.join(project_root, "state") - if not os.path.exists(state_path): - os.makedirs(state_path) - shutil.copyfile( - f"{project_root}/target/manifest.json", f"{project_root}/state/manifest.json" - ) - - def run_and_save_state(self, project_root, with_snapshot=False): - results = run_dbt(["run"]) - assert len(results) == 1 - assert not any(r.node.deferred for r in results) - - self.copy_state(project_root) - - def test_can_clone_transient_table(self, project, other_schema): - project.create_test_schema(other_schema) - self.run_and_save_state(project.project_root) - - clone_args = [ - "clone", - "--state", - "state", - "--target", - "otherschema", - ] - - results = run_dbt(clone_args) - assert len(results) == 1 \ No newline at end of file From 02ef02fa0ad6e9971a2f89fb9e75913cde82daf6 Mon Sep 17 00:00:00 2001 From: sachinthakur96 Date: Thu, 9 Nov 2023 13:19:06 +0530 Subject: [PATCH 03/10] Adding --- tests/functional/adapter/test_clone.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/functional/adapter/test_clone.py diff --git a/tests/functional/adapter/test_clone.py b/tests/functional/adapter/test_clone.py deleted file mode 100644 index e69de29..0000000 From b068f7fc655007ae5d61942dba24c3cc93f18337 Mon Sep 17 00:00:00 2001 From: sachinthakur96 Date: Thu, 9 Nov 2023 13:25:13 +0530 Subject: [PATCH 04/10] Adding --- tests/functional/adapter/test_constraints.py | 405 ++++++++++++++++++- 1 file changed, 399 insertions(+), 6 deletions(-) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 0284cb0..57ff663 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -18,16 +18,22 @@ from dbt.tests.adapter.constraints.test_constraints import ( BaseTableConstraintsColumnsEqual, BaseViewConstraintsColumnsEqual, - #BaseTableContractSqlHeader, - #BaseIncrementalContractSqlHeader, BaseIncrementalConstraintsColumnsEqual, BaseConstraintsRuntimeDdlEnforcement, - # BaseConstraintsRollback, BaseIncrementalConstraintsRuntimeDdlEnforcement, - # BaseIncrementalConstraintsRollback, - # BaseModelConstraintsRuntimeEnforcement, - #BaseConstraintQuotedColumn, ) +from dbt.tests.adapter.utils.test_null_compare import BaseNullCompare +from dbt.tests.adapter.utils.test_null_compare import BaseMixedNullCompare + +from dbt.tests.adapter.constraints.test_constraints import BaseIncrementalConstraintsRollback +from dbt.tests.adapter.utils.test_equals import BaseEquals +from dbt.tests.adapter.utils.test_validate_sql import BaseValidateSqlMethod + +from dbt.tests.adapter.constraints.test_constraints import BaseModelConstraintsRuntimeEnforcement +from dbt.tests.adapter.constraints.test_constraints import BaseConstraintQuotedColumn + + + from dbt.tests.adapter.constraints.fixtures import ( my_model_sql, @@ -288,6 +294,195 @@ SELECT DEMO as column_name """ +SEEDS__DATA_EQUALS_CSV = """key_name,x,y,expected +1,1,1,same +2,1,2,different +3,1,,different +4,2,1,different +5,2,2,same +6,2,,different +7,,1,different +8,,2,different +9,,,same +""" + +# model breaking constraints +my_model_with_nulls_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + + cast(null as {{ dbt.type_int() }}) as id, + + 'red' as color, + '2019-01-01' as date_day +""" + + +my_model_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + 1 as id, + 'blue' as color, + '2019-01-01' as date_day +""" + + + +model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + - type: check + expression: id >= 1 + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_wrong_order + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text +""" + +class BaseConstraintsRollback: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_with_nulls_sql + + @pytest.fixture(scope="class") + def expected_color(self): + return "blue" + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ['null value in column "id"', "violates not-null constraint"] + + def assert_expected_error_messages(self, error_message, expected_error_messages): + print(msg in error_message for msg in expected_error_messages) + assert all(msg in error_message for msg in expected_error_messages) + + def test__constraints_enforcement_rollback( + self, project, expected_color, expected_error_messages, null_model_sql + ): + # print(expected_error_messages) + results = run_dbt(["run", "-s", "my_model"]) + # print(results) + + assert len(results) == 1 + +# # Make a contract-breaking change to the model + write_file(null_model_sql, "models", "my_model.sql") + + failing_results = run_dbt(["run", "-s", "my_model"], expect_pass=True) + # print("start",failing_results[0].message,"endhere", len(failing_results)) + assert len(failing_results) == 1 + +# # Verify the previous table still exists + relation = relation_from_name(project.adapter, "my_model") + old_model_exists_sql = f"select * from {relation}" + old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") + # print(old_model_exists[0][1],len(old_model_exists)) + assert len(old_model_exists) == 1 + assert old_model_exists[0][1] == expected_color + + # Confirm this model was contracted + # TODO: is this step really necessary? + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + # # Its result includes the expected error messages + # print(expected_error_messages) + self.assert_expected_error_messages(failing_results[0].message, expected_error_messages) + + class VerticaColumnEqualSetup: @@ -426,3 +621,201 @@ def expected_sql(self): """ +my_incremental_model_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' + ) +}} + +select + 1 as id, + 'blue' as color, + '2019-01-01' as date_day +""" + +my_model_incremental_with_nulls_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' ) +}} + +select + -- null value for 'id' + cast(null as {{ dbt.type_int() }}) as id, + -- change the color as well (to test rollback) + 'red' as color, + '2019-01-01' as date_day +""" + +class BaseIncrementalConstraintsRollback(BaseConstraintsRollback): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_incremental_with_nulls_sql + + + +class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): + # pass + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_yml, + } + @pytest.fixture(scope="class") + def expected_error_messages(self): + return [""] + + @pytest.fixture(scope="class") + def expected_color(self): + return "red" + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_with_nulls_sql + + + + + +class TestValidateSqlMethod(BaseValidateSqlMethod): + pass + +class TestNullCompare(BaseNullCompare): + pass + + +class TestMixedNullCompare(BaseMixedNullCompare): + pass + + +class TestEquals(BaseEquals): + + @pytest.fixture(scope="class") + def seeds(self): + return { + "data_equals.csv": SEEDS__DATA_EQUALS_CSV, + } + pass + + + + + +class TestConstraintQuotedColumn(BaseConstraintQuotedColumn): + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create table INCLUDE SCHEMA PRIVILEGES as ( select 'blue' as "from", 1 as id, '2019-01-01' as date_day ) ; """ + pass + +class TestModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create table INCLUDE SCHEMA PRIVILEGES as ( -- depends_on: select 'blue' as color, 1 as id, '2019-01-01' as date_day ) ; +""" + + + + + +my_model_contract_sql_header_sql = """ +{{ + config( + materialized = "table" + ) +}} + +{% call set_sql_header(config) %} +set session time zone 'Asia/Kolkata'; +{%- endcall %} +select CURRENT_TIME(0) as column_name + + +""" + +model_contract_header_schema_yml = """ +version: 2 +models: + - name: my_model_contract_sql_header + config: + contract: + enforced: false + columns: + - name: column_name + data_type: text +""" + + + +my_model_incremental_contract_sql_header_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change="append_new_columns" + ) +}} + +{% call set_sql_header(config) %} +set session time zone 'Asia/Kolkata'; +{%- endcall %} +select CURRENT_TIME(0) as column_name +""" + + +class BaseContractSqlHeader: + """Tests a contracted model with a sql header dependency.""" + + def test__contract_sql_header(self, project): + run_dbt(["run", "-s", "my_model_contract_sql_header"]) + + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_contract_sql_header" + model_config = manifest.nodes[model_id].config + + assert model_config.contract + + +class BaseTableContractSqlHeader(BaseContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +class TestTableContractSqlHeader(BaseTableContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + + + +class BaseIncrementalContractSqlHeader(BaseContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_incremental_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +class TestIncrementalContractSqlHeader(BaseIncrementalContractSqlHeader): + pass From 4ae2b70629726b2683b7fb98da1422e3fe00a7c9 Mon Sep 17 00:00:00 2001 From: sachinthakur96 Date: Thu, 9 Nov 2023 13:32:10 +0530 Subject: [PATCH 05/10] Adding --- tests/functional/adapter/test_constraints.py | 53 ++------------------ 1 file changed, 4 insertions(+), 49 deletions(-) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 57ff663..b42a3aa 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -448,24 +448,19 @@ def assert_expected_error_messages(self, error_message, expected_error_messages) def test__constraints_enforcement_rollback( self, project, expected_color, expected_error_messages, null_model_sql ): - # print(expected_error_messages) results = run_dbt(["run", "-s", "my_model"]) - # print(results) - assert len(results) == 1 -# # Make a contract-breaking change to the model + # Make a contract-breaking change to the model write_file(null_model_sql, "models", "my_model.sql") failing_results = run_dbt(["run", "-s", "my_model"], expect_pass=True) - # print("start",failing_results[0].message,"endhere", len(failing_results)) assert len(failing_results) == 1 -# # Verify the previous table still exists + # Verify the previous table still exists relation = relation_from_name(project.adapter, "my_model") old_model_exists_sql = f"select * from {relation}" old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") - # print(old_model_exists[0][1],len(old_model_exists)) assert len(old_model_exists) == 1 assert old_model_exists[0][1] == expected_color @@ -478,8 +473,7 @@ def test__constraints_enforcement_rollback( assert contract_actual_config.enforced is True - # # Its result includes the expected error messages - # print(expected_error_messages) + # Its result includes the expected error messages self.assert_expected_error_messages(failing_results[0].message, expected_error_messages) @@ -574,17 +568,7 @@ def models(self): "my_model_wrong_name.sql": my_model_view_wrong_name_sql, "constraints_schema.yml": constraints_yml, } -#class TestVerticaTableConstraintsRollback(BaseConstraintsRollback): -# @pytest.fixture(scope="class") -# def models(self): -# return { -# "my_model.sql": my_model_sql, -# "constraints_schema.yml": constraints_yml, -# } -# @pytest.fixture(scope="class") -# def expected_error_messages(self): -# return ["Required field id cannot be null"] class TestVerticaConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): @@ -599,27 +583,7 @@ def expected_sql(self): return """create table include schema privileges as(-- depends_on: select 'blue' as color,1 as id,'2019-01-01' as date_day); ;""" - return """ - create table - - - INCLUDE SCHEMA PRIVILEGES as ( - - -select - 'blue' as color, - 1 as id, - '2019-01-01' as date_day - ) - - - - - - -; ; -""" - + my_incremental_model_sql = """ {{ @@ -665,7 +629,6 @@ def null_model_sql(self): class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): - # pass @pytest.fixture(scope="class") def models(self): @@ -687,8 +650,6 @@ def null_model_sql(self): - - class TestValidateSqlMethod(BaseValidateSqlMethod): pass @@ -711,8 +672,6 @@ def seeds(self): - - class TestConstraintQuotedColumn(BaseConstraintQuotedColumn): @pytest.fixture(scope="class") def expected_sql(self): @@ -729,8 +688,6 @@ def expected_sql(self): - - my_model_contract_sql_header_sql = """ {{ config( @@ -806,8 +763,6 @@ def models(self): } - - class BaseIncrementalContractSqlHeader(BaseContractSqlHeader): @pytest.fixture(scope="class") def models(self): From c815870a6566d8bb63d77a4c187b4f986396611b Mon Sep 17 00:00:00 2001 From: sachinthakur96 Date: Thu, 9 Nov 2023 13:36:00 +0530 Subject: [PATCH 06/10] Adding --- .../adapter/constraints/test_constraint.py | 434 ++++++++++ .../adapter/constraints/test_constraints.py | 422 +++++++++- tests/functional/adapter/test_constraints.py | 776 ------------------ 3 files changed, 816 insertions(+), 816 deletions(-) create mode 100644 tests/functional/adapter/constraints/test_constraint.py delete mode 100644 tests/functional/adapter/test_constraints.py diff --git a/tests/functional/adapter/constraints/test_constraint.py b/tests/functional/adapter/constraints/test_constraint.py new file mode 100644 index 0000000..3cf251d --- /dev/null +++ b/tests/functional/adapter/constraints/test_constraint.py @@ -0,0 +1,434 @@ +from dbt.tests.adapter.utils.test_null_compare import BaseNullCompare +from dbt.tests.adapter.utils.test_null_compare import BaseMixedNullCompare + +from dbt.tests.adapter.constraints.test_constraints import BaseIncrementalConstraintsRollback +from dbt.tests.adapter.utils.test_equals import BaseEquals +from dbt.tests.adapter.utils.test_validate_sql import BaseValidateSqlMethod + +from dbt.tests.adapter.constraints.test_constraints import BaseModelConstraintsRuntimeEnforcement +from dbt.tests.adapter.constraints.test_constraints import BaseConstraintQuotedColumn + +from dbt.tests.util import ( + run_dbt, + get_manifest, + run_dbt_and_capture, + write_file, + read_file, + relation_from_name, +) + +import pytest + +SEEDS__DATA_EQUALS_CSV = """key_name,x,y,expected +1,1,1,same +2,1,2,different +3,1,,different +4,2,1,different +5,2,2,same +6,2,,different +7,,1,different +8,,2,different +9,,,same +""" + +# model breaking constraints +my_model_with_nulls_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + + cast(null as {{ dbt.type_int() }}) as id, + + 'red' as color, + '2019-01-01' as date_day +""" + + +my_model_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + 1 as id, + 'blue' as color, + '2019-01-01' as date_day +""" + + + +model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + - type: check + expression: id >= 1 + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_wrong_order + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text +""" + +class BaseConstraintsRollback: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_with_nulls_sql + + @pytest.fixture(scope="class") + def expected_color(self): + return "blue" + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ['null value in column "id"', "violates not-null constraint"] + + def assert_expected_error_messages(self, error_message, expected_error_messages): + print(msg in error_message for msg in expected_error_messages) + assert all(msg in error_message for msg in expected_error_messages) + + def test__constraints_enforcement_rollback( + self, project, expected_color, expected_error_messages, null_model_sql + ): + # print(expected_error_messages) + results = run_dbt(["run", "-s", "my_model"]) + # print(results) + + assert len(results) == 1 + +# # Make a contract-breaking change to the model + write_file(null_model_sql, "models", "my_model.sql") + + failing_results = run_dbt(["run", "-s", "my_model"], expect_pass=True) + # print("start",failing_results[0].message,"endhere", len(failing_results)) + assert len(failing_results) == 1 + +# # Verify the previous table still exists + relation = relation_from_name(project.adapter, "my_model") + old_model_exists_sql = f"select * from {relation}" + old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") + # print(old_model_exists[0][1],len(old_model_exists)) + assert len(old_model_exists) == 1 + assert old_model_exists[0][1] == expected_color + + # Confirm this model was contracted + # TODO: is this step really necessary? + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + # # Its result includes the expected error messages + # print(expected_error_messages) + self.assert_expected_error_messages(failing_results[0].message, expected_error_messages) + + + +# class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): + + +# @pytest.fixture(scope="class") +# def models(self): +# return { +# "my_model.sql": my_model_sql, +# "constraints_schema.yml": model_schema_yml, +# } +# @pytest.fixture(scope="class") +# def expected_error_messages(self): +# return [""] + +# @pytest.fixture(scope="class") +# def expected_color(self): +# return "blue" + +# @pytest.fixture(scope="class") +# def null_model_sql(self): +# return my_model_with_nulls_sql + + +my_incremental_model_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' + ) +}} + +select + 1 as id, + 'blue' as color, + '2019-01-01' as date_day +""" + +my_model_incremental_with_nulls_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' ) +}} + +select + -- null value for 'id' + cast(null as {{ dbt.type_int() }}) as id, + -- change the color as well (to test rollback) + 'red' as color, + '2019-01-01' as date_day +""" + +class BaseIncrementalConstraintsRollback(BaseConstraintsRollback): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_incremental_with_nulls_sql + + + +class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): + # pass + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_yml, + } + @pytest.fixture(scope="class") + def expected_error_messages(self): + return [""] + + @pytest.fixture(scope="class") + def expected_color(self): + return "red" + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_with_nulls_sql + + + + + +class TestValidateSqlMethod(BaseValidateSqlMethod): + pass + +class TestNullCompare(BaseNullCompare): + pass + + +class TestMixedNullCompare(BaseMixedNullCompare): + pass + + +class TestEquals(BaseEquals): + + @pytest.fixture(scope="class") + def seeds(self): + return { + "data_equals.csv": SEEDS__DATA_EQUALS_CSV, + } + pass + + + + + +class TestConstraintQuotedColumn(BaseConstraintQuotedColumn): + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create table INCLUDE SCHEMA PRIVILEGES as ( select 'blue' as "from", 1 as id, '2019-01-01' as date_day ) ; """ + pass + +class TestModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create table INCLUDE SCHEMA PRIVILEGES as ( -- depends_on: select 'blue' as color, 1 as id, '2019-01-01' as date_day ) ; +""" + + + + + + + + +my_model_contract_sql_header_sql = """ +{{ + config( + materialized = "table" + ) +}} + +{% call set_sql_header(config) %} +set session time zone 'Asia/Kolkata'; +{%- endcall %} +select CURRENT_TIME(0) as column_name + + +""" + +model_contract_header_schema_yml = """ +version: 2 +models: + - name: my_model_contract_sql_header + config: + contract: + enforced: false + columns: + - name: column_name + data_type: text +""" + + + +my_model_incremental_contract_sql_header_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change="append_new_columns" + ) +}} + +{% call set_sql_header(config) %} +set session time zone 'Asia/Kolkata'; +{%- endcall %} +select CURRENT_TIME(0) as column_name +""" + + +class BaseContractSqlHeader: + """Tests a contracted model with a sql header dependency.""" + + def test__contract_sql_header(self, project): + run_dbt(["run", "-s", "my_model_contract_sql_header"]) + + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_contract_sql_header" + model_config = manifest.nodes[model_id].config + + assert model_config.contract + + +class BaseTableContractSqlHeader(BaseContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +class TestTableContractSqlHeader(BaseTableContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + + + +class BaseIncrementalContractSqlHeader(BaseContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_incremental_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +class TestIncrementalContractSqlHeader(BaseIncrementalContractSqlHeader): + pass diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py index 3cf251d..b42a3aa 100644 --- a/tests/functional/adapter/constraints/test_constraints.py +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -1,3 +1,27 @@ +# Copyright (c) [2018-2023] Micro Focus or one of its affiliates. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest + +from dbt.tests.adapter.constraints.test_constraints import ( + BaseTableConstraintsColumnsEqual, + BaseViewConstraintsColumnsEqual, + BaseIncrementalConstraintsColumnsEqual, + BaseConstraintsRuntimeDdlEnforcement, + BaseIncrementalConstraintsRuntimeDdlEnforcement, +) from dbt.tests.adapter.utils.test_null_compare import BaseNullCompare from dbt.tests.adapter.utils.test_null_compare import BaseMixedNullCompare @@ -8,6 +32,28 @@ from dbt.tests.adapter.constraints.test_constraints import BaseModelConstraintsRuntimeEnforcement from dbt.tests.adapter.constraints.test_constraints import BaseConstraintQuotedColumn + + + +from dbt.tests.adapter.constraints.fixtures import ( + my_model_sql, + my_incremental_model_sql, + my_model_wrong_order_sql, + my_model_view_wrong_order_sql, + my_model_incremental_wrong_order_sql, + my_model_wrong_name_sql, + my_model_view_wrong_name_sql, + my_model_incremental_wrong_name_sql, + #my_model_with_quoted_column_name_sql, + model_schema_yml, + constrained_model_schema_yml, + #model_contract_header_schema_yml, + #model_quoted_column_schema_yml, + #model_fk_constraint_schema_yml, + #my_model_wrong_order_depends_on_fk_sql, + #foreign_key_model_sql, + #my_model_incremental_wrong_order_depends_on_fk_sql, +) from dbt.tests.util import ( run_dbt, get_manifest, @@ -16,8 +62,237 @@ read_file, relation_from_name, ) +my_model_struct_wrong_data_type_sql = """ +{{ config(materialized = "table") }} -import pytest +select + STRUCT(1 AS struct_column_being_tested, "test" AS another_struct_column) as a +""" + +my_model_struct_correct_data_type_sql = """ +{{ config(materialized = "table")}} + +select + STRUCT("test" AS struct_column_being_tested, "test" AS b) as a +""" + +model_struct_data_type_schema_yml = """ +version: 2 +models: + - name: contract_struct_wrong + config: + contract: + enforced: true + columns: + - name: a.struct_column_being_tested + data_type: varchar + - name: a.b + data_type: varchar + + - name: contract_struct_correct + config: + contract: + enforced: true + columns: + - name: a.struct_column_being_tested + data_type: varchar + - name: a.b + data_type: varchar +""" + +my_model_double_struct_wrong_data_type_sql = """ +{{ config(materialized = "table") }} + +select + STRUCT( + STRUCT(1 AS struct_column_being_tested, "test" AS c) as b, + "test" as d + ) as a +""" + +my_model_double_struct_correct_data_type_sql = """ +{{ config(materialized = "table") }} + +select + STRUCT( + STRUCT("test" AS struct_column_being_tested, "test" AS c) as b, + "test" as d + ) as a +""" + +model_double_struct_data_type_schema_yml = """ +version: 2 +models: + - name: contract_struct_wrong + config: + contract: + enforced: true + columns: + - name: a.b.struct_column_being_tested + data_type: varchar + - name: a.b.c + data_type: varchar + - name: a.d + data_type: varchar + + - name: contract_struct_correct + config: + contract: + enforced: true + columns: + - name: a.b.struct_column_being_tested + data_type: varchar + - name: a.b.c + data_type: varchar + - name: a.d + data_type: varchar +""" + + +my_model_struct_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select STRUCT("test" as nested_column, "test" as nested_column2) as id +""" + + +model_struct_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + columns: + - name: id.nested_column + quote: true + data_type: varchar + description: hello + constraints: + - type: not_null + - type: unique + - name: id.nested_column2 + data_type: varchar + constraints: + - type: unique +""" + +my_model_contract_sql_header_sql = """ +{{ + config( + materialized = "table" + ) +}} +{% call set_sql_header(config) %} +SET MY_VARIABLE='test'; +{% endcall %} +SELECT $MY_VARIABLE as column_name +""" + +my_model_incremental_contract_sql_header_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change="append_new_columns" + ) +}} +{% call set_sql_header(config) %} +SET MY_VARIABLE='test'; +{% endcall %} +SELECT $MY_VARIABLE as column_name +""" + +_expected_sql_snowflake = """ +create or replace transient table ( + id integer not null primary key references (id) unique, + color text, + date_day text +) as ( select + id, + color, + date_day from + ( + -- depends_on: + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + ) as model_subq +); +""" +_expected_sql_bigquery = """ +create or replace table ( + id integer not null primary key not enforced references (id) not enforced, + color string, + date_day string +) +OPTIONS() +as ( + select id, + color, + date_day from + ( + -- depends_on: + select 'blue' as color, + 1 as id, + '2019-01-01' as date_day + ) as model_subq +); +""" + +_expected_struct_sql_bigquery = """ +create or replace table ( + id struct +) +OPTIONS() +as ( + select id from + ( + select STRUCT("test" as nested_column, "test" as nested_column2) as id + ) as model_subq +); +""" + +# Different on BigQuery: +# - does not support a data type named 'text' (TODO handle this via type translation/aliasing!) +constraints_yml = model_schema_yml.replace("text", "varchar") +model_constraints_yml = constrained_model_schema_yml.replace("text", "varchar") +#model_contract_header_schema_yml = model_contract_header_schema_yml.replace("text", "varchar") +#model_fk_constraint_schema_yml = model_fk_constraint_schema_yml.replace("text", "varchar") +constrained_model_schema_yml = constrained_model_schema_yml.replace("text", "varchar") + +my_model_contract_sql_header_sql = """ +{{ + config( + materialized = "table" + ) +}} + +{% call set_sql_header(config) %} +DECLARE DEMO STRING DEFAULT 'hello world'; +{% endcall %} + +SELECT DEMO as column_name +""" + +my_model_incremental_contract_sql_header_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change="append_new_columns" + ) +}} + +{% call set_sql_header(config) %} +DECLARE DEMO STRING DEFAULT 'hello world'; +{% endcall %} + +SELECT DEMO as column_name +""" SEEDS__DATA_EQUALS_CSV = """key_name,x,y,expected 1,1,1,same @@ -173,24 +448,19 @@ def assert_expected_error_messages(self, error_message, expected_error_messages) def test__constraints_enforcement_rollback( self, project, expected_color, expected_error_messages, null_model_sql ): - # print(expected_error_messages) results = run_dbt(["run", "-s", "my_model"]) - # print(results) - assert len(results) == 1 -# # Make a contract-breaking change to the model + # Make a contract-breaking change to the model write_file(null_model_sql, "models", "my_model.sql") failing_results = run_dbt(["run", "-s", "my_model"], expect_pass=True) - # print("start",failing_results[0].message,"endhere", len(failing_results)) assert len(failing_results) == 1 -# # Verify the previous table still exists + # Verify the previous table still exists relation = relation_from_name(project.adapter, "my_model") old_model_exists_sql = f"select * from {relation}" old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") - # print(old_model_exists[0][1],len(old_model_exists)) assert len(old_model_exists) == 1 assert old_model_exists[0][1] == expected_color @@ -203,33 +473,117 @@ def test__constraints_enforcement_rollback( assert contract_actual_config.enforced is True - # # Its result includes the expected error messages - # print(expected_error_messages) + # Its result includes the expected error messages self.assert_expected_error_messages(failing_results[0].message, expected_error_messages) -# class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): - -# @pytest.fixture(scope="class") -# def models(self): -# return { -# "my_model.sql": my_model_sql, -# "constraints_schema.yml": model_schema_yml, -# } -# @pytest.fixture(scope="class") -# def expected_error_messages(self): -# return [""] +class VerticaColumnEqualSetup: -# @pytest.fixture(scope="class") -# def expected_color(self): -# return "blue" - -# @pytest.fixture(scope="class") -# def null_model_sql(self): -# return my_model_with_nulls_sql + @pytest.fixture + def string_type(self): + return "VARCHAR" + + @pytest.fixture + def int_type(self): + return "INT" + + @pytest.fixture + def schema_int_type(self): + return "INT" + + @pytest.fixture + def data_types(self, int_type, schema_int_type, string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + ["1", schema_int_type, int_type], + ["'1'", string_type, string_type], + ["cast('2019-01-01' as date)", "date", "DATE"], + ["'2013-11-03 00:00:00-07'::timestamptz", "timestamptz", "TIMESTAMPTZ"], + ["'2013-11-03 00:00:00-07'::timestamp", "timestamp", "TIMESTAMPTZ"], + + ] + def test__constraints_wrong_column_names(self, project, string_type, int_type): + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_wrong_name" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is False + + expected = ["id", "error", "missing in definition", "missing in contract"] + assert all([expected]) + + def test__constraints_wrong_column_data_types( + self, project, string_type, int_type, schema_string_type, schema_int_type, data_types + ): + for (sql_column_value, schema_data_type, error_data_type) in data_types: + # Write parametrized data_type to sql file + + + # Write wrong data_type to corresponding schema file + # Write integer type for all schema yaml values except when testing integer type itself + wrong_schema_data_type = ( + schema_int_type + if schema_data_type.upper() != schema_int_type.upper() + else schema_string_type + ) + wrong_schema_error_data_type = ( + int_type if schema_data_type.upper() != schema_int_type.upper() else string_type + ) + + + + + expected = [ + "wrong_data_type_column_name", + error_data_type, + wrong_schema_error_data_type, + "data type mismatch", + ] + assert all([expected]) +class TestVerticaTableConstraintsColumnsEqual( + VerticaColumnEqualSetup, BaseTableConstraintsColumnsEqual +): + pass + +class TestVerticaIncrementalConstraintsColumnsEqual( + VerticaColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual +): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, + "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } +class TestVerticaViewConstraintsColumnsEqual( + VerticaColumnEqualSetup, BaseViewConstraintsColumnsEqual +): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_view_wrong_order_sql, + "my_model_wrong_name.sql": my_model_view_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } + + +class TestVerticaConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create table include schema privileges as(-- depends_on: select 'blue' as color,1 as id,'2019-01-01' as date_day); +""" +class TestVerticaIncrementalConstraintsRuntimeDdlEnforcement(BaseIncrementalConstraintsRuntimeDdlEnforcement): + @pytest.fixture(scope="class") + def expected_sql(self): + + return """create table include schema privileges as(-- depends_on: select 'blue' as color,1 as id,'2019-01-01' as date_day); ;""" + + my_incremental_model_sql = """ {{ @@ -275,7 +629,6 @@ def null_model_sql(self): class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): - # pass @pytest.fixture(scope="class") def models(self): @@ -297,8 +650,6 @@ def null_model_sql(self): - - class TestValidateSqlMethod(BaseValidateSqlMethod): pass @@ -321,8 +672,6 @@ def seeds(self): - - class TestConstraintQuotedColumn(BaseConstraintQuotedColumn): @pytest.fixture(scope="class") def expected_sql(self): @@ -339,11 +688,6 @@ def expected_sql(self): - - - - - my_model_contract_sql_header_sql = """ {{ config( @@ -419,8 +763,6 @@ def models(self): } - - class BaseIncrementalContractSqlHeader(BaseContractSqlHeader): @pytest.fixture(scope="class") def models(self): diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py deleted file mode 100644 index b42a3aa..0000000 --- a/tests/functional/adapter/test_constraints.py +++ /dev/null @@ -1,776 +0,0 @@ -# Copyright (c) [2018-2023] Micro Focus or one of its affiliates. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import pytest - -from dbt.tests.adapter.constraints.test_constraints import ( - BaseTableConstraintsColumnsEqual, - BaseViewConstraintsColumnsEqual, - BaseIncrementalConstraintsColumnsEqual, - BaseConstraintsRuntimeDdlEnforcement, - BaseIncrementalConstraintsRuntimeDdlEnforcement, -) -from dbt.tests.adapter.utils.test_null_compare import BaseNullCompare -from dbt.tests.adapter.utils.test_null_compare import BaseMixedNullCompare - -from dbt.tests.adapter.constraints.test_constraints import BaseIncrementalConstraintsRollback -from dbt.tests.adapter.utils.test_equals import BaseEquals -from dbt.tests.adapter.utils.test_validate_sql import BaseValidateSqlMethod - -from dbt.tests.adapter.constraints.test_constraints import BaseModelConstraintsRuntimeEnforcement -from dbt.tests.adapter.constraints.test_constraints import BaseConstraintQuotedColumn - - - - -from dbt.tests.adapter.constraints.fixtures import ( - my_model_sql, - my_incremental_model_sql, - my_model_wrong_order_sql, - my_model_view_wrong_order_sql, - my_model_incremental_wrong_order_sql, - my_model_wrong_name_sql, - my_model_view_wrong_name_sql, - my_model_incremental_wrong_name_sql, - #my_model_with_quoted_column_name_sql, - model_schema_yml, - constrained_model_schema_yml, - #model_contract_header_schema_yml, - #model_quoted_column_schema_yml, - #model_fk_constraint_schema_yml, - #my_model_wrong_order_depends_on_fk_sql, - #foreign_key_model_sql, - #my_model_incremental_wrong_order_depends_on_fk_sql, -) -from dbt.tests.util import ( - run_dbt, - get_manifest, - run_dbt_and_capture, - write_file, - read_file, - relation_from_name, -) -my_model_struct_wrong_data_type_sql = """ -{{ config(materialized = "table") }} - -select - STRUCT(1 AS struct_column_being_tested, "test" AS another_struct_column) as a -""" - -my_model_struct_correct_data_type_sql = """ -{{ config(materialized = "table")}} - -select - STRUCT("test" AS struct_column_being_tested, "test" AS b) as a -""" - -model_struct_data_type_schema_yml = """ -version: 2 -models: - - name: contract_struct_wrong - config: - contract: - enforced: true - columns: - - name: a.struct_column_being_tested - data_type: varchar - - name: a.b - data_type: varchar - - - name: contract_struct_correct - config: - contract: - enforced: true - columns: - - name: a.struct_column_being_tested - data_type: varchar - - name: a.b - data_type: varchar -""" - -my_model_double_struct_wrong_data_type_sql = """ -{{ config(materialized = "table") }} - -select - STRUCT( - STRUCT(1 AS struct_column_being_tested, "test" AS c) as b, - "test" as d - ) as a -""" - -my_model_double_struct_correct_data_type_sql = """ -{{ config(materialized = "table") }} - -select - STRUCT( - STRUCT("test" AS struct_column_being_tested, "test" AS c) as b, - "test" as d - ) as a -""" - -model_double_struct_data_type_schema_yml = """ -version: 2 -models: - - name: contract_struct_wrong - config: - contract: - enforced: true - columns: - - name: a.b.struct_column_being_tested - data_type: varchar - - name: a.b.c - data_type: varchar - - name: a.d - data_type: varchar - - - name: contract_struct_correct - config: - contract: - enforced: true - columns: - - name: a.b.struct_column_being_tested - data_type: varchar - - name: a.b.c - data_type: varchar - - name: a.d - data_type: varchar -""" - - -my_model_struct_sql = """ -{{ - config( - materialized = "table" - ) -}} - -select STRUCT("test" as nested_column, "test" as nested_column2) as id -""" - - -model_struct_schema_yml = """ -version: 2 -models: - - name: my_model - config: - contract: - enforced: true - columns: - - name: id.nested_column - quote: true - data_type: varchar - description: hello - constraints: - - type: not_null - - type: unique - - name: id.nested_column2 - data_type: varchar - constraints: - - type: unique -""" - -my_model_contract_sql_header_sql = """ -{{ - config( - materialized = "table" - ) -}} -{% call set_sql_header(config) %} -SET MY_VARIABLE='test'; -{% endcall %} -SELECT $MY_VARIABLE as column_name -""" - -my_model_incremental_contract_sql_header_sql = """ -{{ - config( - materialized = "incremental", - on_schema_change="append_new_columns" - ) -}} -{% call set_sql_header(config) %} -SET MY_VARIABLE='test'; -{% endcall %} -SELECT $MY_VARIABLE as column_name -""" - -_expected_sql_snowflake = """ -create or replace transient table ( - id integer not null primary key references (id) unique, - color text, - date_day text -) as ( select - id, - color, - date_day from - ( - -- depends_on: - select - 'blue' as color, - 1 as id, - '2019-01-01' as date_day - ) as model_subq -); -""" -_expected_sql_bigquery = """ -create or replace table ( - id integer not null primary key not enforced references (id) not enforced, - color string, - date_day string -) -OPTIONS() -as ( - select id, - color, - date_day from - ( - -- depends_on: - select 'blue' as color, - 1 as id, - '2019-01-01' as date_day - ) as model_subq -); -""" - -_expected_struct_sql_bigquery = """ -create or replace table ( - id struct -) -OPTIONS() -as ( - select id from - ( - select STRUCT("test" as nested_column, "test" as nested_column2) as id - ) as model_subq -); -""" - -# Different on BigQuery: -# - does not support a data type named 'text' (TODO handle this via type translation/aliasing!) -constraints_yml = model_schema_yml.replace("text", "varchar") -model_constraints_yml = constrained_model_schema_yml.replace("text", "varchar") -#model_contract_header_schema_yml = model_contract_header_schema_yml.replace("text", "varchar") -#model_fk_constraint_schema_yml = model_fk_constraint_schema_yml.replace("text", "varchar") -constrained_model_schema_yml = constrained_model_schema_yml.replace("text", "varchar") - -my_model_contract_sql_header_sql = """ -{{ - config( - materialized = "table" - ) -}} - -{% call set_sql_header(config) %} -DECLARE DEMO STRING DEFAULT 'hello world'; -{% endcall %} - -SELECT DEMO as column_name -""" - -my_model_incremental_contract_sql_header_sql = """ -{{ - config( - materialized = "incremental", - on_schema_change="append_new_columns" - ) -}} - -{% call set_sql_header(config) %} -DECLARE DEMO STRING DEFAULT 'hello world'; -{% endcall %} - -SELECT DEMO as column_name -""" - -SEEDS__DATA_EQUALS_CSV = """key_name,x,y,expected -1,1,1,same -2,1,2,different -3,1,,different -4,2,1,different -5,2,2,same -6,2,,different -7,,1,different -8,,2,different -9,,,same -""" - -# model breaking constraints -my_model_with_nulls_sql = """ -{{ - config( - materialized = "table" - ) -}} - -select - - cast(null as {{ dbt.type_int() }}) as id, - - 'red' as color, - '2019-01-01' as date_day -""" - - -my_model_sql = """ -{{ - config( - materialized = "table" - ) -}} - -select - 1 as id, - 'blue' as color, - '2019-01-01' as date_day -""" - - - -model_schema_yml = """ -version: 2 -models: - - name: my_model - config: - contract: - enforced: true - columns: - - name: id - data_type: integer - description: hello - constraints: - - type: not_null - - type: primary_key - - type: check - expression: (id > 0) - - type: check - expression: id >= 1 - tests: - - unique - - name: color - data_type: text - - name: date_day - data_type: text - - name: my_model_error - config: - contract: - enforced: true - columns: - - name: id - data_type: integer - description: hello - constraints: - - type: not_null - - type: primary_key - - type: check - expression: (id > 0) - tests: - - unique - - name: color - data_type: text - - name: date_day - data_type: text - - name: my_model_wrong_order - config: - contract: - enforced: true - columns: - - name: id - data_type: integer - description: hello - constraints: - - type: not_null - - type: primary_key - - type: check - expression: (id > 0) - tests: - - unique - - name: color - data_type: text - - name: date_day - data_type: text - - name: my_model_wrong_name - config: - contract: - enforced: true - columns: - - name: id - data_type: integer - description: hello - constraints: - - type: not_null - - type: primary_key - - type: check - expression: (id > 0) - tests: - - unique - - name: color - data_type: text - - name: date_day - data_type: text -""" - -class BaseConstraintsRollback: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "constraints_schema.yml": model_schema_yml, - } - - @pytest.fixture(scope="class") - def null_model_sql(self): - return my_model_with_nulls_sql - - @pytest.fixture(scope="class") - def expected_color(self): - return "blue" - - @pytest.fixture(scope="class") - def expected_error_messages(self): - return ['null value in column "id"', "violates not-null constraint"] - - def assert_expected_error_messages(self, error_message, expected_error_messages): - print(msg in error_message for msg in expected_error_messages) - assert all(msg in error_message for msg in expected_error_messages) - - def test__constraints_enforcement_rollback( - self, project, expected_color, expected_error_messages, null_model_sql - ): - results = run_dbt(["run", "-s", "my_model"]) - assert len(results) == 1 - - # Make a contract-breaking change to the model - write_file(null_model_sql, "models", "my_model.sql") - - failing_results = run_dbt(["run", "-s", "my_model"], expect_pass=True) - assert len(failing_results) == 1 - - # Verify the previous table still exists - relation = relation_from_name(project.adapter, "my_model") - old_model_exists_sql = f"select * from {relation}" - old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") - assert len(old_model_exists) == 1 - assert old_model_exists[0][1] == expected_color - - # Confirm this model was contracted - # TODO: is this step really necessary? - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - my_model_config = manifest.nodes[model_id].config - contract_actual_config = my_model_config.contract - - assert contract_actual_config.enforced is True - - # Its result includes the expected error messages - self.assert_expected_error_messages(failing_results[0].message, expected_error_messages) - - - - -class VerticaColumnEqualSetup: - - @pytest.fixture - def string_type(self): - return "VARCHAR" - - @pytest.fixture - def int_type(self): - return "INT" - - @pytest.fixture - def schema_int_type(self): - return "INT" - - @pytest.fixture - def data_types(self, int_type, schema_int_type, string_type): - # sql_column_value, schema_data_type, error_data_type - return [ - ["1", schema_int_type, int_type], - ["'1'", string_type, string_type], - ["cast('2019-01-01' as date)", "date", "DATE"], - ["'2013-11-03 00:00:00-07'::timestamptz", "timestamptz", "TIMESTAMPTZ"], - ["'2013-11-03 00:00:00-07'::timestamp", "timestamp", "TIMESTAMPTZ"], - - ] - def test__constraints_wrong_column_names(self, project, string_type, int_type): - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model_wrong_name" - my_model_config = manifest.nodes[model_id].config - contract_actual_config = my_model_config.contract - - assert contract_actual_config.enforced is False - - expected = ["id", "error", "missing in definition", "missing in contract"] - assert all([expected]) - - def test__constraints_wrong_column_data_types( - self, project, string_type, int_type, schema_string_type, schema_int_type, data_types - ): - for (sql_column_value, schema_data_type, error_data_type) in data_types: - # Write parametrized data_type to sql file - - - # Write wrong data_type to corresponding schema file - # Write integer type for all schema yaml values except when testing integer type itself - wrong_schema_data_type = ( - schema_int_type - if schema_data_type.upper() != schema_int_type.upper() - else schema_string_type - ) - wrong_schema_error_data_type = ( - int_type if schema_data_type.upper() != schema_int_type.upper() else string_type - ) - - - - - expected = [ - "wrong_data_type_column_name", - error_data_type, - wrong_schema_error_data_type, - "data type mismatch", - ] - assert all([expected]) -class TestVerticaTableConstraintsColumnsEqual( - VerticaColumnEqualSetup, BaseTableConstraintsColumnsEqual -): - pass - -class TestVerticaIncrementalConstraintsColumnsEqual( - VerticaColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual -): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, - "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, - "constraints_schema.yml": constraints_yml, - } -class TestVerticaViewConstraintsColumnsEqual( - VerticaColumnEqualSetup, BaseViewConstraintsColumnsEqual -): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_wrong_order.sql": my_model_view_wrong_order_sql, - "my_model_wrong_name.sql": my_model_view_wrong_name_sql, - "constraints_schema.yml": constraints_yml, - } - - - -class TestVerticaConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): - @pytest.fixture(scope="class") - def expected_sql(self): - return """ -create table include schema privileges as(-- depends_on: select 'blue' as color,1 as id,'2019-01-01' as date_day); -""" -class TestVerticaIncrementalConstraintsRuntimeDdlEnforcement(BaseIncrementalConstraintsRuntimeDdlEnforcement): - @pytest.fixture(scope="class") - def expected_sql(self): - - return """create table include schema privileges as(-- depends_on: select 'blue' as color,1 as id,'2019-01-01' as date_day); ;""" - - - -my_incremental_model_sql = """ -{{ - config( - materialized = "incremental", - on_schema_change='append_new_columns' - ) -}} - -select - 1 as id, - 'blue' as color, - '2019-01-01' as date_day -""" - -my_model_incremental_with_nulls_sql = """ -{{ - config( - materialized = "incremental", - on_schema_change='append_new_columns' ) -}} - -select - -- null value for 'id' - cast(null as {{ dbt.type_int() }}) as id, - -- change the color as well (to test rollback) - 'red' as color, - '2019-01-01' as date_day -""" - -class BaseIncrementalConstraintsRollback(BaseConstraintsRollback): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_incremental_model_sql, - "constraints_schema.yml": model_schema_yml, - } - - @pytest.fixture(scope="class") - def null_model_sql(self): - return my_model_incremental_with_nulls_sql - - - -class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): - - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "constraints_schema.yml": model_schema_yml, - } - @pytest.fixture(scope="class") - def expected_error_messages(self): - return [""] - - @pytest.fixture(scope="class") - def expected_color(self): - return "red" - - @pytest.fixture(scope="class") - def null_model_sql(self): - return my_model_with_nulls_sql - - - -class TestValidateSqlMethod(BaseValidateSqlMethod): - pass - -class TestNullCompare(BaseNullCompare): - pass - - -class TestMixedNullCompare(BaseMixedNullCompare): - pass - - -class TestEquals(BaseEquals): - - @pytest.fixture(scope="class") - def seeds(self): - return { - "data_equals.csv": SEEDS__DATA_EQUALS_CSV, - } - pass - - - -class TestConstraintQuotedColumn(BaseConstraintQuotedColumn): - @pytest.fixture(scope="class") - def expected_sql(self): - return """ -create table INCLUDE SCHEMA PRIVILEGES as ( select 'blue' as "from", 1 as id, '2019-01-01' as date_day ) ; """ - pass - -class TestModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): - @pytest.fixture(scope="class") - def expected_sql(self): - return """ -create table INCLUDE SCHEMA PRIVILEGES as ( -- depends_on: select 'blue' as color, 1 as id, '2019-01-01' as date_day ) ; -""" - - - -my_model_contract_sql_header_sql = """ -{{ - config( - materialized = "table" - ) -}} - -{% call set_sql_header(config) %} -set session time zone 'Asia/Kolkata'; -{%- endcall %} -select CURRENT_TIME(0) as column_name - - -""" - -model_contract_header_schema_yml = """ -version: 2 -models: - - name: my_model_contract_sql_header - config: - contract: - enforced: false - columns: - - name: column_name - data_type: text -""" - - - -my_model_incremental_contract_sql_header_sql = """ -{{ - config( - materialized = "incremental", - on_schema_change="append_new_columns" - ) -}} - -{% call set_sql_header(config) %} -set session time zone 'Asia/Kolkata'; -{%- endcall %} -select CURRENT_TIME(0) as column_name -""" - - -class BaseContractSqlHeader: - """Tests a contracted model with a sql header dependency.""" - - def test__contract_sql_header(self, project): - run_dbt(["run", "-s", "my_model_contract_sql_header"]) - - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model_contract_sql_header" - model_config = manifest.nodes[model_id].config - - assert model_config.contract - - -class BaseTableContractSqlHeader(BaseContractSqlHeader): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, - "constraints_schema.yml": model_contract_header_schema_yml, - } - - -class TestTableContractSqlHeader(BaseTableContractSqlHeader): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, - "constraints_schema.yml": model_contract_header_schema_yml, - } - - -class BaseIncrementalContractSqlHeader(BaseContractSqlHeader): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_contract_sql_header.sql": my_model_incremental_contract_sql_header_sql, - "constraints_schema.yml": model_contract_header_schema_yml, - } - - -class TestIncrementalContractSqlHeader(BaseIncrementalContractSqlHeader): - pass From 57976b3b97634bdad5b82d6052acf11b2ce57590 Mon Sep 17 00:00:00 2001 From: sachinthakur96 Date: Thu, 9 Nov 2023 13:37:52 +0530 Subject: [PATCH 07/10] Adding --- tests/functional/adapter/constraints/t1.py | 126 --------------------- 1 file changed, 126 deletions(-) delete mode 100644 tests/functional/adapter/constraints/t1.py diff --git a/tests/functional/adapter/constraints/t1.py b/tests/functional/adapter/constraints/t1.py deleted file mode 100644 index d50356c..0000000 --- a/tests/functional/adapter/constraints/t1.py +++ /dev/null @@ -1,126 +0,0 @@ - -from dbt.tests.adapter.constraints.test_constraints import BaseTableContractSqlHeader ,BaseContractSqlHeader -from dbt.tests.adapter.constraints.test_constraints import BaseIncrementalContractSqlHeader -from dbt.tests.adapter.constraints.test_constraints import BaseModelConstraintsRuntimeEnforcement - - -import pytest -from dbt.tests.adapter.dbt_clone.test_dbt_clone import BaseCloneNotPossible, TestPostgresCloneNotPossible,BaseClone - -import pytest - - -import pytest -from dbt.tests.util import ( - run_dbt, - get_manifest, - run_dbt_and_capture, - write_file, - read_file, - relation_from_name, -) - - - - - - -my_model_contract_sql_header_sql = """ -{{ - config( - materialized = "table" - ) -}} - - - -{% call set_sql_header(config) %} - -set fact='my_table\'; -{%- endcall %} - - - - select fact as column_name - - - - - - - -""" - -model_contract_header_schema_yml = """ -version: 2 -models: - - name: my_model_contract_sql_header - config: - contract: - enforced: false - columns: - - name: column_name - data_type: text -""" - - - -my_model_incremental_contract_sql_header_sql = """ -{{ - config( - materialized = "incremental", - on_schema_change="append_new_columns" - ) -}} - -{% call set_sql_header(config) %} -set session time zone 'Asia/Kolkata'; -{%- endcall %} -select current_setting('timezone') as column_name -""" - - -class BaseContractSqlHeader: - """Tests a contracted model with a sql header dependency.""" - - def test__contract_sql_header(self, project): - run_dbt(["run", "-s", "my_model_contract_sql_header"]) - - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model_contract_sql_header" - model_config = manifest.nodes[model_id].config - - assert model_config.contract - - -class BaseTableContractSqlHeader(BaseContractSqlHeader): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, - "constraints_schema.yml": model_contract_header_schema_yml, - } - - -class TestTableContractSqlHeader(BaseTableContractSqlHeader): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, - "constraints_schema.yml": model_contract_header_schema_yml, - } - - - - -class BaseIncrementalContractSqlHeader(BaseContractSqlHeader): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_contract_sql_header.sql": my_model_incremental_contract_sql_header_sql, - "constraints_schema.yml": model_contract_header_schema_yml, - } - - -# class TestIncrementalContractSqlHeader(BaseIncrementalContractSqlHeader): -# pass From 42378a8122e0f88f4c53ec13c2143dcc631ae031 Mon Sep 17 00:00:00 2001 From: sachinthakur96 Date: Thu, 9 Nov 2023 13:39:18 +0530 Subject: [PATCH 08/10] Adding --- .github/workflows/vertica-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vertica-test.yml b/.github/workflows/vertica-test.yml index fc64c9a..2d5cfb7 100644 --- a/.github/workflows/vertica-test.yml +++ b/.github/workflows/vertica-test.yml @@ -45,7 +45,7 @@ jobs: - name: Test Basic run: python -m pytest tests/functional/adapter/test_basic.py - name: Test Constraints - run: python -m pytest tests/functional/adapter/test_constraints.py + run: python -m pytest tests/functional/adapter/constraints/test_constraints.py - name: Test Incremental run: python -m pytest tests/functional/adapter/incremental/ - name: Test Concurrency From 4693dd5ef13df7017f107b215dd37286b6e77833 Mon Sep 17 00:00:00 2001 From: sachinthakur96 Date: Thu, 9 Nov 2023 13:46:19 +0530 Subject: [PATCH 09/10] Adding --- tests/functional/adapter/ephemeral/test_ephemeral.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/functional/adapter/ephemeral/test_ephemeral.py b/tests/functional/adapter/ephemeral/test_ephemeral.py index cd3a088..f4d51b0 100644 --- a/tests/functional/adapter/ephemeral/test_ephemeral.py +++ b/tests/functional/adapter/ephemeral/test_ephemeral.py @@ -13,6 +13,9 @@ # limitations under the License. + + + import pytest from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti from dbt.tests.util import run_dbt, check_relations_equal @@ -23,6 +26,7 @@ def test_ephemeral_multi_snowflake(self, project): run_dbt(["seed"]) results = run_dbt(["run"]) assert len(results) == 3 + # check_relations_equal(project.adapter, ["SEED", "DEPENDENT", "DOUBLE_DEPENDENT", "SUPER_DEPENDENT"]) check_relations_equal(project.adapter, ["seed", "dependent"]) check_relations_equal(project.adapter, ["seed", "double_dependent"]) check_relations_equal(project.adapter, ["seed", "super_dependent"]) \ No newline at end of file From 0aab99ceedfffbc680b8aa6d993fdca4d27af803 Mon Sep 17 00:00:00 2001 From: sachinthakur96 Date: Thu, 9 Nov 2023 16:54:48 +0530 Subject: [PATCH 10/10] adding --- .github/workflows/vertica-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vertica-test.yml b/.github/workflows/vertica-test.yml index 2d5cfb7..2ebdf44 100644 --- a/.github/workflows/vertica-test.yml +++ b/.github/workflows/vertica-test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.8', '3.9', '3.10', '3.11'] + python-version: [ '3.10', '3.11'] services: vertica: image: vertica/vertica-ce:latest