diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index 14038e0c..cad8d8e2 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -104,6 +104,40 @@ jobs: - name: Run BigQuery Tests run: tox -e integration_bigquery + integration-sqlserver: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR + + + - uses: actions/setup-python@v2 + with: + python-version: '3.8.x' + architecture: 'x64' + + - name: Install SQL Server + run: docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=1StrongPwd!!" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2019-latest + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + + - name: Install tox + run: python3 -m pip install tox + + - name: Run SQL Server Tests + run: tox -e integration_sqlserver + + - name: Upload logfile + uses: actions/upload-artifact@v3 + if: always() + with: + name: dbt.log + path: /home/runner/work/dbt_artifacts/dbt_artifacts/integration_test_project/logs/ + sqlfluff-lint-models: name: Lint dbt models using SQLFluff runs-on: ubuntu-latest diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 02843710..30d07c7d 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -91,6 +91,37 @@ jobs: - name: Run BigQuery Tests run: tox -e integration_bigquery + integration-sqlserver: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: '3.8.x' + architecture: 'x64' + + - name: Install SQL Server + run: docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=1StrongPwd!!" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2019-latest + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + + - name: Install tox + run: python3 -m pip install tox + + - name: Run SQL Server Tests + run: tox -e integration_sqlserver + + - name: Upload logfile + uses: actions/upload-artifact@v3 + if: always() + with: + name: dbt.log + path: /home/runner/work/dbt_artifacts/dbt_artifacts/integration_test_project/logs/ + sqlfluff-lint-models: name: Lint dbt models using SQLFluff runs-on: ubuntu-latest diff --git a/README.md b/README.md index 29a00140..eb4466d7 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ The package currently supports - Spark :white_check_mark: - Snowflake adapters :white_check_mark: - Google BigQuery :white_check_mark: +- Microsoft SQL Server :white_check_mark: Models included: diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index cd1bbe79..b2d3da30 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -30,8 +30,8 @@ vars: models: +persist_docs: - relation: true - columns: true + relation: "{{ false if target.name == 'sqlserver' else true }}" + columns: "{{ false if target.name == 'sqlserver' else true }}" seeds: +quote_columns: false diff --git a/integration_test_project/models/tests_and_exposures.yml b/integration_test_project/models/tests_and_exposures.yml index b397edc0..d7a2d609 100644 --- a/integration_test_project/models/tests_and_exposures.yml +++ b/integration_test_project/models/tests_and_exposures.yml @@ -10,10 +10,10 @@ models: - unique exposures: - - name: "ceo's imaginary dashboard" + - name: "ceos_imaginary_dashboard" type: dashboard maturity: high - description: "ceo's favourite dashboard" + description: "ceos favourite dashboard" url: https://bi.tool/dashboards/1 depends_on: @@ -23,7 +23,7 @@ exposures: name: Claire from Data email: data@jaffleshop.com - - name: "CIO dashboard" + - name: "CIO_dashboard" type: dashboard maturity: high description: '{{ doc("clickstream") }}' diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index df40f909..114ecf2f 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -43,3 +43,15 @@ dbt_artifacts: timeout_seconds: 300 priority: interactive retries: 1 + sqlserver: + type: sqlserver + #schema: dbt_artifacts_test_commit_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} + schema: dbttest + driver: 'ODBC Driver 18 for SQL Server' # (The ODBC Driver installed on your system) + server: localhost + port: 1433 + database: master + user: sa + password: 1StrongPwd!! + encrypt: True + trust_cert: True diff --git a/integration_test_project/tests/singular_test.sql b/integration_test_project/tests/singular_test.sql index 67a2508d..f7ff493e 100644 --- a/integration_test_project/tests/singular_test.sql +++ b/integration_test_project/tests/singular_test.sql @@ -1 +1,5 @@ +{% if target.name == 'sqlserver' %} +select 1 as failures from (values(2)) as tab(col) where 1 = 2 +{% else %} select 1 as failures from (select 2) where 1 = 2 +{% endif %} diff --git a/macros/column_identifier.sql b/macros/column_identifier.sql index 9e1d0665..75ad34a4 100644 --- a/macros/column_identifier.sql +++ b/macros/column_identifier.sql @@ -13,3 +13,7 @@ {% macro spark__column_identifier(column_index) -%} col{{ column_index }} {%- endmacro %} + +{% macro sqlserver__column_identifier(column_index) -%} + col{{ column_index }} +{%- endmacro %} diff --git a/macros/insert_into_metadata_table.sql b/macros/insert_into_metadata_table.sql index bc464f6a..658064e3 100644 --- a/macros/insert_into_metadata_table.sql +++ b/macros/insert_into_metadata_table.sql @@ -34,5 +34,14 @@ {%- endmacro %} +{% macro sqlserver__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} + {% set insert_into_table_query %} + insert into {{ schema_name }}.{{ table_name }} + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} +{%- endmacro %} + {% macro default__insert_into_metadata_table(database_name, schema_name, table_name, content) -%} {%- endmacro %} diff --git a/macros/quote_reserved_keywords.sql b/macros/quote_reserved_keywords.sql new file mode 100644 index 00000000..d7890ee0 --- /dev/null +++ b/macros/quote_reserved_keywords.sql @@ -0,0 +1,210 @@ +{% macro quote_reserved_keywords(column_name) -%} + {{ return(adapter.dispatch('quote_reserved_keywords')(column_name)) }} +{%- endmacro %} + +{% macro default__quote_reserved_keywords(column_name) -%} + {{ column_name }} +{%- endmacro %} + +{% macro snowflake__quote_reserved_keywords(column_name) -%} + {{ column_name }} +{%- endmacro %} + +{% macro spark__quote_reserved_keywords(column_name) -%} + {{ column_name }} +{%- endmacro %} + +{% macro sqlserver__quote_reserved_keywords(column_name) -%} + {%- set reserved_keywords = [ + "ADD", + "ALL", + "ALTER", + "AND", + "ANY", + "AS", + "ASC", + "AUTHORIZATION", + "BACKUP", + "BEGIN", + "BETWEEN", + "BREAK", + "BROWSE", + "BULK", + "BY", + "CASCADE", + "CASE", + "CHECK", + "CHECKPOINT", + "CLOSE", + "CLUSTERED", + "COALESCE", + "COLLATE", + "COLUMN", + "COMMIT", + "COMPUTE", + "CONSTRAINT", + "CONTAINS", + "CONTAINSTABLE", + "CONTINUE", + "CONVERT", + "CREATE", + "CROSS", + "CURRENT", + "CURRENT_DATE", + "CURRENT_TIME", + "CURRENT_TIMESTAMP", + "CURRENT_USER", + "CURSOR", + "DATABASE", + "DBCC", + "DEALLOCATE", + "DECLARE", + "DEFAULT", + "DELETE", + "DENY", + "DESC", + "DISK", + "DISTINCT", + "DISTRIBUTED", + "DOUBLE", + "DROP", + "DUMP", + "ELSE", + "END", + "ERRLVL", + "ESCAPE", + "EXCEPT", + "EXEC", + "EXECUTE", + "EXISTS", + "EXIT", + "EXTERNAL", + "FETCH", + "FILE", + "FILLFACTOR", + "FOR", + "FOREIGN", + "FREETEXT", + "FREETEXTTABLE", + "FROM", + "FULL", + "FUNCTION", + "GOTO", + "GRANT", + "GROUP", + "HAVING", + "HOLDLOCK", + "IDENTITY", + "IDENTITYCOL", + "IDENTITY_INSERT", + "IF", + "IN", + "INDEX", + "INNER", + "INSERT", + "INTERSECT", + "INTO", + "IS", + "JOIN", + "KEY", + "KILL", + "LEFT", + "LIKE", + "LINENO", + "LOAD", + "MERGE", + "NATIONAL", + "NOCHECK", + "NONCLUSTERED", + "NOT", + "NULL", + "NULLIF", + "OF", + "OFF", + "OFFSETS", + "ON", + "OPEN", + "OPENDATASOURCE", + "OPENQUERY", + "OPENROWSET", + "OPENXML", + "OPTION", + "OR", + "ORDER", + "OUTER", + "OVER", + "PERCENT", + "PIVOT", + "PLAN", + "PRECISION", + "PRIMARY", + "PRINT", + "PROC", + "PROCEDURE", + "PUBLIC", + "RAISERROR", + "READ", + "READTEXT", + "RECONFIGURE", + "REFERENCES", + "REPLICATION", + "RESTORE", + "RESTRICT", + "RETURN", + "REVERT", + "REVOKE", + "RIGHT", + "ROLLBACK", + "ROWCOUNT", + "ROWGUIDCOL", + "RULE", + "SAVE", + "SCHEMA", + "SECURITYAUDIT", + "SELECT", + "SEMANTICKEYPHRASETABLE", + "SEMANTICSIMILARITYDETAILSTABLE", + "SEMANTICSIMILARITYTABLE", + "SESSION_USER", + "SET", + "SETUSER", + "SHUTDOWN", + "SOME", + "STATISTICS", + "SYSTEM_USER", + "TABLE", + "TABLESAMPLE", + "TEXTSIZE", + "THEN", + "TO", + "TOP", + "TRAN", + "TRANSACTION", + "TRIGGER", + "TRUNCATE", + "TRY_CONVERT", + "TSEQUAL", + "UNION", + "UNIQUE", + "UNPIVOT", + "UPDATE", + "UPDATETEXT", + "USE", + "USER", + "VALUES", + "VARYING", + "VIEW", + "WAITFOR", + "WHEN", + "WHERE", + "WHILE", + "WITH", + "WITHIN GROUP", + "WRITETEXT" + ] -%} + {%- if column_name.upper() in reserved_keywords -%} + "{{ column_name }}" + {%- else -%} + {{ column_name }} + {%- endif -%} +{%- endmacro %} diff --git a/macros/upload_exposures.sql b/macros/upload_exposures.sql index 93f10517..4676f604 100644 --- a/macros/upload_exposures.sql +++ b/macros/upload_exposures.sql @@ -74,3 +74,59 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_exposures_dml_sql(exposures) -%} + + {% if exposures != [] %} + {% set exposure_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(6)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(12)) }} + from (values + {% for exposure in exposures -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ exposure.unique_id | replace("'","''") }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ exposure.name | replace("'","''") }}', {# name #} + '{{ exposure.type }}', {# type #} + '{{ tojson(exposure.owner) }}', {# owner #} + '{{ exposure.maturity }}', {# maturity #} + '{{ exposure.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ exposure.description | replace("'","''") }}', {# description #} + '{{ exposure.url }}', {# url #} + '{{ exposure.package_name }}', {# package_name #} + '{{ tojson(exposure.depends_on.nodes) }}' {# depends_on_nodes #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) as exposure_values( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(6)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(12)) }} + ) + {% endset %} + {{ exposure_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_invocations.sql b/macros/upload_invocations.sql index 253a7c5b..20c25559 100644 --- a/macros/upload_invocations.sql +++ b/macros/upload_invocations.sql @@ -120,3 +120,95 @@ {{ invocation_values }} {% endmacro -%} + +{% macro sqlserver__get_invocations_dml_sql() -%} + {% set invocation_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + nullif({{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, ''), + nullif({{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, ''), + nullif({{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, ''), + nullif({{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, ''), + nullif({{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, ''), + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(17)) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(18)) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(19)) }} + from (values + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_version }}', {# dbt_version #} + '{{ project_name }}', {# project_name #} + '{{ run_started_at }}', {# run_started_at #} + '{{ flags.WHICH }}', {# dbt_command #} + '{{ flags.FULL_REFRESH }}', {# full_refresh_flag #} + '{{ target.profile_name }}', {# target_profile_name #} + '{{ target.name }}', {# target_name #} + '{{ target.schema }}', {# target_schema #} + {{ target.threads }}, {# target_threads #} + + '{{ env_var('DBT_CLOUD_PROJECT_ID', '') }}', {# dbt_cloud_project_id #} + '{{ env_var('DBT_CLOUD_JOB_ID', '') }}', {# dbt_cloud_job_id #} + '{{ env_var('DBT_CLOUD_RUN_ID', '') }}', {# dbt_cloud_run_id #} + '{{ env_var('DBT_CLOUD_RUN_REASON_CATEGORY', '') }}', {# dbt_cloud_run_reason_category #} + '{{ env_var('DBT_CLOUD_RUN_REASON', '') | replace("'","''") }}', {# dbt_cloud_run_reason #} + + {% if var('env_vars', none) %} + {% set env_vars_dict = {} %} + {% for env_variable in var('env_vars') %} + {% do env_vars_dict.update({env_variable: env_var(env_variable)}) %} + {% endfor %} + '{{ tojson(env_vars_dict) }}', {# env_vars #} + {% else %} + null, {# env_vars #} + {% endif %} + + {% if var('dbt_vars', none) %} + {% set dbt_vars_dict = {} %} + {% for dbt_var in var('dbt_vars') %} + {% do dbt_vars_dict.update({dbt_var: var(dbt_var)}) %} + {% endfor %} + '{{ tojson(dbt_vars_dict) }}', {# dbt_vars #} + {% else %} + null, {# dbt_vars #} + {% endif %} + + + '{{ tojson(invocation_args_dict) | replace('\\', '\\\\') }}', {# invocation_args #} + '{{ tojson(dbt_metadata_envs) }}' {# dbt_custom_envs #} + + ) + ) as invocation_values( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(17)) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(18)) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(19)) }} + ) + {% endset %} + {{ invocation_values }} + +{% endmacro -%} diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql index b730a6ab..2338f000 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_model_executions.sql @@ -205,3 +205,90 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_model_executions_dml_sql(models) -%} + {% if models != [] %} + {% set model_execution_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }} + from (values + {% for model in models -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}' {# alias #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) as model_execution_values( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }} + ) + {% endset %} + {{ model_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_models.sql b/macros/upload_models.sql index 8657739b..0b90ed7c 100644 --- a/macros/upload_models.sql +++ b/macros/upload_models.sql @@ -80,3 +80,65 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_models_dml_sql(models) -%} + + {% if models != [] %} + {% set model_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(7)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(12)) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(13)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }} + from (values + {% for model in models -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ model.database }}', {# database #} + '{{ model.schema }}', {# schema #} + '{{ model.name }}', {# name #} + '{{ tojson(model.depends_on.nodes) | replace('\\', '\\\\') }}', {# depends_on_nodes #} + '{{ model.package_name }}', {# package_name #} + '{{ model.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ model.checksum.checksum }}', {# checksum #} + '{{ model.config.materialized }}', {# materialization #} + '{{ tojson(model.tags) }}', {# tags #} + '{{ tojson(model.config.meta) }}', {# meta #} + '{{ model.alias }}' {# alias #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) as model_values ( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(7)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(12)) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(13)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }} + ) + {% endset %} + {{ model_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index b6604499..228f1e43 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -204,3 +204,90 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_seed_executions_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_execution_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }} + from (values + {% for model in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}' {# alias #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) as seed_execution_values( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }} + ) + {% endset %} + {{ seed_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql index dbdc7c80..fd5dbe3b 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_seeds.sql @@ -71,3 +71,56 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_seeds_dml_sql(seeds) -%} + + {% if seeds != [] %} + {% set seed_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(10)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }} + from (values + {% for seed in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ seed.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ seed.database }}', {# database #} + '{{ seed.schema }}', {# schema #} + '{{ seed.name }}', {# name #} + '{{ seed.package_name }}', {# package_name #} + '{{ seed.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ seed.checksum.checksum }}', {# checksum #} + '{{ tojson(seed.config.meta) }}', {# meta #} + '{{ seed.alias }}' {# alias #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) as seed_values( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(10)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }} + ) + {% endset %} + {{ seed_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_snapshot_executions.sql index d91775c2..2990f265 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_snapshot_executions.sql @@ -25,7 +25,7 @@ {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, - {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }} + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }} from values {% for model in snapshots -%} ( @@ -204,3 +204,90 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_snapshot_executions_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_execution_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }} + from (values + {% for model in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}' {# alias #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) as snapshot_execution_values( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }} + ) + {% endset %} + {{ snapshot_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql index 953315f9..baae3fbe 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_snapshots.sql @@ -78,3 +78,62 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_snapshots_dml_sql(snapshots) -%} + + {% if snapshots != [] %} + {% set snapshot_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(7)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(12)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }} + from (values + {% for snapshot in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ snapshot.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ snapshot.database }}', {# database #} + '{{ snapshot.schema }}', {# schema #} + '{{ snapshot.name }}', {# name #} + '{{ tojson(snapshot.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ snapshot.package_name }}', {# package_name #} + '{{ snapshot.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.config.strategy }}', {# strategy #} + '{{ tojson(snapshot.config.meta) }}', {# meta #} + '{{ snapshot.alias }}' {# alias #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) as snapshot_values( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(7)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(12)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }} + ) + {% endset %} + {{ snapshot_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index b29047ba..f411e0ca 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -71,3 +71,56 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_sources_dml_sql(sources) -%} + + {% if sources != [] %} + {% set source_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(11)) }} + from (values + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.database }}', {# database #} + '{{ source.schema }}', {# schema #} + '{{ source.source_name }}', {# source_name #} + '{{ source.loader }}', {# loader #} + '{{ source.name }}', {# name #} + '{{ source.identifier }}', {# identifier #} + '{{ source.loaded_at_field | replace("'","''") }}', {# loaded_at_field #} + '{{ tojson(source.freshness) | replace("'","''") }}' {# freshness #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) as source_values( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(11)) }} + ) + {% endset %} + {{ source_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_test_executions.sql b/macros/upload_test_executions.sql index 3348da4e..cf622bac 100644 --- a/macros/upload_test_executions.sql +++ b/macros/upload_test_executions.sql @@ -125,3 +125,81 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_test_executions_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_execution_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }} + from (values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = test.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ test.thread_id }}', {# thread_id #} + '{{ test.status }}', {# status #} + + {% if test.timing != [] %} + {% for stage in test.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in test.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ test.execution_time }}, {# total_node_runtime #} + null, {# rows_affected not available in Databricks #} + {{ 'null' if test.failures is none else test.failures }} {# failures #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) as test_execution_values( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }} + ) + {% endset %} + {{ test_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_tests.sql b/macros/upload_tests.sql index f08e74c7..a99e2182 100644 --- a/macros/upload_tests.sql +++ b/macros/upload_tests.sql @@ -62,3 +62,47 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_tests_dml_sql(tests) -%} + + {% if tests != [] %} + {% set test_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(5)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(8)) }} + from (values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ test.name }}', {# name #} + '{{ tojson(test.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ test.package_name }}', {# package_name #} + '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} + '{{ tojson(test.tags) }}' {# tags #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) as test_values( + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(5)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(8)) }} + ) + {% endset %} + {{ test_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index c78a063b..2bb697e7 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -39,9 +39,9 @@ latest_models_runs as ( latest_model_stats as ( select node_id - , max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at - , max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime - , max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected + , max(case when was_full_refresh {% if target.type == 'sqlserver' %}= 1{% endif %} then query_completed_at end) as last_full_refresh_run_completed_at + , max(case when was_full_refresh {% if target.type == 'sqlserver' %}= 1{% endif %} then total_node_runtime end) as last_full_refresh_run_total_runtime + , max(case when was_full_refresh {% if target.type == 'sqlserver' %}= 1{% endif %} then rows_affected end) as last_full_refresh_run_rows_affected {% if target.type == 'bigquery' %} , max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed {% endif %} @@ -53,7 +53,12 @@ latest_model_stats as ( {% endif %} from latest_models_runs where run_idx = 1 - group by 1 + group by + {% if target.type == 'sqlserver' -%} + node_id + {%- else -%} + 1 + {%- endif %} ), final as ( diff --git a/models/dim_dbt__models.sql b/models/dim_dbt__models.sql index 2722ce83..0a632209 100644 --- a/models/dim_dbt__models.sql +++ b/models/dim_dbt__models.sql @@ -12,8 +12,8 @@ models as ( command_invocation_id, node_id, run_started_at, - database, - schema, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, name, depends_on_nodes, package_name, diff --git a/models/dim_dbt__seeds.sql b/models/dim_dbt__seeds.sql index d7a461ab..f9b21dd0 100644 --- a/models/dim_dbt__seeds.sql +++ b/models/dim_dbt__seeds.sql @@ -12,8 +12,8 @@ seeds as ( command_invocation_id, node_id, run_started_at, - database, - schema, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, name, package_name, path, diff --git a/models/dim_dbt__snapshots.sql b/models/dim_dbt__snapshots.sql index 13f05d0e..f308e759 100644 --- a/models/dim_dbt__snapshots.sql +++ b/models/dim_dbt__snapshots.sql @@ -12,8 +12,8 @@ snapshots as ( command_invocation_id, node_id, run_started_at, - database, - schema, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, name, depends_on_nodes, package_name, diff --git a/models/dim_dbt__sources.sql b/models/dim_dbt__sources.sql index 84b76d4d..b3f9e291 100644 --- a/models/dim_dbt__sources.sql +++ b/models/dim_dbt__sources.sql @@ -12,8 +12,8 @@ sources as ( command_invocation_id, node_id, run_started_at, - database, - schema, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, source_name, loader, name, diff --git a/models/fct_dbt__model_executions.sql b/models/fct_dbt__model_executions.sql index e902e1cb..70b7aa3f 100644 --- a/models/fct_dbt__model_executions.sql +++ b/models/fct_dbt__model_executions.sql @@ -23,7 +23,7 @@ model_executions as ( bytes_processed, {% endif %} materialization, - schema, -- noqa + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, -- noqa name, alias from base diff --git a/models/fct_dbt__seed_executions.sql b/models/fct_dbt__seed_executions.sql index 021161f6..43d4f856 100644 --- a/models/fct_dbt__seed_executions.sql +++ b/models/fct_dbt__seed_executions.sql @@ -20,7 +20,7 @@ seed_executions as ( total_node_runtime, rows_affected, materialization, - schema, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, name, alias from base diff --git a/models/fct_dbt__snapshot_executions.sql b/models/fct_dbt__snapshot_executions.sql index 54e0cdb1..62d99e96 100644 --- a/models/fct_dbt__snapshot_executions.sql +++ b/models/fct_dbt__snapshot_executions.sql @@ -20,7 +20,7 @@ snapshot_executions as ( total_node_runtime, rows_affected, materialization, - schema, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, name, alias from base diff --git a/models/sources/model_executions.sql b/models/sources/model_executions.sql index 7b051a8c..735f9ead 100644 --- a/models/sources/model_executions.sql +++ b/models/sources/model_executions.sql @@ -18,7 +18,7 @@ select cast(null as {{ type_int() }}) as bytes_processed, {% endif %} cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, cast(null as {{ type_string() }}) as name, cast(null as {{ type_string() }}) as alias from dummy_cte diff --git a/models/sources/models.sql b/models/sources/models.sql index 4cbb1adc..25b26cdc 100644 --- a/models/sources/models.sql +++ b/models/sources/models.sql @@ -7,8 +7,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, cast(null as {{ type_string() }}) as name, cast(null as {{ type_array() }}) as depends_on_nodes, cast(null as {{ type_string() }}) as package_name, diff --git a/models/sources/seed_executions.sql b/models/sources/seed_executions.sql index 1f79c743..78cb6810 100644 --- a/models/sources/seed_executions.sql +++ b/models/sources/seed_executions.sql @@ -15,7 +15,7 @@ select cast(null as {{ type_float() }}) as total_node_runtime, cast(null as {{ type_int() }}) as rows_affected, cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, cast(null as {{ type_string() }}) as name, cast(null as {{ type_string() }}) as alias from dummy_cte diff --git a/models/sources/seeds.sql b/models/sources/seeds.sql index 3967d243..3a5a9710 100644 --- a/models/sources/seeds.sql +++ b/models/sources/seeds.sql @@ -7,8 +7,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, cast(null as {{ type_string() }}) as name, cast(null as {{ type_string() }}) as package_name, cast(null as {{ type_string() }}) as path, diff --git a/models/sources/snapshot_executions.sql b/models/sources/snapshot_executions.sql index 1f79c743..78cb6810 100644 --- a/models/sources/snapshot_executions.sql +++ b/models/sources/snapshot_executions.sql @@ -15,7 +15,7 @@ select cast(null as {{ type_float() }}) as total_node_runtime, cast(null as {{ type_int() }}) as rows_affected, cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, cast(null as {{ type_string() }}) as name, cast(null as {{ type_string() }}) as alias from dummy_cte diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql index 579fea4d..b9a19a57 100644 --- a/models/sources/snapshots.sql +++ b/models/sources/snapshots.sql @@ -7,8 +7,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, cast(null as {{ type_string() }}) as name, cast(null as {{ type_array() }}) as depends_on_nodes, cast(null as {{ type_string() }}) as package_name, diff --git a/models/sources/sources.sql b/models/sources/sources.sql index 13f69ddc..69ef45e2 100644 --- a/models/sources/sources.sql +++ b/models/sources/sources.sql @@ -7,14 +7,14 @@ select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + cast(null as {{ type_string() }}) as {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, cast(null as {{ type_string() }}) as source_name, cast(null as {{ type_string() }}) as loader, cast(null as {{ type_string() }}) as name, cast(null as {{ type_string() }}) as identifier, cast(null as {{ type_string() }}) as loaded_at_field, - {% if target.type == 'snowflake'%} + {% if target.type == 'snowflake' %} cast(null as {{ type_array() }}) as freshness {% else %} cast(null as {{ type_json() }}) as freshness diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index c94c98d0..69c2579e 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -23,7 +23,7 @@ enhanced as ( bytes_processed, {% endif %} materialization, - schema, -- noqa + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, -- noqa name, alias from base diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index 1208b604..962fbce6 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -12,8 +12,8 @@ enhanced as ( command_invocation_id, node_id, run_started_at, - database, - schema, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, name, depends_on_nodes, package_name, diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index 68bcc725..3129ac4d 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -20,7 +20,7 @@ enhanced as ( total_node_runtime, rows_affected, materialization, - schema, -- noqa + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, -- noqa name, alias from base diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 546e5159..c2bc59f4 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -12,8 +12,8 @@ enhanced as ( command_invocation_id, node_id, run_started_at, - database, - schema, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, name, package_name, path, diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index 66e5ea10..9ebba019 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -20,7 +20,7 @@ enhanced as ( total_node_runtime, rows_affected, materialization, - schema, -- noqa + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, -- noqa name, alias from base diff --git a/models/staging/stg_dbt__snapshots.sql b/models/staging/stg_dbt__snapshots.sql index ff1ee7e4..fb5dfca4 100644 --- a/models/staging/stg_dbt__snapshots.sql +++ b/models/staging/stg_dbt__snapshots.sql @@ -12,8 +12,8 @@ enhanced as ( command_invocation_id, node_id, run_started_at, - database, - schema, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, name, depends_on_nodes, package_name, diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index d35a923a..d3e96222 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -12,8 +12,8 @@ enhanced as ( command_invocation_id, node_id, run_started_at, - database, - schema, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('database') }}, + {{ adapter.dispatch('quote_reserved_keywords', 'dbt_artifacts')('schema') }}, source_name, loader, name, diff --git a/tox.ini b/tox.ini index f9b1e76e..d29317ff 100644 --- a/tox.ini +++ b/tox.ini @@ -143,3 +143,10 @@ commands = dbt deps dbt build --exclude snapshot --target spark +[testenv:integration_sqlserver] +changedir = integration_test_project +deps = dbt-sqlserver~=1.3.0 +commands = + dbt deps + dbt build --target sqlserver +