From c9623058349804eb24f44f7f4b32a160b77f5c1e Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 25 Nov 2024 22:50:23 +0100 Subject: [PATCH] dbt: Start using dbt-cratedb2 --- framework/dbt/basic/macros/catalog.sql | 43 --------- framework/dbt/basic/macros/overrides.sql | 116 ----------------------- framework/dbt/basic/macros/relations.sql | 34 ------- framework/dbt/basic/profiles.yml | 4 +- framework/dbt/basic/requirements.txt | 3 +- 5 files changed, 2 insertions(+), 198 deletions(-) delete mode 100644 framework/dbt/basic/macros/catalog.sql delete mode 100644 framework/dbt/basic/macros/relations.sql diff --git a/framework/dbt/basic/macros/catalog.sql b/framework/dbt/basic/macros/catalog.sql deleted file mode 100644 index de2b0f80..00000000 --- a/framework/dbt/basic/macros/catalog.sql +++ /dev/null @@ -1,43 +0,0 @@ -{% macro postgres__get_catalog(information_schema, schemas) -%} - - {%- call statement('catalog', fetch_result=True) -%} - {# - Derived from https://github.com/dbt-labs/dbt-core/blob/main/plugins/postgres/dbt/include/postgres/macros/catalog.sql - #} - {% set database = information_schema.database %} - {{ adapter.verify_database(database) }} - select - '{{ database }}' as table_database, - sch.nspname as table_schema, - tbl.relname as table_name, - case tbl.relkind - when 'v' then 'VIEW' - else 'BASE TABLE' - end as table_type, - tbl_desc.description as table_comment, - col.attname as column_name, - col.attnum as column_index, - pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type, - col_desc.description as column_comment, - pg_get_userbyid(tbl.relowner) as table_owner - from pg_catalog.pg_namespace sch - join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid - join pg_catalog.pg_attribute col on col.attrelid = tbl.oid - left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0) - left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum) - where ( - {%- for schema in schemas -%} - upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables - and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view - and col.attnum > 0 -- negative numbers are used for system columns such as oid - and not col.attisdropped -- column as not been dropped - order by - sch.nspname, - tbl.relname, - col.attnum - {%- endcall -%} - {{ return(load_result('catalog').table) }} -{%- endmacro %} diff --git a/framework/dbt/basic/macros/overrides.sql b/framework/dbt/basic/macros/overrides.sql index e062636e..e69de29b 100644 --- a/framework/dbt/basic/macros/overrides.sql +++ b/framework/dbt/basic/macros/overrides.sql @@ -1,116 +0,0 @@ -{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %} - {% set sql = "" %} - {% if full_refresh %} - {{ adapter.drop_relation(old_relation) }} - {% set sql = create_csv_table(model, agate_table) %} - {% else %} - {{ adapter.truncate_relation(old_relation) }} - {% set sql = "delete from " ~ old_relation %} - {% endif %} - - {{ return(sql) }} -{% endmacro %} - -{% macro generate_schema_name(custom_schema_name, node) -%} - - {%- set default_schema = target.schema -%} - {%- if custom_schema_name is none -%} - - {{ default_schema }} - - {%- else -%} - - {{ custom_schema_name | trim }} - - {%- endif -%} - -{%- endmacro %} - -{% macro postgres__create_schema(relation) -%} - {%- call statement('create_schema') -%} - /* schemas are not created in CrateDB */ - DROP TABLE IF EXISTS thisschemadefinitelydoesnotexits.thiswouldnotexist - /* but we need to run something to not have just EOF */ - {% endcall %} -{% endmacro %} - -{% macro postgres__create_table_as(temporary, relation, sql) -%} - {%- set unlogged = config.get('unlogged', default=false) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - - create table {{ relation }} - as ( - {{ sql|replace('"crate".', "") }} - ); -{%- endmacro %} - -{% macro postgres__drop_schema(relation) -%} - {% if relation.database -%} - {{ adapter.verify_database(relation.database) }} - {%- endif -%} - {%- call statement('drop_schema') -%} - /* schemas are not dropped in CrateDB */ - {%- endcall -%} -{% endmacro %} - -{% macro default__drop_relation(relation) -%} - {% call statement('drop_relation', auto_begin=False) -%} - drop {{ relation.type }} if exists "{{ relation.schema }}"."{{ relation.identifier }}" - {%- endcall %} -{% endmacro %} - -{% macro default__drop_schema(relation) -%} - {%- call statement('drop_schema') -%} - /* schemas are not dropped in CrateDB */ - {% endcall %} -{% endmacro %} - -{% macro default__create_view_as(relation, sql) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - create view "{{ relation.schema }}"."{{ relation.identifier }}" as - {{ sql|replace('"crate".', "") }} - ; -{%- endmacro %} - -{% macro postgres__rename_relation(from_relation, to_relation) -%} - {% do drop_relation(to_relation) %} - {% set schema_query = "SELECT table_type FROM information_schema.tables WHERE table_schema = '{}' AND table_name = '{}'".format(from_relation.schema, from_relation.identifier) %} - {% set results = run_query(schema_query) %} - {% if execute %} - {% set results_list = results.columns[0].values() %} - {% else %} - {% set results_list = [] %} - {% endif %} - {% for relation_type in results_list %} - {% if relation_type == 'VIEW' %} - {% set view_query = "SELECT view_definition FROM information_schema.views WHERE table_schema = '{}' AND table_name = '{}'".format(from_relation.schema, from_relation.identifier) %} - {% set view_definitions = run_query(view_query) %} - {% if execute %} - {% set view_definitions_list = view_definitions.columns[0].values() %} - {% else %} - {% set view_definitions_list = [] %} - {% endif %} - {% for view_definition in view_definitions_list %} - {% call statement('drop_view') -%} - DROP VIEW IF EXISTS {{ to_relation.schema }}.{{ to_relation.identifier }}; - {%- endcall %} - {% call statement('create_view') -%} - CREATE VIEW {{ to_relation.schema }}.{{ to_relation.identifier }} AS {{ view_definition }} - {%- endcall %} - {% call statement('drop_view') -%} - DROP VIEW IF EXISTS {{ from_relation.schema }}.{{ from_relation.identifier }}; - {%- endcall %} - {% endfor %} - {% else %} - {% call statement('rename_table') -%} - ALTER TABLE {{ from_relation.schema }}.{{ from_relation.identifier }} - RENAME TO {{ to_relation.identifier }} - {%- endcall %} - {% endif %} - {% endfor %} -{% endmacro %} - diff --git a/framework/dbt/basic/macros/relations.sql b/framework/dbt/basic/macros/relations.sql deleted file mode 100644 index 1f1583e9..00000000 --- a/framework/dbt/basic/macros/relations.sql +++ /dev/null @@ -1,34 +0,0 @@ -{% macro postgres_get_relations () -%} - {%- call statement('relations', fetch_result=True) -%} - select 'mock' as referenced_schema,'referenced ' as referenced_name, - 'mock' as dependent_schema,'dependent' as dependent_name; - {%- endcall -%} - {{ return(load_result('relations').table) }} -{% endmacro %} - -{% macro postgres__list_relations_without_caching(schema_relation) %} - {% call statement('list_relations_without_caching', fetch_result=True) -%} - select - '{{ schema_relation.database }}' as database, - tablename as name, - schemaname as schema, - 'table' as type - from pg_tables - where schemaname ilike '{{ schema_relation.schema }}' - union all - select - '{{ schema_relation.database }}' as database, - viewname as name, - schemaname as schema, - 'view' as type - from pg_views - where schemaname ilike '{{ schema_relation.schema }}' - {% endcall %} - {{ return(load_result('list_relations_without_caching').table) }} -{% endmacro %} - -{% macro default__truncate_relation(relation) -%} - {% call statement('truncate_relation') -%} - delete from {{ relation }} - {%- endcall %} -{% endmacro %} diff --git a/framework/dbt/basic/profiles.yml b/framework/dbt/basic/profiles.yml index a381e50e..87e68f08 100644 --- a/framework/dbt/basic/profiles.yml +++ b/framework/dbt/basic/profiles.yml @@ -1,14 +1,12 @@ cratedb_localhost: outputs: dev: - type: postgres + type: cratedb host: localhost user: crate pass: crate port: 5432 dbname: crate schema: doc - search_path: doc - threads: 1 catalog: crate target: dev diff --git a/framework/dbt/basic/requirements.txt b/framework/dbt/basic/requirements.txt index 67711ec8..21bf3dc9 100644 --- a/framework/dbt/basic/requirements.txt +++ b/framework/dbt/basic/requirements.txt @@ -1,2 +1 @@ -dbt-postgres<1.7 -snowplow-tracker<0.13 +dbt-cratedb2>=0.0.1