diff --git a/dbt/adapters/vertica/__init__.py b/dbt/adapters/vertica/__init__.py index a03e756..8e99d8e 100644 --- a/dbt/adapters/vertica/__init__.py +++ b/dbt/adapters/vertica/__init__.py @@ -16,7 +16,7 @@ from dbt.adapters.vertica.connections import verticaConnectionManager from dbt.adapters.vertica.connections import verticaCredentials from dbt.adapters.vertica.impl import verticaAdapter -from dbt.adapters.vertica.column import verticaColumn +from dbt.adapters.vertica.column import VerticaColumn from dbt.adapters.base import AdapterPlugin from dbt.include import vertica diff --git a/dbt/adapters/vertica/column.py b/dbt/adapters/vertica/column.py index a9d57c7..404f5a9 100644 --- a/dbt/adapters/vertica/column.py +++ b/dbt/adapters/vertica/column.py @@ -15,164 +15,16 @@ from dataclasses import dataclass -import re -from typing import Dict, ClassVar, Any, Optional +from typing import Dict, ClassVar -from dbt.exceptions import DbtRuntimeError from dbt.adapters.base.column import Column @dataclass(init=False) -class verticaColumn(Column): - TYPE_LABELS = { +class VerticaColumn(Column): + TYPE_LABELS: ClassVar[Dict[str, str]] = { "STRING": "VARCHAR", "TIMESTAMP": "TIMESTAMP", "FLOAT": "FLOAT", "INTEGER": "INT", "BOOLEAN": "BOOLEAN", - "string": "VERCHAR", } - column: str - dtype: str - char_size: Optional[int] = None - numeric_precision: Optional[Any] = None - numeric_scale: Optional[Any] = None - - @classmethod - def translate_type(cls, dtype: str) -> str: - return cls.TYPE_LABELS.get(dtype.upper(), dtype) - - @classmethod - def create(cls, name, label_or_dtype: str) -> "Column": - column_type = cls.translate_type(label_or_dtype) - return cls(name, column_type) - - @property - def name(self) -> str: - return self.column - - @property - def quoted(self) -> str: - return '"{}"'.format(self.column) - - @property - def data_type(self) -> str: - if self.is_string(): - return self.string_type(self.string_size()) - elif self.is_numeric(): - return self.numeric_type(self.dtype, self.numeric_precision, self.numeric_scale) - else: - return self.dtype - - def is_string(self) -> bool: - return self.dtype.lower() in ["text", "character varying", "character", "varchar"] - - def is_number(self): - return any([self.is_integer(), self.is_numeric(), self.is_float()]) - - def is_float(self): - return self.dtype.lower() in [ - # floats - "real", - "float4", - "float", - "double precision", - "float8", - "double", - ] - - def is_integer(self) -> bool: - return self.dtype.lower() in [ - # real types - "smallint", - "integer", - "bigint", - "smallserial", - "serial", - "bigserial", - # aliases - "int2", - "int4", - "int8", - "serial2", - "serial4", - "serial8", - ] - - def is_numeric(self) -> bool: - return self.dtype.lower() in ["numeric", "decimal"] - - def string_size(self) -> int: - if not self.is_string(): - raise DbtRuntimeError("Called string_size() on non-string field!") - - if self.dtype == "text" or self.char_size is None: - # char_size should never be None. Handle it reasonably just in case - return 256 - else: - return int(self.char_size) - - def can_expand_to(self, other_column: "Column") -> bool: - """returns True if this column can be expanded to the size of the - other column""" - if not self.is_string() or not other_column.is_string(): - return False - - return other_column.string_size() > self.string_size() - - def literal(self, value: Any) -> str: - return "{}::{}".format(value, self.data_type) - - @classmethod - def string_type(cls, size: int) -> str: - return "character varying({})".format(size) - - @classmethod - def numeric_type(cls, dtype: str, precision: Any, scale: Any) -> str: - # This could be decimal(...), numeric(...), number(...) - # Just use whatever was fed in here -- don't try to get too clever - if precision is None or scale is None: - return dtype - else: - return "{}({},{})".format(dtype, precision, scale) - - def __repr__(self) -> str: - return "".format(self.name, self.data_type) - - @classmethod - def from_description(cls, name: str, raw_data_type: str) -> "Column": - match = re.match(r"([^(]+)(\([^)]+\))?", raw_data_type) - if match is None: - raise DbtRuntimeError(f'Could not interpret data type "{raw_data_type}"') - data_type, size_info = match.groups() - char_size = None - numeric_precision = None - numeric_scale = None - if size_info is not None: - # strip out the parentheses - size_info = size_info[1:-1] - parts = size_info.split(",") - if len(parts) == 1: - try: - char_size = int(parts[0]) - except ValueError: - raise DbtRuntimeError( - f'Could not interpret data_type "{raw_data_type}": ' - f'could not convert "{parts[0]}" to an integer' - ) - elif len(parts) == 2: - try: - numeric_precision = int(parts[0]) - except ValueError: - raise DbtRuntimeError( - f'Could not interpret data_type "{raw_data_type}": ' - f'could not convert "{parts[0]}" to an integer' - ) - try: - numeric_scale = int(parts[1]) - except ValueError: - raise DbtRuntimeError( - f'Could not interpret data_type "{raw_data_type}": ' - f'could not convert "{parts[1]}" to an integer' - ) - - return cls(name, data_type, char_size, numeric_precision, numeric_scale) \ No newline at end of file diff --git a/dbt/adapters/vertica/impl.py b/dbt/adapters/vertica/impl.py index 8acad00..e17b894 100644 --- a/dbt/adapters/vertica/impl.py +++ b/dbt/adapters/vertica/impl.py @@ -16,8 +16,8 @@ from dbt.adapters.sql import SQLAdapter from dbt.adapters.vertica import verticaConnectionManager #from dbt.adapters.vertica import VerticaRelation -#from dbt.adapters.vertica import VerticaColumn -from typing import Mapping, Any, Optional, List, Union, Dict +from dbt.adapters.vertica import VerticaColumn +from typing import Optional, List, Union, Dict from dbt.adapters.base import available from dbt.exceptions import ( @@ -29,11 +29,6 @@ from dbt.adapters.base.meta import available from dbt.adapters.sql import SQLAdapter # type: ignore -from dbt.adapters.sql.impl import ( - LIST_SCHEMAS_MACRO_NAME, - LIST_RELATIONS_MACRO_NAME, -) - from dbt.adapters.base.impl import AdapterConfig,ConstraintSupport from dbt.contracts.graph.nodes import ConstraintType @@ -54,7 +49,7 @@ class VerticaConfig(AdapterConfig): class verticaAdapter(SQLAdapter): ConnectionManager = verticaConnectionManager # Relation = VerticaRelation - #Column = VerticaColumn + Column = VerticaColumn AdapterSpecificConfigs = VerticaConfig CONSTRAINT_SUPPORT = { @@ -84,6 +79,10 @@ def convert_number_type(cls, agate_table, col_idx): decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) return "numeric(18,{})".format(decimals) if decimals else "integer" + @classmethod + def convert_datetime_type(cls, agate_table, col_idx): + return "timestamp" + @available def standardize_grants_dict(self, grants_table: agate.Table) -> dict: """ diff --git a/dbt/include/vertica/macros/utils/array_append.sql b/dbt/include/vertica/macros/utils/array_append.sql new file mode 100644 index 0000000..5d00583 --- /dev/null +++ b/dbt/include/vertica/macros/utils/array_append.sql @@ -0,0 +1,4 @@ +{# new_element must be the same data type as elements in array to match postgres functionality #} +{% macro vertica__array_append(array, new_element) -%} + {{ array_concat(array, array_construct([new_element])) }} +{%- endmacro %} diff --git a/dbt/include/vertica/macros/utils/array_concat.sql b/dbt/include/vertica/macros/utils/array_concat.sql deleted file mode 100644 index 7fe9d74..0000000 --- a/dbt/include/vertica/macros/utils/array_concat.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro array_concat(array_1, array_2) -%} - array_concat({{ array_1 }}, {{ array_2 }}) -{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/vertica/macros/utils/array_construct.sql b/dbt/include/vertica/macros/utils/array_construct.sql index c10d9fa..f8b9a17 100644 --- a/dbt/include/vertica/macros/utils/array_construct.sql +++ b/dbt/include/vertica/macros/utils/array_construct.sql @@ -1,4 +1,4 @@ -{% macro array_construct(inputs, data_type) -%} +{% macro vertica__array_construct(inputs, data_type) -%} {% if inputs|length > 0 %} [ {{ inputs|join(' , ') }} ] {% else %} diff --git a/dbt/include/vertica/macros/utils/bool_or.sql b/dbt/include/vertica/macros/utils/bool_or.sql deleted file mode 100644 index 881b29e..0000000 --- a/dbt/include/vertica/macros/utils/bool_or.sql +++ /dev/null @@ -1,5 +0,0 @@ -{% macro bool_or(expression) -%} - - logical_or({{ expression }}) - -{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/vertica/macros/utils/cast_bool_to_text.sql b/dbt/include/vertica/macros/utils/cast_bool_to_text.sql new file mode 100644 index 0000000..47185d9 --- /dev/null +++ b/dbt/include/vertica/macros/utils/cast_bool_to_text.sql @@ -0,0 +1,3 @@ +{% macro vertica__cast_bool_to_text(field) %} + case when {{ field }} then 'true' when {{ field }} is null then null else 'false' end +{% endmacro %} diff --git a/dbt/include/vertica/macros/utils/concat.sql b/dbt/include/vertica/macros/utils/concat.sql deleted file mode 100644 index 0669290..0000000 --- a/dbt/include/vertica/macros/utils/concat.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% macro concat(fields) -%} - {{ return(adapter.dispatch('concat', 'dbt')(fields)) }} -{%- endmacro %} - -{% macro default__concat(fields) -%} - {{ fields|join(' || ') }} -{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/vertica/macros/utils/data_types.sql b/dbt/include/vertica/macros/utils/data_types.sql deleted file mode 100644 index ef240c3..0000000 --- a/dbt/include/vertica/macros/utils/data_types.sql +++ /dev/null @@ -1,11 +0,0 @@ - {%- macro type_string() -%} - {{ return(adapter.dispatch('type_string', 'dbt')()) }} - {%- endmacro -%} - - {% macro default__type_string() %} - {{ return(api.Column.translate_type("string")) }} - {% endmacro %} - -{% macro vertica__type_string() %} - {{ return(api.Column.translate_type("string")) }} - {% endmacro %} diff --git a/dbt/include/vertica/macros/utils/dateadd.sql b/dbt/include/vertica/macros/utils/dateadd.sql new file mode 100644 index 0000000..6dfadef --- /dev/null +++ b/dbt/include/vertica/macros/utils/dateadd.sql @@ -0,0 +1,9 @@ +{% macro vertica__dateadd(datepart, interval, from_date_or_timestamp) %} + + timestampadd( + {{ datepart }}, + {{ interval }}, + {{ from_date_or_timestamp }} + ) + +{% endmacro %} diff --git a/dbt/include/vertica/macros/utils/datediff.sql b/dbt/include/vertica/macros/utils/datediff.sql index 9e7f97f..5c7f34a 100644 --- a/dbt/include/vertica/macros/utils/datediff.sql +++ b/dbt/include/vertica/macros/utils/datediff.sql @@ -1,4 +1,4 @@ -{% macro datediff(first_date, second_date, datepart) -%} +{% macro vertica__datediff(first_date, second_date, datepart) -%} {% if dbt_version[0] == 1 and dbt_version[2] >= 2 %} {{ return(dbt.datediff(first_date, second_date, datepart)) }} diff --git a/dbt/include/vertica/macros/utils/except.sql b/dbt/include/vertica/macros/utils/except.sql index d54fc0f..0f5fffa 100644 --- a/dbt/include/vertica/macros/utils/except.sql +++ b/dbt/include/vertica/macros/utils/except.sql @@ -1,4 +1,4 @@ -{% macro except() %} +{% macro vertica__except() %} except distinct diff --git a/dbt/include/vertica/macros/utils/hash.sql b/dbt/include/vertica/macros/utils/hash.sql new file mode 100644 index 0000000..4ef5793 --- /dev/null +++ b/dbt/include/vertica/macros/utils/hash.sql @@ -0,0 +1,3 @@ +{% macro vertica__hash(field) -%} + md5(cast({{ field }} as varchar)) +{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/vertica/macros/utils/intersect.sql b/dbt/include/vertica/macros/utils/intersect.sql index f9aa379..e29c094 100644 --- a/dbt/include/vertica/macros/utils/intersect.sql +++ b/dbt/include/vertica/macros/utils/intersect.sql @@ -1,4 +1,4 @@ -{% macro intersect() %} +{% macro vertica__intersect() %} intersect distinct diff --git a/dbt/include/vertica/macros/utils/right.sql b/dbt/include/vertica/macros/utils/right.sql index 9e5948b..f898f0c 100644 --- a/dbt/include/vertica/macros/utils/right.sql +++ b/dbt/include/vertica/macros/utils/right.sql @@ -1,4 +1,4 @@ -{% macro right(string_text, length_expression) %} +{% macro vertica__right(string_text, length_expression) %} case when {{ length_expression }} = 0 then ''