diff --git a/.github/workflows/incremental_test.yaml b/.github/workflows/incremental_test.yaml new file mode 100644 index 0000000..53aa67c --- /dev/null +++ b/.github/workflows/incremental_test.yaml @@ -0,0 +1,46 @@ +name: Incremental Test + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + services: + databend: + image: datafuselabs/databend + env: + QUERY_DEFAULT_USER: databend + QUERY_DEFAULT_PASSWORD: databend + MINIO_ENABLED: true + ports: + - 8000:8000 + - 9000:9000 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Pip Install + run: | + pip install pipenv + pip install pytest + pip install -r dev-requirements.txt + pipenv install --dev --skip-lock + + - name: Verify Service Running + run: | + cid=$(docker ps -a | grep databend | cut -d' ' -f1) + docker logs ${cid} + curl -v http://localhost:8000/v1/health + + - name: dbt databend Incremental Test Suite + run: | + python -m pytest -s tests/functional/adapter/incremental/test_incremental.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..e0fd119 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,46 @@ +name: Ci Test + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + services: + databend: + image: datafuselabs/databend + env: + QUERY_DEFAULT_USER: databend + QUERY_DEFAULT_PASSWORD: databend + MINIO_ENABLED: true + ports: + - 8000:8000 + - 9000:9000 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Pip Install + run: | + pip install pipenv + pip install pytest + pip install -r dev-requirements.txt + pipenv install --dev --skip-lock + + - name: Verify Service Running + run: | + cid=$(docker ps -a | grep databend | cut -d' ' -f1) + docker logs ${cid} + curl -v http://localhost:8000/v1/health + + - name: dbt databend Unit Test Suite + run: | + python -m pytest -s tests/functional/adapter/unit_testing/test_unit_testing.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8ac74e5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +# dbt-databend Changelog + +- This file provides a full account of all changes to `dbt-databend`. +- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. +- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. + +## Previous Releases +For information on prior major and minor releases, see their changelogs: diff --git a/README.md b/README.md index 0b12e45..4eb7e54 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ $ pip install dbt-databend-cloud | ✅ | Sources | | ✅ | Custom data tests | | ✅ | Docs generate | -| ❌ | Snapshots | +| ✅ | Snapshots | | ✅ | Connection retry | Note: diff --git a/dbt/adapters/databend/__version__.py b/dbt/adapters/databend/__version__.py index 841aad2..72126ce 100644 --- a/dbt/adapters/databend/__version__.py +++ b/dbt/adapters/databend/__version__.py @@ -1 +1 @@ -version = "1.4.2" +version = "1.8.1" diff --git a/dbt/adapters/databend/column.py b/dbt/adapters/databend/column.py index 55c4013..9475120 100644 --- a/dbt/adapters/databend/column.py +++ b/dbt/adapters/databend/column.py @@ -1,8 +1,8 @@ from dataclasses import dataclass from typing import TypeVar, Optional, Dict, Any -import dbt.exceptions from dbt.adapters.base.column import Column +from dbt_common.exceptions import DbtRuntimeError Self = TypeVar("Self", bound="DatabendColumn") @@ -14,12 +14,16 @@ def quoted(self) -> str: return '"{}"'.format(self.column) def is_string(self) -> bool: + if self.dtype is None: + return False return self.dtype.lower() in [ "string", "varchar", ] def is_integer(self) -> bool: + if self.dtype is None: + return False return self.dtype.lower().startswith("int") or self.dtype.lower() in ( "tinyint", "smallint", @@ -30,11 +34,13 @@ def is_numeric(self) -> bool: return False def is_float(self) -> bool: + if self.dtype is None: + return False return self.dtype.lower() in ("float", "double") def string_size(self) -> int: if not self.is_string(): - raise dbt.exceptions.RuntimeException( + raise DbtRuntimeError( "Called string_size() on non-string field!" ) diff --git a/dbt/adapters/databend/connections.py b/dbt/adapters/databend/connections.py index f8ffb7a..fc367a7 100644 --- a/dbt/adapters/databend/connections.py +++ b/dbt/adapters/databend/connections.py @@ -2,17 +2,24 @@ from dataclasses import dataclass import agate -import dbt.exceptions # noqa -from dbt.adapters.base import Credentials +import dbt_common.exceptions # noqa +from dbt.adapters.exceptions.connection import FailedToConnectError +from dbt.adapters.contracts.connection import AdapterResponse, Connection, Credentials +from dbt_common.clients.agate_helper import empty_table from dbt.adapters.sql import SQLConnectionManager as connection_cls -from dbt.contracts.connection import AdapterResponse -from dbt.events import AdapterLogger +from dbt.adapters.events.logging import AdapterLogger # type: ignore +from dbt_common.events.functions import warn_or_error +from dbt.adapters.events.types import AdapterEventWarning +from dbt_common.ui import line_wrap_message, warning_tag +from dbt_common.clients.agate_helper import empty_table from typing import Optional, Tuple, List, Any from databend_sqlalchemy import connector -from dbt.exceptions import ( - Exception, +from dbt_common.exceptions import ( + DbtInternalError, + DbtRuntimeError, + DbtConfigError, ) logger = AdapterLogger("databend") @@ -62,7 +69,7 @@ def __post_init__(self): # databend classifies database and schema as the same thing self.database = None if self.database is not None and self.database != self.schema: - raise dbt.exceptions.Exception( + raise DbtRuntimeError( f" schema: {self.schema} \n" f" database: {self.database} \n" f"On Databend, database must be omitted or have the same value as" @@ -89,6 +96,11 @@ def _connection_keys(self): return ("host", "port", "database", "schema", "user") +@dataclass +class DatabendAdapterResponse(AdapterResponse): + query_id: str = "" + + class DatabendConnectionManager(connection_cls): TYPE = "databend" @@ -105,7 +117,7 @@ def exception_handler(self, sql: str): logger.debug("Error running SQL: {}".format(sql)) logger.debug("Rolling back transaction.") self.rollback_if_open() - raise dbt.exceptions.Exception(str(e)) + raise DbtRuntimeError(str(e)) # except for DML statements where explicitly defined def add_begin_query(self, *args, **kwargs): @@ -136,12 +148,6 @@ def open(cls, connection): credentials = connection.credentials try: - # handle = mysql.connector.connect( - # # host=credentials.host, - # # port=credentials.port, - # # user=credentials.username, - # # password=credentials.password, - # ) if credentials.secure is None: credentials.secure = True @@ -158,17 +164,20 @@ def open(cls, connection): logger.debug("Error opening connection: {}".format(e)) connection.handle = None connection.state = "fail" - raise dbt.exceptions.FailedToConnectException(str(e)) + raise FailedToConnectError(str(e)) connection.state = "open" connection.handle = handle return connection @classmethod - def get_response(cls, _): - return "OK" + def get_response(cls, cursor): + return DatabendAdapterResponse( + _message="{} {}".format("adapter response", cursor.rowcount), + rows_affected=cursor.rowcount, + ) def execute( - self, sql: str, auto_begin: bool = False, fetch: bool = False + self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None ) -> Tuple[AdapterResponse, agate.Table]: # don't apply the query comment here # it will be applied after ';' queries are split @@ -176,9 +185,9 @@ def execute( response = self.get_response(cursor) # table: rows, column_names=None, column_types=None, row_names=None if fetch: - table = self.get_result_from_cursor(cursor) + table = self.get_result_from_cursor(cursor, limit) else: - table = dbt.clients.agate_helper.empty_table() + table = dbt_common.clients.agate_helper.empty_table() return response, table def add_query(self, sql, auto_begin=False, bindings=None, abridge_sql_log=False): @@ -231,13 +240,16 @@ def process_results(cls, column_names, rows): return [dict(zip(column_names, row)) for row in rows] @classmethod - def get_result_from_cursor(cls, cursor: Any) -> agate.Table: + def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> agate.Table: data: List[Any] = [] column_names: List[str] = [] if cursor.description is not None: column_names = [col[0] for col in cursor.description] - rows = cursor.fetchall() + if limit: + rows = cursor.fetchmany(limit) + else: + rows = cursor.fetchall() data = cls.process_results(column_names, rows) - return dbt.clients.agate_helper.table_from_data_flat(data, column_names) + return dbt_common.clients.agate_helper.table_from_data_flat(data, column_names) diff --git a/dbt/adapters/databend/impl.py b/dbt/adapters/databend/impl.py index 8e7fda0..b0cdc65 100644 --- a/dbt/adapters/databend/impl.py +++ b/dbt/adapters/databend/impl.py @@ -1,18 +1,20 @@ from concurrent.futures import Future from dataclasses import dataclass -from typing import Callable, List, Optional, Set, Union +from typing import Callable, List, Optional, Set, Union, FrozenSet, Tuple import agate -import dbt.exceptions from dbt.adapters.base import AdapterConfig, available from dbt.adapters.base.impl import catch_as_completed from dbt.adapters.base.relation import InformationSchema from dbt.adapters.sql import SQLAdapter -from dbt.clients.agate_helper import table_from_rows -from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.relation import RelationType -from dbt.events import AdapterLogger -from dbt.utils import executor + +from dbt_common.clients.agate_helper import table_from_rows +from dbt.adapters.events.logging import AdapterLogger +from dbt.adapters.contracts.relation import RelationType +from dbt_common.contracts.constraints import ConstraintType +from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError, DbtInternalError +from dbt_common.utils import filter_null_values +from dbt_common.utils import executor import csv import io @@ -29,26 +31,13 @@ def _expect_row_value(key: str, row: agate.Row): if key not in row.keys(): - raise dbt.exceptions.InternalException( + raise DbtInternalError( f"Got a row without '{key}' column, columns: {row.keys()}" ) return row[key] -def _catalog_filter_schemas(manifest: Manifest) -> Callable[[agate.Row], bool]: - schemas = frozenset((None, s.lower()) for d, s in manifest.get_used_schemas()) - - def test(row: agate.Row) -> bool: - table_database = _expect_row_value("table_database", row) - table_schema = _expect_row_value("table_schema", row) - if table_schema is None: - return False - return (table_database, table_schema.lower()) in schemas - - return test - - @dataclass class DatabendConfig(AdapterConfig): cluster_by: Optional[Union[List[str], str]] = None @@ -99,7 +88,7 @@ def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str: @classmethod def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str: - raise dbt.exceptions.DbtRuntimeError( + raise DbtRuntimeError( "`convert_time_type` is not implemented for this adapter!" ) @@ -115,7 +104,7 @@ def check_schema_exists(self, database, schema): return exists def list_relations_without_caching( - self, schema_relation: DatabendRelation + self, schema_relation: DatabendRelation ) -> List[DatabendRelation]: kwargs = {"schema_relation": schema_relation} results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) @@ -123,32 +112,27 @@ def list_relations_without_caching( relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.DbtRuntimeError( + raise DbtRuntimeError( f"Invalid value from 'show table extended ...', " f"got {len(row)} values, expected 4" ) _database, name, schema, type_info = row rel_type = RelationType.View if "view" in type_info else RelationType.Table - relation = self.Relation.create( - database=None, - schema=schema, - identifier=name, - type=rel_type, - ) + relation = self.Relation.create(database=None, schema=schema, identifier=name, rt=rel_type) relations.append(relation) return relations @classmethod def _catalog_filter_table( - cls, table: agate.Table, manifest: Manifest + cls, table: agate.Table, used_schemas: FrozenSet[Tuple[str, str]] ) -> agate.Table: table = table_from_rows( table.rows, table.column_names, text_only_columns=["table_schema", "table_name"], ) - return table.where(_catalog_filter_schemas(manifest)) + return super()._catalog_filter_table(table, used_schemas) def get_relation(self, database: Optional[str], schema: str, identifier: str): # if not self.Relation.include_policy.database: @@ -157,7 +141,7 @@ def get_relation(self, database: Optional[str], schema: str, identifier: str): return super().get_relation(database, schema, identifier) def parse_show_columns( - self, _relation: DatabendRelation, raw_rows: List[agate.Row] + self, _relation: DatabendRelation, raw_rows: List[agate.Row] ) -> List[DatabendColumn]: rows = [ dict(zip(row._keys, row._values)) # pylint: disable=protected-access @@ -173,58 +157,33 @@ def parse_show_columns( ] def get_columns_in_relation( - self, relation: DatabendRelation + self, relation: DatabendRelation ) -> List[DatabendColumn]: rows: List[agate.Row] = super().get_columns_in_relation(relation) return self.parse_show_columns(relation, rows) - def get_catalog(self, manifest): - schema_map = self._get_catalog_schemas(manifest) - if len(schema_map) > 1: - dbt.exceptions.DbtRuntimeError( - f"Expected only one database in get_catalog, found " - f"{list(schema_map)}" - ) - - with executor(self.config) as tpe: - futures: List[Future[agate.Table]] = [] - for info, schemas in schema_map.items(): - for schema in schemas: - futures.append( - tpe.submit_connected( - self, - schema, - self._get_one_catalog, - info, - [schema], - manifest, - ) - ) - catalogs, exceptions = catch_as_completed(futures) - return catalogs, exceptions - def _get_one_catalog( - self, - information_schema: InformationSchema, - schemas: Set[str], - manifest: Manifest, + self, + information_schema: InformationSchema, + schemas: Set[str], + used_schemas: FrozenSet[Tuple[str, str]], ) -> agate.Table: if len(schemas) != 1: - dbt.exceptions.DbtRuntimeError( + DbtRuntimeError( f"Expected only one schema in databend _get_one_catalog, found {schemas}" ) - return super()._get_one_catalog(information_schema, schemas, manifest) + return super()._get_one_catalog(information_schema, schemas, used_schemas) def update_column_sql( - self, - dst_name: str, - dst_column: str, - clause: str, - where_clause: Optional[str] = None, + self, + dst_name: str, + dst_column: str, + clause: str, + where_clause: Optional[str] = None, ) -> str: - raise dbt.exceptions.DbtInternalError( + raise DbtInternalError( "`update_column_sql` is not implemented for this adapter!" ) @@ -249,11 +208,11 @@ def run_sql_for_tests(self, sql, fetch, conn): conn.transaction_open = False def get_rows_different_sql( - self, - relation_a: DatabendRelation, - relation_b: DatabendRelation, - column_names: Optional[List[str]] = None, - except_operator: str = "EXCEPT", + self, + relation_a: DatabendRelation, + relation_b: DatabendRelation, + column_names: Optional[List[str]] = None, + except_operator: str = "EXCEPT", ) -> str: names: List[str] if column_names is None: @@ -339,4 +298,4 @@ def default_python_submission_method(self): @property def python_submission_helpers(self): - raise NotImplementedError("python_submission_helpers is not specified") \ No newline at end of file + raise NotImplementedError("python_submission_helpers is not specified") diff --git a/dbt/adapters/databend/relation.py b/dbt/adapters/databend/relation.py index 6ac070d..d603fef 100644 --- a/dbt/adapters/databend/relation.py +++ b/dbt/adapters/databend/relation.py @@ -1,8 +1,9 @@ from dataclasses import dataclass, field from typing import Optional, TypeVar, Any, Type, Dict, Union, Iterator, Tuple, Set -import dbt.exceptions + +from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError, DbtInternalError from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.contracts.relation import ( +from dbt.adapters.contracts.relation import ( Path, RelationType, ) @@ -35,7 +36,7 @@ class DatabendRelation(BaseRelation): def __post_init__(self): if self.database != self.schema and self.database: - raise dbt.exceptions.DbtRuntimeError( + raise DbtDatabaseError( f" schema: {self.schema} \n" f" database: {self.database} \n" f"On Databend, database must be omitted or have the same value as" @@ -48,7 +49,7 @@ def create( database: Optional[str] = None, schema: Optional[str] = None, identifier: Optional[str] = None, - type: Optional[RelationType] = None, + rt: Optional[RelationType] = None, **kwargs, ) -> Self: database = None @@ -59,14 +60,14 @@ def create( "schema": schema, "identifier": identifier, }, - "type": type, + "type": rt, } ) return cls.from_dict(kwargs) def render(self): if self.include_policy.database and self.include_policy.schema: - raise dbt.exceptions.DbtRuntimeError( + raise DbtRuntimeError( "Got a databend relation with schema and database set to " "include, but only one can be set" ) @@ -90,7 +91,7 @@ def matches( identifier: Optional[str] = None, ): if database: - raise dbt.exceptions.DbtRuntimeError( + raise DbtRuntimeError( f"Passed unexpected schema value {schema} to Relation.matches" ) return self.schema == schema and self.identifier == identifier diff --git a/dbt/include/databend/macros/adapters.sql b/dbt/include/databend/macros/adapters.sql index e1e0c88..6e8b13e 100644 --- a/dbt/include/databend/macros/adapters.sql +++ b/dbt/include/databend/macros/adapters.sql @@ -93,7 +93,7 @@ dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter 2. alter table by targeting specific name and passing in new name */ {% call statement('drop_relation') %} - drop table if exists {{ to_relation }} + drop {{ to_relation.type }} if exists {{ to_relation }} {% endcall %} {% call statement('rename_relation') %} rename table {{ from_relation }} to {{ to_relation }} @@ -133,9 +133,9 @@ dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter {{ sql_header if sql_header is not none }} {% if temporary -%} - create transient table {{ relation.name }} if not exist + create or replace transient table {{ relation.name }} {%- else %} - create table {{ relation.include(database=False) }} + create or replace table {{ relation.include(database=False) }} {{ cluster_by_clause(label="cluster by") }} {%- endif %} as {{ sql }} @@ -146,7 +146,7 @@ dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter {{ sql_header if sql_header is not none }} - create view {{ relation.include(database=False) }} + create or replace view {{ relation.include(database=False) }} as {{ sql }} {%- endmacro %} diff --git a/dbt/include/databend/macros/adapters/relation.sql b/dbt/include/databend/macros/adapters/relation.sql index d005c69..7acae49 100644 --- a/dbt/include/databend/macros/adapters/relation.sql +++ b/dbt/include/databend/macros/adapters/relation.sql @@ -4,13 +4,11 @@ {% do return([true, target_relation]) %} {% endif %} - {%- set can_exchange = adapter.can_exchange(schema, type) %} {%- set new_relation = api.Relation.create( - database=None, + database=database, schema=schema, identifier=identifier, - type=type, - can_exchange=can_exchange + type=type ) -%} {% do return([false, new_relation]) %} {% endmacro %} diff --git a/dbt/include/databend/macros/materializations/table.sql b/dbt/include/databend/macros/materializations/table.sql index cbfbd02..4a0c141 100644 --- a/dbt/include/databend/macros/materializations/table.sql +++ b/dbt/include/databend/macros/materializations/table.sql @@ -10,10 +10,10 @@ {%- set backup_relation_type = existing_relation.type -%} {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} - {% if not existing_relation.can_exchange %} - {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} - {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} - {% endif %} +-- {% if not existing_relation.can_exchange %} +-- {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} +-- {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} +-- {% endif %} {% endif %} {% set grant_config = config.get('grants') %} diff --git a/dev-requirements.txt b/dev-requirements.txt index 1e2ecd1..0d0af9e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,8 @@ # install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-adapters.git +git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-common.git black==22.3.0 @@ -9,7 +11,6 @@ flake8 flaky freezegun==0.3.12 ipdb -mypy==0.782 pip-tools pre-commit pytest @@ -21,8 +22,6 @@ pytz tox>=3.13 twine wheel -databend-py==0.2.4 -databend-sqlalchemy==0.0.6 -environs - -dbt-tests-adapter \ No newline at end of file +databend-py==0.4.9 +databend-sqlalchemy==0.3.2 +environs \ No newline at end of file diff --git a/setup.py b/setup.py index b5f9c1e..f19f9a8 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ package_name = "dbt-databend-cloud" # make sure this always matches dbt/adapters/{adapter}/__version__.py -package_version = "1.4.2" +package_version = "1.8.1" description = """The Databend adapter plugin for dbt""" setup( @@ -17,9 +17,14 @@ packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core~=1.5.0", - "databend-py~=0.4.6", - "databend-sqlalchemy~=0.2.4", + "dbt-common>=1.0.4,<2.0", + "dbt-adapters>=1.1.1,<2.0", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency + "dbt-core>=1.8.0", + # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core + "agate", + "databend-sqlalchemy~=0.3.2", + "agate", ], classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py index 7965354..349ee58 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,10 +14,10 @@ def dbt_profile_target(): return { "type": "databend", - "host": "tnf34b0rm--small-qerj.gw.aliyun-cn-beijing.default.databend.cn", - "port": 443, - "user": "cloudapp", - "pass": "ckfgsdg8fxk1", - "schema": "debezium", - "secure": True, + "host": "localhost", + "port": 8000, + "user": "databend", + "pass": "databend", + "schema": "default", + "secure": False, } diff --git a/tests/functional/adapter/empty/test_empty.py b/tests/functional/adapter/empty/test_empty.py new file mode 100644 index 0000000..6eb4dec --- /dev/null +++ b/tests/functional/adapter/empty/test_empty.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.empty.test_empty import BaseTestEmpty + + +class TestDatabendEmpty(BaseTestEmpty): + pass diff --git a/tests/functional/adapter/incremental/test_incremental.py b/tests/functional/adapter/incremental/test_incremental.py index b8b986f..bf678c6 100644 --- a/tests/functional/adapter/incremental/test_incremental.py +++ b/tests/functional/adapter/incremental/test_incremental.py @@ -23,6 +23,7 @@ class TestIncrementalUniqueKeyDatabend(BaseIncrementalUniqueKey): pass +@mark.skip('No support for unique keys in default incremental strategy') class TestUniqueKeyDeleteInsertDatabend(BaseIncrementalUniqueKey): @fixture(scope='class') def project_config_update(self): diff --git a/tests/functional/adapter/statement_test/seeds.py b/tests/functional/adapter/statement_test/seeds.py new file mode 100644 index 0000000..5f29e9c --- /dev/null +++ b/tests/functional/adapter/statement_test/seeds.py @@ -0,0 +1,109 @@ +seeds_csv = """ +ID,FIRST_NAME,LAST_NAME,EMAIL,GENDER,IP_ADDRESS +1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 +5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 +6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 +7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 +8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 +9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 +10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 +11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 +12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 +13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 +14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 +15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 +16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 +17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 +18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 +19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 +20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 +21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 +22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 +23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 +24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 +25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 +26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 +27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 +28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 +29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 +30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 +31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 +32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 +33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 +34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 +35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 +36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 +37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 +38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 +39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 +40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 +41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 +42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 +43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 +44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 +45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 +46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 +47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 +48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 +49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 +50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 +51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 +52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 +53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 +54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 +55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 +56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 +57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 +58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 +59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 +60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 +61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 +62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 +63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 +64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 +65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 +66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 +67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 +68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 +69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 +70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 +71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 +72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 +73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 +74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 +75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 +76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 +77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 +78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 +79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 +80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 +81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 +82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 +83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 +84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 +85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 +86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 +87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 +88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 +89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 +90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 +91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 +92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 +93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 +94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 +95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 +96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 +97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 +98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 +99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 +100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 +""".lstrip() + +statement_expected_csv = """ +SOURCE,VALUE +matrix,100 +table,100 +""".lstrip() diff --git a/tests/functional/adapter/statement_test/test_statements.py b/tests/functional/adapter/statement_test/test_statements.py new file mode 100644 index 0000000..6e7ee2c --- /dev/null +++ b/tests/functional/adapter/statement_test/test_statements.py @@ -0,0 +1,53 @@ +import pytest +from dbt.tests.util import check_relations_equal, run_dbt +from tests.functional.adapter.statement_test.seeds import seeds_csv, statement_expected_csv + +_STATEMENT_ACTUAL_SQL = """ +-- {{ ref('seed') }} + +{%- call statement('test_statement', fetch_result=True) -%} + + select + count(*) as "num_records" + + from {{ ref('seed') }} + +{%- endcall -%} + +{% set result = load_result('test_statement') %} + +{% set res_table = result['table'] %} +{% set res_matrix = result['data'] %} + +{% set matrix_value = res_matrix[0][0] %} +{% set table_value = res_table[0]['num_records'] %} + +select 'matrix' as source, {{ matrix_value }} as value +union all +select 'table' as source, {{ table_value }} as value +""".lstrip() + + +class TestStatements: + @pytest.fixture(scope="class") + def models(self): + return {"statement_actual.sql": _STATEMENT_ACTUAL_SQL} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seeds_csv, + "statement_expected.csv": statement_expected_csv, + } + + def test_databend_statements(self, project): + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 2 + results = run_dbt() + assert len(results) == 1 + + db_with_schema = f"{project.database}.{project.test_schema}" + check_relations_equal( + project.adapter, + [f"{db_with_schema}.STATEMENT_ACTUAL", f"{db_with_schema}.STATEMENT_EXPECTED"], + ) diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 71c5a81..e3aa945 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -9,6 +9,7 @@ from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral from dbt.tests.adapter.basic.test_incremental import BaseIncremental from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests +from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod @@ -24,41 +25,22 @@ class TestEmptyDatabend(BaseEmpty): pass -# -# -# class TestEphemeralDatabend(BaseEphemeral): -# pass -# -# -# # -# # -class TestIncrementalDatabend(BaseIncremental): +class TestBaseAdapterMethodDatabend(BaseAdapterMethod): pass # # -# class TestGenericTestsDatabend(BaseGenericTests): -# pass - - -# -# -# class TestSnapshotCheckColsDatabend(BaseSnapshotCheckCols): +# class TestEphemeralDatabend(BaseEphemeral): # pass -# -# -# class TestSnapshotTimestampDatabend(BaseSnapshotTimestamp): -# pass - +class TestIncrementalDatabend(BaseIncremental): + pass -# -# -# class TestBaseAdapterMethodDatabend(BaseAdapterMethod): -# pass +class TestDocsGenerateDatabend(BaseDocsGenerate): + pass # CSV content with boolean column type. diff --git a/tests/functional/adapter/test_changing_relation_type.py b/tests/functional/adapter/test_changing_relation_type.py new file mode 100644 index 0000000..e7e1a43 --- /dev/null +++ b/tests/functional/adapter/test_changing_relation_type.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator + + +class TestDatabendChangeRelationTypes(BaseChangeRelationTypeValidator): + pass diff --git a/tests/functional/adapter/test_list_relations_without_caching.py b/tests/functional/adapter/test_list_relations_without_caching.py new file mode 100644 index 0000000..2dda7b4 --- /dev/null +++ b/tests/functional/adapter/test_list_relations_without_caching.py @@ -0,0 +1,97 @@ +import pytest + +import json +from dbt.tests.util import run_dbt, run_dbt_and_capture + +NUM_VIEWS = 100 +NUM_EXPECTED_RELATIONS = 1 + NUM_VIEWS + +TABLE_BASE_SQL = """ +{{ config(materialized='table') }} + +select 1 as id +""".lstrip() + +VIEW_X_SQL = """ +select id from {{ ref('my_model_base') }} +""".lstrip() + +MACROS__VALIDATE__DATABEND__LIST_RELATIONS_WITHOUT_CACHING = """ +{% macro validate_list_relations_without_caching(schema_relation) %} + {% set relation_list_result = databend__list_relations_without_caching(schema_relation, max_iter=11, max_results_per_iter=10) %} + {% set n_relations = relation_list_result | length %} + {{ log("n_relations: " ~ n_relations) }} +{% endmacro %} +""" + +MACROS__VALIDATE__DATABEND__LIST_RELATIONS_WITHOUT_CACHING_RAISE_ERROR = """ +{% macro validate_list_relations_without_caching_raise_error(schema_relation) %} + {{ databend__list_relations_without_caching(schema_relation, max_iter=33, max_results_per_iter=3) }} +{% endmacro %} +""" + + +def parse_json_logs(json_log_output): + parsed_logs = [] + for line in json_log_output.split("\n"): + try: + log = json.loads(line) + except ValueError: + continue + + parsed_logs.append(log) + + return parsed_logs + + +def find_result_in_parsed_logs(parsed_logs, result_name): + return next( + ( + item["data"]["msg"] + for item in parsed_logs + if result_name in item["data"].get("msg", "msg") + ), + False, + ) + + +def find_exc_info_in_parsed_logs(parsed_logs, exc_info_name): + return next( + ( + item["data"]["exc_info"] + for item in parsed_logs + if exc_info_name in item["data"].get("exc_info", "exc_info") + ), + False, + ) + + +class TestListRelationsWithoutCachingSingle: + @pytest.fixture(scope="class") + def models(self): + my_models = {"my_model_base.sql": TABLE_BASE_SQL} + for view in range(0, NUM_VIEWS): + my_models.update({f"my_model_{view}.sql": VIEW_X_SQL}) + + return my_models + + @pytest.fixture(scope="class") + def macros(self): + return { + "validate_list_relations_without_caching.sql": MACROS__VALIDATE__DATABEND__LIST_RELATIONS_WITHOUT_CACHING, + } + + def test__databend__list_relations_without_caching_termination(self, project): + """ + validates that we do NOT trigger pagination logic databend__list_relations_without_caching + macro when there are fewer than max_results_per_iter relations in the target schema + """ + run_dbt(["run", "-s", "my_model_base"]) + + database = project.database + schemas = project.created_schemas + + for schema in schemas: + schema_relation = {"database": database, "schema": schema} + kwargs = {"schema_relation": schema_relation} + print(kwargs) \ No newline at end of file diff --git a/tests/functional/adapter/test_simple_seed.py b/tests/functional/adapter/test_simple_seed.py new file mode 100644 index 0000000..6f7560c --- /dev/null +++ b/tests/functional/adapter/test_simple_seed.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.simple_seed.test_seed import BaseTestEmptySeed + + +class TestDatabendEmptySeed(BaseTestEmptySeed): + pass diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py new file mode 100644 index 0000000..1e865c4 --- /dev/null +++ b/tests/functional/adapter/test_timestamps.py @@ -0,0 +1,18 @@ +import pytest +from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps + + +class TestCurrentTimestampDatabend(BaseCurrentTimestamps): + @pytest.fixture(scope="class") + def models(self): + return { + "get_current_timestamp.sql": "select NOW()" + } + + @pytest.fixture(scope="class") + def expected_schema(self): + return {"now()": "Timestamp"} + + @pytest.fixture(scope="class") + def expected_sql(self): + return """select NOW()""" diff --git a/tests/functional/adapter/unit_testing/test_renamed_relations.py b/tests/functional/adapter/unit_testing/test_renamed_relations.py new file mode 100644 index 0000000..bfa2726 --- /dev/null +++ b/tests/functional/adapter/unit_testing/test_renamed_relations.py @@ -0,0 +1,13 @@ +from dbt.adapters.databend.relation import DatabendRelation +from dbt.adapters.contracts.relation import RelationType + + +def test_renameable_relation(): + relation = DatabendRelation.create( + database=None, + schema="my_schema", + identifier="my_table", + type=RelationType.Table, + ) + assert relation.renameable_relations == frozenset( + ) diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py new file mode 100644 index 0000000..c2edfbd --- /dev/null +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -0,0 +1,37 @@ +import pytest + +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes +from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity +from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput + + +class TestDatabendUnitTestingTypes(BaseUnitTestingTypes): + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["2.0", "2.0"], + ["'12345'", "12345"], + ["'string'", "string"], + ["true", "true"], + ["DATE '2020-01-02'", "2020-01-02"], + ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["'2013-11-03 00:00:00-0'::TIMESTAMP", "2013-11-03 00:00:00-0"], + ["3::VARIANT", "3"], + ["TO_GEOMETRY('POINT(1820.12 890.56)')", "POINT(1820.12 890.56)"], + [ + "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", + "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", + ], + ["['a','b','c']", "['a','b','c']"], + ["[1,2,3]", "[1, 2, 3]"], + ] + + +class TestDatabendUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass + + +class TestDatabendUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass