diff --git a/ibis-server/app/main.py b/ibis-server/app/main.py index 0876904fb..af4ff6988 100644 --- a/ibis-server/app/main.py +++ b/ibis-server/app/main.py @@ -76,3 +76,8 @@ def custom_http_error_handler(request, exc: CustomHttpError): with logger.contextualize(correlation_id=request.headers.get("X-Correlation-ID")): logger.opt(exception=exc).error("Request failed") return PlainTextResponse(str(exc), status_code=exc.status_code) + + +@app.exception_handler(NotImplementedError) +def not_implemented_error_handler(request, exc: NotImplementedError): + return PlainTextResponse(str(exc), status_code=501) diff --git a/ibis-server/app/mdl/rewriter.py b/ibis-server/app/mdl/rewriter.py index 51e8666cb..e3ebfd0e6 100644 --- a/ibis-server/app/mdl/rewriter.py +++ b/ibis-server/app/mdl/rewriter.py @@ -72,6 +72,8 @@ def _get_read_dialect(cls, experiment) -> str | None: def _get_write_dialect(cls, data_source: DataSource) -> str: if data_source == DataSource.canner: return "trino" + elif data_source == DataSource.local_file: + return "duckdb" return data_source.name diff --git a/ibis-server/app/model/__init__.py b/ibis-server/app/model/__init__.py index 98e3a081f..fb385f6ce 100644 --- a/ibis-server/app/model/__init__.py +++ b/ibis-server/app/model/__init__.py @@ -51,6 +51,10 @@ class QueryTrinoDTO(QueryDTO): connection_info: ConnectionUrl | TrinoConnectionInfo = connection_info_field +class QueryLocalFileDTO(QueryDTO): + connection_info: LocalFileConnectionInfo = connection_info_field + + class BigQueryConnectionInfo(BaseModel): project_id: SecretStr dataset_id: SecretStr @@ -133,6 +137,13 @@ class TrinoConnectionInfo(BaseModel): password: SecretStr | None = None +class LocalFileConnectionInfo(BaseModel): + url: SecretStr + format: str = Field( + description="File format", default="csv", examples=["csv", "parquet", "json"] + ) + + ConnectionInfo = ( BigQueryConnectionInfo | CannerConnectionInfo @@ -142,6 +153,7 @@ class TrinoConnectionInfo(BaseModel): | PostgresConnectionInfo | SnowflakeConnectionInfo | TrinoConnectionInfo + | LocalFileConnectionInfo ) diff --git a/ibis-server/app/model/connector.py b/ibis-server/app/model/connector.py index 0640f5825..307c40f09 100644 --- a/ibis-server/app/model/connector.py +++ b/ibis-server/app/model/connector.py @@ -30,6 +30,8 @@ def __init__(self, data_source: DataSource, connection_info: ConnectionInfo): self._connector = CannerConnector(connection_info) elif data_source == DataSource.bigquery: self._connector = BigQueryConnector(connection_info) + elif data_source == DataSource.local_file: + self._connector = DuckDBConnector(connection_info) else: self._connector = SimpleConnector(data_source, connection_info) @@ -144,6 +146,19 @@ def query(self, sql: str, limit: int) -> pd.DataFrame: raise e +class DuckDBConnector: + def __init__(self, _connection_info: ConnectionInfo): + import duckdb + + self.connection = duckdb.connect() + + def query(self, sql: str, limit: int) -> pd.DataFrame: + return self.connection.execute(sql).fetch_df().head(limit) + + def dry_run(self, sql: str) -> None: + self.connection.execute(sql) + + @cache def _get_pg_type_names(connection: BaseBackend) -> dict[int, str]: cur = connection.raw_sql("SELECT oid, typname FROM pg_type") diff --git a/ibis-server/app/model/data_source.py b/ibis-server/app/model/data_source.py index ba282e973..ab31feb67 100644 --- a/ibis-server/app/model/data_source.py +++ b/ibis-server/app/model/data_source.py @@ -20,6 +20,7 @@ QueryCannerDTO, QueryClickHouseDTO, QueryDTO, + QueryLocalFileDTO, QueryMSSqlDTO, QueryMySqlDTO, QueryPostgresDTO, @@ -39,6 +40,7 @@ class DataSource(StrEnum): postgres = auto() snowflake = auto() trino = auto() + local_file = auto() def get_connection(self, info: ConnectionInfo) -> BaseBackend: try: @@ -62,6 +64,7 @@ class DataSourceExtension(Enum): postgres = QueryPostgresDTO snowflake = QuerySnowflakeDTO trino = QueryTrinoDTO + local_file = QueryLocalFileDTO def __init__(self, dto: QueryDTO): self.dto = dto @@ -70,6 +73,10 @@ def get_connection(self, info: ConnectionInfo) -> BaseBackend: try: if hasattr(info, "connection_url"): return ibis.connect(info.connection_url.get_secret_value()) + if self.name == "local_file": + raise NotImplementedError( + "Local file connection is not implemented to get ibis backend" + ) return getattr(self, f"get_{self.name}_connection")(info) except KeyError: raise NotImplementedError(f"Unsupported data source: {self}") diff --git a/ibis-server/app/model/metadata/dto.py b/ibis-server/app/model/metadata/dto.py index 6d6706929..85ec3f8d9 100644 --- a/ibis-server/app/model/metadata/dto.py +++ b/ibis-server/app/model/metadata/dto.py @@ -66,6 +66,9 @@ class TableProperties(BaseModel): schema_: str | None = Field(alias="schema", default=None) catalog: str | None table: str | None # only table name without schema or catalog + path: str | None = Field( + alias="path", default=None + ) # the full path of the table for file-based table class Table(BaseModel): diff --git a/ibis-server/app/model/metadata/factory.py b/ibis-server/app/model/metadata/factory.py index b27a4a946..ad6dcb50f 100644 --- a/ibis-server/app/model/metadata/factory.py +++ b/ibis-server/app/model/metadata/factory.py @@ -5,6 +5,7 @@ from app.model.metadata.metadata import Metadata from app.model.metadata.mssql import MSSQLMetadata from app.model.metadata.mysql import MySQLMetadata +from app.model.metadata.object_storage import LocalFileMetadata from app.model.metadata.postgres import PostgresMetadata from app.model.metadata.snowflake import SnowflakeMetadata from app.model.metadata.trino import TrinoMetadata @@ -18,6 +19,7 @@ DataSource.postgres: PostgresMetadata, DataSource.trino: TrinoMetadata, DataSource.snowflake: SnowflakeMetadata, + DataSource.local_file: LocalFileMetadata, } diff --git a/ibis-server/app/model/metadata/object_storage.py b/ibis-server/app/model/metadata/object_storage.py new file mode 100644 index 000000000..744523ee4 --- /dev/null +++ b/ibis-server/app/model/metadata/object_storage.py @@ -0,0 +1,156 @@ +import os + +import duckdb +import opendal +from loguru import logger + +from app.model import LocalFileConnectionInfo +from app.model.metadata.dto import ( + Column, + RustWrenEngineColumnType, + Table, + TableProperties, +) +from app.model.metadata.metadata import Metadata + + +class ObjectStorageMetadata(Metadata): + def __init__(self, connection_info): + super().__init__(connection_info) + + def get_table_list(self) -> list[Table]: + op = opendal.Operator("fs", root=self.connection_info.url.get_secret_value()) + conn = self._get_connection() + unique_tables = {} + for file in op.list("/"): + if file.path != "/": + stat = op.stat(file.path) + if stat.mode.is_dir(): + # if the file is a directory, use the directory name as the table name + table_name = os.path.basename(os.path.normpath(file.path)) + full_path = f"{self.connection_info.url.get_secret_value()}/{table_name}/*.{self.connection_info.format}" + else: + # if the file is a file, use the file name as the table name + table_name = os.path.splitext(os.path.basename(file.path))[0] + full_path = ( + f"{self.connection_info.url.get_secret_value()}/{file.path}" + ) + + # read the file with the target format if unreadable, skip the file + df = self._read_df(conn, full_path) + if df is None: + continue + columns = [] + try: + for col in df.columns: + duckdb_type = df[col].dtypes[0] + columns.append( + Column( + name=col, + type=self._to_column_type(duckdb_type.__str__()), + notNull=False, + ) + ) + except Exception as e: + logger.debug(f"Failed to read column types: {e}") + continue + + unique_tables[table_name] = Table( + name=table_name, + description=None, + columns=[], + properties=TableProperties( + table=table_name, + schema=None, + catalog=None, + path=full_path, + ), + primaryKey=None, + ) + unique_tables[table_name].columns = columns + + return list(unique_tables.values()) + + def get_constraints(self): + return [] + + def get_version(self): + raise NotImplementedError("Subclasses must implement `get_version` method") + + def _read_df(self, conn, path): + if self.connection_info.format == "parquet": + try: + return conn.read_parquet(path) + except Exception as e: + logger.debug(f"Failed to read parquet file: {e}") + return None + elif self.connection_info.format == "csv": + try: + logger.debug(f"Reading csv file: {path}") + return conn.read_csv(path) + except Exception as e: + logger.debug(f"Failed to read csv file: {e}") + return None + elif self.connection_info.format == "json": + try: + return conn.read_json(path) + except Exception as e: + logger.debug(f"Failed to read json file: {e}") + return None + else: + raise NotImplementedError( + f"Unsupported format: {self.connection_info.format}" + ) + + def _to_column_type(self, col_type: str) -> RustWrenEngineColumnType: + if col_type.startswith("DECIMAL"): + return RustWrenEngineColumnType.DECIMAL + + # TODO: support struct + if col_type.startswith("STRUCT"): + return RustWrenEngineColumnType.UNKNOWN + + # TODO: support array + if col_type.endswith("[]"): + return RustWrenEngineColumnType.UNKNOWN + + # refer to https://duckdb.org/docs/sql/data_types/overview#general-purpose-data-types + switcher = { + "BIGINT": RustWrenEngineColumnType.INT64, + "BIT": RustWrenEngineColumnType.INT2, + "BLOB": RustWrenEngineColumnType.BYTES, + "BOOLEAN": RustWrenEngineColumnType.BOOL, + "DATE": RustWrenEngineColumnType.DATE, + "DOUBLE": RustWrenEngineColumnType.DOUBLE, + "FLOAT": RustWrenEngineColumnType.FLOAT, + "INTEGER": RustWrenEngineColumnType.INT, + # TODO: Wren engine does not support HUGEINT. Map to INT64 for now. + "HUGEINT": RustWrenEngineColumnType.INT64, + "INTERVAL": RustWrenEngineColumnType.INTERVAL, + "JSON": RustWrenEngineColumnType.JSON, + "SMALLINT": RustWrenEngineColumnType.INT2, + "TIME": RustWrenEngineColumnType.TIME, + "TIMESTAMP": RustWrenEngineColumnType.TIMESTAMP, + "TIMESTAMP WITH TIME ZONE": RustWrenEngineColumnType.TIMESTAMPTZ, + "TINYINT": RustWrenEngineColumnType.INT2, + "UBIGINT": RustWrenEngineColumnType.INT64, + # TODO: Wren engine does not support UHUGEINT. Map to INT64 for now. + "UHUGEINT": RustWrenEngineColumnType.INT64, + "UINTEGER": RustWrenEngineColumnType.INT, + "USMALLINT": RustWrenEngineColumnType.INT2, + "UTINYINT": RustWrenEngineColumnType.INT2, + "UUID": RustWrenEngineColumnType.UUID, + "VARCHAR": RustWrenEngineColumnType.STRING, + } + return switcher.get(col_type, RustWrenEngineColumnType.UNKNOWN) + + def _get_connection(self): + return duckdb.connect() + + +class LocalFileMetadata(ObjectStorageMetadata): + def __init__(self, connection_info: LocalFileConnectionInfo): + super().__init__(connection_info) + + def get_version(self): + return "Local File System" diff --git a/ibis-server/poetry.lock b/ibis-server/poetry.lock index 2a281b29d..269da2720 100644 --- a/ibis-server/poetry.lock +++ b/ibis-server/poetry.lock @@ -823,6 +823,67 @@ docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"] ssh = ["paramiko (>=2.4.3)"] websockets = ["websocket-client (>=1.3.0)"] +[[package]] +name = "duckdb" +version = "1.1.3" +description = "DuckDB in-process database" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "duckdb-1.1.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:1c0226dc43e2ee4cc3a5a4672fddb2d76fd2cf2694443f395c02dd1bea0b7fce"}, + {file = "duckdb-1.1.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:7c71169fa804c0b65e49afe423ddc2dc83e198640e3b041028da8110f7cd16f7"}, + {file = "duckdb-1.1.3-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:872d38b65b66e3219d2400c732585c5b4d11b13d7a36cd97908d7981526e9898"}, + {file = "duckdb-1.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25fb02629418c0d4d94a2bc1776edaa33f6f6ccaa00bd84eb96ecb97ae4b50e9"}, + {file = "duckdb-1.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e3f5cd604e7c39527e6060f430769b72234345baaa0987f9500988b2814f5e4"}, + {file = "duckdb-1.1.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08935700e49c187fe0e9b2b86b5aad8a2ccd661069053e38bfaed3b9ff795efd"}, + {file = "duckdb-1.1.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f9b47036945e1db32d70e414a10b1593aec641bd4c5e2056873d971cc21e978b"}, + {file = "duckdb-1.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:35c420f58abc79a68a286a20fd6265636175fadeca1ce964fc8ef159f3acc289"}, + {file = "duckdb-1.1.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:4f0e2e5a6f5a53b79aee20856c027046fba1d73ada6178ed8467f53c3877d5e0"}, + {file = "duckdb-1.1.3-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:911d58c22645bfca4a5a049ff53a0afd1537bc18fedb13bc440b2e5af3c46148"}, + {file = "duckdb-1.1.3-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:c443d3d502335e69fc1e35295fcfd1108f72cb984af54c536adfd7875e79cee5"}, + {file = "duckdb-1.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a55169d2d2e2e88077d91d4875104b58de45eff6a17a59c7dc41562c73df4be"}, + {file = "duckdb-1.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d0767ada9f06faa5afcf63eb7ba1befaccfbcfdac5ff86f0168c673dd1f47aa"}, + {file = "duckdb-1.1.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51c6d79e05b4a0933672b1cacd6338f882158f45ef9903aef350c4427d9fc898"}, + {file = "duckdb-1.1.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:183ac743f21c6a4d6adfd02b69013d5fd78e5e2cd2b4db023bc8a95457d4bc5d"}, + {file = "duckdb-1.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:a30dd599b8090ea6eafdfb5a9f1b872d78bac318b6914ada2d35c7974d643640"}, + {file = "duckdb-1.1.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:a433ae9e72c5f397c44abdaa3c781d94f94f4065bcbf99ecd39433058c64cb38"}, + {file = "duckdb-1.1.3-cp312-cp312-macosx_12_0_universal2.whl", hash = "sha256:d08308e0a46c748d9c30f1d67ee1143e9c5ea3fbcccc27a47e115b19e7e78aa9"}, + {file = "duckdb-1.1.3-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5d57776539211e79b11e94f2f6d63de77885f23f14982e0fac066f2885fcf3ff"}, + {file = "duckdb-1.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e59087dbbb63705f2483544e01cccf07d5b35afa58be8931b224f3221361d537"}, + {file = "duckdb-1.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ebf5f60ddbd65c13e77cddb85fe4af671d31b851f125a4d002a313696af43f1"}, + {file = "duckdb-1.1.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4ef7ba97a65bd39d66f2a7080e6fb60e7c3e41d4c1e19245f90f53b98e3ac32"}, + {file = "duckdb-1.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f58db1b65593ff796c8ea6e63e2e144c944dd3d51c8d8e40dffa7f41693d35d3"}, + {file = "duckdb-1.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:e86006958e84c5c02f08f9b96f4bc26990514eab329b1b4f71049b3727ce5989"}, + {file = "duckdb-1.1.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:0897f83c09356206ce462f62157ce064961a5348e31ccb2a557a7531d814e70e"}, + {file = "duckdb-1.1.3-cp313-cp313-macosx_12_0_universal2.whl", hash = "sha256:cddc6c1a3b91dcc5f32493231b3ba98f51e6d3a44fe02839556db2b928087378"}, + {file = "duckdb-1.1.3-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:1d9ab6143e73bcf17d62566e368c23f28aa544feddfd2d8eb50ef21034286f24"}, + {file = "duckdb-1.1.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f073d15d11a328f2e6d5964a704517e818e930800b7f3fa83adea47f23720d3"}, + {file = "duckdb-1.1.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5724fd8a49e24d730be34846b814b98ba7c304ca904fbdc98b47fa95c0b0cee"}, + {file = "duckdb-1.1.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51e7dbd968b393343b226ab3f3a7b5a68dee6d3fe59be9d802383bf916775cb8"}, + {file = "duckdb-1.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00cca22df96aa3473fe4584f84888e2cf1c516e8c2dd837210daec44eadba586"}, + {file = "duckdb-1.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:77f26884c7b807c7edd07f95cf0b00e6d47f0de4a534ac1706a58f8bc70d0d31"}, + {file = "duckdb-1.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4748635875fc3c19a7320a6ae7410f9295557450c0ebab6d6712de12640929a"}, + {file = "duckdb-1.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b74e121ab65dbec5290f33ca92301e3a4e81797966c8d9feef6efdf05fc6dafd"}, + {file = "duckdb-1.1.3-cp37-cp37m-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c619e4849837c8c83666f2cd5c6c031300cd2601e9564b47aa5de458ff6e69d"}, + {file = "duckdb-1.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0ba6baa0af33ded836b388b09433a69b8bec00263247f6bf0a05c65c897108d3"}, + {file = "duckdb-1.1.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:ecb1dc9062c1cc4d2d88a5e5cd8cc72af7818ab5a3c0f796ef0ffd60cfd3efb4"}, + {file = "duckdb-1.1.3-cp38-cp38-macosx_12_0_universal2.whl", hash = "sha256:5ace6e4b1873afdd38bd6cc8fcf90310fb2d454f29c39a61d0c0cf1a24ad6c8d"}, + {file = "duckdb-1.1.3-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:a1fa0c502f257fa9caca60b8b1478ec0f3295f34bb2efdc10776fc731b8a6c5f"}, + {file = "duckdb-1.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6411e21a2128d478efbd023f2bdff12464d146f92bc3e9c49247240448ace5a6"}, + {file = "duckdb-1.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5336939d83837af52731e02b6a78a446794078590aa71fd400eb17f083dda3e"}, + {file = "duckdb-1.1.3-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f549af9f7416573ee48db1cf8c9d27aeed245cb015f4b4f975289418c6cf7320"}, + {file = "duckdb-1.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:2141c6b28162199999075d6031b5d63efeb97c1e68fb3d797279d31c65676269"}, + {file = "duckdb-1.1.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:09c68522c30fc38fc972b8a75e9201616b96ae6da3444585f14cf0d116008c95"}, + {file = "duckdb-1.1.3-cp39-cp39-macosx_12_0_universal2.whl", hash = "sha256:8ee97ec337794c162c0638dda3b4a30a483d0587deda22d45e1909036ff0b739"}, + {file = "duckdb-1.1.3-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:a1f83c7217c188b7ab42e6a0963f42070d9aed114f6200e3c923c8899c090f16"}, + {file = "duckdb-1.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1aa3abec8e8995a03ff1a904b0e66282d19919f562dd0a1de02f23169eeec461"}, + {file = "duckdb-1.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80158f4c7c7ada46245837d5b6869a336bbaa28436fbb0537663fa324a2750cd"}, + {file = "duckdb-1.1.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:647f17bd126170d96a38a9a6f25fca47ebb0261e5e44881e3782989033c94686"}, + {file = "duckdb-1.1.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:252d9b17d354beb9057098d4e5d5698e091a4f4a0d38157daeea5fc0ec161670"}, + {file = "duckdb-1.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:eeacb598120040e9591f5a4edecad7080853aa8ac27e62d280f151f8c862afa3"}, + {file = "duckdb-1.1.3.tar.gz", hash = "sha256:68c3a46ab08836fe041d15dcbf838f74a990d551db47cb24ab1c4576fc19351c"}, +] + [[package]] name = "email-validator" version = "2.2.0" @@ -1341,85 +1402,85 @@ test = ["objgraph", "psutil"] [[package]] name = "grpcio" -version = "1.68.1" +version = "1.69.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.8" files = [ - {file = "grpcio-1.68.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:d35740e3f45f60f3c37b1e6f2f4702c23867b9ce21c6410254c9c682237da68d"}, - {file = "grpcio-1.68.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:d99abcd61760ebb34bdff37e5a3ba333c5cc09feda8c1ad42547bea0416ada78"}, - {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f8261fa2a5f679abeb2a0a93ad056d765cdca1c47745eda3f2d87f874ff4b8c9"}, - {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0feb02205a27caca128627bd1df4ee7212db051019a9afa76f4bb6a1a80ca95e"}, - {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:919d7f18f63bcad3a0f81146188e90274fde800a94e35d42ffe9eadf6a9a6330"}, - {file = "grpcio-1.68.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:963cc8d7d79b12c56008aabd8b457f400952dbea8997dd185f155e2f228db079"}, - {file = "grpcio-1.68.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ccf2ebd2de2d6661e2520dae293298a3803a98ebfc099275f113ce1f6c2a80f1"}, - {file = "grpcio-1.68.1-cp310-cp310-win32.whl", hash = "sha256:2cc1fd04af8399971bcd4f43bd98c22d01029ea2e56e69c34daf2bf8470e47f5"}, - {file = "grpcio-1.68.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2e743e51cb964b4975de572aa8fb95b633f496f9fcb5e257893df3be854746"}, - {file = "grpcio-1.68.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:55857c71641064f01ff0541a1776bfe04a59db5558e82897d35a7793e525774c"}, - {file = "grpcio-1.68.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4b177f5547f1b995826ef529d2eef89cca2f830dd8b2c99ffd5fde4da734ba73"}, - {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:3522c77d7e6606d6665ec8d50e867f13f946a4e00c7df46768f1c85089eae515"}, - {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d1fae6bbf0816415b81db1e82fb3bf56f7857273c84dcbe68cbe046e58e1ccd"}, - {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:298ee7f80e26f9483f0b6f94cc0a046caf54400a11b644713bb5b3d8eb387600"}, - {file = "grpcio-1.68.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cbb5780e2e740b6b4f2d208e90453591036ff80c02cc605fea1af8e6fc6b1bbe"}, - {file = "grpcio-1.68.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ddda1aa22495d8acd9dfbafff2866438d12faec4d024ebc2e656784d96328ad0"}, - {file = "grpcio-1.68.1-cp311-cp311-win32.whl", hash = "sha256:b33bd114fa5a83f03ec6b7b262ef9f5cac549d4126f1dc702078767b10c46ed9"}, - {file = "grpcio-1.68.1-cp311-cp311-win_amd64.whl", hash = "sha256:7f20ebec257af55694d8f993e162ddf0d36bd82d4e57f74b31c67b3c6d63d8b2"}, - {file = "grpcio-1.68.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:8829924fffb25386995a31998ccbbeaa7367223e647e0122043dfc485a87c666"}, - {file = "grpcio-1.68.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3aed6544e4d523cd6b3119b0916cef3d15ef2da51e088211e4d1eb91a6c7f4f1"}, - {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:4efac5481c696d5cb124ff1c119a78bddbfdd13fc499e3bc0ca81e95fc573684"}, - {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ab2d912ca39c51f46baf2a0d92aa265aa96b2443266fc50d234fa88bf877d8e"}, - {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c87ce2a97434dffe7327a4071839ab8e8bffd0054cc74cbe971fba98aedd60"}, - {file = "grpcio-1.68.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e4842e4872ae4ae0f5497bf60a0498fa778c192cc7a9e87877abd2814aca9475"}, - {file = "grpcio-1.68.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:255b1635b0ed81e9f91da4fcc8d43b7ea5520090b9a9ad9340d147066d1d3613"}, - {file = "grpcio-1.68.1-cp312-cp312-win32.whl", hash = "sha256:7dfc914cc31c906297b30463dde0b9be48e36939575eaf2a0a22a8096e69afe5"}, - {file = "grpcio-1.68.1-cp312-cp312-win_amd64.whl", hash = "sha256:a0c8ddabef9c8f41617f213e527254c41e8b96ea9d387c632af878d05db9229c"}, - {file = "grpcio-1.68.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:a47faedc9ea2e7a3b6569795c040aae5895a19dde0c728a48d3c5d7995fda385"}, - {file = "grpcio-1.68.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:390eee4225a661c5cd133c09f5da1ee3c84498dc265fd292a6912b65c421c78c"}, - {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:66a24f3d45c33550703f0abb8b656515b0ab777970fa275693a2f6dc8e35f1c1"}, - {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c08079b4934b0bf0a8847f42c197b1d12cba6495a3d43febd7e99ecd1cdc8d54"}, - {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8720c25cd9ac25dd04ee02b69256d0ce35bf8a0f29e20577427355272230965a"}, - {file = "grpcio-1.68.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:04cfd68bf4f38f5bb959ee2361a7546916bd9a50f78617a346b3aeb2b42e2161"}, - {file = "grpcio-1.68.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c28848761a6520c5c6071d2904a18d339a796ebe6b800adc8b3f474c5ce3c3ad"}, - {file = "grpcio-1.68.1-cp313-cp313-win32.whl", hash = "sha256:77d65165fc35cff6e954e7fd4229e05ec76102d4406d4576528d3a3635fc6172"}, - {file = "grpcio-1.68.1-cp313-cp313-win_amd64.whl", hash = "sha256:a8040f85dcb9830d8bbb033ae66d272614cec6faceee88d37a88a9bd1a7a704e"}, - {file = "grpcio-1.68.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:eeb38ff04ab6e5756a2aef6ad8d94e89bb4a51ef96e20f45c44ba190fa0bcaad"}, - {file = "grpcio-1.68.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8a3869a6661ec8f81d93f4597da50336718bde9eb13267a699ac7e0a1d6d0bea"}, - {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2c4cec6177bf325eb6faa6bd834d2ff6aa8bb3b29012cceb4937b86f8b74323c"}, - {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12941d533f3cd45d46f202e3667be8ebf6bcb3573629c7ec12c3e211d99cfccf"}, - {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80af6f1e69c5e68a2be529990684abdd31ed6622e988bf18850075c81bb1ad6e"}, - {file = "grpcio-1.68.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e8dbe3e00771bfe3d04feed8210fc6617006d06d9a2679b74605b9fed3e8362c"}, - {file = "grpcio-1.68.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:83bbf5807dc3ee94ce1de2dfe8a356e1d74101e4b9d7aa8c720cc4818a34aded"}, - {file = "grpcio-1.68.1-cp38-cp38-win32.whl", hash = "sha256:8cb620037a2fd9eeee97b4531880e439ebfcd6d7d78f2e7dcc3726428ab5ef63"}, - {file = "grpcio-1.68.1-cp38-cp38-win_amd64.whl", hash = "sha256:52fbf85aa71263380d330f4fce9f013c0798242e31ede05fcee7fbe40ccfc20d"}, - {file = "grpcio-1.68.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:cb400138e73969eb5e0535d1d06cae6a6f7a15f2cc74add320e2130b8179211a"}, - {file = "grpcio-1.68.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a1b988b40f2fd9de5c820f3a701a43339d8dcf2cb2f1ca137e2c02671cc83ac1"}, - {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:96f473cdacfdd506008a5d7579c9f6a7ff245a9ade92c3c0265eb76cc591914f"}, - {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:37ea3be171f3cf3e7b7e412a98b77685eba9d4fd67421f4a34686a63a65d99f9"}, - {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ceb56c4285754e33bb3c2fa777d055e96e6932351a3082ce3559be47f8024f0"}, - {file = "grpcio-1.68.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dffd29a2961f3263a16d73945b57cd44a8fd0b235740cb14056f0612329b345e"}, - {file = "grpcio-1.68.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:025f790c056815b3bf53da850dd70ebb849fd755a4b1ac822cb65cd631e37d43"}, - {file = "grpcio-1.68.1-cp39-cp39-win32.whl", hash = "sha256:1098f03dedc3b9810810568060dea4ac0822b4062f537b0f53aa015269be0a76"}, - {file = "grpcio-1.68.1-cp39-cp39-win_amd64.whl", hash = "sha256:334ab917792904245a028f10e803fcd5b6f36a7b2173a820c0b5b076555825e1"}, - {file = "grpcio-1.68.1.tar.gz", hash = "sha256:44a8502dd5de653ae6a73e2de50a401d84184f0331d0ac3daeb044e66d5c5054"}, + {file = "grpcio-1.69.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:2060ca95a8db295ae828d0fc1c7f38fb26ccd5edf9aa51a0f44251f5da332e97"}, + {file = "grpcio-1.69.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2e52e107261fd8fa8fa457fe44bfadb904ae869d87c1280bf60f93ecd3e79278"}, + {file = "grpcio-1.69.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:316463c0832d5fcdb5e35ff2826d9aa3f26758d29cdfb59a368c1d6c39615a11"}, + {file = "grpcio-1.69.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:26c9a9c4ac917efab4704b18eed9082ed3b6ad19595f047e8173b5182fec0d5e"}, + {file = "grpcio-1.69.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90b3646ced2eae3a0599658eeccc5ba7f303bf51b82514c50715bdd2b109e5ec"}, + {file = "grpcio-1.69.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3b75aea7c6cb91b341c85e7c1d9db1e09e1dd630b0717f836be94971e015031e"}, + {file = "grpcio-1.69.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5cfd14175f9db33d4b74d63de87c64bb0ee29ce475ce3c00c01ad2a3dc2a9e51"}, + {file = "grpcio-1.69.0-cp310-cp310-win32.whl", hash = "sha256:9031069d36cb949205293cf0e243abd5e64d6c93e01b078c37921493a41b72dc"}, + {file = "grpcio-1.69.0-cp310-cp310-win_amd64.whl", hash = "sha256:cc89b6c29f3dccbe12d7a3b3f1b3999db4882ae076c1c1f6df231d55dbd767a5"}, + {file = "grpcio-1.69.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:8de1b192c29b8ce45ee26a700044717bcbbd21c697fa1124d440548964328561"}, + {file = "grpcio-1.69.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:7e76accf38808f5c5c752b0ab3fd919eb14ff8fafb8db520ad1cc12afff74de6"}, + {file = "grpcio-1.69.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:d5658c3c2660417d82db51e168b277e0ff036d0b0f859fa7576c0ffd2aec1442"}, + {file = "grpcio-1.69.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5494d0e52bf77a2f7eb17c6da662886ca0a731e56c1c85b93505bece8dc6cf4c"}, + {file = "grpcio-1.69.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ed866f9edb574fd9be71bf64c954ce1b88fc93b2a4cbf94af221e9426eb14d6"}, + {file = "grpcio-1.69.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c5ba38aeac7a2fe353615c6b4213d1fbb3a3c34f86b4aaa8be08baaaee8cc56d"}, + {file = "grpcio-1.69.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f79e05f5bbf551c4057c227d1b041ace0e78462ac8128e2ad39ec58a382536d2"}, + {file = "grpcio-1.69.0-cp311-cp311-win32.whl", hash = "sha256:bf1f8be0da3fcdb2c1e9f374f3c2d043d606d69f425cd685110dd6d0d2d61258"}, + {file = "grpcio-1.69.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb9302afc3a0e4ba0b225cd651ef8e478bf0070cf11a529175caecd5ea2474e7"}, + {file = "grpcio-1.69.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:fc18a4de8c33491ad6f70022af5c460b39611e39578a4d84de0fe92f12d5d47b"}, + {file = "grpcio-1.69.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:0f0270bd9ffbff6961fe1da487bdcd594407ad390cc7960e738725d4807b18c4"}, + {file = "grpcio-1.69.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:dc48f99cc05e0698e689b51a05933253c69a8c8559a47f605cff83801b03af0e"}, + {file = "grpcio-1.69.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e925954b18d41aeb5ae250262116d0970893b38232689c4240024e4333ac084"}, + {file = "grpcio-1.69.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87d222569273720366f68a99cb62e6194681eb763ee1d3b1005840678d4884f9"}, + {file = "grpcio-1.69.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b62b0f41e6e01a3e5082000b612064c87c93a49b05f7602fe1b7aa9fd5171a1d"}, + {file = "grpcio-1.69.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:db6f9fd2578dbe37db4b2994c94a1d9c93552ed77dca80e1657bb8a05b898b55"}, + {file = "grpcio-1.69.0-cp312-cp312-win32.whl", hash = "sha256:b192b81076073ed46f4b4dd612b8897d9a1e39d4eabd822e5da7b38497ed77e1"}, + {file = "grpcio-1.69.0-cp312-cp312-win_amd64.whl", hash = "sha256:1227ff7836f7b3a4ab04e5754f1d001fa52a730685d3dc894ed8bc262cc96c01"}, + {file = "grpcio-1.69.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:a78a06911d4081a24a1761d16215a08e9b6d4d29cdbb7e427e6c7e17b06bcc5d"}, + {file = "grpcio-1.69.0-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:dc5a351927d605b2721cbb46158e431dd49ce66ffbacb03e709dc07a491dde35"}, + {file = "grpcio-1.69.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:3629d8a8185f5139869a6a17865d03113a260e311e78fbe313f1a71603617589"}, + {file = "grpcio-1.69.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9a281878feeb9ae26db0622a19add03922a028d4db684658f16d546601a4870"}, + {file = "grpcio-1.69.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cc614e895177ab7e4b70f154d1a7c97e152577ea101d76026d132b7aaba003b"}, + {file = "grpcio-1.69.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:1ee76cd7e2e49cf9264f6812d8c9ac1b85dda0eaea063af07292400f9191750e"}, + {file = "grpcio-1.69.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:0470fa911c503af59ec8bc4c82b371ee4303ececbbdc055f55ce48e38b20fd67"}, + {file = "grpcio-1.69.0-cp313-cp313-win32.whl", hash = "sha256:b650f34aceac8b2d08a4c8d7dc3e8a593f4d9e26d86751ebf74ebf5107d927de"}, + {file = "grpcio-1.69.0-cp313-cp313-win_amd64.whl", hash = "sha256:028337786f11fecb5d7b7fa660475a06aabf7e5e52b5ac2df47414878c0ce7ea"}, + {file = "grpcio-1.69.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:b7f693db593d6bf285e015d5538bf1c86cf9c60ed30b6f7da04a00ed052fe2f3"}, + {file = "grpcio-1.69.0-cp38-cp38-macosx_10_14_universal2.whl", hash = "sha256:8b94e83f66dbf6fd642415faca0608590bc5e8d30e2c012b31d7d1b91b1de2fd"}, + {file = "grpcio-1.69.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:b634851b92c090763dde61df0868c730376cdb73a91bcc821af56ae043b09596"}, + {file = "grpcio-1.69.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bf5f680d3ed08c15330d7830d06bc65f58ca40c9999309517fd62880d70cb06e"}, + {file = "grpcio-1.69.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:200e48a6e7b00f804cf00a1c26292a5baa96507c7749e70a3ec10ca1a288936e"}, + {file = "grpcio-1.69.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:45a4704339b6e5b24b0e136dea9ad3815a94f30eb4f1e1d44c4ac484ef11d8dd"}, + {file = "grpcio-1.69.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:85d347cb8237751b23539981dbd2d9d8f6e9ff90082b427b13022b948eb6347a"}, + {file = "grpcio-1.69.0-cp38-cp38-win32.whl", hash = "sha256:60e5de105dc02832dc8f120056306d0ef80932bcf1c0e2b4ca3b676de6dc6505"}, + {file = "grpcio-1.69.0-cp38-cp38-win_amd64.whl", hash = "sha256:282f47d0928e40f25d007f24eb8fa051cb22551e3c74b8248bc9f9bea9c35fe0"}, + {file = "grpcio-1.69.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:dd034d68a2905464c49479b0c209c773737a4245d616234c79c975c7c90eca03"}, + {file = "grpcio-1.69.0-cp39-cp39-macosx_10_14_universal2.whl", hash = "sha256:01f834732c22a130bdf3dc154d1053bdbc887eb3ccb7f3e6285cfbfc33d9d5cc"}, + {file = "grpcio-1.69.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a7f4ed0dcf202a70fe661329f8874bc3775c14bb3911d020d07c82c766ce0eb1"}, + {file = "grpcio-1.69.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd7ea241b10bc5f0bb0f82c0d7896822b7ed122b3ab35c9851b440c1ccf81588"}, + {file = "grpcio-1.69.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f03dc9b4da4c0dc8a1db7a5420f575251d7319b7a839004d8916257ddbe4816"}, + {file = "grpcio-1.69.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ca71d73a270dff052fe4edf74fef142d6ddd1f84175d9ac4a14b7280572ac519"}, + {file = "grpcio-1.69.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5ccbed100dc43704e94ccff9e07680b540d64e4cc89213ab2832b51b4f68a520"}, + {file = "grpcio-1.69.0-cp39-cp39-win32.whl", hash = "sha256:1514341def9c6ec4b7f0b9628be95f620f9d4b99331b7ef0a1845fd33d9b579c"}, + {file = "grpcio-1.69.0-cp39-cp39-win_amd64.whl", hash = "sha256:c1fea55d26d647346acb0069b08dca70984101f2dc95066e003019207212e303"}, + {file = "grpcio-1.69.0.tar.gz", hash = "sha256:936fa44241b5379c5afc344e1260d467bee495747eaf478de825bab2791da6f5"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.68.1)"] +protobuf = ["grpcio-tools (>=1.69.0)"] [[package]] name = "grpcio-status" -version = "1.68.1" +version = "1.69.0" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.8" files = [ - {file = "grpcio_status-1.68.1-py3-none-any.whl", hash = "sha256:66f3d8847f665acfd56221333d66f7ad8927903d87242a482996bdb45e8d28fd"}, - {file = "grpcio_status-1.68.1.tar.gz", hash = "sha256:e1378d036c81a1610d7b4c7a146cd663dd13fcc915cf4d7d053929dba5bbb6e1"}, + {file = "grpcio_status-1.69.0-py3-none-any.whl", hash = "sha256:d6b2a3c9562c03a817c628d7ba9a925e209c228762d6d7677ae5c9401a542853"}, + {file = "grpcio_status-1.69.0.tar.gz", hash = "sha256:595ef84e5178d6281caa732ccf68ff83259241608d26b0e9c40a5e66eee2a2d2"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.68.1" +grpcio = ">=1.69.0" protobuf = ">=5.26.1,<6.0dev" [[package]] @@ -1597,13 +1658,13 @@ visualization = ["graphviz (>=0.16,<1)"] [[package]] name = "identify" -version = "2.6.3" +version = "2.6.5" description = "File identification library for Python" optional = false python-versions = ">=3.9" files = [ - {file = "identify-2.6.3-py2.py3-none-any.whl", hash = "sha256:9edba65473324c2ea9684b1f944fe3191db3345e50b6d04571d10ed164f8d7bd"}, - {file = "identify-2.6.3.tar.gz", hash = "sha256:62f5dae9b5fef52c84cc188514e9ea4f3f636b1d8799ab5ebc475471f9e47a02"}, + {file = "identify-2.6.5-py2.py3-none-any.whl", hash = "sha256:14181a47091eb75b337af4c23078c9d09225cd4c48929f521f3bf16b09d02566"}, + {file = "identify-2.6.5.tar.gz", hash = "sha256:c10b33f250e5bba374fae86fb57f3adcebf1161bce7cdf92031915fd480c13bc"}, ] [package.extras] @@ -2016,6 +2077,32 @@ rsa = ["cryptography (>=3.0.0)"] signals = ["blinker (>=1.4.0)"] signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] +[[package]] +name = "opendal" +version = "0.45.13" +description = "Apache OpenDAL™ Python Binding" +optional = false +python-versions = ">=3.10" +files = [ + {file = "opendal-0.45.13-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:3afe389215249b1d067cace6b8d1259ab1a2a74bc963d1c7e47dac5e85c8ffc5"}, + {file = "opendal-0.45.13-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a0062482d348617abdc89515fa9cea5c17ae8ac28694b8b5a704530eb91c90e"}, + {file = "opendal-0.45.13-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5cb06d73cc93a13e1a4faa2f369ffe64726f459e53358058720d67efec6dd5fd"}, + {file = "opendal-0.45.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:122942d8185b441774d566c6970b0012ffde9370282c4384c84e6eaa793d4891"}, + {file = "opendal-0.45.13-cp310-cp310-win_amd64.whl", hash = "sha256:e451e1ae63343d07fa57225417e898639240083d2a53ecd7dbafa72254f058bd"}, + {file = "opendal-0.45.13-cp311-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:4b8faf7f780849c6bd777528080864c6c5e46e61488fafb8d2dcb0f9e4e88845"}, + {file = "opendal-0.45.13-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1987cc0ac3e05ebcd963b431a0c05607d9a9d7ed204ba8053258f812434b262"}, + {file = "opendal-0.45.13-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9fb89b30ee3cb8fd432ada45be0ae9dad9d7483e1e1db22deb3074ef61a194ca"}, + {file = "opendal-0.45.13-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:606e99a147de8a0a41285fb240e0280fea6a7436fe3f7341815157bce69104d6"}, + {file = "opendal-0.45.13-cp311-abi3-win_amd64.whl", hash = "sha256:ab187174dede49a7e9821a4d4792676a62c887279465d33624009e720a086667"}, + {file = "opendal-0.45.13.tar.gz", hash = "sha256:ed818dd564beeace57a040f65415525838ad78c20bdffdbe0ba54281e7f17064"}, +] + +[package.extras] +benchmark = ["boto3", "boto3-stubs[essential]", "gevent", "greenify", "greenlet", "pydantic"] +docs = ["pdoc"] +lint = ["ruff"] +test = ["pytest", "pytest-asyncio", "python-dotenv"] + [[package]] name = "orjson" version = "3.10.13" @@ -2394,7 +2481,6 @@ files = [ {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, - {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, @@ -2648,13 +2734,13 @@ setuptools = "*" [[package]] name = "pygments" -version = "2.18.0" +version = "2.19.1" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" files = [ - {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, - {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, + {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, + {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, ] [package.extras] @@ -3101,23 +3187,23 @@ files = [ [[package]] name = "setuptools" -version = "75.6.0" +version = "75.7.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" files = [ - {file = "setuptools-75.6.0-py3-none-any.whl", hash = "sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d"}, - {file = "setuptools-75.6.0.tar.gz", hash = "sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6"}, + {file = "setuptools-75.7.0-py3-none-any.whl", hash = "sha256:84fb203f278ebcf5cd08f97d3fb96d3fbed4b629d500b29ad60d11e00769b183"}, + {file = "setuptools-75.7.0.tar.gz", hash = "sha256:886ff7b16cd342f1d1defc16fc98c9ce3fde69e087a4e1983d7ab634e5f41f4f"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.7.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (>=1.12,<1.14)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "shellingham" @@ -3677,13 +3763,13 @@ test = ["aiohttp (>=3.10.5)", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", [[package]] name = "virtualenv" -version = "20.28.0" +version = "20.28.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" files = [ - {file = "virtualenv-20.28.0-py3-none-any.whl", hash = "sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0"}, - {file = "virtualenv-20.28.0.tar.gz", hash = "sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa"}, + {file = "virtualenv-20.28.1-py3-none-any.whl", hash = "sha256:412773c85d4dab0409b83ec36f7a6499e72eaf08c80e81e9576bca61831c71cb"}, + {file = "virtualenv-20.28.1.tar.gz", hash = "sha256:5d34ab240fdb5d21549b76f9e8ff3af28252f5499fb6d6f031adac4e5a8c5329"}, ] [package.dependencies] @@ -4155,4 +4241,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.12" -content-hash = "6a69629481455ea31e3e9249169d0c629bb8028da63ae6302ee949b8d04c47a2" +content-hash = "c48a9477b1f9587dfa52f28ffb0c38101c9038ad8a01d1027da080ee19af31d0" diff --git a/ibis-server/pyproject.toml b/ibis-server/pyproject.toml index 09c6c7665..399de9b54 100644 --- a/ibis-server/pyproject.toml +++ b/ibis-server/pyproject.toml @@ -48,6 +48,8 @@ trino = ">=0.321,<1" psycopg2 = ">=2.8.4,<3" clickhouse-connect = "0.8.11" asgi-lifespan = "2.1.0" +duckdb = "1.1.3" +opendal = ">=0.45" [tool.pytest.ini_options] addopts = ["--strict-markers"] @@ -61,6 +63,7 @@ markers = [ "postgres: mark a test as a postgres test", "snowflake: mark a test as a snowflake test", "trino: mark a test as a trino test", + "local_file: mark a test as a local file test", "beta: mark a test as a test for beta versions of the engine", ] diff --git a/ibis-server/tests/resource/test_file_source/invalid b/ibis-server/tests/resource/test_file_source/invalid new file mode 100644 index 000000000..3c546eb27 --- /dev/null +++ b/ibis-server/tests/resource/test_file_source/invalid @@ -0,0 +1 @@ +dummy file \ No newline at end of file diff --git a/ibis-server/tests/resource/test_file_source/type-test-csv/t1.csv b/ibis-server/tests/resource/test_file_source/type-test-csv/t1.csv new file mode 100644 index 000000000..53a49593d --- /dev/null +++ b/ibis-server/tests/resource/test_file_source/type-test-csv/t1.csv @@ -0,0 +1,2 @@ +c_bigint,c_bit,c_blob,c_boolean,c_date,c_double,c_float,c_integer,c_hugeint,c_interval,c_json,c_smallint,c_time,c_timestamp,c_timestamptz,c_tinyint,c_ubigint,c_uhugeint,c_uinteger,c_usmallint,c_utinyint,c_uuid,c_varchar +100,00000000000000000000000000000001,hello,true,2020-01-01,1.1,1.1,1,1,1 day,"{""a"": 1}",1,12:00:00,2020-01-01 12:00:00,2020-01-01 12:00:00+08,1,1,1,1,1,1,123e4567-e89b-12d3-a456-426614174000,hello diff --git a/ibis-server/tests/resource/test_file_source/type-test-csv/t2.csv b/ibis-server/tests/resource/test_file_source/type-test-csv/t2.csv new file mode 100644 index 000000000..53a49593d --- /dev/null +++ b/ibis-server/tests/resource/test_file_source/type-test-csv/t2.csv @@ -0,0 +1,2 @@ +c_bigint,c_bit,c_blob,c_boolean,c_date,c_double,c_float,c_integer,c_hugeint,c_interval,c_json,c_smallint,c_time,c_timestamp,c_timestamptz,c_tinyint,c_ubigint,c_uhugeint,c_uinteger,c_usmallint,c_utinyint,c_uuid,c_varchar +100,00000000000000000000000000000001,hello,true,2020-01-01,1.1,1.1,1,1,1 day,"{""a"": 1}",1,12:00:00,2020-01-01 12:00:00,2020-01-01 12:00:00+08,1,1,1,1,1,1,123e4567-e89b-12d3-a456-426614174000,hello diff --git a/ibis-server/tests/resource/test_file_source/type-test-json/t1.json b/ibis-server/tests/resource/test_file_source/type-test-json/t1.json new file mode 100644 index 000000000..fed01ee75 --- /dev/null +++ b/ibis-server/tests/resource/test_file_source/type-test-json/t1.json @@ -0,0 +1 @@ +{"c_bigint":100,"c_bit":"00000000000000000000000000000001","c_blob":"hello","c_boolean":true,"c_date":"2020-01-01","c_double":1.1,"c_float":1.100000023841858,"c_integer":1,"c_hugeint":1.0,"c_interval":"1 day","c_json":{"a":1},"c_smallint":1,"c_time":"12:00:00","c_timestamp":"2020-01-01 12:00:00","c_timestamptz":"2020-01-01 12:00:00+08","c_tinyint":1,"c_ubigint":1,"c_uhugeint":1.0,"c_uinteger":1,"c_usmallint":1,"c_utinyint":1,"c_uuid":"123e4567-e89b-12d3-a456-426614174000","c_varchar":"hello"} diff --git a/ibis-server/tests/resource/test_file_source/type-test-json/t2.json b/ibis-server/tests/resource/test_file_source/type-test-json/t2.json new file mode 100644 index 000000000..fed01ee75 --- /dev/null +++ b/ibis-server/tests/resource/test_file_source/type-test-json/t2.json @@ -0,0 +1 @@ +{"c_bigint":100,"c_bit":"00000000000000000000000000000001","c_blob":"hello","c_boolean":true,"c_date":"2020-01-01","c_double":1.1,"c_float":1.100000023841858,"c_integer":1,"c_hugeint":1.0,"c_interval":"1 day","c_json":{"a":1},"c_smallint":1,"c_time":"12:00:00","c_timestamp":"2020-01-01 12:00:00","c_timestamptz":"2020-01-01 12:00:00+08","c_tinyint":1,"c_ubigint":1,"c_uhugeint":1.0,"c_uinteger":1,"c_usmallint":1,"c_utinyint":1,"c_uuid":"123e4567-e89b-12d3-a456-426614174000","c_varchar":"hello"} diff --git a/ibis-server/tests/resource/test_file_source/type-test-parquet/t1.parquet b/ibis-server/tests/resource/test_file_source/type-test-parquet/t1.parquet new file mode 100644 index 000000000..68a76fa9b Binary files /dev/null and b/ibis-server/tests/resource/test_file_source/type-test-parquet/t1.parquet differ diff --git a/ibis-server/tests/resource/test_file_source/type-test-parquet/t2.parquet b/ibis-server/tests/resource/test_file_source/type-test-parquet/t2.parquet new file mode 100644 index 000000000..68a76fa9b Binary files /dev/null and b/ibis-server/tests/resource/test_file_source/type-test-parquet/t2.parquet differ diff --git a/ibis-server/tests/resource/test_file_source/type-test.csv b/ibis-server/tests/resource/test_file_source/type-test.csv new file mode 100644 index 000000000..53a49593d --- /dev/null +++ b/ibis-server/tests/resource/test_file_source/type-test.csv @@ -0,0 +1,2 @@ +c_bigint,c_bit,c_blob,c_boolean,c_date,c_double,c_float,c_integer,c_hugeint,c_interval,c_json,c_smallint,c_time,c_timestamp,c_timestamptz,c_tinyint,c_ubigint,c_uhugeint,c_uinteger,c_usmallint,c_utinyint,c_uuid,c_varchar +100,00000000000000000000000000000001,hello,true,2020-01-01,1.1,1.1,1,1,1 day,"{""a"": 1}",1,12:00:00,2020-01-01 12:00:00,2020-01-01 12:00:00+08,1,1,1,1,1,1,123e4567-e89b-12d3-a456-426614174000,hello diff --git a/ibis-server/tests/resource/test_file_source/type-test.json b/ibis-server/tests/resource/test_file_source/type-test.json new file mode 100644 index 000000000..fed01ee75 --- /dev/null +++ b/ibis-server/tests/resource/test_file_source/type-test.json @@ -0,0 +1 @@ +{"c_bigint":100,"c_bit":"00000000000000000000000000000001","c_blob":"hello","c_boolean":true,"c_date":"2020-01-01","c_double":1.1,"c_float":1.100000023841858,"c_integer":1,"c_hugeint":1.0,"c_interval":"1 day","c_json":{"a":1},"c_smallint":1,"c_time":"12:00:00","c_timestamp":"2020-01-01 12:00:00","c_timestamptz":"2020-01-01 12:00:00+08","c_tinyint":1,"c_ubigint":1,"c_uhugeint":1.0,"c_uinteger":1,"c_usmallint":1,"c_utinyint":1,"c_uuid":"123e4567-e89b-12d3-a456-426614174000","c_varchar":"hello"} diff --git a/ibis-server/tests/resource/test_file_source/type-test.parquet b/ibis-server/tests/resource/test_file_source/type-test.parquet new file mode 100644 index 000000000..68a76fa9b Binary files /dev/null and b/ibis-server/tests/resource/test_file_source/type-test.parquet differ diff --git a/ibis-server/tests/routers/v2/connector/test_clickhouse.py b/ibis-server/tests/routers/v2/connector/test_clickhouse.py index b2d832039..5a0523afa 100644 --- a/ibis-server/tests/routers/v2/connector/test_clickhouse.py +++ b/ibis-server/tests/routers/v2/connector/test_clickhouse.py @@ -527,6 +527,7 @@ async def test_metadata_list_tables(client, clickhouse: ClickHouseContainer): "catalog": None, "schema": "test", "table": "orders", + "path": None, } assert len(result["columns"]) == 9 assert result["columns"][8] == { diff --git a/ibis-server/tests/routers/v2/connector/test_local_file.py b/ibis-server/tests/routers/v2/connector/test_local_file.py new file mode 100644 index 000000000..a4c394d70 --- /dev/null +++ b/ibis-server/tests/routers/v2/connector/test_local_file.py @@ -0,0 +1,448 @@ +import base64 + +import orjson +import pytest + +pytestmark = pytest.mark.local_file + + +base_url = "/v2/connector/local_file" +manifest = { + "catalog": "my_calalog", + "schema": "my_schema", + "models": [ + { + "name": "Orders", + "tableReference": { + "table": "tests/resource/tpch/data/orders.parquet", + }, + "columns": [ + {"name": "orderkey", "expression": "o_orderkey", "type": "integer"}, + {"name": "custkey", "expression": "o_custkey", "type": "integer"}, + { + "name": "orderstatus", + "expression": "o_orderstatus", + "type": "varchar", + }, + { + "name": "totalprice", + "expression": "o_totalprice", + "type": "float", + }, + {"name": "orderdate", "expression": "o_orderdate", "type": "date"}, + { + "name": "order_cust_key", + "expression": "concat(o_orderkey, '_', o_custkey)", + "type": "varchar", + }, + ], + "primaryKey": "orderkey", + }, + { + "name": "Customer", + "tableReference": { + "table": "tests/resource/tpch/data/customer.parquet", + }, + "columns": [ + { + "name": "custkey", + "type": "integer", + "expression": "c_custkey", + }, + { + "name": "orders", + "type": "Orders", + "relationship": "CustomerOrders", + }, + { + "name": "sum_totalprice", + "type": "float", + "isCalculated": True, + "expression": "sum(orders.totalprice)", + }, + ], + "primaryKey": "custkey", + }, + ], + "relationships": [ + { + "name": "CustomerOrders", + "models": ["Customer", "Orders"], + "joinType": "ONE_TO_MANY", + "condition": "Customer.custkey = Orders.custkey", + } + ], +} + + +@pytest.fixture(scope="module") +def manifest_str(): + return base64.b64encode(orjson.dumps(manifest)).decode("utf-8") + + +@pytest.fixture(scope="module") +def connection_info() -> dict[str, str]: + return { + "url": "tests/resource/tpch/data", + "format": "parquet", + } + + +async def test_query(client, manifest_str, connection_info): + response = await client.post( + f"{base_url}/query", + json={ + "manifestStr": manifest_str, + "sql": 'SELECT * FROM "Orders" LIMIT 1', + "connectionInfo": connection_info, + }, + ) + assert response.status_code == 200 + result = response.json() + assert len(result["columns"]) == len(manifest["models"][0]["columns"]) + assert len(result["data"]) == 1 + assert result["data"][0] == [ + 1, + 370, + "O", + "172799.49", + "1996-01-02 00:00:00.000000", + "1_370", + ] + assert result["dtypes"] == { + "orderkey": "int32", + "custkey": "int32", + "orderstatus": "object", + "totalprice": "float64", + "orderdate": "object", + "order_cust_key": "object", + } + + +async def test_query_with_limit(client, manifest_str, connection_info): + response = await client.post( + f"{base_url}/query", + params={"limit": 1}, + json={ + "manifestStr": manifest_str, + "sql": 'SELECT * FROM "Orders" limit 2', + "connectionInfo": connection_info, + }, + ) + assert response.status_code == 200 + result = response.json() + assert len(result["data"]) == 1 + + +async def test_query_calculated_field(client, manifest_str, connection_info): + response = await client.post( + f"{base_url}/query", + json={ + "manifestStr": manifest_str, + "sql": 'SELECT custkey, sum_totalprice FROM "Customer" WHERE custkey = 370', + "connectionInfo": connection_info, + }, + ) + assert response.status_code == 200 + result = response.json() + assert len(result["columns"]) == 2 + assert len(result["data"]) == 1 + assert result["data"][0] == [ + 370, + "2860895.79", + ] + assert result["dtypes"] == { + "custkey": "int32", + "sum_totalprice": "float64", + } + + +async def test_dry_run(client, manifest_str, connection_info): + response = await client.post( + f"{base_url}/query", + params={"dryRun": True}, + json={ + "manifestStr": manifest_str, + "sql": 'SELECT * FROM "Orders" LIMIT 1', + "connectionInfo": connection_info, + }, + ) + assert response.status_code == 204 + + response = await client.post( + f"{base_url}/query", + params={"dryRun": True}, + json={ + "manifestStr": manifest_str, + "sql": 'SELECT * FROM "NotFound" LIMIT 1', + "connectionInfo": connection_info, + }, + ) + assert response.status_code == 422 + assert response.text is not None + + +async def test_metadata_list_tables(client, connection_info): + response = await client.post( + url=f"{base_url}/metadata/tables", + json={ + "connectionInfo": connection_info, + }, + ) + assert response.status_code == 200 + + result = next(filter(lambda x: x["name"] == "orders", response.json())) + assert result["name"] == "orders" + assert result["primaryKey"] is None + assert result["description"] is None + assert result["properties"] == { + "catalog": None, + "schema": None, + "table": "orders", + "path": "tests/resource/tpch/data/orders.parquet", + } + assert len(result["columns"]) == 9 + assert result["columns"][8] == { + "name": "o_comment", + "nestedColumns": None, + "type": "STRING", + "notNull": False, + "description": None, + "properties": None, + } + + +async def test_metadata_list_constraints(client, connection_info): + response = await client.post( + url=f"{base_url}/metadata/constraints", + json={ + "connectionInfo": connection_info, + }, + ) + assert response.status_code == 200 + + +async def test_metadata_db_version(client, connection_info): + response = await client.post( + url=f"{base_url}/metadata/version", + json={ + "connectionInfo": connection_info, + }, + ) + assert response.status_code == 200 + assert "Local File System" in response.text + + +async def test_unsupported_format(client): + response = await client.post( + url=f"{base_url}/metadata/tables", + json={ + "connectionInfo": { + "url": "tests/resource/tpch/data", + "format": "unsupported", + }, + }, + ) + assert response.status_code == 501 + assert response.text == "Unsupported format: unsupported" + + +async def test_list_parquet_files(client): + response = await client.post( + url=f"{base_url}/metadata/tables", + json={ + "connectionInfo": { + "url": "tests/resource/test_file_source", + "format": "parquet", + }, + }, + ) + assert response.status_code == 200 + result = response.json() + assert len(result) == 2 + table_names = [table["name"] for table in result] + assert "type-test-parquet" in table_names + assert "type-test" in table_names + columns = result[0]["columns"] + assert len(columns) == 23 + assert columns[0]["name"] == "c_bigint" + assert columns[0]["type"] == "INT64" + assert columns[1]["name"] == "c_bit" + assert columns[1]["type"] == "STRING" + assert columns[2]["name"] == "c_blob" + assert columns[2]["type"] == "BYTES" + assert columns[3]["name"] == "c_boolean" + assert columns[3]["type"] == "BOOL" + assert columns[4]["name"] == "c_date" + assert columns[4]["type"] == "DATE" + assert columns[5]["name"] == "c_double" + assert columns[5]["type"] == "DOUBLE" + assert columns[6]["name"] == "c_float" + assert columns[6]["type"] == "FLOAT" + assert columns[7]["name"] == "c_integer" + assert columns[7]["type"] == "INT" + assert columns[8]["name"] == "c_hugeint" + assert columns[8]["type"] == "DOUBLE" + assert columns[9]["name"] == "c_interval" + assert columns[9]["type"] == "INTERVAL" + assert columns[10]["name"] == "c_json" + assert columns[10]["type"] == "JSON" + assert columns[11]["name"] == "c_smallint" + assert columns[11]["type"] == "INT2" + assert columns[12]["name"] == "c_time" + assert columns[12]["type"] == "TIME" + assert columns[13]["name"] == "c_timestamp" + assert columns[13]["type"] == "TIMESTAMP" + assert columns[14]["name"] == "c_timestamptz" + assert columns[14]["type"] == "TIMESTAMPTZ" + assert columns[15]["name"] == "c_tinyint" + assert columns[15]["type"] == "INT2" + assert columns[16]["name"] == "c_ubigint" + assert columns[16]["type"] == "INT64" + assert columns[17]["name"] == "c_uhugeint" + assert columns[17]["type"] == "DOUBLE" + assert columns[18]["name"] == "c_uinteger" + assert columns[18]["type"] == "INT" + assert columns[19]["name"] == "c_usmallint" + assert columns[19]["type"] == "INT2" + assert columns[20]["name"] == "c_utinyint" + assert columns[20]["type"] == "INT2" + assert columns[21]["name"] == "c_uuid" + assert columns[21]["type"] == "UUID" + assert columns[22]["name"] == "c_varchar" + assert columns[22]["type"] == "STRING" + + +async def test_list_csv_files(client): + response = await client.post( + url=f"{base_url}/metadata/tables", + json={ + "connectionInfo": { + "url": "tests/resource/test_file_source", + "format": "csv", + }, + }, + ) + assert response.status_code == 200 + result = response.json() + assert len(result) == 3 + table_names = [table["name"] for table in result] + assert "type-test-csv" in table_names + assert "type-test" in table_names + # `invalid` will be considered as a one column csv file + assert "invalid" in table_names + columns = result[0]["columns"] + assert columns[0]["name"] == "c_bigint" + assert columns[0]["type"] == "INT64" + assert columns[1]["name"] == "c_bit" + assert columns[1]["type"] == "STRING" + assert columns[2]["name"] == "c_blob" + assert columns[2]["type"] == "STRING" + assert columns[3]["name"] == "c_boolean" + assert columns[3]["type"] == "BOOL" + assert columns[4]["name"] == "c_date" + assert columns[4]["type"] == "DATE" + assert columns[5]["name"] == "c_double" + assert columns[5]["type"] == "DOUBLE" + assert columns[6]["name"] == "c_float" + assert columns[6]["type"] == "DOUBLE" + assert columns[7]["name"] == "c_integer" + assert columns[7]["type"] == "INT64" + assert columns[8]["name"] == "c_hugeint" + assert columns[8]["type"] == "INT64" + assert columns[9]["name"] == "c_interval" + assert columns[9]["type"] == "STRING" + assert columns[10]["name"] == "c_json" + assert columns[10]["type"] == "STRING" + assert columns[11]["name"] == "c_smallint" + assert columns[11]["type"] == "INT64" + assert columns[12]["name"] == "c_time" + assert columns[12]["type"] == "TIME" + assert columns[13]["name"] == "c_timestamp" + assert columns[13]["type"] == "TIMESTAMP" + assert columns[14]["name"] == "c_timestamptz" + assert columns[14]["type"] == "TIMESTAMP" + assert columns[15]["name"] == "c_tinyint" + assert columns[15]["type"] == "INT64" + assert columns[16]["name"] == "c_ubigint" + assert columns[16]["type"] == "INT64" + assert columns[17]["name"] == "c_uhugeint" + assert columns[17]["type"] == "INT64" + assert columns[18]["name"] == "c_uinteger" + assert columns[18]["type"] == "INT64" + assert columns[19]["name"] == "c_usmallint" + assert columns[19]["type"] == "INT64" + assert columns[20]["name"] == "c_utinyint" + assert columns[20]["type"] == "INT64" + assert columns[21]["name"] == "c_uuid" + assert columns[21]["type"] == "STRING" + assert columns[22]["name"] == "c_varchar" + assert columns[22]["type"] == "STRING" + + +async def test_list_json_files(client): + response = await client.post( + url=f"{base_url}/metadata/tables", + json={ + "connectionInfo": { + "url": "tests/resource/test_file_source", + "format": "json", + }, + }, + ) + assert response.status_code == 200 + result = response.json() + assert len(result) == 2 + table_names = [table["name"] for table in result] + assert "type-test-json" in table_names + assert "type-test" in table_names + + columns = result[0]["columns"] + assert columns[0]["name"] == "c_bigint" + assert columns[0]["type"] == "INT64" + # `c_bit` is a string in json which value is `00000000000000000000000000000001` + # It's considered as a UUID by DuckDB json reader. + assert columns[1]["name"] == "c_bit" + assert columns[1]["type"] == "UUID" + assert columns[2]["name"] == "c_blob" + assert columns[2]["type"] == "STRING" + assert columns[3]["name"] == "c_boolean" + assert columns[3]["type"] == "BOOL" + assert columns[4]["name"] == "c_date" + assert columns[4]["type"] == "DATE" + assert columns[5]["name"] == "c_double" + assert columns[5]["type"] == "DOUBLE" + assert columns[6]["name"] == "c_float" + assert columns[6]["type"] == "DOUBLE" + assert columns[7]["name"] == "c_integer" + assert columns[7]["type"] == "INT64" + assert columns[8]["name"] == "c_hugeint" + assert columns[8]["type"] == "DOUBLE" + assert columns[9]["name"] == "c_interval" + assert columns[9]["type"] == "STRING" + assert columns[10]["name"] == "c_json" + assert columns[10]["type"] == "UNKNOWN" + assert columns[11]["name"] == "c_smallint" + assert columns[11]["type"] == "INT64" + assert columns[12]["name"] == "c_time" + assert columns[12]["type"] == "TIME" + assert columns[13]["name"] == "c_timestamp" + assert columns[13]["type"] == "TIMESTAMP" + assert columns[14]["name"] == "c_timestamptz" + assert columns[14]["type"] == "STRING" + assert columns[15]["name"] == "c_tinyint" + assert columns[15]["type"] == "INT64" + assert columns[16]["name"] == "c_ubigint" + assert columns[16]["type"] == "INT64" + assert columns[17]["name"] == "c_uhugeint" + assert columns[17]["type"] == "DOUBLE" + assert columns[18]["name"] == "c_uinteger" + assert columns[18]["type"] == "INT64" + assert columns[19]["name"] == "c_usmallint" + assert columns[19]["type"] == "INT64" + assert columns[20]["name"] == "c_utinyint" + assert columns[20]["type"] == "INT64" + assert columns[21]["name"] == "c_uuid" + assert columns[21]["type"] == "UUID" + assert columns[22]["name"] == "c_varchar" + assert columns[22]["type"] == "STRING" diff --git a/ibis-server/tests/routers/v2/connector/test_mssql.py b/ibis-server/tests/routers/v2/connector/test_mssql.py index 5ed8e0767..8c22a70b9 100644 --- a/ibis-server/tests/routers/v2/connector/test_mssql.py +++ b/ibis-server/tests/routers/v2/connector/test_mssql.py @@ -380,6 +380,7 @@ async def test_metadata_list_tables(client, mssql: SqlServerContainer): "catalog": "tempdb", "schema": "dbo", "table": "orders", + "path": None, } assert len(result["columns"]) == 9 assert result["columns"][8] == { diff --git a/ibis-server/tests/routers/v2/connector/test_mysql.py b/ibis-server/tests/routers/v2/connector/test_mysql.py index 6138e97f4..01f3cb695 100644 --- a/ibis-server/tests/routers/v2/connector/test_mysql.py +++ b/ibis-server/tests/routers/v2/connector/test_mysql.py @@ -362,6 +362,7 @@ async def test_metadata_list_tables(client, mysql: MySqlContainer): "catalog": "", "schema": "test", "table": "orders", + "path": None, } assert len(result["columns"]) == 9 o_comment_column = next( diff --git a/ibis-server/tests/routers/v2/connector/test_postgres.py b/ibis-server/tests/routers/v2/connector/test_postgres.py index 53001867f..8916713c5 100644 --- a/ibis-server/tests/routers/v2/connector/test_postgres.py +++ b/ibis-server/tests/routers/v2/connector/test_postgres.py @@ -566,6 +566,7 @@ async def test_metadata_list_tables(client, postgres: PostgresContainer): "catalog": "test", "schema": "public", "table": "orders", + "path": None, } assert len(result["columns"]) == 9 assert result["columns"][8] == { diff --git a/ibis-server/tests/routers/v2/connector/test_snowflake.py b/ibis-server/tests/routers/v2/connector/test_snowflake.py index 728b78a33..688505628 100644 --- a/ibis-server/tests/routers/v2/connector/test_snowflake.py +++ b/ibis-server/tests/routers/v2/connector/test_snowflake.py @@ -287,6 +287,7 @@ async def test_metadata_list_tables(client): "catalog": "SNOWFLAKE_SAMPLE_DATA", "schema": "TPCH_SF1", "table": "ORDERS", + "path": None, } assert len(table["columns"]) == 9 column = next(filter(lambda c: c["name"] == "O_COMMENT", table["columns"])) diff --git a/ibis-server/tests/routers/v2/connector/test_trino.py b/ibis-server/tests/routers/v2/connector/test_trino.py index 25cfb40dd..5b14c2432 100644 --- a/ibis-server/tests/routers/v2/connector/test_trino.py +++ b/ibis-server/tests/routers/v2/connector/test_trino.py @@ -391,6 +391,7 @@ async def test_metadata_list_tables(client, trino: TrinoContainer): "catalog": "memory", "schema": "default", "table": "orders", + "path": None, } assert len(table["columns"]) == 9 column = next(filter(lambda c: c["name"] == "comment", table["columns"])) diff --git a/ibis-server/tests/routers/v3/connector/local_file/__init__.py b/ibis-server/tests/routers/v3/connector/local_file/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ibis-server/tests/routers/v3/connector/local_file/conftest.py b/ibis-server/tests/routers/v3/connector/local_file/conftest.py new file mode 100644 index 000000000..62f4b7aaf --- /dev/null +++ b/ibis-server/tests/routers/v3/connector/local_file/conftest.py @@ -0,0 +1,25 @@ +import pathlib + +import pytest + +pytestmark = pytest.mark.local_file + +base_url = "/v3/connector/local_file" + + +def pytest_collection_modifyitems(items): + current_file_dir = pathlib.Path(__file__).resolve().parent + for item in items: + try: + pathlib.Path(item.fspath).relative_to(current_file_dir) + item.add_marker(pytestmark) + except ValueError: + pass + + +@pytest.fixture(scope="module") +def connection_info() -> dict[str, str]: + return { + "url": "tests/resource/tpch/data", + "format": "parquet", + } diff --git a/ibis-server/tests/routers/v3/connector/local_file/test_query.py b/ibis-server/tests/routers/v3/connector/local_file/test_query.py new file mode 100644 index 000000000..19c924086 --- /dev/null +++ b/ibis-server/tests/routers/v3/connector/local_file/test_query.py @@ -0,0 +1,180 @@ +import base64 + +import orjson +import pytest + +from tests.routers.v3.connector.local_file.conftest import base_url + +manifest = { + "catalog": "my_calalog", + "schema": "my_schema", + "models": [ + { + "name": "Orders", + "tableReference": { + "table": "tests/resource/tpch/data/orders.parquet", + }, + "columns": [ + {"name": "orderkey", "expression": "o_orderkey", "type": "integer"}, + {"name": "custkey", "expression": "o_custkey", "type": "integer"}, + { + "name": "orderstatus", + "expression": "o_orderstatus", + "type": "varchar", + }, + { + "name": "totalprice", + "expression": "o_totalprice", + "type": "float", + }, + {"name": "orderdate", "expression": "o_orderdate", "type": "date"}, + ], + "primaryKey": "orderkey", + }, + { + "name": "Customer", + "tableReference": { + "table": "tests/resource/tpch/data/customer.parquet", + }, + "columns": [ + { + "name": "custkey", + "type": "integer", + "expression": "c_custkey", + }, + { + "name": "Orders", + "type": "Orders", + "relationship": "CustomerOrders", + }, + { + "name": "sum_totalprice", + "type": "float", + "isCalculated": True, + "expression": 'sum("Orders".totalprice)', + }, + ], + "primaryKey": "custkey", + }, + ], + "relationships": [ + { + "name": "CustomerOrders", + "models": ["Customer", "Orders"], + "joinType": "ONE_TO_MANY", + "condition": '"Customer".custkey = "Orders".custkey', + } + ], +} + + +@pytest.fixture(scope="module") +def manifest_str(): + return base64.b64encode(orjson.dumps(manifest)).decode("utf-8") + + +async def test_query(client, manifest_str): + response = await client.post( + f"{base_url}/query", + json={ + "manifestStr": manifest_str, + "sql": 'SELECT orderkey, custkey, orderstatus, totalprice, orderdate FROM "Orders" LIMIT 1', + "connectionInfo": { + "url": "tests/resource/tpch", + "format": "parquet", + }, + }, + ) + assert response.status_code == 200 + result = response.json() + assert len(result["columns"]) == len(manifest["models"][0]["columns"]) + assert len(result["data"]) == 1 + assert result["data"][0] == [ + 1, + 370, + "O", + "172799.49", + "1996-01-02 00:00:00.000000", + ] + assert result["dtypes"] == { + "orderkey": "int32", + "custkey": "int32", + "orderstatus": "object", + "totalprice": "float64", + "orderdate": "object", + } + + +async def test_query_with_limit(client, manifest_str): + response = await client.post( + f"{base_url}/query", + params={"limit": 1}, + json={ + "manifestStr": manifest_str, + "sql": 'SELECT * FROM "Orders" limit 2', + "connectionInfo": { + "url": "tests/resource/tpch", + "format": "parquet", + }, + }, + ) + assert response.status_code == 200 + result = response.json() + assert len(result["data"]) == 1 + + +async def test_query_calculated_field(client, manifest_str): + response = await client.post( + f"{base_url}/query", + json={ + "manifestStr": manifest_str, + "sql": 'SELECT custkey, sum_totalprice FROM "Customer" WHERE custkey = 370', + "connectionInfo": { + "url": "tests/resource/tpch", + "format": "parquet", + }, + }, + ) + assert response.status_code == 200 + result = response.json() + assert len(result["columns"]) == 2 + assert len(result["data"]) == 1 + assert result["data"][0] == [ + 370, + "2860895.79", + ] + assert result["dtypes"] == { + "custkey": "int32", + "sum_totalprice": "float64", + } + + +async def test_dry_run(client, manifest_str): + response = await client.post( + f"{base_url}/query", + params={"dryRun": True}, + json={ + "manifestStr": manifest_str, + "sql": 'SELECT * FROM "Orders" LIMIT 1', + "connectionInfo": { + "url": "tests/resource/tpch", + "format": "parquet", + }, + }, + ) + assert response.status_code == 204 + + response = await client.post( + f"{base_url}/query", + params={"dryRun": True}, + json={ + "manifestStr": manifest_str, + "sql": 'SELECT * FROM "NotFound" LIMIT 1', + "connectionInfo": { + "url": "tests/resource/tpch", + "format": "parquet", + }, + }, + ) + assert response.status_code == 422 + assert response.text is not None diff --git a/wren-core-py/tests/test_modeling_core.py b/wren-core-py/tests/test_modeling_core.py index a6563082e..0bd85f808 100644 --- a/wren-core-py/tests/test_modeling_core.py +++ b/wren-core-py/tests/test_modeling_core.py @@ -90,7 +90,7 @@ def test_session_context(): rewritten_sql = session_context.transform_sql(sql) assert ( rewritten_sql - == "SELECT customer.c_custkey, customer.c_name FROM (SELECT customer.c_custkey AS c_custkey, customer.c_name AS c_name FROM main.customer) AS customer" + == "SELECT customer.c_custkey, customer.c_name FROM (SELECT __source.c_custkey AS c_custkey, __source.c_name AS c_name FROM main.customer AS __source) AS customer" ) session_context = SessionContext(manifest_str, "tests/functions.csv") @@ -98,7 +98,7 @@ def test_session_context(): rewritten_sql = session_context.transform_sql(sql) assert ( rewritten_sql - == "SELECT add_two(customer.c_custkey) FROM (SELECT customer.c_custkey FROM (SELECT customer.c_custkey AS c_custkey FROM main.customer) AS customer) AS customer" + == "SELECT add_two(customer.c_custkey) FROM (SELECT customer.c_custkey FROM (SELECT __source.c_custkey AS c_custkey FROM main.customer AS __source) AS customer) AS customer" ) @@ -113,7 +113,7 @@ def test_read_function_list(): ) assert ( rewritten_sql - == "SELECT add_two(customer.c_custkey) FROM (SELECT customer.c_custkey FROM (SELECT customer.c_custkey AS c_custkey FROM main.customer) AS customer) AS customer" + == "SELECT add_two(customer.c_custkey) FROM (SELECT customer.c_custkey FROM (SELECT __source.c_custkey AS c_custkey FROM main.customer AS __source) AS customer) AS customer" ) session_context = SessionContext(manifest_str, None) diff --git a/wren-core/core/src/logical_plan/analyze/model_generation.rs b/wren-core/core/src/logical_plan/analyze/model_generation.rs index ffd933766..96f89d444 100644 --- a/wren-core/core/src/logical_plan/analyze/model_generation.rs +++ b/wren-core/core/src/logical_plan/analyze/model_generation.rs @@ -1,6 +1,16 @@ use std::fmt::Debug; use std::sync::Arc; +use crate::logical_plan::analyze::plan::{ + CalculationPlanNode, ModelPlanNode, ModelSourceNode, PartialModelPlanNode, +}; +use crate::logical_plan::utils::{ + create_remote_table_source, eliminate_ambiguous_columns, rebase_column, +}; +use crate::mdl::manifest::Model; +use crate::mdl::utils::quoted; +use crate::mdl::{AnalyzedWrenMDL, SessionStateRef}; +use crate::DataFusionError; use datafusion::common::alias::AliasGenerator; use datafusion::common::config::ConfigOptions; use datafusion::common::tree_node::{Transformed, TransformedResult}; @@ -11,15 +21,7 @@ use datafusion::optimizer::analyzer::AnalyzerRule; use datafusion::physical_plan::internal_err; use datafusion::sql::TableReference; -use crate::logical_plan::analyze::plan::{ - CalculationPlanNode, ModelPlanNode, ModelSourceNode, PartialModelPlanNode, -}; -use crate::logical_plan::utils::{ - create_remote_table_source, eliminate_ambiguous_columns, rebase_column, -}; -use crate::mdl::manifest::Model; -use crate::mdl::utils::quoted; -use crate::mdl::{AnalyzedWrenMDL, SessionStateRef}; +pub const SOURCE_ALIAS: &str = "__source"; /// [ModelGenerationRule] is responsible for generating the model plan node. pub struct ModelGenerationRule { @@ -89,6 +91,11 @@ impl ModelGenerationRule { .get_model(&model_plan.model_name) .expect("Model not found"), ); + let mut required_exprs = model_plan.required_exprs.clone(); + required_exprs.iter_mut().try_for_each(|expr| { + *expr = rebase_column(expr, SOURCE_ALIAS)?; + Ok::<(), DataFusionError>(()) + })?; // support table reference let table_scan = match &model_plan.original_table_scan { Some(LogicalPlan::TableScan(original_scan)) => { @@ -101,9 +108,10 @@ impl ModelGenerationRule { )?, None, original_scan.filters.clone(), - ).expect("Failed to create table scan") - .project(model_plan.required_exprs.clone())? - .build() + )? + .alias(SOURCE_ALIAS)? + .project(required_exprs)? + .build() } Some(_) => Err(datafusion::error::DataFusionError::Internal( "ModelPlanNode should have a TableScan as original_table_scan" @@ -117,8 +125,9 @@ impl ModelGenerationRule { &self.analyzed_wren_mdl.wren_mdl(), Arc::clone(&self.session_state))?, None, - ).expect("Failed to create table scan") - .project(model_plan.required_exprs.clone())? + )? + .alias(SOURCE_ALIAS)? + .project(required_exprs)? .build() }, }?; diff --git a/wren-core/core/src/mdl/mod.rs b/wren-core/core/src/mdl/mod.rs index a871c337b..b85d4ed6a 100644 --- a/wren-core/core/src/mdl/mod.rs +++ b/wren-core/core/src/mdl/mod.rs @@ -561,11 +561,11 @@ mod test { ) .await?; let expected = "SELECT profile.totalcost FROM (SELECT totalcost.totalcost FROM \ - (SELECT __relation__2.p_custkey AS p_custkey, sum(CAST(__relation__2.o_totalprice AS BIGINT)) AS totalcost \ - FROM (SELECT __relation__1.c_custkey, orders.o_custkey, orders.o_totalprice, __relation__1.p_custkey \ - FROM (SELECT orders.o_custkey AS o_custkey, orders.o_totalprice AS o_totalprice FROM orders) AS orders \ - RIGHT JOIN (SELECT customer.c_custkey, profile.p_custkey FROM (SELECT customer.c_custkey AS c_custkey FROM customer) AS customer \ - RIGHT JOIN (SELECT profile.p_custkey AS p_custkey FROM profile) AS profile ON customer.c_custkey = profile.p_custkey) AS __relation__1 \ + (SELECT __relation__2.p_custkey AS p_custkey, sum(CAST(__relation__2.o_totalprice AS BIGINT)) AS totalcost FROM \ + (SELECT __relation__1.c_custkey, orders.o_custkey, orders.o_totalprice, __relation__1.p_custkey FROM \ + (SELECT __source.o_custkey AS o_custkey, __source.o_totalprice AS o_totalprice FROM orders AS __source) AS orders RIGHT JOIN \ + (SELECT customer.c_custkey, profile.p_custkey FROM (SELECT __source.c_custkey AS c_custkey FROM customer AS __source) AS customer RIGHT JOIN \ + (SELECT __source.p_custkey AS p_custkey FROM profile AS __source) AS profile ON customer.c_custkey = profile.p_custkey) AS __relation__1 \ ON orders.o_custkey = __relation__1.c_custkey) AS __relation__2 GROUP BY __relation__2.p_custkey) AS totalcost) AS profile"; assert_eq!(result, expected); @@ -577,16 +577,17 @@ mod test { sql, ) .await?; - assert_eq!(result, "SELECT profile.totalcost FROM (SELECT __relation__1.p_sex, __relation__1.totalcost \ - FROM (SELECT totalcost.p_custkey, profile.p_sex, totalcost.totalcost FROM \ - (SELECT __relation__2.p_custkey AS p_custkey, sum(CAST(__relation__2.o_totalprice AS BIGINT)) AS totalcost FROM \ - (SELECT __relation__1.c_custkey, orders.o_custkey, orders.o_totalprice, __relation__1.p_custkey FROM \ - (SELECT orders.o_custkey AS o_custkey, orders.o_totalprice AS o_totalprice FROM orders) AS orders RIGHT JOIN \ - (SELECT customer.c_custkey, profile.p_custkey FROM (SELECT customer.c_custkey AS c_custkey FROM customer) AS customer \ - RIGHT JOIN (SELECT profile.p_custkey AS p_custkey FROM profile) AS profile ON customer.c_custkey = profile.p_custkey) AS __relation__1 \ - ON orders.o_custkey = __relation__1.c_custkey) AS __relation__2 GROUP BY __relation__2.p_custkey) AS totalcost RIGHT JOIN \ - (SELECT profile.p_custkey AS p_custkey, profile.p_sex AS p_sex FROM profile) AS profile \ - ON totalcost.p_custkey = profile.p_custkey) AS __relation__1) AS profile WHERE profile.p_sex = 'M'"); + assert_eq!(result, + "SELECT profile.totalcost FROM (SELECT __relation__1.p_sex, __relation__1.totalcost FROM \ + (SELECT totalcost.p_custkey, profile.p_sex, totalcost.totalcost FROM (SELECT __relation__2.p_custkey AS p_custkey, \ + sum(CAST(__relation__2.o_totalprice AS BIGINT)) AS totalcost FROM (SELECT __relation__1.c_custkey, orders.o_custkey, \ + orders.o_totalprice, __relation__1.p_custkey FROM (SELECT __source.o_custkey AS o_custkey, __source.o_totalprice AS o_totalprice \ + FROM orders AS __source) AS orders RIGHT JOIN (SELECT customer.c_custkey, profile.p_custkey FROM \ + (SELECT __source.c_custkey AS c_custkey FROM customer AS __source) AS customer RIGHT JOIN \ + (SELECT __source.p_custkey AS p_custkey FROM profile AS __source) AS profile ON customer.c_custkey = profile.p_custkey) AS __relation__1 \ + ON orders.o_custkey = __relation__1.c_custkey) AS __relation__2 GROUP BY __relation__2.p_custkey) AS totalcost RIGHT JOIN \ + (SELECT __source.p_custkey AS p_custkey, __source.p_sex AS p_sex FROM profile AS __source) AS profile \ + ON totalcost.p_custkey = profile.p_custkey) AS __relation__1) AS profile WHERE profile.p_sex = 'M'"); Ok(()) } @@ -616,8 +617,8 @@ mod test { .await?; assert_eq!(actual, "SELECT \"Customer\".\"Custkey\", \"Customer\".\"Name\" FROM \ - (SELECT customer.\"Custkey\" AS \"Custkey\", \ - customer.\"Name\" AS \"Name\" FROM datafusion.public.customer) AS \"Customer\""); + (SELECT __source.\"Custkey\" AS \"Custkey\", __source.\"Name\" AS \"Name\" FROM \ + datafusion.public.customer AS __source) AS \"Customer\""); Ok(()) } @@ -653,9 +654,8 @@ mod test { r#"select add_two("Custkey") from "Customer""#, ) .await?; - assert_eq!(actual, "SELECT add_two(\"Customer\".\"Custkey\") FROM \ - (SELECT \"Customer\".\"Custkey\" FROM (SELECT customer.\"Custkey\" AS \"Custkey\" \ - FROM datafusion.public.customer) AS \"Customer\") AS \"Customer\""); + assert_eq!(actual, "SELECT add_two(\"Customer\".\"Custkey\") FROM (SELECT \"Customer\".\"Custkey\" \ + FROM (SELECT __source.\"Custkey\" AS \"Custkey\" FROM datafusion.public.customer AS __source) AS \"Customer\") AS \"Customer\""); let actual = transform_sql_with_ctx( &ctx, @@ -664,10 +664,9 @@ mod test { r#"select median("Custkey") from "CTest"."STest"."Customer" group by "Name""#, ) .await?; - assert_eq!(actual, "SELECT median(\"Customer\".\"Custkey\") FROM \ - (SELECT \"Customer\".\"Custkey\", \"Customer\".\"Name\" FROM \ - (SELECT customer.\"Custkey\" AS \"Custkey\", customer.\"Name\" AS \"Name\" \ - FROM datafusion.public.customer) AS \"Customer\") AS \"Customer\" GROUP BY \"Customer\".\"Name\""); + assert_eq!(actual, "SELECT median(\"Customer\".\"Custkey\") FROM (SELECT \"Customer\".\"Custkey\", \"Customer\".\"Name\" \ + FROM (SELECT __source.\"Custkey\" AS \"Custkey\", __source.\"Name\" AS \"Name\" FROM datafusion.public.customer AS __source) AS \"Customer\") AS \"Customer\" \ + GROUP BY \"Customer\".\"Name\""); // TODO: support window functions analysis // let actual = transform_sql_with_ctx( @@ -725,10 +724,10 @@ mod test { ) .await?; assert_eq!(actual, - "SELECT artist.\"名字\", artist.name_append, artist.\"group\", artist.subscribe_plus, artist.subscribe FROM \ - (SELECT artist.\"名字\" AS \"名字\", artist.\"名字\" || artist.\"名字\" AS name_append, \ - artist.\"組別\" AS \"group\", CAST(artist.\"訂閱數\" AS BIGINT) + 1 AS subscribe_plus, artist.\"訂閱數\" AS subscribe FROM artist) \ - AS artist"); + "SELECT artist.\"名字\", artist.name_append, artist.\"group\", artist.subscribe_plus, artist.subscribe \ + FROM (SELECT __source.\"名字\" AS \"名字\", __source.\"名字\" || __source.\"名字\" AS name_append, __source.\"組別\" AS \"group\", \ + CAST(__source.\"訂閱數\" AS BIGINT) + 1 AS subscribe_plus, __source.\"訂閱數\" AS subscribe FROM artist AS __source) AS artist" +); ctx.sql(&actual).await?.show().await?; let sql = r#"select group from wren.test.artist"#; @@ -740,7 +739,7 @@ mod test { ) .await?; assert_eq!(actual, - "SELECT artist.\"group\" FROM (SELECT artist.\"group\" FROM (SELECT artist.\"組別\" AS \"group\" FROM artist) AS artist) AS artist"); + "SELECT artist.\"group\" FROM (SELECT artist.\"group\" FROM (SELECT __source.\"組別\" AS \"group\" FROM artist AS __source) AS artist) AS artist"); ctx.sql(&actual).await?.show().await?; let sql = r#"select subscribe_plus from wren.test.artist"#; @@ -752,7 +751,7 @@ mod test { ) .await?; assert_eq!(actual, - "SELECT artist.subscribe_plus FROM (SELECT artist.subscribe_plus FROM (SELECT CAST(artist.\"訂閱數\" AS BIGINT) + 1 AS subscribe_plus FROM artist) AS artist) AS artist"); + "SELECT artist.subscribe_plus FROM (SELECT artist.subscribe_plus FROM (SELECT CAST(__source.\"訂閱數\" AS BIGINT) + 1 AS subscribe_plus FROM artist AS __source) AS artist) AS artist"); ctx.sql(&actual).await?.show().await } @@ -844,7 +843,7 @@ mod test { .await?; assert_eq!(actual, "SELECT artist.\"串接名字\" FROM (SELECT artist.\"串接名字\" FROM \ - (SELECT artist.\"名字\" || artist.\"名字\" AS \"串接名字\" FROM artist) AS artist) AS artist"); + (SELECT __source.\"名字\" || __source.\"名字\" AS \"串接名字\" FROM artist AS __source) AS artist) AS artist"); let sql = r#"select * from wren.test.artist"#; let actual = transform_sql_with_ctx( @@ -855,7 +854,7 @@ mod test { ) .await?; assert_eq!(actual, - "SELECT artist.\"串接名字\" FROM (SELECT artist.\"名字\" || artist.\"名字\" AS \"串接名字\" FROM artist) AS artist"); + "SELECT artist.\"串接名字\" FROM (SELECT __source.\"名字\" || __source.\"名字\" AS \"串接名字\" FROM artist AS __source) AS artist"); let sql = r#"select "名字" from wren.test.artist"#; let _ = transform_sql_with_ctx( @@ -914,7 +913,7 @@ mod test { .await?; assert_eq!(actual, "SELECT CAST(current_date() AS TIMESTAMP) > artist.\"出道時間\" FROM \ - (SELECT artist.\"出道時間\" FROM (SELECT artist.\"出道時間\" AS \"出道時間\" FROM artist) AS artist) AS artist"); + (SELECT artist.\"出道時間\" FROM (SELECT __source.\"出道時間\" AS \"出道時間\" FROM artist AS __source) AS artist) AS artist"); Ok(()) } @@ -1063,8 +1062,9 @@ mod test { ) .await?; assert_eq!(actual, - "SELECT count(*) FROM (SELECT artist.cast_timestamptz FROM (SELECT CAST(artist.\"出道時間\" AS TIMESTAMP WITH TIME ZONE) AS cast_timestamptz \ - FROM artist) AS artist) AS artist WHERE CAST(artist.cast_timestamptz AS TIMESTAMP) > CAST('2011-01-01 21:00:00' AS TIMESTAMP)"); + "SELECT count(*) FROM (SELECT artist.cast_timestamptz FROM \ + (SELECT CAST(__source.\"出道時間\" AS TIMESTAMP WITH TIME ZONE) AS cast_timestamptz \ + FROM artist AS __source) AS artist) AS artist WHERE CAST(artist.cast_timestamptz AS TIMESTAMP) > CAST('2011-01-01 21:00:00' AS TIMESTAMP)"); Ok(()) } @@ -1145,10 +1145,10 @@ mod test { ) .await?; assert_eq!(actual, - "SELECT CAST(timestamp_table.timestamp_col AS TIMESTAMP WITH TIME ZONE) = timestamp_table.timestamptz_col FROM \ - (SELECT timestamp_table.timestamp_col, timestamp_table.timestamptz_col FROM \ - (SELECT timestamp_table.timestamp_col AS timestamp_col, timestamp_table.timestamptz_col AS timestamptz_col \ - FROM datafusion.public.timestamp_table) AS timestamp_table) AS timestamp_table"); + "SELECT CAST(timestamp_table.timestamp_col AS TIMESTAMP WITH TIME ZONE) = timestamp_table.timestamptz_col \ + FROM (SELECT timestamp_table.timestamp_col, timestamp_table.timestamptz_col FROM \ + (SELECT __source.timestamp_col AS timestamp_col, __source.timestamptz_col AS timestamptz_col \ + FROM datafusion.public.timestamp_table AS __source) AS timestamp_table) AS timestamp_table"); let sql = r#"select timestamptz_col > cast('2011-01-01 18:00:00' as TIMESTAMP WITH TIME ZONE) from wren.test.timestamp_table"#; let actual = transform_sql_with_ctx( @@ -1160,9 +1160,8 @@ mod test { .await?; // assert the simplified literal will be casted to the timestamp tz assert_eq!(actual, - "SELECT timestamp_table.timestamptz_col > CAST(CAST('2011-01-01 18:00:00' AS TIMESTAMP) AS TIMESTAMP WITH TIME ZONE) \ - FROM (SELECT timestamp_table.timestamptz_col FROM (SELECT timestamp_table.timestamptz_col AS timestamptz_col \ - FROM datafusion.public.timestamp_table) AS timestamp_table) AS timestamp_table"); + "SELECT timestamp_table.timestamptz_col > CAST(CAST('2011-01-01 18:00:00' AS TIMESTAMP) AS TIMESTAMP WITH TIME ZONE) FROM (SELECT timestamp_table.timestamptz_col FROM (SELECT __source.timestamptz_col AS timestamptz_col FROM datafusion.public.timestamp_table AS __source) AS timestamp_table) AS timestamp_table" +); let sql = r#"select timestamptz_col > '2011-01-01 18:00:00' from wren.test.timestamp_table"#; let actual = transform_sql_with_ctx( @@ -1175,8 +1174,8 @@ mod test { // assert the string literal will be casted to the timestamp tz assert_eq!(actual, "SELECT timestamp_table.timestamptz_col > CAST('2011-01-01 18:00:00' AS TIMESTAMP WITH TIME ZONE) \ - FROM (SELECT timestamp_table.timestamptz_col FROM (SELECT timestamp_table.timestamptz_col AS timestamptz_col \ - FROM datafusion.public.timestamp_table) AS timestamp_table) AS timestamp_table"); + FROM (SELECT timestamp_table.timestamptz_col FROM (SELECT __source.timestamptz_col AS timestamptz_col \ + FROM datafusion.public.timestamp_table AS __source) AS timestamp_table) AS timestamp_table"); let sql = r#"select timestamp_col > cast('2011-01-01 18:00:00' as TIMESTAMP WITH TIME ZONE) from wren.test.timestamp_table"#; let actual = transform_sql_with_ctx( @@ -1188,9 +1187,9 @@ mod test { .await?; // assert the simplified literal won't be casted to the timestamp tz assert_eq!(actual, - "SELECT timestamp_table.timestamp_col > CAST('2011-01-01 18:00:00' AS TIMESTAMP) FROM \ - (SELECT timestamp_table.timestamp_col FROM (SELECT timestamp_table.timestamp_col AS timestamp_col \ - FROM datafusion.public.timestamp_table) AS timestamp_table) AS timestamp_table"); + "SELECT timestamp_table.timestamp_col > CAST('2011-01-01 18:00:00' AS TIMESTAMP) \ + FROM (SELECT timestamp_table.timestamp_col FROM (SELECT __source.timestamp_col AS timestamp_col \ + FROM datafusion.public.timestamp_table AS __source) AS timestamp_table) AS timestamp_table"); } Ok(()) } @@ -1213,7 +1212,7 @@ mod test { let actual = transform_sql_with_ctx(&ctx, Arc::clone(&analyzed_mdl), &[], sql).await?; assert_eq!(actual, "SELECT list_table.list_col[1] FROM (SELECT list_table.list_col FROM \ - (SELECT list_table.list_col AS list_col FROM list_table) AS list_table) AS list_table"); + (SELECT __source.list_col AS list_col FROM list_table AS __source) AS list_table) AS list_table"); Ok(()) } @@ -1247,16 +1246,19 @@ mod test { let sql = "select struct_col.float_field from wren.test.struct_table"; let actual = transform_sql_with_ctx(&ctx, Arc::clone(&analyzed_mdl), &[], sql).await?; - assert_eq!(actual, "SELECT struct_table.struct_col.float_field FROM \ - (SELECT struct_table.struct_col FROM (SELECT struct_table.struct_col AS struct_col \ - FROM struct_table) AS struct_table) AS struct_table"); + assert_eq!( + actual, + "SELECT struct_table.struct_col.float_field FROM \ + (SELECT struct_table.struct_col FROM (SELECT __source.struct_col AS struct_col \ + FROM struct_table AS __source) AS struct_table) AS struct_table" + ); let sql = "select struct_array_col[1].float_field from wren.test.struct_table"; let actual = transform_sql_with_ctx(&ctx, Arc::clone(&analyzed_mdl), &[], sql).await?; assert_eq!(actual, "SELECT struct_table.struct_array_col[1].float_field FROM \ - (SELECT struct_table.struct_array_col FROM (SELECT struct_table.struct_array_col AS struct_array_col \ - FROM struct_table) AS struct_table) AS struct_table"); + (SELECT struct_table.struct_array_col FROM (SELECT __source.struct_array_col AS struct_array_col \ + FROM struct_table AS __source) AS struct_table) AS struct_table"); let sql = "select {float_field: 1.0, time_field: timestamp '2021-01-01 00:00:00'}"; @@ -1353,9 +1355,10 @@ mod test { transform_sql_with_ctx(&ctx, Arc::clone(&analyzed_mdl), &[], sql).await?; assert_eq!( result, - "SELECT customer.c_custkey, count(DISTINCT customer.c_name) FROM (SELECT customer.c_custkey, customer.c_name \ - FROM (SELECT customer.c_custkey AS c_custkey, customer.c_name AS c_name \ - FROM customer) AS customer) AS customer GROUP BY customer.c_custkey" + "SELECT customer.c_custkey, count(DISTINCT customer.c_name) FROM \ + (SELECT customer.c_custkey, customer.c_name FROM \ + (SELECT __source.c_custkey AS c_custkey, __source.c_name AS c_name FROM customer AS __source) AS customer) AS customer \ + GROUP BY customer.c_custkey" ); Ok(()) }