From 169dafafad6963cbb0345be98527b96a40148409 Mon Sep 17 00:00:00 2001 From: Alberto Schiabel Date: Wed, 8 Jan 2025 15:24:32 +0100 Subject: [PATCH] feat(schema-engine-wasm): wasm-compatible `sql-schema-describer`, `schema-connector` (#5106) * feat(schema-engine-wasm): wasm-compatible sql-schema-describer * fix(schema-engine-wasm): wasm compilation, move "SqlSchemaDescriberBackend" to sqlite-native impl * fix(schema-engine-wasm): tests * feat(schema-engine-wasm): wasm-compatible schema-connector * fix(schema-engine-wasm): tests * fix(schema-engine-wasm): tests * fix(sql-schema-describer): tests * fix(schema-engine-wasm): Cargo.toml features of schema-connector and sql-schema-describer * chore(quaint): workaround for mongodb-schema-connector --------- Co-authored-by: jkomyno <12381818+jkomyno@users.noreply.github.com> --- .github/workflows/test-schema-engine.yml | 4 +- quaint/src/lib.rs | 3 + schema-engine/cli/Cargo.toml | 4 +- .../connectors/schema-connector/Cargo.toml | 23 +++++++- .../src/introspection_context.rs | 5 ++ .../sql-schema-connector/Cargo.toml | 8 ++- .../src/flavour/sqlite/connection.rs | 1 + schema-engine/core/Cargo.toml | 6 +- .../sql-introspection-tests/Cargo.toml | 8 ++- schema-engine/sql-migration-tests/Cargo.toml | 4 +- schema-engine/sql-schema-describer/Cargo.toml | 22 ++++++- .../sql-schema-describer/src/sqlite.rs | 58 ++----------------- .../src/sqlite/native/mod.rs | 51 ++++++++++++++++ .../tests/test_api/mod.rs | 4 ++ 14 files changed, 132 insertions(+), 69 deletions(-) create mode 100644 schema-engine/sql-schema-describer/src/sqlite/native/mod.rs diff --git a/.github/workflows/test-schema-engine.yml b/.github/workflows/test-schema-engine.yml index 88c9ba43f745..d62def028f04 100644 --- a/.github/workflows/test-schema-engine.yml +++ b/.github/workflows/test-schema-engine.yml @@ -133,7 +133,7 @@ jobs: CLICOLOR_FORCE: 1 TEST_DATABASE_URL: ${{ matrix.database.url }} - - run: cargo nextest run -p sql-schema-describer + - run: cargo nextest run -p sql-schema-describer --features all-native if: ${{ !matrix.database.single_threaded }} env: CLICOLOR_FORCE: 1 @@ -179,7 +179,7 @@ jobs: # # Single threaded tests (excluding Vitess) # - - run: cargo nextest run -p sql-schema-describer --test-threads=1 + - run: cargo nextest run -p sql-schema-describer --features all-native --test-threads=1 if: ${{ !matrix.database.is_vitess && matrix.database.single_threaded }} env: CLICOLOR_FORCE: 1 diff --git a/quaint/src/lib.rs b/quaint/src/lib.rs index ab73ef7e66aa..ca0e1f5152c7 100644 --- a/quaint/src/lib.rs +++ b/quaint/src/lib.rs @@ -104,6 +104,9 @@ //! # } //! ``` +// TODO: remove once `quaint` is no longer a transitive dependency of `mongodb-schema-connector`. +#![allow(dead_code)] + #[cfg(not(any(feature = "sqlite", feature = "postgresql", feature = "mysql", feature = "mssql")))] compile_error!("one of 'sqlite', 'postgresql', 'mysql' or 'mssql' features must be enabled"); diff --git a/schema-engine/cli/Cargo.toml b/schema-engine/cli/Cargo.toml index 65718f8c8cd5..e1111d087c4f 100644 --- a/schema-engine/cli/Cargo.toml +++ b/schema-engine/cli/Cargo.toml @@ -4,7 +4,9 @@ version = "0.1.0" edition = "2021" [dependencies] -schema-connector = { path = "../connectors/schema-connector" } +schema-connector = { path = "../connectors/schema-connector", features = [ + "all-native", +] } schema-core = { path = "../core" } user-facing-errors = { path = "../../libs/user-facing-errors", features = [ "all-native", diff --git a/schema-engine/connectors/schema-connector/Cargo.toml b/schema-engine/connectors/schema-connector/Cargo.toml index 5c40185f2a3b..9388a9673605 100644 --- a/schema-engine/connectors/schema-connector/Cargo.toml +++ b/schema-engine/connectors/schema-connector/Cargo.toml @@ -3,13 +3,32 @@ name = "schema-connector" version = "0.1.0" edition = "2021" +[features] +postgresql = ["psl/postgresql", "quaint/postgresql"] +postgresql-native = ["postgresql", "quaint/postgresql-native", "quaint/pooled"] +sqlite = ["psl/sqlite", "quaint/sqlite"] +sqlite-native = ["sqlite", "quaint/sqlite-native", "quaint/pooled", "quaint/expose-drivers"] +mysql = ["psl/mysql", "quaint/mysql"] +mysql-native = ["mysql", "quaint/mysql-native", "quaint/pooled"] +mssql = ["psl/mssql", "quaint/mssql"] +mssql-native = ["mssql", "quaint/mssql-native", "quaint/pooled"] +cockroachdb = ["psl/cockroachdb", "quaint/postgresql"] +cockroachdb-native = ["cockroachdb", "quaint/postgresql-native", "quaint/pooled"] +all-native = [ + "postgresql-native", + "sqlite-native", + "mysql-native", + "mssql-native", + "cockroachdb-native", +] + [dependencies] psl.workspace = true -quaint = { workspace = true, features = ["all-native", "pooled"] } +quaint.workspace = true serde.workspace = true serde_json.workspace = true user-facing-errors = { path = "../../../libs/user-facing-errors", features = [ - "all-native", + "quaint", ] } chrono.workspace = true diff --git a/schema-engine/connectors/schema-connector/src/introspection_context.rs b/schema-engine/connectors/schema-connector/src/introspection_context.rs index d19e7b7acf83..bfd7078ea983 100644 --- a/schema-engine/connectors/schema-connector/src/introspection_context.rs +++ b/schema-engine/connectors/schema-connector/src/introspection_context.rs @@ -109,10 +109,15 @@ impl IntrospectionContext { /// The SQL family we're using currently. pub fn sql_family(&self) -> SqlFamily { match self.datasource().active_provider { + #[cfg(feature = "postgresql")] "postgresql" => SqlFamily::Postgres, + #[cfg(feature = "cockroachdb")] "cockroachdb" => SqlFamily::Postgres, + #[cfg(feature = "sqlite")] "sqlite" => SqlFamily::Sqlite, + #[cfg(feature = "mssql")] "sqlserver" => SqlFamily::Mssql, + #[cfg(feature = "mysql")] "mysql" => SqlFamily::Mysql, name => unreachable!("The name `{}` for the datamodel connector is not known", name), } diff --git a/schema-engine/connectors/sql-schema-connector/Cargo.toml b/schema-engine/connectors/sql-schema-connector/Cargo.toml index a336495e7436..4bd4d2b677da 100644 --- a/schema-engine/connectors/sql-schema-connector/Cargo.toml +++ b/schema-engine/connectors/sql-schema-connector/Cargo.toml @@ -21,8 +21,12 @@ uuid.workspace = true indexmap.workspace = true prisma-value = { path = "../../../libs/prisma-value" } -schema-connector = { path = "../schema-connector" } -sql-schema-describer = { path = "../../sql-schema-describer" } +schema-connector = { path = "../schema-connector", features = [ + "all-native", +] } +sql-schema-describer = { path = "../../sql-schema-describer", features = [ + "all-native", +] } datamodel-renderer = { path = "../../datamodel-renderer" } sql-ddl = { path = "../../../libs/sql-ddl" } user-facing-errors = { path = "../../../libs/user-facing-errors", features = [ diff --git a/schema-engine/connectors/sql-schema-connector/src/flavour/sqlite/connection.rs b/schema-engine/connectors/sql-schema-connector/src/flavour/sqlite/connection.rs index 995a86e87c97..a34cca5cb9e1 100644 --- a/schema-engine/connectors/sql-schema-connector/src/flavour/sqlite/connection.rs +++ b/schema-engine/connectors/sql-schema-connector/src/flavour/sqlite/connection.rs @@ -24,6 +24,7 @@ impl Connection { } pub(super) async fn describe_schema(&mut self) -> ConnectorResult { + // Note: this relies on quaint::connector::rusqlite::Connection, which is exposed by `quaint/expose-drivers`, and is not Wasm-compatible. describer::SqlSchemaDescriber::new(&self.0) .describe_impl() .await diff --git a/schema-engine/core/Cargo.toml b/schema-engine/core/Cargo.toml index 6fb22d8a98cb..c16d81b2df19 100644 --- a/schema-engine/core/Cargo.toml +++ b/schema-engine/core/Cargo.toml @@ -5,11 +5,13 @@ version = "0.1.0" [dependencies] psl = { workspace = true, features = ["all"] } -schema-connector = { path = "../connectors/schema-connector" } +schema-connector = { path = "../connectors/schema-connector", features = [ + "all-native", +] } mongodb-schema-connector = { path = "../connectors/mongodb-schema-connector" } sql-schema-connector = { path = "../connectors/sql-schema-connector" } user-facing-errors = { path = "../../libs/user-facing-errors", features = [ - "all-native", + "quaint", ] } async-trait.workspace = true diff --git a/schema-engine/sql-introspection-tests/Cargo.toml b/schema-engine/sql-introspection-tests/Cargo.toml index d0891b0bbaaa..d612d787329a 100644 --- a/schema-engine/sql-introspection-tests/Cargo.toml +++ b/schema-engine/sql-introspection-tests/Cargo.toml @@ -4,9 +4,13 @@ version = "0.1.0" edition = "2021" [dependencies] -schema-connector = { path = "../connectors/schema-connector" } +schema-connector = { path = "../connectors/schema-connector", features = [ + "all-native", +] } sql-schema-connector = { path = "../connectors/sql-schema-connector" } -sql-schema-describer = { path = "../sql-schema-describer" } +sql-schema-describer = { path = "../sql-schema-describer", features = [ + "all-native", +] } psl = { workspace = true, features = ["all"] } test-macros = { path = "../../libs/test-macros" } user-facing-errors = { path = "../../libs/user-facing-errors", features = [ diff --git a/schema-engine/sql-migration-tests/Cargo.toml b/schema-engine/sql-migration-tests/Cargo.toml index 6a345a365486..b2b3646ef2ef 100644 --- a/schema-engine/sql-migration-tests/Cargo.toml +++ b/schema-engine/sql-migration-tests/Cargo.toml @@ -7,7 +7,9 @@ edition = "2021" psl = { workspace = true, features = ["all"] } schema-core = { path = "../core" } sql-schema-connector = { path = "../connectors/sql-schema-connector" } -sql-schema-describer = { path = "../sql-schema-describer" } +sql-schema-describer = { path = "../sql-schema-describer", features = [ + "all-native", +] } user-facing-errors = { path = "../../libs/user-facing-errors", features = [ "all-native", ] } diff --git a/schema-engine/sql-schema-describer/Cargo.toml b/schema-engine/sql-schema-describer/Cargo.toml index 17b8eae63684..34f708dca0a8 100644 --- a/schema-engine/sql-schema-describer/Cargo.toml +++ b/schema-engine/sql-schema-describer/Cargo.toml @@ -3,6 +3,25 @@ edition = "2021" name = "sql-schema-describer" version = "0.1.0" +[features] +postgresql = ["psl/postgresql", "quaint/postgresql"] +postgresql-native = ["postgresql", "quaint/postgresql-native", "quaint/pooled"] +sqlite = ["psl/sqlite", "quaint/sqlite"] +sqlite-native = ["sqlite", "quaint/sqlite-native", "quaint/pooled", "quaint/expose-drivers"] +mysql = ["psl/mysql", "quaint/mysql"] +mysql-native = ["mysql", "quaint/mysql-native", "quaint/pooled"] +mssql = ["psl/mssql", "quaint/mssql"] +mssql-native = ["mssql", "quaint/mssql-native", "quaint/pooled"] +cockroachdb = ["psl/cockroachdb", "quaint/postgresql"] +cockroachdb-native = ["cockroachdb", "quaint/postgresql-native", "quaint/pooled"] +all-native = [ + "postgresql-native", + "sqlite-native", + "mysql-native", + "mssql-native", + "cockroachdb-native", +] + [dependencies] prisma-value = { path = "../../libs/prisma-value" } psl = { workspace = true, features = ["all"] } @@ -20,9 +39,6 @@ tracing.workspace = true tracing-error = "0.2" tracing-futures.workspace = true quaint = { workspace = true, features = [ - "all-native", - "pooled", - "expose-drivers", "fmt-sql", ] } diff --git a/schema-engine/sql-schema-describer/src/sqlite.rs b/schema-engine/sql-schema-describer/src/sqlite.rs index c130a7aa68ed..658207283786 100644 --- a/schema-engine/sql-schema-describer/src/sqlite.rs +++ b/schema-engine/sql-schema-describer/src/sqlite.rs @@ -2,19 +2,20 @@ use crate::{ getters::Getter, ids::*, parsers::Parser, Column, ColumnArity, ColumnType, ColumnTypeFamily, DefaultValue, - DescriberResult, ForeignKeyAction, Lazy, PrismaValue, Regex, SQLSortOrder, SqlMetadata, SqlSchema, - SqlSchemaDescriberBackend, + DescriberResult, ForeignKeyAction, Lazy, PrismaValue, Regex, SQLSortOrder, SqlSchema, }; use either::Either; use indexmap::IndexMap; use quaint::{ ast::{Value, ValueType}, - connector::{ColumnType as QuaintColumnType, GetRow, ToColumnNames}, prelude::ResultRow, }; use std::{any::type_name, borrow::Cow, collections::BTreeMap, convert::TryInto, fmt::Debug, path::Path}; use tracing::trace; +#[cfg(feature = "sqlite-native")] +pub(crate) mod native; + #[async_trait::async_trait] pub trait Connection { async fn query_raw<'a>( @@ -24,31 +25,6 @@ pub trait Connection { ) -> quaint::Result; } -#[async_trait::async_trait] -impl Connection for std::sync::Mutex { - async fn query_raw<'a>( - &'a self, - sql: &'a str, - params: &'a [quaint::prelude::Value<'a>], - ) -> quaint::Result { - let conn = self.lock().unwrap(); - let mut stmt = conn.prepare_cached(sql)?; - let column_types = stmt.columns().iter().map(QuaintColumnType::from).collect::>(); - let mut rows = stmt.query(quaint::connector::rusqlite::params_from_iter(params.iter()))?; - let column_names = rows.to_column_names(); - let mut converted_rows = Vec::new(); - while let Some(row) = rows.next()? { - converted_rows.push(row.get_result_row().unwrap()); - } - - Ok(quaint::prelude::ResultSet::new( - column_names, - column_types, - converted_rows, - )) - } -} - #[async_trait::async_trait] impl Connection for quaint::single::Quaint { async fn query_raw<'a>( @@ -70,32 +46,6 @@ impl Debug for SqlSchemaDescriber<'_> { } } -#[async_trait::async_trait] -impl SqlSchemaDescriberBackend for SqlSchemaDescriber<'_> { - async fn list_databases(&self) -> DescriberResult> { - Ok(self.get_databases().await?) - } - - async fn get_metadata(&self, _schema: &str) -> DescriberResult { - let mut sql_schema = SqlSchema::default(); - let table_count = self.get_table_names(&mut sql_schema).await?.len(); - let size_in_bytes = self.get_size().await?; - - Ok(SqlMetadata { - table_count, - size_in_bytes, - }) - } - - async fn describe(&self, _schemas: &[&str]) -> DescriberResult { - self.describe_impl().await - } - - async fn version(&self) -> DescriberResult> { - Ok(Some(quaint::connector::sqlite_version().to_owned())) - } -} - impl Parser for SqlSchemaDescriber<'_> {} impl<'a> SqlSchemaDescriber<'a> { diff --git a/schema-engine/sql-schema-describer/src/sqlite/native/mod.rs b/schema-engine/sql-schema-describer/src/sqlite/native/mod.rs new file mode 100644 index 000000000000..df17da7a1769 --- /dev/null +++ b/schema-engine/sql-schema-describer/src/sqlite/native/mod.rs @@ -0,0 +1,51 @@ +use crate::{sqlite::Connection, DescriberResult, SqlMetadata, SqlSchema, SqlSchemaDescriberBackend}; + +use quaint::{ + connector::{rusqlite, ColumnType as QuaintColumnType, GetRow, ToColumnNames}, + prelude::{ResultSet, Value}, +}; + +use super::SqlSchemaDescriber; + +#[async_trait::async_trait] +impl Connection for std::sync::Mutex { + async fn query_raw<'a>(&'a self, sql: &'a str, params: &'a [Value<'a>]) -> quaint::Result { + let conn = self.lock().unwrap(); + let mut stmt = conn.prepare_cached(sql)?; + let column_types = stmt.columns().iter().map(QuaintColumnType::from).collect::>(); + let mut rows = stmt.query(rusqlite::params_from_iter(params.iter()))?; + let column_names = rows.to_column_names(); + let mut converted_rows = Vec::new(); + while let Some(row) = rows.next()? { + converted_rows.push(row.get_result_row().unwrap()); + } + + Ok(ResultSet::new(column_names, column_types, converted_rows)) + } +} + +#[async_trait::async_trait] +impl SqlSchemaDescriberBackend for SqlSchemaDescriber<'_> { + async fn list_databases(&self) -> DescriberResult> { + Ok(self.get_databases().await?) + } + + async fn get_metadata(&self, _schema: &str) -> DescriberResult { + let mut sql_schema = SqlSchema::default(); + let table_count = self.get_table_names(&mut sql_schema).await?.len(); + let size_in_bytes = self.get_size().await?; + + Ok(SqlMetadata { + table_count, + size_in_bytes, + }) + } + + async fn describe(&self, _schemas: &[&str]) -> DescriberResult { + self.describe_impl().await + } + + async fn version(&self) -> DescriberResult> { + Ok(Some(quaint::connector::sqlite_version().to_owned())) + } +} diff --git a/schema-engine/sql-schema-describer/tests/test_api/mod.rs b/schema-engine/sql-schema-describer/tests/test_api/mod.rs index 1d14b2735d39..69b3227b75e7 100644 --- a/schema-engine/sql-schema-describer/tests/test_api/mod.rs +++ b/schema-engine/sql-schema-describer/tests/test_api/mod.rs @@ -80,6 +80,7 @@ impl TestApi { async fn describe_impl(&self, schemas: &[&str]) -> Result { match self.sql_family() { + #[cfg(any(feature = "postgresql", feature = "cockroachdb"))] SqlFamily::Postgres => { use postgres::Circumstances; sql_schema_describer::postgres::SqlSchemaDescriber::new( @@ -93,11 +94,13 @@ impl TestApi { .describe(schemas) .await } + #[cfg(feature = "sqlite")] SqlFamily::Sqlite => { sql_schema_describer::sqlite::SqlSchemaDescriber::new(&self.database) .describe_impl() .await } + #[cfg(feature = "mysql")] SqlFamily::Mysql => { use mysql::Circumstances; sql_schema_describer::mysql::SqlSchemaDescriber::new( @@ -115,6 +118,7 @@ impl TestApi { .describe(schemas) .await } + #[cfg(feature = "mssql")] SqlFamily::Mssql => { sql_schema_describer::mssql::SqlSchemaDescriber::new(&self.database) .describe(schemas)