feat: sqlite JSON support (#5088)

* feat: sqlite JSON support * chore: unneeded clone * fix: fix tests * fix: correct snapshot * fix: pass sqlite Json args as strings * fix: update json arg test * fix: disable unwanted adaptors * DRIVER_ADAPTERS_BRANCH=feat/sqlite-json chore: re-trigger CI * DRIVER_ADAPTERS_BRANCH=feat/sqlite-json chore: re-trigger CI * chore: address comments * chore: address comments fix * chore: bring back json_support test with a new assertion * chore: remove clone * DRIVER_ADAPTERS_BRANCH=feat/sqlite-json chore: re-trigger CI * fix: fix edge cases around plain JSON values * DRIVER_ADAPTERS_BRANCH=feat/sqlite-json chore: re-trigger CI * feat: conditionally generate the ArrayContains filter * DRIVER_ADAPTERS_BRANCH=feat/sqlite-json chore: re-trigger CI * fix: correct copy paste errors * DRIVER_ADAPTERS_BRANCH=feat/sqlite-json chore: re-trigger CI * fix: null handling for unknown number for d1 * DRIVER_ADAPTERS_BRANCH=feat/sqlite-json chore: re-trigger CI * chore: minor cleanups * DRIVER_ADAPTERS_BRANCH=feat/sqlite-json chore: re-trigger CI --------- Co-authored-by: jkomyno <[email protected]> Co-authored-by: Alberto Schiabel <[email protected]>
prisma · Dec 18, 2024 · 3dc72ed · 3dc72ed
1 parent 11f085a
commit 3dc72ed
Show file tree

Hide file tree

Showing 29 changed files with 355 additions and 98 deletions.
diff --git a/psl/psl-core/src/builtin_connectors/cockroach_datamodel_connector.rs b/psl/psl-core/src/builtin_connectors/cockroach_datamodel_connector.rs
@@ -43,9 +43,9 @@ const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Connector
     Json |
     JsonFiltering |
     JsonFilteringArrayPath |
+    JsonArrayContains |
     NamedPrimaryKeys |
     NamedForeignKeys |
-    SqlQueryRaw |
     RelationFieldsInArbitraryOrder |
     ScalarLists |
     UpdateableId |

diff --git a/psl/psl-core/src/builtin_connectors/mongodb.rs b/psl/psl-core/src/builtin_connectors/mongodb.rs
@@ -27,7 +27,6 @@ const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Connector
     CompositeTypes |
     FullTextIndex |
     SortOrderInFullTextIndex |
-    MongoDbQueryRaw |
     DefaultValueAuto |
     TwoWayEmbeddedManyToManyRelation |
     UndefinedType |

diff --git a/psl/psl-core/src/builtin_connectors/mssql_datamodel_connector.rs b/psl/psl-core/src/builtin_connectors/mssql_datamodel_connector.rs
@@ -37,7 +37,6 @@ const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Connector
     NamedDefaultValues |
     NamedForeignKeys |
     NamedPrimaryKeys |
-    SqlQueryRaw |
     ReferenceCycleDetection |
     UpdateableId |
     PrimaryKeySortOrderDefinition |

diff --git a/psl/psl-core/src/builtin_connectors/mysql_datamodel_connector.rs b/psl/psl-core/src/builtin_connectors/mysql_datamodel_connector.rs
@@ -38,11 +38,11 @@ pub const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Conne
     JsonFiltering |
     JsonFilteringJsonPath |
     JsonFilteringAlphanumeric |
+    JsonArrayContains |
     CreateManyWriteableAutoIncId |
     AutoIncrement |
     CompoundIds |
     AnyId |
-    SqlQueryRaw |
     NamedForeignKeys |
     AdvancedJsonNullability |
     IndexColumnLengthPrefixing |

diff --git a/psl/psl-core/src/builtin_connectors/postgres_datamodel_connector.rs b/psl/psl-core/src/builtin_connectors/postgres_datamodel_connector.rs
@@ -47,10 +47,10 @@ pub const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Conne
     JsonFilteringArrayPath |
     JsonFilteringAlphanumeric |
     JsonFilteringAlphanumericFieldRef |
+    JsonArrayContains |
     MultiSchema |
     NamedForeignKeys |
     NamedPrimaryKeys |
-    SqlQueryRaw |
     RelationFieldsInArbitraryOrder |
     ScalarLists |
     JsonLists |

diff --git a/psl/psl-core/src/builtin_connectors/sqlite_datamodel_connector.rs b/psl/psl-core/src/builtin_connectors/sqlite_datamodel_connector.rs
@@ -14,7 +14,6 @@ pub const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Conne
     AnyId |
     AutoIncrement |
     CompoundIds |
-    SqlQueryRaw |
     RelationFieldsInArbitraryOrder |
     UpdateableId |
     ImplicitManyToManyRelation |
@@ -30,7 +29,11 @@ pub const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Conne
     UpdateReturning |
     SupportsFiltersOnRelationsWithoutJoins |
     CreateMany |
-    CreateManyWriteableAutoIncId
+    CreateManyWriteableAutoIncId |
+    Json |
+    JsonFiltering |
+    JsonFilteringJsonPath |
+    AdvancedJsonNullability
 });
 
 pub struct SqliteDatamodelConnector;

diff --git a/psl/psl-core/src/datamodel_connector.rs b/psl/psl-core/src/datamodel_connector.rs
@@ -302,6 +302,16 @@ pub enum Flavour {
     Sqlite,
 }
 
+impl Flavour {
+    pub fn is_sql(&self) -> bool {
+        !self.is_mongo()
+    }
+
+    pub fn is_mongo(&self) -> bool {
+        matches!(self, Flavour::Mongo)
+    }
+}
+
 impl FromStr for Flavour {
     type Err = String;
     fn from_str(s: &str) -> Result<Self, Self::Err> {

diff --git a/psl/psl-core/src/datamodel_connector/capabilities.rs b/psl/psl-core/src/datamodel_connector/capabilities.rs
@@ -83,10 +83,9 @@ capabilities!(
     JsonFilteringArrayPath, // Connector supports filtering json fields using array path (eg: postgres).
     JsonFilteringAlphanumeric, // Connector supports alphanumeric json filters (gt, gte, lt, lte...).
     JsonFilteringAlphanumericFieldRef, // Connector supports alphanumeric json filters against a json field reference.
+    JsonArrayContains, // Connector supports the contains operator for json fields.
     CompoundIds,
     AnyId, // Any (or combination of) uniques and not only id fields can constitute an id for a model.
-    SqlQueryRaw,
-    MongoDbQueryRaw,
     NativeFullTextSearch,
     NativeFullTextSearchWithoutIndex,
     NativeFullTextSearchWithIndex,

diff --git a/psl/psl/tests/base/base_types.rs b/psl/psl/tests/base/base_types.rs
@@ -184,7 +184,7 @@ fn json_type_must_work_for_some_connectors() {
         .assert_has_scalar_field("json")
         .assert_scalar_type(ScalarType::Json);
 
-    let error = parse_unwrap_err(&format!("{SQLITE_SOURCE}\n{dml}"));
+    let error = parse_unwrap_err(&format!("{MSSQL_SOURCE}\n{dml}"));
 
     let expectation = expect![[r#"
         [1;91merror[0m: [1mError validating field `json` in model `User`: Field `json` in model `User` can't be of type Json. The current connector does not support the Json type.[0m

diff --git a/psl/psl/tests/capabilities/sqlite.rs b/psl/psl/tests/capabilities/sqlite.rs
@@ -36,6 +36,22 @@ fn enum_support() {
     expectation.assert_eq(&error);
 }
 
+#[test]
+fn json_support() {
+    let dml = indoc! {r#"
+        datasource db {
+          provider = "sqlite"
+          url = "file:test.db"
+        }
+        model User {
+          id   Int @id
+          data Json
+        }
+    "#};
+
+    assert_valid(dml);
+}
+
 #[test]
 fn scalar_list_support() {
     let dml = indoc! {r#"
@@ -108,35 +124,6 @@ fn unique_index_names_support() {
     expectation.assert_eq(&error);
 }
 
-#[test]
-fn json_support() {
-    let dml = indoc! {r#"
-        datasource db {
-          provider = "sqlite"
-          url = "file:test.db"
-        }
-
-        model User {
-          id   Int @id
-          data Json
-        }
-    "#};
-
-    let error = parse_unwrap_err(dml);
-
-    let expectation = expect![[r#"
-        [1;91merror[0m: [1mError validating field `data` in model `User`: Field `data` in model `User` can't be of type Json. The current connector does not support the Json type.[0m
-          [1;94m-->[0m  [4mschema.prisma:8[0m
-        [1;94m   | [0m
-        [1;94m 7 | [0m  id   Int @id
-        [1;94m 8 | [0m  [1;91mdata Json[0m
-        [1;94m 9 | [0m}
-        [1;94m   | [0m
-    "#]];
-
-    expectation.assert_eq(&error);
-}
-
 #[test]
 fn non_unique_relation_criteria_support() {
     let dml = indoc! {r#"

diff --git a/psl/psl/tests/common/mod.rs b/psl/psl/tests/common/mod.rs
@@ -65,13 +65,6 @@ pub(crate) fn assert_valid(schema: &str) {
     }
 }
 
-pub(crate) const SQLITE_SOURCE: &str = r#"
-    datasource db {
-        provider = "sqlite"
-        url      = "file:dev.db"
-    }
-"#;
-
 pub(crate) const POSTGRES_SOURCE: &str = r#"
     datasource db {
         provider = "postgres"
@@ -85,3 +78,10 @@ pub(crate) const MYSQL_SOURCE: &str = r#"
         url      = "mysql://localhost:3306"
     }
 "#;
+
+pub(crate) const MSSQL_SOURCE: &str = r#"
+    datasource db {
+        provider = "sqlserver"
+        url      = "jdbc:sqlserver://localhost:3306"
+    }
+"#;
diff --git a/quaint/src/connector/sqlite/native/conversion.rs b/quaint/src/connector/sqlite/native/conversion.rs
@@ -119,7 +119,7 @@ impl TypeIdentifier for &Column<'_> {
     }
 
     fn is_json(&self) -> bool {
-        false
+        matches!(self.decl_type(), Some("JSONB") | Some("jsonb"))
     }
 
     fn is_enum(&self) -> bool {
@@ -148,6 +148,7 @@ impl GetRow for SqliteRow<'_> {
                     c if c.is_datetime() => Value::null_datetime(),
                     c if c.is_date() => Value::null_date(),
                     c if c.is_bool() => Value::null_boolean(),
+                    c if c.is_json() => Value::null_json(),
                     c => match c.decl_type() {
                         Some(n) => {
                             let msg = format!("Value {n} not supported");
@@ -186,6 +187,7 @@ impl GetRow for SqliteRow<'_> {
                                 return Err(Error::builder(kind).build());
                             }
                         }
+                        c if c.is_json() => Value::json(serde_json::Value::Number(serde_json::Number::from(i))),
                         // NOTE: When SQLite does not know what type the return is (for example at explicit values and RETURNING statements) we will 'assume' int64
                         _ => Value::int64(i),
                     }
@@ -196,6 +198,9 @@ impl GetRow for SqliteRow<'_> {
 
                     Value::numeric(BigDecimal::from_str(&f.to_string()).unwrap())
                 }
+                ValueRef::Real(f) if column.is_json() => Value::json(serde_json::Value::Number(
+                    serde_json::Number::from_f64(f).expect("JSON real should always be convertible to serde Number"),
+                )),
                 ValueRef::Real(f) => Value::double(f),
                 ValueRef::Text(bytes) if column.is_datetime() => {
                     let parse_res = std::str::from_utf8(bytes).map_err(|_| {
@@ -222,6 +227,20 @@ impl GetRow for SqliteRow<'_> {
                             })
                     })?
                 }
+                ValueRef::Text(bytes) if column.is_json() => {
+                    let json_str = std::str::from_utf8(bytes).map_err(|_| {
+                        Error::builder(ErrorKind::ConversionError(
+                            "Failed to read contents of SQLite JSON column as UTF-8".into(),
+                        ))
+                        .build()
+                    })?;
+
+                    let json_value: serde_json::Value = serde_json::from_str(json_str).map_err(|serde_error| {
+                        Error::builder(ErrorKind::ConversionError(serde_error.to_string().into())).build()
+                    })?;
+
+                    Value::json(json_value)
+                }
                 ValueRef::Text(bytes) => Value::text(String::from_utf8(bytes.to_vec())?),
                 ValueRef::Blob(bytes) => Value::bytes(bytes.to_owned()),
             };

diff --git a/quaint/src/visitor/mysql.rs b/quaint/src/visitor/mysql.rs
@@ -407,7 +407,7 @@ impl<'a> Visitor<'a> for Mysql<'a> {
         }
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_extract(&mut self, json_extract: JsonExtract<'a>) -> visitor::Result {
         if json_extract.extract_as_string {
             self.write("JSON_UNQUOTE(")?;
@@ -431,7 +431,7 @@ impl<'a> Visitor<'a> for Mysql<'a> {
         Ok(())
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_array_contains(&mut self, left: Expression<'a>, right: Expression<'a>, not: bool) -> visitor::Result {
         self.write("JSON_CONTAINS(")?;
         self.visit_expression(left)?;
@@ -446,7 +446,7 @@ impl<'a> Visitor<'a> for Mysql<'a> {
         Ok(())
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_type_equals(&mut self, left: Expression<'a>, json_type: JsonType<'a>, not: bool) -> visitor::Result {
         self.write("(")?;
         self.write("JSON_TYPE")?;
@@ -558,7 +558,7 @@ impl<'a> Visitor<'a> for Mysql<'a> {
         Ok(())
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_extract_last_array_item(&mut self, extract: JsonExtractLastArrayElem<'a>) -> visitor::Result {
         self.write("JSON_EXTRACT(")?;
         self.visit_expression(*extract.expr.clone())?;
@@ -571,7 +571,7 @@ impl<'a> Visitor<'a> for Mysql<'a> {
         Ok(())
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_extract_first_array_item(&mut self, extract: JsonExtractFirstArrayElem<'a>) -> visitor::Result {
         self.write("JSON_EXTRACT(")?;
         self.visit_expression(*extract.expr)?;
@@ -582,7 +582,7 @@ impl<'a> Visitor<'a> for Mysql<'a> {
         Ok(())
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_unquote(&mut self, json_unquote: JsonUnquote<'a>) -> visitor::Result {
         self.write("JSON_UNQUOTE(")?;
         self.visit_expression(*json_unquote.expr)?;

diff --git a/quaint/src/visitor/postgres.rs b/quaint/src/visitor/postgres.rs
@@ -406,7 +406,7 @@ impl<'a> Visitor<'a> for Postgres<'a> {
         Ok(())
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_extract(&mut self, json_extract: JsonExtract<'a>) -> visitor::Result {
         match json_extract.path {
             JsonPath::String(_) => panic!("JSON path string notation is not supported for Postgres"),
@@ -445,7 +445,7 @@ impl<'a> Visitor<'a> for Postgres<'a> {
         Ok(())
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_unquote(&mut self, json_unquote: JsonUnquote<'a>) -> visitor::Result {
         self.write("(")?;
         self.visit_expression(*json_unquote.expr)?;
@@ -472,7 +472,7 @@ impl<'a> Visitor<'a> for Postgres<'a> {
         Ok(())
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_extract_last_array_item(&mut self, extract: JsonExtractLastArrayElem<'a>) -> visitor::Result {
         self.write("(")?;
         self.visit_expression(*extract.expr)?;
@@ -482,7 +482,7 @@ impl<'a> Visitor<'a> for Postgres<'a> {
         Ok(())
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_extract_first_array_item(&mut self, extract: JsonExtractFirstArrayElem<'a>) -> visitor::Result {
         self.write("(")?;
         self.visit_expression(*extract.expr)?;
@@ -492,7 +492,7 @@ impl<'a> Visitor<'a> for Postgres<'a> {
         Ok(())
     }
 
-    #[cfg(any(feature = "postgresql", feature = "mysql"))]
+    #[cfg(any(feature = "postgresql", feature = "mysql", feature = "sqlite"))]
     fn visit_json_type_equals(&mut self, left: Expression<'a>, json_type: JsonType<'a>, not: bool) -> visitor::Result {
         self.write("JSONB_TYPEOF")?;
         self.write("(")?;