feat: add json_strip_nulls (databendlabs#12602)

Cargo.toml

@@ -114,7 +114,7 @@ opendal = { version = "0.39", features = [
 ] }
 ethnum = { version = "1.3.2" }
 ordered-float = { version = "3.6.0", default-features = false }
-jsonb = { git = "https://github.com/datafuselabs/jsonb", rev = "cb4b1eb" }
+jsonb = { git = "https://github.com/datafuselabs/jsonb", rev = "77a5920" }
 
 # openraft = { version = "0.8.2", features = ["compat-07"] }
 # For debugging

src/query/functions/src/scalars/variant.rs

@@ -71,6 +71,7 @@ use jsonb::is_object;
 use jsonb::jsonpath::parse_json_path;
 use jsonb::object_keys;
 use jsonb::parse_value;
+use jsonb::strip_nulls;
 use jsonb::to_bool;
 use jsonb::to_f64;
 use jsonb::to_i64;
@@ -918,6 +919,23 @@ pub fn register(registry: &mut FunctionRegistry) {
         }),
     );
 
+    registry.register_passthrough_nullable_1_arg(
+        "json_strip_nulls",
+        |_, _| FunctionDomain::MayThrow,
+        vectorize_with_builder_1_arg::<VariantType, VariantType>(|val, output, ctx| {
+            if let Some(validity) = &ctx.validity {
+                if !validity.get_bit(output.len()) {
+                    output.commit_row();
+                    return;
+                }
+            }
+            if let Err(err) = strip_nulls(val, &mut output.data) {
+                ctx.set_error(output.len(), err.to_string());
+            };
+            output.commit_row();
+        }),
+    );
+
     registry.register_function_factory("json_object", |_, args_type| {
         Some(Arc::new(Function {
             signature: FunctionSignature {

src/query/functions/tests/it/scalars/testdata/function_list.txt

@@ -1837,6 +1837,8 @@ Functions overloads:
 1 json_path_query_first(Variant NULL, String NULL) :: Variant NULL
 0 json_pretty(Variant) :: String
 1 json_pretty(Variant NULL) :: String NULL
+0 json_strip_nulls(Variant) :: Variant
+1 json_strip_nulls(Variant NULL) :: Variant NULL
 0 json_to_string(Variant) :: String
 1 json_to_string(Variant NULL) :: String NULL
 0 l2_distance(Array(Float32), Array(Float32)) :: Float32

src/query/functions/tests/it/scalars/testdata/variant.txt

@@ -2435,3 +2435,39 @@ output         : '{
 }'
 
 
+ast            : json_strip_nulls(parse_json('true'))
+raw expr       : json_strip_nulls(parse_json('true'))
+checked expr   : json_strip_nulls<Variant>(parse_json<String>("true"))
+optimized expr : 0x2000000040000000
+output type    : Variant
+output domain  : Undefined
+output         : true
+
+
+ast            : json_strip_nulls(parse_json('null'))
+raw expr       : json_strip_nulls(parse_json('null'))
+checked expr   : json_strip_nulls<Variant>(parse_json<String>("null"))
+optimized expr : 0x2000000000000000
+output type    : Variant
+output domain  : Undefined
+output         : null
+
+
+ast            : json_strip_nulls(parse_json('[1, 2, 3, null]'))
+raw expr       : json_strip_nulls(parse_json('[1, 2, 3, null]'))
+checked expr   : json_strip_nulls<Variant>(parse_json<String>("[1, 2, 3, null]"))
+optimized expr : 0x8000000420000002200000022000000200000000500150025003
+output type    : Variant
+output domain  : Undefined
+output         : [1,2,3,null]
+
+
+ast            : json_strip_nulls(parse_json('{"a":null, "b": {"c": 1, "d": null}, "c": [{"a": 1, "b": null}]}'))
+raw expr       : json_strip_nulls(parse_json('{"a":null, "b": {"c": 1, "d": null}, "c": [{"a": 1, "b": null}]}'))
+checked expr   : json_strip_nulls<Variant>(parse_json<String>("{\"a\":null, \"b\": {\"c\": 1, \"d\": null}, \"c\": [{\"a\": 1, \"b\": null}]}"))
+optimized expr : 0x4000000210000001100000015000000f500000176263400000011000000120000002635001800000015000000f400000011000000120000002615001
+output type    : Variant
+output domain  : Undefined
+output         : {"b":{"c":1},"c":[{"a":1}]}
+
+

src/query/functions/tests/it/scalars/variant.rs

@@ -43,6 +43,7 @@ fn test_variant() {
     test_json_path_query_first(file);
     test_json_to_string(file);
     test_json_pretty(file);
+    test_json_strip_nulls(file);
 }
 
 fn test_parse_json(file: &mut impl Write) {
@@ -711,3 +712,18 @@ fn test_json_pretty(file: &mut impl Write) {
         &[],
     );
 }
+
+fn test_json_strip_nulls(file: &mut impl Write) {
+    run_ast(file, r#"json_strip_nulls(parse_json('true'))"#, &[]);
+    run_ast(file, r#"json_strip_nulls(parse_json('null'))"#, &[]);
+    run_ast(
+        file,
+        r#"json_strip_nulls(parse_json('[1, 2, 3, null]'))"#,
+        &[],
+    );
+    run_ast(
+        file,
+        r#"json_strip_nulls(parse_json('{"a":null, "b": {"c": 1, "d": null}, "c": [{"a": 1, "b": null}]}'))"#,
+        &[],
+    );
+}

tests/sqllogictests/suites/query/02_function/02_0065_function_json

@@ -0,0 +1,19 @@
+query T
+SELECT json_strip_nulls(parse_json('null'))
+----
+null
+
+query T
+SELECT json_strip_nulls(parse_json('true'))
+----
+true
+
+query T
+SELECT json_strip_nulls(parse_json('[1,2,3,null]'))
+----
+[1,2,3,null]
+
+query T
+SELECT json_strip_nulls(parse_json('{"a":1,"b":null,"c":{"a":1,"b":null},"d":[{"a":1,"b":null},{"a":2,"b":3}]}'))
+----
+{"a":1,"c":{"a":1},"d":[{"a":1},{"a":2,"b":3}]}