Enable json array traversal to only top level elements

sourcenetwork · Dec 16, 2024 · 4eae9c5 · 4eae9c5
1 parent 73e425c
commit 4eae9c5
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 38 deletions.
diff --git a/client/json.go b/client/json.go
@@ -80,7 +80,7 @@ func TraverseJSON(j JSON, visitor JSONVisitor, opts ...traverseJSONOption) error
 	for _, opt := range opts {
 		opt(&options)
 	}
-	if shouldVisitPath(options.PathPrefix, nil) {
+	if shouldVisitPath(options.pathPrefix, nil) {
 		return j.accept(visitor, []string{}, options)
 	}
 	return nil
@@ -92,7 +92,7 @@ type traverseJSONOption func(*traverseJSONOptions)
 // Only nodes with paths that start with the prefix will be visited.
 func TraverseJSONWithPrefix(prefix []string) traverseJSONOption {
 	return func(opts *traverseJSONOptions) {
-		opts.PathPrefix = prefix
+		opts.pathPrefix = prefix
 	}
 }
 
@@ -101,22 +101,24 @@ func TraverseJSONWithPrefix(prefix []string) traverseJSONOption {
 // be called for objects or arrays and proceed with theirs children.
 func TraverseJSONOnlyLeaves() traverseJSONOption {
 	return func(opts *traverseJSONOptions) {
-		opts.OnlyLeaves = true
+		opts.onlyLeaves = true
 	}
 }
 
 // TraverseJSONVisitArrayElements returns a traverseJSONOption that sets the traversal to visit array elements.
 // When this option is set, the visitor function will be called for each element of an array.
-func TraverseJSONVisitArrayElements() traverseJSONOption {
+// If recurseElements is true, the visitor function will be called for each array element of type object or array.
+func TraverseJSONVisitArrayElements(recurseElements bool) traverseJSONOption {
 	return func(opts *traverseJSONOptions) {
-		opts.VisitArrayElements = true
+		opts.visitArrayElements = true
+		opts.recurseVisitedArrayElements = recurseElements
 	}
 }
 
 // TraverseJSONWithArrayIndexInPath returns a traverseJSONOption that includes array indices in the path.
 func TraverseJSONWithArrayIndexInPath() traverseJSONOption {
 	return func(opts *traverseJSONOptions) {
-		opts.IncludeArrayIndexInPath = true
+		opts.includeArrayIndexInPath = true
 	}
 }
 
@@ -127,14 +129,16 @@ type JSONVisitor func(value JSON) error
 
 // traverseJSONOptions configures how the JSON tree is traversed.
 type traverseJSONOptions struct {
-	// OnlyLeaves when true visits only leaf nodes (not objects or arrays)
-	OnlyLeaves bool
-	// PathPrefix when set visits only paths that start with this prefix
-	PathPrefix []string
-	// VisitArrayElements when true visits array elements
-	VisitArrayElements bool
-	// IncludeArrayIndexInPath when true includes array indices in the path
-	IncludeArrayIndexInPath bool
+	// onlyLeaves when true visits only leaf nodes (not objects or arrays)
+	onlyLeaves bool
+	// pathPrefix when set visits only paths that start with this prefix
+	pathPrefix []string
+	// visitArrayElements when true visits array elements
+	visitArrayElements bool
+	// recurseVisitedArrayElements when true visits array elements recursively
+	recurseVisitedArrayElements bool
+	// includeArrayIndexInPath when true includes array indices in the path
+	includeArrayIndexInPath bool
 }
 
 type jsonVoid struct{}
@@ -217,15 +221,15 @@ func (obj jsonObject) Unwrap() any {
 
 func (obj jsonObject) accept(visitor JSONVisitor, path []string, opts traverseJSONOptions) error {
 	obj.path = path
-	if !opts.OnlyLeaves && len(path) >= len(opts.PathPrefix) {
+	if !opts.onlyLeaves && len(path) >= len(opts.pathPrefix) {
 		if err := visitor(obj); err != nil {
 			return err
 		}
 	}
 
 	for k, v := range obj.val {
 		newPath := append(path, k)
-		if !shouldVisitPath(opts.PathPrefix, newPath) {
+		if !shouldVisitPath(opts.pathPrefix, newPath) {
 			continue
 		}
 
@@ -260,21 +264,24 @@ func (arr jsonArray) Unwrap() any {
 
 func (arr jsonArray) accept(visitor JSONVisitor, path []string, opts traverseJSONOptions) error {
 	arr.path = path
-	if !opts.OnlyLeaves {
+	if !opts.onlyLeaves {
 		if err := visitor(arr); err != nil {
 			return err
 		}
 	}
 
-	if opts.VisitArrayElements {
+	if opts.visitArrayElements {
 		for i := range arr.val {
+			if !opts.recurseVisitedArrayElements && isCompositeJSON(arr.val[i]) {
+				continue
+			}
 			var newPath []string
-			if opts.IncludeArrayIndexInPath {
+			if opts.includeArrayIndexInPath {
 				newPath = append(path, strconv.Itoa(i))
 			} else {
 				newPath = path
 			}
-			if !shouldVisitPath(opts.PathPrefix, newPath) {
+			if !shouldVisitPath(opts.pathPrefix, newPath) {
 				continue
 			}
 
@@ -605,3 +612,12 @@ func shouldVisitPath(prefix, path []string) bool {
 	}
 	return true
 }
+
+func isCompositeJSON(v JSON) bool {
+	_, isObject := v.Object()
+	if isObject {
+		return true
+	}
+	_, isArray := v.Array()
+	return isArray
+}
diff --git a/client/json_traverse_test.go b/client/json_traverse_test.go
@@ -109,7 +109,7 @@ func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) {
 		{
 			name: "VisitArrayElements",
 			options: []traverseJSONOption{
-				TraverseJSONVisitArrayElements(),
+				TraverseJSONVisitArrayElements(true),
 			},
 			expected: []traverseNode{
 				{path: "", value: json},
@@ -131,10 +131,30 @@ func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) {
 				{path: "array", value: newJSONNumber(5, nil)},
 			},
 		},
+		{
+			name: "VisitArrayElements without recursion",
+			options: []traverseJSONOption{
+				TraverseJSONVisitArrayElements(false),
+			},
+			expected: []traverseNode{
+				{path: "", value: json},
+				{path: "string", value: newJSONString("value", nil)},
+				{path: "number", value: newJSONNumber(42, nil)},
+				{path: "bool", value: newJSONBool(true, nil)},
+				{path: "null", value: newJSONNull(nil)},
+				{path: "object", value: json.Value().(map[string]JSON)["object"]},
+				{path: "object/nested", value: newJSONString("inside", nil)},
+				{path: "object/deep", value: json.Value().(map[string]JSON)["object"].Value().(map[string]JSON)["deep"]},
+				{path: "object/deep/level", value: newJSONNumber(3, nil)},
+				{path: "array", value: json.Value().(map[string]JSON)["array"]},
+				{path: "array", value: newJSONNumber(1, nil)},
+				{path: "array", value: newJSONString("two", nil)},
+			},
+		},
 		{
 			name: "VisitArrayElementsWithIndex",
 			options: []traverseJSONOption{
-				TraverseJSONVisitArrayElements(),
+				TraverseJSONVisitArrayElements(true),
 				TraverseJSONWithArrayIndexInPath(),
 			},
 			expected: []traverseNode{
@@ -161,7 +181,7 @@ func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) {
 			name: "CombinedOptions",
 			options: []traverseJSONOption{
 				TraverseJSONOnlyLeaves(),
-				TraverseJSONVisitArrayElements(),
+				TraverseJSONVisitArrayElements(true),
 				TraverseJSONWithPrefix([]string{"array"}),
 				TraverseJSONWithArrayIndexInPath(),
 			},

diff --git a/client/secondary_indexes.md b/client/secondary_indexes.md
@@ -153,7 +153,7 @@ The system can represent the "iPhone" value as a `JSON` type with its complete p
 
 For JSON fields, DefraDB uses inverted indexes with the following key format:
 ```
-<collection_id>/<index_id>(/<json_path><json_value>)+/<doc_id>
+<collection_id>/<index_id>(/<json_path>/<json_value>)+/<doc_id>
 ```
 
 The term "inverted" comes from how these indexes reverse the typical document-to-value relationship. Instead of starting with a document and finding its values, we start with a value and can quickly find all documents containing that value at any path.

diff --git a/internal/db/index.go b/internal/db/index.go
@@ -129,7 +129,7 @@ func (g *JSONFieldGenerator) Generate(value client.NormalValue, f func(client.No
 			return err
 		}
 		return f(val)
-	}, client.TraverseJSONOnlyLeaves(), client.TraverseJSONVisitArrayElements()) // TODO: add option to traverse array elements
+	}, client.TraverseJSONOnlyLeaves(), client.TraverseJSONVisitArrayElements(false))
 }
 
 // getFieldGenerator returns appropriate generator for the field type

diff --git a/internal/encoding/json_test.go b/internal/encoding/json_test.go
@@ -59,7 +59,7 @@ func TestJSONEncodingAndDecoding_ShouldEncodeAndDecodeBack(t *testing.T) {
 		jsons = append(jsons, value)
 		pathMap[p] = jsons
 		return nil
-	}, client.TraverseJSONOnlyLeaves(), client.TraverseJSONVisitArrayElements())
+	}, client.TraverseJSONOnlyLeaves(), client.TraverseJSONVisitArrayElements(true))
 	assert.NoError(t, err)
 
 	for path, jsons := range pathMap {

diff --git a/tests/integration/index/json_unique_array_test.go b/tests/integration/index/json_unique_array_test.go
@@ -98,18 +98,6 @@ func TestJSONArrayUniqueIndex_ShouldAllowOnlyUniqueValuesAndUseThemForFetching(t
 					"bae-54e76159-66c6-56be-ad65-7ff83edda058",
 					errors.NewKV("custom", map[string]any{"numbers": 3})).Error(),
 			},
-			testUtils.CreateDoc{
-				DocMap: map[string]any{
-					"name": "Chris",
-					"custom": map[string]any{
-						// existing nested value
-						"numbers": []any{9, []int{3}},
-					},
-				},
-				ExpectedError: db.NewErrCanNotIndexNonUniqueFields(
-					"bae-8dba1343-148c-590c-a942-dd6c80f204fb",
-					errors.NewKV("custom", map[string]any{"numbers": []any{9, []int{3}}})).Error(),
-			},
 			testUtils.Request{
 				Request: req,
 				Results: map[string]any{