diff --git a/client/json.go b/client/json.go index fd3e3dbad3..b22a0793ea 100644 --- a/client/json.go +++ b/client/json.go @@ -14,12 +14,91 @@ import ( "encoding/json" "io" "strconv" + "strings" "github.com/valyala/fastjson" "golang.org/x/exp/constraints" ) +// JSONPathPart represents a part of a JSON path. +// Json path can be either a property of an object or an index of an element in an array. +// For example, consider the following JSON: +// +// { +// "custom": { +// "name": "John" +// }, +// "0": { +// "val": 1 +// }, +// [ +// { +// "val": 2 +// } +// ] +// } +// +// The path to a top-level document is empty. +// The path to subtree { "name": "John" } can be described as "custom". +// The path to value "John" can be described as "custom.name". +// The paths to both values 1 and 2 can be described as "0.val": +// - for value 1 it's "0" property of the object and "val" property of the object +// - for value 2 it's "0" index of the array and "val" property of the object +// That's why we need to distinguish between properties and indices in the path. +type JSONPathPart struct { + value any +} + +// Property returns the property name if the part is a property, and a boolean indicating if the part is a property. +func (p JSONPathPart) Property() (string, bool) { + v, ok := p.value.(string) + return v, ok +} + +// Index returns the index if the part is an index, and a boolean indicating if the part is an index. +func (p JSONPathPart) Index() (uint64, bool) { + v, ok := p.value.(uint64) + return v, ok +} + +// JSONPath represents a path to a JSON value in a JSON tree. +type JSONPath []JSONPathPart + +// Parts returns the parts of the JSON path. +func (p JSONPath) Parts() []JSONPathPart { + return p +} + +// AppendProperty appends a property part to the JSON path. +func (p JSONPath) AppendProperty(part string) JSONPath { + return append(p, JSONPathPart{value: part}) +} + +// AppendIndex appends an index part to the JSON path. +func (p JSONPath) AppendIndex(part uint64) JSONPath { + return append(p, JSONPathPart{value: part}) +} + +// String returns the string representation of the JSON path. +func (p JSONPath) String() string { + var sb strings.Builder + for i, part := range p { + if prop, ok := part.Property(); ok { + if i > 0 { + sb.WriteByte('.') + } + sb.WriteString(prop) + } else if index, ok := part.Index(); ok { + sb.WriteByte('[') + sb.WriteString(strconv.FormatUint(index, 10)) + sb.WriteByte(']') + } + } + return sb.String() +} + // JSON represents a JSON value that can be any valid JSON type: object, array, number, string, boolean, or null. +// It can also represent a subtree of a JSON tree. // It provides type-safe access to the underlying value through various accessor methods. type JSON interface { json.Marshaler @@ -58,18 +137,18 @@ type JSON interface { // Returns an error if marshaling fails. Marshal(w io.Writer) error - // GetPath returns the path of the JSON value in the JSON tree. - GetPath() []string + // GetPath returns the path of the JSON value (or subtree) in the JSON tree. + GetPath() JSONPath // visit calls the visitor function for the JSON value at the given path. - visit(visitor JSONVisitor, path []string, opts traverseJSONOptions) error + visit(visitor JSONVisitor, path JSONPath, opts traverseJSONOptions) error } // MakeVoidJSON creates a JSON value that represents a void value with just a path. // This is necessary purely for creating a json path prefix for storage queries. // All other json values will be encoded with some value after the path which makes // them unsuitable to build a path prefix. -func MakeVoidJSON(path []string) JSON { +func MakeVoidJSON(path JSONPath) JSON { return jsonBase[any]{path: path} } @@ -81,7 +160,7 @@ func TraverseJSON(j JSON, visitor JSONVisitor, opts ...traverseJSONOption) error opt(&options) } if shouldVisitPath(options.pathPrefix, nil) { - return j.visit(visitor, []string{}, options) + return j.visit(visitor, JSONPath{}, options) } return nil } @@ -90,7 +169,7 @@ type traverseJSONOption func(*traverseJSONOptions) // TraverseJSONWithPrefix returns a traverseJSONOption that sets the path prefix for the traversal. // Only nodes with paths that start with the prefix will be visited. -func TraverseJSONWithPrefix(prefix []string) traverseJSONOption { +func TraverseJSONWithPrefix(prefix JSONPath) traverseJSONOption { return func(opts *traverseJSONOptions) { opts.pathPrefix = prefix } @@ -131,7 +210,7 @@ type traverseJSONOptions struct { // onlyLeaves when true visits only leaf nodes (not objects or arrays) onlyLeaves bool // pathPrefix when set visits only paths that start with this prefix - pathPrefix []string + pathPrefix JSONPath // visitArrayElements when true visits array elements visitArrayElements bool // recurseVisitedArrayElements when true visits array elements recursively @@ -166,14 +245,14 @@ func (v jsonVoid) IsNull() bool { return false } -func (v jsonVoid) visit(visitor JSONVisitor, path []string, opts traverseJSONOptions) error { +func (v jsonVoid) visit(visitor JSONVisitor, path JSONPath, opts traverseJSONOptions) error { return nil } type jsonBase[T any] struct { jsonVoid val T - path []string + path JSONPath } func (v jsonBase[T]) Value() any { @@ -192,7 +271,7 @@ func (v jsonBase[T]) MarshalJSON() ([]byte, error) { return json.Marshal(v.val) } -func (v jsonBase[T]) GetPath() []string { +func (v jsonBase[T]) GetPath() JSONPath { return v.path } @@ -218,7 +297,7 @@ func (obj jsonObject) Unwrap() any { return result } -func (obj jsonObject) visit(visitor JSONVisitor, path []string, opts traverseJSONOptions) error { +func (obj jsonObject) visit(visitor JSONVisitor, path JSONPath, opts traverseJSONOptions) error { obj.path = path if !opts.onlyLeaves && len(path) >= len(opts.pathPrefix) { if err := visitor(obj); err != nil { @@ -227,7 +306,7 @@ func (obj jsonObject) visit(visitor JSONVisitor, path []string, opts traverseJSO } for k, v := range obj.val { - newPath := append(path, k) + newPath := path.AppendProperty(k) if !shouldVisitPath(opts.pathPrefix, newPath) { continue } @@ -261,7 +340,7 @@ func (arr jsonArray) Unwrap() any { return result } -func (arr jsonArray) visit(visitor JSONVisitor, path []string, opts traverseJSONOptions) error { +func (arr jsonArray) visit(visitor JSONVisitor, path JSONPath, opts traverseJSONOptions) error { arr.path = path if !opts.onlyLeaves { if err := visitor(arr); err != nil { @@ -274,9 +353,9 @@ func (arr jsonArray) visit(visitor JSONVisitor, path []string, opts traverseJSON if !opts.recurseVisitedArrayElements && isCompositeJSON(arr.val[i]) { continue } - var newPath []string + var newPath JSONPath if opts.includeArrayIndexInPath { - newPath = append(path, strconv.Itoa(i)) + newPath = path.AppendIndex(uint64(i)) } else { newPath = path } @@ -302,7 +381,7 @@ func (n jsonNumber) Number() (float64, bool) { return n.val, true } -func (n jsonNumber) visit(visitor JSONVisitor, path []string, opts traverseJSONOptions) error { +func (n jsonNumber) visit(visitor JSONVisitor, path JSONPath, opts traverseJSONOptions) error { n.path = path return visitor(n) } @@ -317,7 +396,7 @@ func (s jsonString) String() (string, bool) { return s.val, true } -func (n jsonString) visit(visitor JSONVisitor, path []string, opts traverseJSONOptions) error { +func (n jsonString) visit(visitor JSONVisitor, path JSONPath, opts traverseJSONOptions) error { n.path = path return visitor(n) } @@ -332,7 +411,7 @@ func (b jsonBool) Bool() (bool, bool) { return b.val, true } -func (n jsonBool) visit(visitor JSONVisitor, path []string, opts traverseJSONOptions) error { +func (n jsonBool) visit(visitor JSONVisitor, path JSONPath, opts traverseJSONOptions) error { n.path = path return visitor(n) } @@ -347,32 +426,32 @@ func (n jsonNull) IsNull() bool { return true } -func (n jsonNull) visit(visitor JSONVisitor, path []string, opts traverseJSONOptions) error { +func (n jsonNull) visit(visitor JSONVisitor, path JSONPath, opts traverseJSONOptions) error { n.path = path return visitor(n) } -func newJSONObject(val map[string]JSON, path []string) jsonObject { +func newJSONObject(val map[string]JSON, path JSONPath) jsonObject { return jsonObject{jsonBase[map[string]JSON]{val: val, path: path}} } -func newJSONArray(val []JSON, path []string) jsonArray { +func newJSONArray(val []JSON, path JSONPath) jsonArray { return jsonArray{jsonBase[[]JSON]{val: val, path: path}} } -func newJSONNumber(val float64, path []string) jsonNumber { +func newJSONNumber(val float64, path JSONPath) jsonNumber { return jsonNumber{jsonBase[float64]{val: val, path: path}} } -func newJSONString(val string, path []string) jsonString { +func newJSONString(val string, path JSONPath) jsonString { return jsonString{jsonBase[string]{val: val, path: path}} } -func newJSONBool(val bool, path []string) jsonBool { +func newJSONBool(val bool, path JSONPath) jsonBool { return jsonBool{jsonBase[bool]{val: val, path: path}} } -func newJSONNull(path []string) jsonNull { +func newJSONNull(path JSONPath) jsonNull { return jsonNull{jsonBase[any]{path: path}} } @@ -426,12 +505,12 @@ func NewJSON(v any) (JSON, error) { // - slice of any above type // - []any // Returns error if the input cannot be converted to JSON. -func NewJSONWithPath(v any, path []string) (JSON, error) { +func NewJSONWithPath(v any, path JSONPath) (JSON, error) { return newJSON(v, path) } // newJSON is an internal function that creates a new JSON value with parent and property name -func newJSON(v any, path []string) (JSON, error) { +func newJSON(v any, path JSONPath) (JSON, error) { if v == nil { return newJSONNull(path), nil } else { @@ -505,10 +584,10 @@ func newJSON(v any, path []string) (JSON, error) { return nil, NewErrInvalidJSONPayload(v) } -func newJsonArrayFromAnyArray(arr []any, path []string) (JSON, error) { +func newJsonArrayFromAnyArray(arr []any, path JSONPath) (JSON, error) { result := make([]JSON, len(arr)) for i := range arr { - jsonVal, err := newJSON(arr[i], path) + jsonVal, err := newJSON(arr[i], path.AppendIndex(uint64(i))) if err != nil { return nil, err } @@ -517,46 +596,46 @@ func newJsonArrayFromAnyArray(arr []any, path []string) (JSON, error) { return newJSONArray(result, path), nil } -func newJSONBoolArray(v []bool, path []string) JSON { +func newJSONBoolArray(v []bool, path JSONPath) JSON { arr := make([]JSON, len(v)) for i := range v { - arr[i] = newJSONBool(v[i], path) + arr[i] = newJSONBool(v[i], path.AppendIndex(uint64(i))) } return newJSONArray(arr, path) } -func newJSONNumberArray[T constraints.Integer | constraints.Float](v []T, path []string) JSON { +func newJSONNumberArray[T constraints.Integer | constraints.Float](v []T, path JSONPath) JSON { arr := make([]JSON, len(v)) for i := range v { - arr[i] = newJSONNumber(float64(v[i]), path) + arr[i] = newJSONNumber(float64(v[i]), path.AppendIndex(uint64(i))) } return newJSONArray(arr, path) } -func newJSONStringArray(v []string, path []string) JSON { +func newJSONStringArray(v []string, path JSONPath) JSON { arr := make([]JSON, len(v)) for i := range v { - arr[i] = newJSONString(v[i], path) + arr[i] = newJSONString(v[i], path.AppendIndex(uint64(i))) } return newJSONArray(arr, path) } // newJSONFromFastJSON is an internal function that creates a new JSON value with parent and property name -func newJSONFromFastJSON(v *fastjson.Value, path []string) JSON { +func newJSONFromFastJSON(v *fastjson.Value, path JSONPath) JSON { switch v.Type() { case fastjson.TypeObject: fastObj := v.GetObject() obj := make(map[string]JSON, fastObj.Len()) fastObj.Visit(func(k []byte, v *fastjson.Value) { key := string(k) - obj[key] = newJSONFromFastJSON(v, append(path, key)) + obj[key] = newJSONFromFastJSON(v, path.AppendProperty(key)) }) return newJSONObject(obj, path) case fastjson.TypeArray: fastArr := v.GetArray() arr := make([]JSON, len(fastArr)) for i := range fastArr { - arr[i] = NewJSONFromFastJSON(fastArr[i]) + arr[i] = newJSONFromFastJSON(fastArr[i], path.AppendIndex(uint64(i))) } return newJSONArray(arr, path) case fastjson.TypeNumber: @@ -585,10 +664,10 @@ func NewJSONFromMap(data map[string]any) (JSON, error) { return newJSONFromMap(data, nil) } -func newJSONFromMap(data map[string]any, path []string) (JSON, error) { +func newJSONFromMap(data map[string]any, path JSONPath) (JSON, error) { obj := make(map[string]JSON, len(data)) for k, v := range data { - jsonVal, err := newJSON(v, append(path, k)) + jsonVal, err := newJSON(v, path.AppendProperty(k)) if err != nil { return nil, err } @@ -597,7 +676,7 @@ func newJSONFromMap(data map[string]any, path []string) (JSON, error) { return newJSONObject(obj, path), nil } -func shouldVisitPath(prefix, path []string) bool { +func shouldVisitPath(prefix, path JSONPath) bool { if len(prefix) == 0 { return true } diff --git a/client/json_test.go b/client/json_test.go index 811097e709..a006a4efd1 100644 --- a/client/json_test.go +++ b/client/json_test.go @@ -504,7 +504,7 @@ func TestNewJSONAndMarshalJSON(t *testing.T) { }, } - path := []string{"some", "path"} + path := JSONPath{}.AppendProperty("some").AppendProperty("path") for _, tt := range tests { for _, withPath := range []bool{true, false} { @@ -621,19 +621,19 @@ func TestNewJSONFromMap_WithPaths(t *testing.T) { } } -func traverseAndAssertPaths(t *testing.T, j JSON, parentPath []string) { +func traverseAndAssertPaths(t *testing.T, j JSON, parentPath JSONPath) { assert.Equal(t, parentPath, j.GetPath(), "Expected path %v, got %v", parentPath, j.GetPath()) if obj, isObj := j.Object(); isObj { for k, v := range obj { - newPath := append(parentPath, k) + newPath := parentPath.AppendProperty(k) traverseAndAssertPaths(t, v, newPath) } } if arr, isArr := j.Array(); isArr { - for _, v := range arr { - traverseAndAssertPaths(t, v, parentPath) + for i, v := range arr { + traverseAndAssertPaths(t, v, parentPath.AppendIndex(uint64(i))) } } } diff --git a/client/json_traverse_test.go b/client/json_traverse_test.go index ff9103b1ab..c3cf06268a 100644 --- a/client/json_traverse_test.go +++ b/client/json_traverse_test.go @@ -38,6 +38,22 @@ func getArrayValue(j JSON) []JSON { panic("expected array value") } +// Creates a path from mixed string/integer values +func makeJSONPath[T string | int | uint64](parts ...T) JSONPath { + path := JSONPath{} + for _, part := range parts { + switch val := any(part).(type) { + case string: + path = path.AppendProperty(val) + case int: + path = path.AppendIndex(uint64(val)) + case uint64: + path = path.AppendIndex(val) + } + } + return path +} + func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) { // Create a complex JSON structure for testing json := newJSONObject(map[string]JSON{ @@ -79,9 +95,9 @@ func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) { {path: "bool", value: newJSONBool(true, nil)}, {path: "null", value: newJSONNull(nil)}, {path: "object", value: getObjectValue(json)["object"]}, - {path: "object/nested", value: newJSONString("inside", nil)}, - {path: "object/deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, - {path: "object/deep/level", value: newJSONNumber(3, nil)}, + {path: "object.nested", value: newJSONString("inside", nil)}, + {path: "object.deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, + {path: "object.deep.level", value: newJSONNumber(3, nil)}, {path: "array", value: getObjectValue(json)["array"]}, }, }, @@ -95,30 +111,30 @@ func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) { {path: "number", value: newJSONNumber(42, nil)}, {path: "bool", value: newJSONBool(true, nil)}, {path: "null", value: newJSONNull(nil)}, - {path: "object/nested", value: newJSONString("inside", nil)}, - {path: "object/deep/level", value: newJSONNumber(3, nil)}, + {path: "object.nested", value: newJSONString("inside", nil)}, + {path: "object.deep.level", value: newJSONNumber(3, nil)}, }, }, { name: "WithPrefix_Object", options: []traverseJSONOption{ - TraverseJSONWithPrefix([]string{"object"}), + TraverseJSONWithPrefix(makeJSONPath("object")), }, expected: []traverseNode{ {path: "object", value: getObjectValue(json)["object"]}, - {path: "object/nested", value: newJSONString("inside", nil)}, - {path: "object/deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, - {path: "object/deep/level", value: newJSONNumber(3, nil)}, + {path: "object.nested", value: newJSONString("inside", nil)}, + {path: "object.deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, + {path: "object.deep.level", value: newJSONNumber(3, nil)}, }, }, { name: "WithPrefix_Deep", options: []traverseJSONOption{ - TraverseJSONWithPrefix([]string{"object", "deep"}), + TraverseJSONWithPrefix(makeJSONPath("object", "deep")), }, expected: []traverseNode{ - {path: "object/deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, - {path: "object/deep/level", value: newJSONNumber(3, nil)}, + {path: "object.deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, + {path: "object.deep.level", value: newJSONNumber(3, nil)}, }, }, { @@ -133,14 +149,14 @@ func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) { {path: "bool", value: newJSONBool(true, nil)}, {path: "null", value: newJSONNull(nil)}, {path: "object", value: getObjectValue(json)["object"]}, - {path: "object/nested", value: newJSONString("inside", nil)}, - {path: "object/deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, - {path: "object/deep/level", value: newJSONNumber(3, nil)}, + {path: "object.nested", value: newJSONString("inside", nil)}, + {path: "object.deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, + {path: "object.deep.level", value: newJSONNumber(3, nil)}, {path: "array", value: getObjectValue(json)["array"]}, {path: "array", value: newJSONNumber(1, nil)}, {path: "array", value: newJSONString("two", nil)}, {path: "array", value: getArrayValue(getObjectValue(json)["array"])[2]}, - {path: "array/key", value: newJSONString("value", nil)}, + {path: "array.key", value: newJSONString("value", nil)}, {path: "array", value: getArrayValue(getObjectValue(json)["array"])[3]}, {path: "array", value: newJSONNumber(4, nil)}, {path: "array", value: newJSONNumber(5, nil)}, @@ -158,9 +174,9 @@ func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) { {path: "bool", value: newJSONBool(true, nil)}, {path: "null", value: newJSONNull(nil)}, {path: "object", value: getObjectValue(json)["object"]}, - {path: "object/nested", value: newJSONString("inside", nil)}, - {path: "object/deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, - {path: "object/deep/level", value: newJSONNumber(3, nil)}, + {path: "object.nested", value: newJSONString("inside", nil)}, + {path: "object.deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, + {path: "object.deep.level", value: newJSONNumber(3, nil)}, {path: "array", value: getObjectValue(json)["array"]}, {path: "array", value: newJSONNumber(1, nil)}, {path: "array", value: newJSONString("two", nil)}, @@ -179,17 +195,17 @@ func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) { {path: "bool", value: newJSONBool(true, nil)}, {path: "null", value: newJSONNull(nil)}, {path: "object", value: getObjectValue(json)["object"]}, - {path: "object/nested", value: newJSONString("inside", nil)}, - {path: "object/deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, - {path: "object/deep/level", value: newJSONNumber(3, nil)}, + {path: "object.nested", value: newJSONString("inside", nil)}, + {path: "object.deep", value: getObjectValue(getObjectValue(json)["object"])["deep"]}, + {path: "object.deep.level", value: newJSONNumber(3, nil)}, {path: "array", value: getObjectValue(json)["array"]}, - {path: "array/0", value: newJSONNumber(1, nil)}, - {path: "array/1", value: newJSONString("two", nil)}, - {path: "array/2", value: getArrayValue(getObjectValue(json)["array"])[2]}, - {path: "array/2/key", value: newJSONString("value", nil)}, - {path: "array/3", value: getArrayValue(getObjectValue(json)["array"])[3]}, - {path: "array/3/0", value: newJSONNumber(4, nil)}, - {path: "array/3/1", value: newJSONNumber(5, nil)}, + {path: "array[0]", value: newJSONNumber(1, nil)}, + {path: "array[1]", value: newJSONString("two", nil)}, + {path: "array[2]", value: getArrayValue(getObjectValue(json)["array"])[2]}, + {path: "array[2].key", value: newJSONString("value", nil)}, + {path: "array[3]", value: getArrayValue(getObjectValue(json)["array"])[3]}, + {path: "array[3][0]", value: newJSONNumber(4, nil)}, + {path: "array[3][1]", value: newJSONNumber(5, nil)}, }, }, { @@ -197,15 +213,15 @@ func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) { options: []traverseJSONOption{ TraverseJSONOnlyLeaves(), TraverseJSONVisitArrayElements(true), - TraverseJSONWithPrefix([]string{"array"}), + TraverseJSONWithPrefix(makeJSONPath("array")), TraverseJSONWithArrayIndexInPath(), }, expected: []traverseNode{ - {path: "array/0", value: newJSONNumber(1, nil)}, - {path: "array/1", value: newJSONString("two", nil)}, - {path: "array/2/key", value: newJSONString("value", nil)}, - {path: "array/3/0", value: newJSONNumber(4, nil)}, - {path: "array/3/1", value: newJSONNumber(5, nil)}, + {path: "array[0]", value: newJSONNumber(1, nil)}, + {path: "array[1]", value: newJSONString("two", nil)}, + {path: "array[2].key", value: newJSONString("value", nil)}, + {path: "array[3][0]", value: newJSONNumber(4, nil)}, + {path: "array[3][1]", value: newJSONNumber(5, nil)}, }, }, } @@ -214,7 +230,7 @@ func TestTraverseJSON_ShouldVisitAccordingToConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { visited := []traverseNode{} err := TraverseJSON(json, func(value JSON) error { - key := joinPath(value.GetPath()) + key := value.GetPath().String() visited = append(visited, traverseNode{path: key, value: value}) return nil }, tt.options...) @@ -399,44 +415,44 @@ func TestTraverseJSON_WithError(t *testing.T) { func TestShouldVisitPath(t *testing.T) { tests := []struct { name string - prefix []string - path []string + prefix JSONPath + path JSONPath expected bool }{ { name: "EmptyPrefix", - prefix: []string{}, - path: []string{"a", "b"}, + prefix: JSONPath{}, + path: makeJSONPath("a", "b"), expected: true, }, { name: "ExactMatch", - prefix: []string{"a", "b"}, - path: []string{"a", "b"}, + prefix: makeJSONPath("a", "b"), + path: makeJSONPath("a", "b"), expected: true, }, { name: "PrefixMatch", - prefix: []string{"a"}, - path: []string{"a", "b"}, + prefix: makeJSONPath("a"), + path: makeJSONPath("a", "b"), expected: true, }, { name: "NoMatch", - prefix: []string{"a", "b"}, - path: []string{"a", "c"}, + prefix: makeJSONPath("a", "b"), + path: makeJSONPath("a", "c"), expected: false, }, { name: "PathTooShort", - prefix: []string{"a", "b"}, - path: []string{"a"}, + prefix: makeJSONPath("a", "b"), + path: makeJSONPath("a"), expected: true, }, { name: "PathLonger", - prefix: []string{"a", "b"}, - path: []string{"a", "b", "c"}, + prefix: makeJSONPath("a", "b"), + path: makeJSONPath("a", "b", "c"), expected: true, }, } @@ -448,15 +464,3 @@ func TestShouldVisitPath(t *testing.T) { }) } } - -// Helper function to join path segments -func joinPath(path []string) string { - if len(path) == 0 { - return "" - } - result := path[0] - for i := 1; i < len(path); i++ { - result += "/" + path[i] - } - return result -} diff --git a/client/normal_array.go b/client/normal_array.go index b560a4bd9a..7f3605792f 100644 --- a/client/normal_array.go +++ b/client/normal_array.go @@ -120,6 +120,18 @@ func (v normalDocumentArray) Equal(other NormalValue) bool { return areNormalArraysEqual(v.val, other.DocumentArray) } +type normalJSONArray struct { + baseArrayNormalValue[[]JSON] +} + +func (v normalJSONArray) JSONArray() ([]JSON, bool) { + return v.val, true +} + +func (v normalJSONArray) Equal(other NormalValue) bool { + return areNormalArraysEqual(v.val, other.JSONArray) +} + // NewNormalBoolArray creates a new NormalValue that represents a `[]bool` value. func NewNormalBoolArray(val []bool) NormalValue { return normalBoolArray{newBaseArrayNormalValue(val)} @@ -155,6 +167,11 @@ func NewNormalDocumentArray(val []*Document) NormalValue { return normalDocumentArray{newBaseArrayNormalValue(val)} } +// NewNormalJSONArray creates a new NormalValue that represents a `[]JSON` value. +func NewNormalJSONArray(val []JSON) NormalValue { + return normalJSONArray{newBaseArrayNormalValue(val)} +} + func normalizeNumArr[R int64 | float64, T constraints.Integer | constraints.Float](val []T) []R { var v any = val if arr, ok := v.([]R); ok { diff --git a/client/normal_new.go b/client/normal_new.go index 8eb1b9f24c..671f00eac7 100644 --- a/client/normal_new.go +++ b/client/normal_new.go @@ -134,6 +134,8 @@ func NewNormalValue(val any) (NormalValue, error) { return NewNormalTimeArray(v), nil case []*Document: return NewNormalDocumentArray(v), nil + case []JSON: + return NewNormalJSONArray(v), nil case []immutable.Option[bool]: return NewNormalNillableBoolArray(v), nil diff --git a/client/normal_util.go b/client/normal_util.go index 44631fe45c..64845a3404 100644 --- a/client/normal_util.go +++ b/client/normal_util.go @@ -43,6 +43,9 @@ func ToArrayOfNormalValues(val NormalValue) ([]NormalValue, error) { if v, ok := val.DocumentArray(); ok { return toNormalArray(v, NewNormalDocument), nil } + if v, ok := val.JSONArray(); ok { + return toNormalArray(v, NewNormalJSON), nil + } if v, ok := val.NillableBoolArray(); ok { return toNormalArray(v, NewNormalNillableBool), nil } diff --git a/client/normal_value.go b/client/normal_value.go index 3dc66a83fd..536f1daf78 100644 --- a/client/normal_value.go +++ b/client/normal_value.go @@ -123,6 +123,10 @@ type NormalValue interface { // The second return flag is true if the value is a [[]*Document]. // Otherwise it will return nil and false. DocumentArray() ([]*Document, bool) + // JSONArray returns the value as a JSON array. + // The second return flag is true if the value is a JSON array. + // Otherwise it will return nil and false. + JSONArray() ([]JSON, bool) // NillableBoolArray returns the value as nillable array of bool elements. // The second return flag is true if the value is [immutable.Option[[]bool]]. diff --git a/client/normal_value_test.go b/client/normal_value_test.go index 551ef2e300..4bdba2f6aa 100644 --- a/client/normal_value_test.go +++ b/client/normal_value_test.go @@ -47,6 +47,7 @@ const ( BytesArray nType = "BytesArray" TimeArray nType = "TimeArray" DocumentArray nType = "DocumentArray" + JSONArray nType = "JSONArray" NillableBoolArray nType = "NillableBoolArray" NillableIntArray nType = "NillableIntArray" @@ -135,6 +136,7 @@ func TestNormalValue_NewValueAndTypeAssertion(t *testing.T) { BytesArray: func(v NormalValue) (any, bool) { return v.BytesArray() }, TimeArray: func(v NormalValue) (any, bool) { return v.TimeArray() }, DocumentArray: func(v NormalValue) (any, bool) { return v.DocumentArray() }, + JSONArray: func(v NormalValue) (any, bool) { return v.JSONArray() }, BoolNillableArray: func(v NormalValue) (any, bool) { return v.BoolNillableArray() }, IntNillableArray: func(v NormalValue) (any, bool) { return v.IntNillableArray() }, @@ -188,6 +190,7 @@ func TestNormalValue_NewValueAndTypeAssertion(t *testing.T) { BytesArray: func(v any) NormalValue { return NewNormalBytesArray(v.([][]byte)) }, TimeArray: func(v any) NormalValue { return NewNormalTimeArray(v.([]time.Time)) }, DocumentArray: func(v any) NormalValue { return NewNormalDocumentArray(v.([]*Document)) }, + JSONArray: func(v any) NormalValue { return NewNormalJSONArray(v.([]JSON)) }, NillableBoolArray: func(v any) NormalValue { return NewNormalNillableBoolArray(v.([]immutable.Option[bool])) @@ -407,6 +410,11 @@ func TestNormalValue_NewValueAndTypeAssertion(t *testing.T) { input: []*Document{{}, {}}, isArray: true, }, + { + nType: JSONArray, + input: []JSON{newJSONNumber(3, nil), newJSONString("test", nil)}, + isArray: true, + }, { nType: NillableBoolArray, input: []immutable.Option[bool]{immutable.Some(true)}, @@ -1479,6 +1487,14 @@ func TestNormalValue_ToArrayOfNormalValues(t *testing.T) { input: NewNormalDocumentArray([]*Document{doc1, doc2}), expected: []NormalValue{NewNormalDocument(doc1), NewNormalDocument(doc2)}, }, + { + name: "json elements", + input: NewNormalJSONArray([]JSON{newJSONBool(true, nil), newJSONString("test", nil)}), + expected: []NormalValue{ + NewNormalJSON(newJSONBool(true, nil)), + NewNormalJSON(newJSONString("test", nil)), + }, + }, { name: "nillable bool elements", input: NewNormalNillableBoolArray([]immutable.Option[bool]{ diff --git a/client/normal_void.go b/client/normal_void.go index a9078e5328..3c886d4827 100644 --- a/client/normal_void.go +++ b/client/normal_void.go @@ -129,6 +129,10 @@ func (NormalVoid) DocumentArray() ([]*Document, bool) { return nil, false } +func (NormalVoid) JSONArray() ([]JSON, bool) { + return nil, false +} + func (NormalVoid) NillableBoolArray() ([]immutable.Option[bool], bool) { return nil, false } diff --git a/docs/data_format_changes/i3368-json-array-encoding.md b/docs/data_format_changes/i3368-json-array-encoding.md new file mode 100644 index 0000000000..8392a8fb8b --- /dev/null +++ b/docs/data_format_changes/i3368-json-array-encoding.md @@ -0,0 +1,3 @@ +# JSON array encoding + +JSON array elements are now encoded in a different way. diff --git a/internal/connor/none.go b/internal/connor/none.go index 16613b3e46..742d4dc977 100644 --- a/internal/connor/none.go +++ b/internal/connor/none.go @@ -1,12 +1,52 @@ package connor +import "github.com/sourcenetwork/immutable" + // none is an operator which allows the evaluation of // a number of conditions over a list of values // matching if all of them do not match. func none(condition, data any) (bool, error) { - m, err := anyOp(condition, data) - if err != nil { - return false, err + switch t := data.(type) { + case []any: + return noneSlice(condition, t) + + case []string: + return noneSlice(condition, t) + + case []immutable.Option[string]: + return noneSlice(condition, t) + + case []int64: + return noneSlice(condition, t) + + case []immutable.Option[int64]: + return noneSlice(condition, t) + + case []bool: + return noneSlice(condition, t) + + case []immutable.Option[bool]: + return noneSlice(condition, t) + + case []float64: + return noneSlice(condition, t) + + case []immutable.Option[float64]: + return noneSlice(condition, t) + + default: + return false, nil + } +} + +func noneSlice[T any](condition any, data []T) (bool, error) { + for _, c := range data { + m, err := eq(condition, c) + if err != nil { + return false, err + } else if m { + return false, nil + } } - return !m, nil + return true, nil } diff --git a/internal/db/fetcher/indexer_iterators.go b/internal/db/fetcher/indexer_iterators.go index 8df1b114a5..3fba778344 100644 --- a/internal/db/fetcher/indexer_iterators.go +++ b/internal/db/fetcher/indexer_iterators.go @@ -50,6 +50,10 @@ const ( opAny = "__any" ) +func isArrayCondition(op string) bool { + return op == compOpAny || op == compOpAll || op == compOpNone +} + // indexIterator is an iterator over index keys. // It is used to iterate over the index keys that match a specific condition. // For example, iteration over condition _eq and _gt will have completely different logic. @@ -487,7 +491,7 @@ func (f *IndexFetcher) createIndexIterator() (indexIterator, error) { type fieldFilterCond struct { op string arrOp string - jsonPath []string + jsonPath client.JSONPath val client.NormalValue kind client.FieldKind } @@ -515,16 +519,32 @@ func (f *IndexFetcher) determineFieldFilterConditions() ([]fieldFilterCond, erro condMap := indexFilterCond.(map[connor.FilterKey]any) - jsonPath := []string{} + jsonPath := client.JSONPath{} if fieldDef.Kind == client.FieldKind_NILLABLE_JSON { jsonPathLoop: for { for key, filterVal := range condMap { prop, ok := key.(*mapper.ObjectProperty) if !ok { + // if filter contains an array condition, we need to append index 0 to the json path + // to limit the search only to array elements + op, ok := key.(*mapper.Operator) + if ok && isArrayCondition(op.Operation) { + if op.Operation == compOpNone { + // if the array condition is _none it doesn't make sense to use index because + // values picked by the index is random guessing. For example if we have doc1 + // with array of [3, 5, 1] and doc2 with [7, 4, 8] the index first fetches + // value 1 of doc1, let it go through the filter and then fetches value 3 of doc1 + // again, skips it (because it cached doc1 id) and fetches value 4 of doc2, and + // so on until it exhaust all prefixes in ascending order. + // It might be even less effective than just scanning all documents. + return nil, nil + } + jsonPath = jsonPath.AppendIndex(0) + } break jsonPathLoop } - jsonPath = append(jsonPath, prop.Name) + jsonPath = jsonPath.AppendProperty(prop.Name) condMap = filterVal.(map[connor.FilterKey]any) } } @@ -539,22 +559,7 @@ func (f *IndexFetcher) determineFieldFilterConditions() ([]fieldFilterCond, erro var err error if len(jsonPath) > 0 { - var jsonVal client.JSON - if cond.op == compOpAny || cond.op == compOpAll || cond.op == compOpNone { - subCondMap := filterVal.(map[connor.FilterKey]any) - for subKey, subVal := range subCondMap { - cond.arrOp = cond.op - cond.op = subKey.(*mapper.Operator).Operation - jsonVal, err = client.NewJSONWithPath(subVal, jsonPath) - // the sub condition is supposed to have only 1 record - break - } - } else { - jsonVal, err = client.NewJSONWithPath(filterVal, jsonPath) - } - if err == nil { - cond.val = client.NewNormalJSON(jsonVal) - } + err = setJSONFilterCondition(&cond, filterVal, jsonPath) } else if filterVal == nil { cond.val, err = client.NewNormalNil(cond.kind) } else if !f.indexedFields[i].Kind.IsArray() { @@ -594,6 +599,72 @@ func (f *IndexFetcher) determineFieldFilterConditions() ([]fieldFilterCond, erro return result, nil } +// setJSONFilterCondition sets up the given condition struct based on the filter value and JSON path so that +// it can be used to fetch the indexed data. +func setJSONFilterCondition(cond *fieldFilterCond, filterVal any, jsonPath client.JSONPath) error { + if isArrayCondition(cond.op) { + subCondMap := filterVal.(map[connor.FilterKey]any) + for subKey, subVal := range subCondMap { + cond.arrOp = cond.op + cond.op = subKey.(*mapper.Operator).Operation + jsonVal, err := client.NewJSONWithPath(subVal, jsonPath) + if err != nil { + return err + } + cond.val = client.NewNormalJSON(jsonVal) + // the array sub condition (_any, _all or _none) is supposed to have only 1 record + break + } + } else if cond.op == opIn { + // values in _in operator should not be considered as array elements just because they happened + // to be written as an array in the filter. We need to convert them to normal values and + // treat them individually. + var jsonVals []client.JSON + if anyArr, ok := filterVal.([]any); ok { + // if filter value is []any we convert each value separately because JSON might have + // array elements of different types. That's why we can't just pass it directly to + // client.ToArrayOfNormalValues + jsonVals = make([]client.JSON, 0, len(anyArr)) + for _, val := range anyArr { + jsonVal, err := client.NewJSONWithPath(val, jsonPath) + if err != nil { + return err + } + jsonVals = append(jsonVals, jsonVal) + } + } else { + normValue, err := client.NewNormalValue(filterVal) + if err != nil { + return err + } + normArr, err := client.ToArrayOfNormalValues(normValue) + if err != nil { + return err + } + jsonVals = make([]client.JSON, 0, len(normArr)) + for _, val := range normArr { + jsonVal, err := client.NewJSONWithPath(val.Unwrap(), jsonPath) + if err != nil { + return err + } + jsonVals = append(jsonVals, jsonVal) + } + } + normJSONs, err := client.NewNormalValue(jsonVals) + if err != nil { + return err + } + cond.val = normJSONs + } else { + jsonVal, err := client.NewJSONWithPath(filterVal, jsonPath) + if err != nil { + return err + } + cond.val = client.NewNormalJSON(jsonVal) + } + return nil +} + // isUniqueFetchByFullKey checks if the only index key can be fetched by the full index key. // // This method ignores the first condition (unless it's nil) because it's expected to be called only diff --git a/internal/db/index.go b/internal/db/index.go index dfaaf20415..f2dba478d1 100644 --- a/internal/db/index.go +++ b/internal/db/index.go @@ -129,7 +129,17 @@ func (g *JSONFieldGenerator) Generate(value client.NormalValue, f func(client.No return err } return f(val) - }, client.TraverseJSONOnlyLeaves(), client.TraverseJSONVisitArrayElements(false)) + }, + // we don't want to traverse intermediate nodes, because we encode only values that can be filtered on + client.TraverseJSONOnlyLeaves(), + // we want to include array elements' indexes in json path, because we want to differentiate + // between array elements in order to be able to run array-specific queries like _all, _any and _none + client.TraverseJSONWithArrayIndexInPath(), + // we want to traverse array elements, but not recurse into them, because we don't have any way + // to query nested arrays elements. + // this effectively means that we traverse only leave array elements (string, float, bool, null) + client.TraverseJSONVisitArrayElements(false), + ) } // getFieldGenerator returns appropriate generator for the field type diff --git a/internal/db/indexed_docs_test.go b/internal/db/indexed_docs_test.go index d9e4327c35..e436f30b38 100644 --- a/internal/db/indexed_docs_test.go +++ b/internal/db/indexed_docs_test.go @@ -1663,7 +1663,7 @@ func TestJSONIndex_IfDocIsDeleted_ShouldRemoveAllRelatedIndexes(t *testing.T) { require.Equal(t, 1, f.countIndexPrefixes(testUsersColIndexCustom), "Unexpected num of indexes after delete") // make sure the second doc is still indexed - obj2Height, err := client.NewJSONWithPath(178, []string{"height"}) + obj2Height, err := client.NewJSONWithPath(178, client.JSONPath{}.AppendProperty("height")) require.NoError(t, err, "Failed to create JSON with path") key2 := newIndexKeyBuilder(f).Col(usersColName).Fields(usersCustomFieldName). Values(client.NewNormalJSON(obj2Height)).Doc(doc2).Build() @@ -1743,7 +1743,7 @@ func TestJSONUniqueIndex_IfDocIsDeleted_ShouldRemoveAllRelatedIndexes(t *testing require.Equal(t, 1, f.countIndexPrefixes(testUsersColIndexCustom), "Unexpected num of indexes after delete") // make sure the second doc is still indexed - obj2Height, err := client.NewJSONWithPath(178, []string{"height"}) + obj2Height, err := client.NewJSONWithPath(178, client.JSONPath{}.AppendProperty("height")) require.NoError(t, err, "Failed to create JSON with path") key2 := newIndexKeyBuilder(f).Col(usersColName).Fields(usersCustomFieldName). Values(client.NewNormalJSON(obj2Height)).Unique().Doc(doc2).Build() diff --git a/internal/encoding/errors.go b/internal/encoding/errors.go index 26d2daeb93..838dfd5660 100644 --- a/internal/encoding/errors.go +++ b/internal/encoding/errors.go @@ -94,9 +94,8 @@ func NewErrVarintOverflow(b []byte, value uint64) error { return errors.New(errVarintOverflow, errors.NewKV("Buffer", b), errors.NewKV("Value", value)) } -// NewErrInvalidJSONPayload returns a new error indicating that the buffer contains -// an invalid JSON payload. -func NewErrInvalidJSONPayload(b []byte, path []string, err ...error) error { +// NewErrInvalidJSONPayload returns a new error indicating that the buffer +func NewErrInvalidJSONPayload(b []byte, path string, err ...error) error { kvs := []errors.KV{errors.NewKV("Buffer", b), errors.NewKV("Path", path)} if len(err) > 0 { kvs = append(kvs, errors.NewKV("Error", err[0])) diff --git a/internal/encoding/json.go b/internal/encoding/json.go index 9c53f11237..be2f96f202 100644 --- a/internal/encoding/json.go +++ b/internal/encoding/json.go @@ -104,14 +104,14 @@ func decodeJSON(b []byte, ascending bool) ([]byte, client.JSON, error) { case Null: b = decodeNull(b) default: - err = NewErrInvalidJSONPayload(b, path) + err = NewErrInvalidJSONPayload(b, path.String()) } if err != nil { return b, nil, err } - result, err := client.NewJSON(jsonValue) + result, err := client.NewJSONWithPath(jsonValue, path) if err != nil { return b, nil, err @@ -120,8 +120,8 @@ func decodeJSON(b []byte, ascending bool) ([]byte, client.JSON, error) { return b, result, nil } -func decodeJSONPath(b []byte) ([]byte, []string, error) { - var path []string +func decodeJSONPath(b []byte) ([]byte, client.JSONPath, error) { + var path client.JSONPath for { if len(b) == 0 { break @@ -130,12 +130,24 @@ func decodeJSONPath(b []byte) ([]byte, []string, error) { b = b[1:] break } - rem, part, err := DecodeBytesAscending(b) - if err != nil { - return b, nil, NewErrInvalidJSONPath(b, err) + + if PeekType(b) == Bytes { + remainder, part, err := DecodeBytesAscending(b) + if err != nil { + return b, nil, NewErrInvalidJSONPath(b, err) + } + path = path.AppendProperty(string(part)) + b = remainder + } else { + // a part of the path can be either a property or an index, so if the type of the underlying + // encoded value is not Bytes it must be Uvarint. + rem, part, err := DecodeUvarintAscending(b) + if err != nil { + return b, nil, NewErrInvalidJSONPath(b, err) + } + path = path.AppendIndex(part) + b = rem } - path = append(path, string(part)) - b = rem } return b, path, nil } @@ -143,8 +155,15 @@ func decodeJSONPath(b []byte) ([]byte, []string, error) { func encodeJSONPath(b []byte, v client.JSON) []byte { b = append(b, jsonMarker) for _, part := range v.GetPath() { - pathBytes := unsafeConvertStringToBytes(part) - b = EncodeBytesAscending(b, pathBytes) + if prop, ok := part.Property(); ok { + pathBytes := unsafeConvertStringToBytes(prop) + b = EncodeBytesAscending(b, pathBytes) + } else if _, ok := part.Index(); ok { + // the given json value is an array element and we want all array elements to be + // distinguishable. That's why we add a constant 0 prefix. + // We ignore the actual array index value because we have no way of using it at the moment. + b = EncodeUvarintAscending(b, 0) + } } b = append(b, ascendingBytesEscapes.escapedTerm) return b diff --git a/internal/encoding/json_test.go b/internal/encoding/json_test.go index 22aa7741dc..6912b9fdaf 100644 --- a/internal/encoding/json_test.go +++ b/internal/encoding/json_test.go @@ -12,7 +12,6 @@ package encoding import ( "fmt" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -55,7 +54,7 @@ func TestJSONEncodingAndDecoding_ShouldEncodeAndDecodeBack(t *testing.T) { pathMap := make(map[string][]client.JSON) err = client.TraverseJSON(testJSON, func(value client.JSON) error { - p := strings.Join(value.GetPath(), "/") + p := value.GetPath().String() jsons := pathMap[p] jsons = append(jsons, value) pathMap[p] = jsons @@ -94,7 +93,7 @@ func TestJSONEncodingAndDecoding_ShouldEncodeAndDecodeBack(t *testing.T) { } func TestJSONEncodingDecoding_WithVoidValue_ShouldEncodeAndDecodeOnlyPath(t *testing.T) { - void := client.MakeVoidJSON([]string{"path", "to", "void"}) + void := client.MakeVoidJSON(client.JSONPath{}.AppendProperty("path").AppendProperty("to").AppendProperty("void")) encoded := EncodeJSONAscending(nil, void) remaining, decodedPath, err := decodeJSONPath(encoded[1:]) // skip the marker diff --git a/tests/integration/index/json_array_test.go b/tests/integration/index/json_array_test.go index 41e4babde5..3d0892545d 100644 --- a/tests/integration/index/json_array_test.go +++ b/tests/integration/index/json_array_test.go @@ -105,7 +105,7 @@ func TestJSONArrayIndex_WithDifferentElementValuesAndTypes_ShouldFetchCorrectlyU }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithIndexFetches(2), + Asserter: testUtils.NewExplainAsserter().WithIndexFetches(1), }, }, } @@ -113,7 +113,7 @@ func TestJSONArrayIndex_WithDifferentElementValuesAndTypes_ShouldFetchCorrectlyU testUtils.ExecuteTestCase(t, test) } -func TestJSONArrayIndex_WithNestedArrays_ShouldNotConsiderThem(t *testing.T) { +func TestJSONArrayIndex_WithAnyEqFilter_ShouldNotConsiderThem(t *testing.T) { req := `query { User(filter: {custom: {numbers: {_any: {_eq: 4}}}}) { name @@ -140,7 +140,7 @@ func TestJSONArrayIndex_WithNestedArrays_ShouldNotConsiderThem(t *testing.T) { DocMap: map[string]any{ "name": "Islam", "custom": map[string]any{ - "numbers": []any{0, []int{2, 6}, 9}, + "numbers": []any{0, []int{2}, 4}, }, }, }, @@ -160,15 +160,82 @@ func TestJSONArrayIndex_WithNestedArrays_ShouldNotConsiderThem(t *testing.T) { }, }, }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Shahzad", + "custom": map[string]any{ + "numbers": 4, + }, + }, + }, testUtils.Request{ Request: req, Results: map[string]any{ - "User": []map[string]any{}, + "User": []map[string]any{ + {"name": "Islam"}, + }, }, }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithIndexFetches(0), + Asserter: testUtils.NewExplainAsserter().WithIndexFetches(1), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestJSONArrayIndex_WithAnyAndComparisonFilter_ShouldNotConsiderThem(t *testing.T) { + req := `query { + User(filter: {custom: {numbers: {_any: {_gt: 4}}}}) { + name + } + }` + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + custom: JSON @index + }`, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "John", + "custom": map[string]any{ + "numbers": []any{3, 5, 7}, + }, + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Islam", + "custom": map[string]any{ + "numbers": []any{0, []int{6}, 4}, + }, + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Andy", + "custom": map[string]any{ + "numbers": 5, + }, + }, + }, + testUtils.Request{ + Request: req, + Results: map[string]any{ + "User": []map[string]any{ + {"name": "John"}, + }, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithIndexFetches(5), }, }, } @@ -176,7 +243,7 @@ func TestJSONArrayIndex_WithNestedArrays_ShouldNotConsiderThem(t *testing.T) { testUtils.ExecuteTestCase(t, test) } -func TestJSONArrayIndex_WithNoneFilterOnDifferentElementValues_ShouldFetchCorrectlyUsingIndex(t *testing.T) { +func TestJSONArrayIndex_WithNoneEqFilter_ShouldFetchCorrectlyUsingIndex(t *testing.T) { req := `query { User(filter: {custom: {numbers: {_none: {_eq: 4}}}}) { name @@ -195,7 +262,7 @@ func TestJSONArrayIndex_WithNoneFilterOnDifferentElementValues_ShouldFetchCorrec DocMap: map[string]any{ "name": "John", "custom": map[string]any{ - "numbers": []int{3, 5, 7}, + "numbers": []int{3}, }, }, }, @@ -223,8 +290,6 @@ func TestJSONArrayIndex_WithNoneFilterOnDifferentElementValues_ShouldFetchCorrec }, }, }, - // TODO: This document should be part of the query result, but it needs additional work - // with json encoding https://github.com/sourcenetwork/defradb/issues/3329 testUtils.CreateDoc{ DocMap: map[string]any{ "name": "Andy", @@ -238,14 +303,99 @@ func TestJSONArrayIndex_WithNoneFilterOnDifferentElementValues_ShouldFetchCorrec Results: map[string]any{ "User": []map[string]any{ {"name": "Islam"}, + {"name": "John"}, {"name": "Fred"}, + }, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + // We don't use index for _none operator + Asserter: testUtils.NewExplainAsserter().WithIndexFetches(0), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestJSONArrayIndex_WithNoneEqAndComparisonFilter_ShouldFetchCorrectlyUsingIndex(t *testing.T) { + req := `query { + User(filter: {custom: {numbers: {_none: {_gt: 4}}}}) { + name + } + }` + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + custom: JSON @index + }`, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "John", + "custom": map[string]any{ + "numbers": []int{3}, + }, + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Shahzad", + "custom": map[string]any{ + "numbers": []int{3, 8}, + }, + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Islam", + "custom": map[string]any{ + "numbers": []any{2, nil}, + }, + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Fred", + "custom": map[string]any{ + "numbers": []any{1, []int{5}}, + }, + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Andy", + "custom": map[string]any{ + "numbers": 5, + }, + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Bruno", + "custom": map[string]any{ + "numbers": nil, + }, + }, + }, + testUtils.Request{ + Request: req, + Results: map[string]any{ + "User": []map[string]any{ + {"name": "Islam"}, {"name": "John"}, + {"name": "Fred"}, }, }, }, testUtils.Request{ - Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithIndexFetches(9), + Request: makeExplainQuery(req), + // We don't use index for _none operator + Asserter: testUtils.NewExplainAsserter().WithIndexFetches(0), }, }, } @@ -253,7 +403,7 @@ func TestJSONArrayIndex_WithNoneFilterOnDifferentElementValues_ShouldFetchCorrec testUtils.ExecuteTestCase(t, test) } -func TestJSONArrayIndex_WithAllFilterOnDifferentElementValues_ShouldFetchCorrectlyUsingIndex(t *testing.T) { +func TestJSONArrayIndex_WithAllEqFilter_ShouldFetchCorrectlyUsingIndex(t *testing.T) { req := `query { User(filter: {custom: {numbers: {_all: {_eq: 4}}}}) { name @@ -324,6 +474,72 @@ func TestJSONArrayIndex_WithAllFilterOnDifferentElementValues_ShouldFetchCorrect }, }, }, + testUtils.Request{ + Request: makeExplainQuery(req), + // 4 docs have the value 4 in the numbers array + Asserter: testUtils.NewExplainAsserter().WithIndexFetches(4), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestJSONArrayIndex_WithAllEqAndComparisonFilter_ShouldFetchCorrectlyUsingIndex(t *testing.T) { + req := `query { + User(filter: {custom: {numbers: {_all: {_gt: 4}}}}) { + name + } + }` + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + custom: JSON @index + }`, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "John", + "custom": map[string]any{ + "numbers": []int{3, 7}, + }, + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Shahzad", + "custom": map[string]any{ + "numbers": []any{5, []int{6}}, + }, + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Andy", + "custom": map[string]any{ + "numbers": []any{7, 8}, + }, + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Islam", + "custom": map[string]any{ + "numbers": 8, + }, + }, + }, + testUtils.Request{ + Request: req, + Results: map[string]any{ + "User": []map[string]any{ + {"name": "Andy"}, + }, + }, + }, testUtils.Request{ Request: makeExplainQuery(req), Asserter: testUtils.NewExplainAsserter().WithIndexFetches(5), diff --git a/tests/integration/index/json_unique_array_test.go b/tests/integration/index/json_unique_array_test.go index 6e34c8dd13..05ebfcfe2f 100644 --- a/tests/integration/index/json_unique_array_test.go +++ b/tests/integration/index/json_unique_array_test.go @@ -50,6 +50,15 @@ func TestJSONArrayUniqueIndex_ShouldAllowOnlyUniqueValuesAndUseThemForFetching(t }, }, }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Andy", + "custom": map[string]any{ + // existing non-array-element value + "numbers": 3, + }, + }, + }, testUtils.CreateDoc{ DocMap: map[string]any{ "name": "Islam", @@ -86,18 +95,6 @@ func TestJSONArrayUniqueIndex_ShouldAllowOnlyUniqueValuesAndUseThemForFetching(t "bae-bde18215-f623-568e-868d-1156c30e45d3", errors.NewKV("custom", map[string]any{"numbers": []any{6, nil}})).Error(), }, - testUtils.CreateDoc{ - DocMap: map[string]any{ - "name": "Andy", - "custom": map[string]any{ - // existing non-array-element value - "numbers": 3, - }, - }, - ExpectedError: db.NewErrCanNotIndexNonUniqueFields( - "bae-54e76159-66c6-56be-ad65-7ff83edda058", - errors.NewKV("custom", map[string]any{"numbers": 3})).Error(), - }, testUtils.Request{ Request: req, Results: map[string]any{ diff --git a/tests/integration/query/json/with_none_test.go b/tests/integration/query/json/with_none_test.go index 1a65f7de60..2b33791198 100644 --- a/tests/integration/query/json/with_none_test.go +++ b/tests/integration/query/json/with_none_test.go @@ -107,9 +107,6 @@ func TestQueryJSON_WithNoneFilterAndNestedArray_ShouldFilter(t *testing.T) { Results: map[string]any{ "Users": []map[string]any{ {"name": "Shahzad"}, - {"name": "John"}, - {"name": "Islam"}, - {"name": "Bruno"}, }, }, },