feat: JSON type filter (#3122)

## Relevant issue(s)

Resolves #3106

## Description

This PR enables filtering on JSON field types.

## Tasks

- [x] I made sure the code is well commented, particularly
hard-to-understand areas.
- [x] I made sure the repository-held documentation is changed
accordingly.
- [x] I made sure the pull request title adheres to the conventional
commit style (the subset used in the project can be found in
[tools/configs/chglog/config.yml](tools/configs/chglog/config.yml)).
- [x] I made sure to discuss its limitations such as threats to
validity, vulnerability to mistake and misuse, robustness to
invalidation of assumptions, resource requirements, ...

## How has this been tested?

Added integration tests.

Specify the platform(s) on which this was tested:
- MacOS

internal/connor/all.go

@@ -11,6 +11,9 @@ import (
 // matching if all of them match.
 func all(condition, data any) (bool, error) {
 	switch t := data.(type) {
+	case []any:
+		return allSlice(condition, t)
+
 	case []string:
 		return allSlice(condition, t)
 

internal/connor/any.go

@@ -11,6 +11,9 @@ import (
 // matching if any of them match.
 func anyOp(condition, data any) (bool, error) {
 	switch t := data.(type) {
+	case []any:
+		return anySlice(condition, t)
+
 	case []string:
 		return anySlice(condition, t)
 

internal/planner/mapper/mapper.go

@@ -1286,7 +1286,7 @@ func ToFilter(source request.Filter, mapping *core.DocumentMapping) *Filter {
 	conditions := make(map[connor.FilterKey]any, len(source.Conditions))
 
 	for sourceKey, sourceClause := range source.Conditions {
-		key, clause := toFilterMap(sourceKey, sourceClause, mapping)
+		key, clause := toFilterKeyValue(sourceKey, sourceClause, mapping)
 		conditions[key] = clause
 	}
 
@@ -1296,87 +1296,102 @@ func ToFilter(source request.Filter, mapping *core.DocumentMapping) *Filter {
 	}
 }
 
-// toFilterMap converts a consumer-defined filter key-value into a filter clause
-// keyed by field index.
+// toFilterKeyValue converts a consumer-defined filter key-value into a filter clause
+// keyed by connor.FilterKey.
 //
-// Return key will either be an int (field index), or a string (operator).
-func toFilterMap(
+// The returned key will be one of the following:
+// - Operator: if the sourceKey is one of the defined filter operators
+// - PropertyIndex: if the sourceKey exists in the document mapping
+// - ObjectProperty: if the sourceKey does not match one of the above
+func toFilterKeyValue(
 	sourceKey string,
 	sourceClause any,
 	mapping *core.DocumentMapping,
 ) (connor.FilterKey, any) {
+	var returnKey connor.FilterKey
 	if strings.HasPrefix(sourceKey, "_") && sourceKey != request.DocIDFieldName {
-		key := &Operator{
+		returnKey = &Operator{
 			Operation: sourceKey,
 		}
 		// if the operator is simple (not compound) then
 		// it does not require further expansion
 		if connor.IsOpSimple(sourceKey) {
-			return key, sourceClause
-		}
-		switch typedClause := sourceClause.(type) {
-		case []any:
-			// If the clause is an array then we need to convert any inner maps.
-			returnClauses := []any{}
-			for _, innerSourceClause := range typedClause {
-				var returnClause any
-				switch typedInnerSourceClause := innerSourceClause.(type) {
-				case map[string]any:
-					innerMapClause := map[connor.FilterKey]any{}
-					for innerSourceKey, innerSourceValue := range typedInnerSourceClause {
-						rKey, rValue := toFilterMap(innerSourceKey, innerSourceValue, mapping)
-						innerMapClause[rKey] = rValue
-					}
-					returnClause = innerMapClause
-				default:
-					returnClause = innerSourceClause
-				}
-				returnClauses = append(returnClauses, returnClause)
-			}
-			return key, returnClauses
-		case map[string]any:
-			innerMapClause := map[connor.FilterKey]any{}
-			for innerSourceKey, innerSourceValue := range typedClause {
-				rKey, rValue := toFilterMap(innerSourceKey, innerSourceValue, mapping)
-				innerMapClause[rKey] = rValue
-			}
-			return key, innerMapClause
-		default:
-			return key, typedClause
+			return returnKey, sourceClause
 		}
-	} else {
+	} else if mapping != nil && len(mapping.IndexesByName[sourceKey]) > 0 {
 		// If there are multiple properties of the same name we can just take the first as
 		// we have no other reasonable way of identifying which property they mean if multiple
 		// consumer specified requestables are available.  Aggregate dependencies should not
 		// impact this as they are added after selects.
-		index := mapping.FirstIndexOfName(sourceKey)
-		key := &PropertyIndex{
-			Index: index,
+		returnKey = &PropertyIndex{
+			Index: mapping.FirstIndexOfName(sourceKey),
 		}
-		switch typedClause := sourceClause.(type) {
-		case map[string]any:
-			returnClause := map[connor.FilterKey]any{}
-			for innerSourceKey, innerSourceValue := range typedClause {
-				var innerMapping *core.DocumentMapping
-				// innerSourceValue may refer to a child mapping or
-				// an inline array if we don't have a child mapping
-				_, ok := innerSourceValue.(map[string]any)
-				if ok && index < len(mapping.ChildMappings) {
-					// If the innerSourceValue is also a map, then we should parse the nested clause
-					// using the child mapping, as this key must refer to a host property in a join
-					// and deeper keys must refer to properties on the child items.
-					innerMapping = mapping.ChildMappings[index]
-				} else {
-					innerMapping = mapping
-				}
-				rKey, rValue := toFilterMap(innerSourceKey, innerSourceValue, innerMapping)
-				returnClause[rKey] = rValue
+	} else {
+		returnKey = &ObjectProperty{
+			Name: sourceKey,
+		}
+	}
+
+	switch typedClause := sourceClause.(type) {
+	case []any:
+		return returnKey, toFilterList(typedClause, mapping)
+
+	case map[string]any:
+		return returnKey, toFilterMap(returnKey, typedClause, mapping)
+
+	default:
+		return returnKey, typedClause
+	}
+}
+
+func toFilterMap(
+	sourceKey connor.FilterKey,
+	sourceClause map[string]any,
+	mapping *core.DocumentMapping,
+) map[connor.FilterKey]any {
+	innerMapClause := make(map[connor.FilterKey]any)
+	for innerSourceKey, innerSourceValue := range sourceClause {
+		var innerMapping *core.DocumentMapping
+		switch t := sourceKey.(type) {
+		case *PropertyIndex:
+			_, ok := innerSourceValue.(map[string]any)
+			if ok && mapping != nil && t.Index < len(mapping.ChildMappings) {
+				// If the innerSourceValue is also a map, then we should parse the nested clause
+				// using the child mapping, as this key must refer to a host property in a join
+				// and deeper keys must refer to properties on the child items.
+				innerMapping = mapping.ChildMappings[t.Index]
+			} else {
+				innerMapping = mapping
 			}
-			return key, returnClause
-		default:
-			return key, sourceClause
+		case *ObjectProperty:
+			// Object properties can never refer to mapped document fields.
+			// Set the mapping to null for any nested filter values so
+			// that we don't filter any fields outside of this object.
+			innerMapping = nil
+		case *Operator:
+			innerMapping = mapping
+		}
+		rKey, rValue := toFilterKeyValue(innerSourceKey, innerSourceValue, innerMapping)
+		innerMapClause[rKey] = rValue
+	}
+	return innerMapClause
+}
+
+func toFilterList(sourceClause []any, mapping *core.DocumentMapping) []any {
+	returnClauses := make([]any, len(sourceClause))
+	for i, innerSourceClause := range sourceClause {
+		// innerSourceClause must be a map because only compound
+		// operators (_and, _or) can reach this function and should
+		// have already passed GQL type validation
+		typedInnerSourceClause := innerSourceClause.(map[string]any)
+		innerMapClause := make(map[connor.FilterKey]any)
+		for innerSourceKey, innerSourceValue := range typedInnerSourceClause {
+			rKey, rValue := toFilterKeyValue(innerSourceKey, innerSourceValue, mapping)
+			innerMapClause[rKey] = rValue
 		}
+		returnClauses[i] = innerMapClause
 	}
+	return returnClauses
 }
 
 func toLimit(limit immutable.Option[uint64], offset immutable.Option[uint64]) *Limit {

internal/planner/mapper/targetable.go

@@ -21,6 +21,7 @@ import (
 var (
 	_ connor.FilterKey = (*PropertyIndex)(nil)
 	_ connor.FilterKey = (*Operator)(nil)
+	_ connor.FilterKey = (*ObjectProperty)(nil)
 )
 
 // PropertyIndex is a FilterKey that represents a property in a document.
@@ -71,6 +72,34 @@ func (k *Operator) Equal(other connor.FilterKey) bool {
 	return false
 }
 
+// ObjectProperty is a FilterKey that represents a property in an object.
+//
+// This is used to target properties of an object when the fields
+// are not explicitly mapped, such as with JSON.
+type ObjectProperty struct {
+	// The name of the property on object.
+	Name string
+}
+
+func (k *ObjectProperty) GetProp(data any) any {
+	if data == nil {
+		return nil
+	}
+	object := data.(map[string]any)
+	return object[k.Name]
+}
+
+func (k *ObjectProperty) GetOperatorOrDefault(defaultOp string) string {
+	return defaultOp
+}
+
+func (k *ObjectProperty) Equal(other connor.FilterKey) bool {
+	if otherKey, isOk := other.(*ObjectProperty); isOk && *k == *otherKey {
+		return true
+	}
+	return false
+}
+
 // Filter represents a series of conditions that may reduce the number of
 // records that a request returns.
 type Filter struct {
@@ -144,6 +173,14 @@ func filterObjectToMap(mapping *core.DocumentMapping, obj map[connor.FilterKey]a
 			default:
 				outmap[keyType.Operation] = v
 			}
+
+		case *ObjectProperty:
+			switch subObj := v.(type) {
+			case map[connor.FilterKey]any:
+				outmap[keyType.Name] = filterObjectToMap(mapping, subObj)
+			case nil:
+				outmap[keyType.Name] = nil
+			}
 		}
 	}
 	return outmap

internal/request/graphql/schema/generate.go

@@ -1202,43 +1202,18 @@ func (g *Generator) genTypeFilterArgInput(obj *gql.Object) *gql.InputObject {
 			}
 
 			// generate basic filter operator blocks
-			// @todo: Extract object field loop into its own utility func
 			for f, field := range obj.Fields() {
-				if _, ok := request.ReservedFields[f]; ok && f != request.DocIDFieldName {
+				_, ok := request.ReservedFields[f]
+				if ok && f != request.DocIDFieldName {
 					continue
 				}
-				// scalars (leafs)
-				if gql.IsLeafType(field.Type) {
-					var operatorName string
-					if list, isList := field.Type.(*gql.List); isList {
-						if notNull, isNotNull := list.OfType.(*gql.NonNull); isNotNull {
-							operatorName = "NotNull" + notNull.OfType.Name() + "ListOperatorBlock"
-						} else {
-							operatorName = list.OfType.Name() + "ListOperatorBlock"
-						}
-					} else {
-						operatorName = field.Type.Name() + "OperatorBlock"
-					}
-					operatorType, isFilterable := g.manager.schema.TypeMap()[operatorName]
-					if !isFilterable {
-						continue
-					}
-					fields[field.Name] = &gql.InputObjectFieldConfig{
-						Type: operatorType,
-					}
-				} else { // objects (relations)
-					fieldType := field.Type
-					if l, isList := field.Type.(*gql.List); isList {
-						// We want the FilterArg for the object, not the list of objects.
-						fieldType = l.OfType
-					}
-					filterType, isFilterable := g.manager.schema.TypeMap()[genTypeName(fieldType, filterInputNameSuffix)]
-					if !isFilterable {
-						filterType = &gql.InputObjectField{}
-					}
-					fields[field.Name] = &gql.InputObjectFieldConfig{
-						Type: filterType,
-					}
+				operatorName := genFilterOperatorName(field.Type)
+				filterType, isFilterable := g.manager.schema.TypeMap()[operatorName]
+				if !isFilterable {
+					continue
+				}
+				fields[field.Name] = &gql.InputObjectFieldConfig{
+					Type: filterType,
 				}
 			}
 
@@ -1408,6 +1383,35 @@ func isNumericArray(list *gql.List) bool {
 		list.OfType == gql.Float
 }
 
+func genFilterOperatorName(fieldType gql.Type) string {
+	list, isList := fieldType.(*gql.List)
+	if isList {
+		fieldType = list.OfType
+	}
+	if !gql.IsLeafType(fieldType) {
+		return genTypeName(fieldType, filterInputNameSuffix)
+	}
+	notNull, isNotNull := fieldType.(*gql.NonNull)
+	if isNotNull {
+		fieldType = notNull.OfType
+	}
+	switch {
+	case fieldType.Name() == "JSON":
+		return fieldType.Name()
+
+	case isList && isNotNull:
+		// todo: There's a potential to have a name clash
+		// https://github.com/sourcenetwork/defradb/issues/3123
+		return "NotNull" + fieldType.Name() + "ListOperatorBlock"
+
+	case isList:
+		return fieldType.Name() + "ListOperatorBlock"
+
+	default:
+		return fieldType.Name() + "OperatorBlock"
+	}
+}
+
 /* Example
 
 typeDefs := ` ... `

internal/request/graphql/schema/manager.go

@@ -193,15 +193,13 @@ func defaultTypes(
 	floatOpBlock := schemaTypes.FloatOperatorBlock()
 	booleanOpBlock := schemaTypes.BooleanOperatorBlock()
 	stringOpBlock := schemaTypes.StringOperatorBlock()
-	jsonOpBlock := schemaTypes.JSONOperatorBlock(jsonScalarType)
 	blobOpBlock := schemaTypes.BlobOperatorBlock(blobScalarType)
 	dateTimeOpBlock := schemaTypes.DateTimeOperatorBlock()
 
 	notNullIntOpBlock := schemaTypes.NotNullIntOperatorBlock()
 	notNullFloatOpBlock := schemaTypes.NotNullFloatOperatorBlock()
 	notNullBooleanOpBlock := schemaTypes.NotNullBooleanOperatorBlock()
 	notNullStringOpBlock := schemaTypes.NotNullStringOperatorBlock()
-	notNullJSONOpBlock := schemaTypes.NotNullJSONOperatorBlock(jsonScalarType)
 	notNullBlobOpBlock := schemaTypes.NotNullBlobOperatorBlock(blobScalarType)
 
 	return []gql.Type{
@@ -228,7 +226,6 @@ func defaultTypes(
 		floatOpBlock,
 		booleanOpBlock,
 		stringOpBlock,
-		jsonOpBlock,
 		blobOpBlock,
 		dateTimeOpBlock,
 
@@ -237,7 +234,6 @@ func defaultTypes(
 		notNullFloatOpBlock,
 		notNullBooleanOpBlock,
 		notNullStringOpBlock,
-		notNullJSONOpBlock,
 		notNullBlobOpBlock,
 
 		// Filter scalar list blocks