feat: JSON type coercion (#3098)

## Relevant issue(s)

Resolves #3097

## Description

This PR fixes JSON type coercion and makes it much simpler to filter,
create, and update JSON fields.

## Tasks

- [x] I made sure the code is well commented, particularly
hard-to-understand areas.
- [x] I made sure the repository-held documentation is changed
accordingly.
- [x] I made sure the pull request title adheres to the conventional
commit style (the subset used in the project can be found in
[tools/configs/chglog/config.yml](tools/configs/chglog/config.yml)).
- [x] I made sure to discuss its limitations such as threats to
validity, vulnerability to mistake and misuse, robustness to
invalidation of assumptions, resource requirements, ...

## How has this been tested?

Added / updated unit and integration tests.

Specify the platform(s) on which this was tested:
- MacOS

client/document.go

@@ -341,7 +341,7 @@ func validateFieldSchema(val any, field FieldDefinition) (NormalValue, error) {
 		if err != nil {
 			return nil, err
 		}
-		return NewNormalString(v), nil
+		return NewNormalJSON(&JSON{v}), nil
 	}
 
 	return nil, NewErrUnhandledType("FieldKind", field.Kind)
@@ -417,16 +417,70 @@ func getDateTime(v any) (time.Time, error) {
 	return time.Parse(time.RFC3339, s)
 }
 
-func getJSON(v any) (string, error) {
-	s, err := getString(v)
-	if err != nil {
-		return "", err
-	}
-	val, err := fastjson.Parse(s)
-	if err != nil {
-		return "", NewErrInvalidJSONPaylaod(s)
+// getJSON converts the given value to a valid JSON value.
+//
+// If the value is of type *fastjson.Value it needs to be
+// manually parsed. All other values are valid JSON.
+func getJSON(v any) (any, error) {
+	val, ok := v.(*fastjson.Value)
+	if !ok {
+		return v, nil
+	}
+	switch val.Type() {
+	case fastjson.TypeArray:
+		arr, err := val.Array()
+		if err != nil {
+			return nil, err
+		}
+		out := make([]any, len(arr))
+		for i, v := range arr {
+			c, err := getJSON(v)
+			if err != nil {
+				return nil, err
+			}
+			out[i] = c
+		}
+		return out, nil
+
+	case fastjson.TypeObject:
+		obj, err := val.Object()
+		if err != nil {
+			return nil, err
+		}
+		out := make(map[string]any)
+		obj.Visit(func(key []byte, v *fastjson.Value) {
+			c, e := getJSON(v)
+			out[string(key)] = c
+			err = errors.Join(err, e)
+		})
+		return out, err
+
+	case fastjson.TypeFalse:
+		return false, nil
+
+	case fastjson.TypeTrue:
+		return true, nil
+
+	case fastjson.TypeNumber:
+		out, err := val.Int64()
+		if err == nil {
+			return out, nil
+		}
+		return val.Float64()
+
+	case fastjson.TypeString:
+		out, err := val.StringBytes()
+		if err != nil {
+			return nil, err
+		}
+		return string(out), nil
+
+	case fastjson.TypeNull:
+		return nil, nil
+
+	default:
+		return nil, NewErrInvalidJSONPayload(v)
 	}
-	return val.String(), nil
 }
 
 func getArray[T any](

client/document_test.go

@@ -161,7 +161,16 @@ func TestNewFromJSON_WithValidJSONFieldValue_NoError(t *testing.T) {
 	objWithJSONField := []byte(`{
 		"Name": "John",
 		"Age": 26,
-		"Custom": "{\"tree\":\"maple\", \"age\": 260}"
+		"Custom": {
+			"string": "maple", 
+			"int": 260,
+			"float": 3.14,
+			"false": false,
+			"true": true,
+			"null": null,
+			"array": ["one", 1],
+			"object": {"one": 1}
+		}
 	}`)
 	doc, err := NewDocFromJSON(objWithJSONField, def)
 	if err != nil {
@@ -183,28 +192,37 @@ func TestNewFromJSON_WithValidJSONFieldValue_NoError(t *testing.T) {
 	assert.Equal(t, doc.values[doc.fields["Name"]].IsDocument(), false)
 	assert.Equal(t, doc.values[doc.fields["Age"]].Value(), int64(26))
 	assert.Equal(t, doc.values[doc.fields["Age"]].IsDocument(), false)
-	assert.Equal(t, doc.values[doc.fields["Custom"]].Value(), "{\"tree\":\"maple\",\"age\":260}")
+	assert.Equal(t, doc.values[doc.fields["Custom"]].Value(), map[string]any{
+		"string": "maple",
+		"int":    int64(260),
+		"float":  float64(3.14),
+		"false":  false,
+		"true":   true,
+		"null":   nil,
+		"array":  []any{"one", int64(1)},
+		"object": map[string]any{"one": int64(1)},
+	})
 	assert.Equal(t, doc.values[doc.fields["Custom"]].IsDocument(), false)
 }
 
 func TestNewFromJSON_WithInvalidJSONFieldValue_Error(t *testing.T) {
 	objWithJSONField := []byte(`{
 		"Name": "John",
 		"Age": 26,
-		"Custom": "{\"tree\":\"maple, \"age\": 260}"
+		"Custom": {"tree":"maple, "age": 260}
 	}`)
 	_, err := NewDocFromJSON(objWithJSONField, def)
-	require.ErrorContains(t, err, "invalid JSON payload. Payload: {\"tree\":\"maple, \"age\": 260}")
+	require.ErrorContains(t, err, "cannot parse JSON")
 }
 
-func TestNewFromJSON_WithInvalidJSONFieldValueSimpleString_Error(t *testing.T) {
+func TestNewFromJSON_WithJSONFieldValueSimpleString_Succeed(t *testing.T) {
 	objWithJSONField := []byte(`{
 		"Name": "John",
 		"Age": 26,
 		"Custom": "blah"
 	}`)
 	_, err := NewDocFromJSON(objWithJSONField, def)
-	require.ErrorContains(t, err, "invalid JSON payload. Payload: blah")
+	require.NoError(t, err)
 }
 
 func TestIsJSONArray(t *testing.T) {

client/errors.go

@@ -166,7 +166,7 @@ func NewErrCRDTKindMismatch(cType, kind string) error {
 	return errors.New(fmt.Sprintf(errCRDTKindMismatch, cType, kind))
 }
 
-func NewErrInvalidJSONPaylaod(payload string) error {
+func NewErrInvalidJSONPayload(payload any) error {
 	return errors.New(errInvalidJSONPayload, errors.NewKV("Payload", payload))
 }
 

client/normal_new.go

@@ -64,6 +64,8 @@ func NewNormalValue(val any) (NormalValue, error) {
 		return NewNormalTime(v), nil
 	case *Document:
 		return NewNormalDocument(v), nil
+	case *JSON:
+		return NewNormalJSON(v), nil
 
 	case immutable.Option[bool]:
 		return NewNormalNillableBool(v), nil

client/normal_scalar.go

@@ -17,6 +17,13 @@ import (
 	"golang.org/x/exp/constraints"
 )
 
+// JSON contains a valid JSON value.
+//
+// The inner type can be any valid normal value or normal value array.
+type JSON struct {
+	inner any
+}
+
 // NormalValue is dummy implementation of NormalValue to be embedded in other types.
 type baseNormalValue[T any] struct {
 	NormalVoid
@@ -118,6 +125,18 @@ func (v normalDocument) Document() (*Document, bool) {
 	return v.val, true
 }
 
+type normalJSON struct {
+	baseNormalValue[*JSON]
+}
+
+func (v normalJSON) JSON() (*JSON, bool) {
+	return v.val, true
+}
+
+func (v normalJSON) Unwrap() any {
+	return v.val.inner
+}
+
 func newNormalInt(val int64) NormalValue {
 	return normalInt{newBaseNormalValue(val)}
 }
@@ -161,6 +180,11 @@ func NewNormalDocument(val *Document) NormalValue {
 	return normalDocument{baseNormalValue[*Document]{val: val}}
 }
 
+// NewNormalJSON creates a new NormalValue that represents a `JSON` value.
+func NewNormalJSON(val *JSON) NormalValue {
+	return normalJSON{baseNormalValue[*JSON]{val: val}}
+}
+
 func areNormalScalarsEqual[T comparable](val T, f func() (T, bool)) bool {
 	if otherVal, ok := f(); ok {
 		return val == otherVal

client/normal_value.go

@@ -62,6 +62,9 @@ type NormalValue interface {
 	// Document returns the value as a [*Document]. The second return flag is true if the value is a [*Document].
 	// Otherwise it will return nil and false.
 	Document() (*Document, bool)
+	// JSON returns the value as JSON. The second return flag is true if the value is JSON.
+	// Otherwise it will return nil and false.
+	JSON() (*JSON, bool)
 
 	// NillableBool returns the value as a nillable bool.
 	// The second return flag is true if the value is [immutable.Option[bool]].

client/normal_value_test.go

@@ -30,6 +30,7 @@ const (
 	BytesType    nType = "Bytes"
 	TimeType     nType = "Time"
 	DocumentType nType = "Document"
+	JSONType     nType = "JSON"
 
 	NillableBoolType     nType = "NillableBool"
 	NillableIntType      nType = "NillableInt"
@@ -76,6 +77,11 @@ const (
 // If it is and contains a value, it returns the contained value.
 // Otherwise, it returns the input itself.
 func extractValue(input any) any {
+	// unwrap JSON inner values
+	if v, ok := input.(*JSON); ok {
+		return v.inner
+	}
+
 	inputVal := reflect.ValueOf(input)
 
 	// Check if the type is Option[T] by seeing if it has the HasValue and Value methods.
@@ -112,6 +118,7 @@ func TestNormalValue_NewValueAndTypeAssertion(t *testing.T) {
 		BytesType:    func(v NormalValue) (any, bool) { return v.Bytes() },
 		TimeType:     func(v NormalValue) (any, bool) { return v.Time() },
 		DocumentType: func(v NormalValue) (any, bool) { return v.Document() },
+		JSONType:     func(v NormalValue) (any, bool) { return v.JSON() },
 
 		NillableBoolType:     func(v NormalValue) (any, bool) { return v.NillableBool() },
 		NillableIntType:      func(v NormalValue) (any, bool) { return v.NillableInt() },
@@ -164,6 +171,7 @@ func TestNormalValue_NewValueAndTypeAssertion(t *testing.T) {
 		BytesType:    func(v any) NormalValue { return NewNormalBytes(v.([]byte)) },
 		TimeType:     func(v any) NormalValue { return NewNormalTime(v.(time.Time)) },
 		DocumentType: func(v any) NormalValue { return NewNormalDocument(v.(*Document)) },
+		JSONType:     func(v any) NormalValue { return NewNormalJSON(v.(*JSON)) },
 
 		NillableBoolType:     func(v any) NormalValue { return NewNormalNillableBool(v.(immutable.Option[bool])) },
 		NillableIntType:      func(v any) NormalValue { return NewNormalNillableInt(v.(immutable.Option[int64])) },
@@ -283,6 +291,10 @@ func TestNormalValue_NewValueAndTypeAssertion(t *testing.T) {
 			nType: DocumentType,
 			input: &Document{},
 		},
+		{
+			nType: JSONType,
+			input: &JSON{nil},
+		},
 		{
 			nType:      NillableBoolType,
 			input:      immutable.Some(true),
@@ -830,6 +842,53 @@ func TestNormalValue_NewNormalValueFromAnyArray(t *testing.T) {
 	}
 }
 
+func TestNormalValue_NewNormalJSON(t *testing.T) {
+	var expect *JSON
+	var actual *JSON
+
+	expect = &JSON{nil}
+	normal := NewNormalJSON(expect)
+
+	actual, _ = normal.JSON()
+	assert.Equal(t, expect, actual)
+
+	expect = &JSON{"hello"}
+	normal = NewNormalJSON(expect)
+
+	actual, _ = normal.JSON()
+	assert.Equal(t, expect, actual)
+
+	expect = &JSON{true}
+	normal = NewNormalJSON(expect)
+
+	actual, _ = normal.JSON()
+	assert.Equal(t, expect, actual)
+
+	expect = &JSON{int64(10)}
+	normal = NewNormalJSON(expect)
+
+	actual, _ = normal.JSON()
+	assert.Equal(t, expect, actual)
+
+	expect = &JSON{float64(3.14)}
+	normal = NewNormalJSON(expect)
+
+	actual, _ = normal.JSON()
+	assert.Equal(t, expect, actual)
+
+	expect = &JSON{map[string]any{"one": 1}}
+	normal = NewNormalJSON(expect)
+
+	actual, _ = normal.JSON()
+	assert.Equal(t, expect, actual)
+
+	expect = &JSON{[]any{1, "two"}}
+	normal = NewNormalJSON(expect)
+
+	actual, _ = normal.JSON()
+	assert.Equal(t, expect, actual)
+}
+
 func TestNormalValue_NewNormalInt(t *testing.T) {
 	i64 := int64(2)
 	v := NewNormalInt(i64)

client/normal_void.go

@@ -65,6 +65,10 @@ func (NormalVoid) Document() (*Document, bool) {
 	return nil, false
 }
 
+func (NormalVoid) JSON() (*JSON, bool) {
+	return nil, false
+}
+
 func (NormalVoid) NillableBool() (immutable.Option[bool], bool) {
 	return immutable.None[bool](), false
 }

docs/data_format_changes/i3097-json-type-coercion.md

@@ -0,0 +1,3 @@
+# JSON type coercion
+
+JSON types are now stored as parsed values instead of strings.

go.mod

@@ -48,7 +48,7 @@ require (
 	github.com/sourcenetwork/badger/v4 v4.2.1-0.20231113215945-a63444ca5276
 	github.com/sourcenetwork/corelog v0.0.8
 	github.com/sourcenetwork/go-libp2p-pubsub-rpc v0.0.14
-	github.com/sourcenetwork/graphql-go v0.7.10-0.20240924172903-a4088313b20d
+	github.com/sourcenetwork/graphql-go v0.7.10-0.20241003221550-224346887b4a
 	github.com/sourcenetwork/immutable v0.3.0
 	github.com/sourcenetwork/sourcehub v0.2.1-0.20240704194128-f43f5e427274
 	github.com/spf13/cobra v1.8.1

go.sum

@@ -1398,8 +1398,8 @@ github.com/sourcenetwork/corelog v0.0.8 h1:jCo0mFBpWrfhUCGzzN3uUtPGyQv3jnITdPO1s
 github.com/sourcenetwork/corelog v0.0.8/go.mod h1:cMabHgs3kARgYTQeQYSOmaGGP8XMU6sZrHd8LFrL3zA=
 github.com/sourcenetwork/go-libp2p-pubsub-rpc v0.0.14 h1:620zKV4rOn7U5j/WsPkk4SFj0z9/pVV4bBx0BpZQgro=
 github.com/sourcenetwork/go-libp2p-pubsub-rpc v0.0.14/go.mod h1:jUoQv592uUX1u7QBjAY4C+l24X9ArhPfifOqXpDHz4U=
-github.com/sourcenetwork/graphql-go v0.7.10-0.20240924172903-a4088313b20d h1:P5y4g1ONf8HK36L86/8zDYjY7rRLM7AaqlQDRHOBMH8=
-github.com/sourcenetwork/graphql-go v0.7.10-0.20240924172903-a4088313b20d/go.mod h1:rkahXkgRH/3vZErN1Bx+qt1+w+CV5fgaJyKKWgISe4U=
+github.com/sourcenetwork/graphql-go v0.7.10-0.20241003221550-224346887b4a h1:wF7VhX0XKuNqhbbDI7RjoIBKFvdKQKucpZlNr/BGE40=
+github.com/sourcenetwork/graphql-go v0.7.10-0.20241003221550-224346887b4a/go.mod h1:rkahXkgRH/3vZErN1Bx+qt1+w+CV5fgaJyKKWgISe4U=
 github.com/sourcenetwork/immutable v0.3.0 h1:gHPtGvLrTBTK5YpDAhMU+u+S8v1F6iYmc3nbZLryMdc=
 github.com/sourcenetwork/immutable v0.3.0/go.mod h1:GD7ceuh/HD7z6cdIwzKK2ctzgZ1qqYFJpsFp+8qYnbI=
 github.com/sourcenetwork/raccoondb v0.2.1-0.20240606193653-1e91e9be9234 h1:8dA9bVC1A0ChJygtsUfNsek3oR0GnwpLoYpmEo4t2mk=