From ba79b3b5597a8b2b5753d9254a2bdee51ff83f8d Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Wed, 24 Jan 2024 10:39:18 -0800 Subject: [PATCH] feat: Add JSON scalar (#2254) ## Relevant issue(s) Resolves #2243 ## Description This PR adds a JSON scalar type to the schema system ## Tasks - [x] I made sure the code is well commented, particularly hard-to-understand areas. - [x] I made sure the repository-held documentation is changed accordingly. - [x] I made sure the pull request title adheres to the conventional commit style (the subset used in the project can be found in [tools/configs/chglog/config.yml](tools/configs/chglog/config.yml)). - [x] I made sure to discuss its limitations such as threats to validity, vulnerability to mistake and misuse, robustness to invalidation of assumptions, resource requirements, ... ## How has this been tested? make test Specify the platform(s) on which this was tested: - MacOS --------- Co-authored-by: Shahzad Lone --- client/descriptions.go | 5 +- client/document.go | 2 +- request/graphql/schema/collection.go | 3 + request/graphql/schema/descriptions.go | 3 + request/graphql/schema/types/scalars.go | 53 +++++++ request/graphql/schema/types/scalars_test.go | 83 +++++++++++ .../updates/add/field/kind/invalid_test.go | 24 --- .../updates/add/field/kind/json_test.go | 137 ++++++++++++++++++ 8 files changed, 284 insertions(+), 26 deletions(-) create mode 100644 tests/integration/schema/updates/add/field/kind/json_test.go diff --git a/client/descriptions.go b/client/descriptions.go index cb312e8fac..ca88f9362e 100644 --- a/client/descriptions.go +++ b/client/descriptions.go @@ -183,6 +183,8 @@ func (f FieldKind) String() string { return "[String!]" case FieldKind_BLOB: return "Blob" + case FieldKind_JSON: + return "JSON" default: return fmt.Sprint(uint8(f)) } @@ -204,7 +206,7 @@ const ( FieldKind_STRING FieldKind = 11 FieldKind_STRING_ARRAY FieldKind = 12 FieldKind_BLOB FieldKind = 13 - _ FieldKind = 14 // safe to repurpose (was never used) + FieldKind_JSON FieldKind = 14 _ FieldKind = 15 // safe to repurpose (was never used) // Embedded object, but accessed via foreign keys @@ -242,6 +244,7 @@ var FieldKindStringToEnumMapping = map[string]FieldKind{ "[String]": FieldKind_NILLABLE_STRING_ARRAY, "[String!]": FieldKind_STRING_ARRAY, "Blob": FieldKind_BLOB, + "JSON": FieldKind_JSON, } // RelationType describes the type of relation between two types. diff --git a/client/document.go b/client/document.go index 93e06df27e..7345f10d09 100644 --- a/client/document.go +++ b/client/document.go @@ -177,7 +177,7 @@ func NewDocsFromJSON(obj []byte, sd SchemaDescription) ([]*Document, error) { // the typed value again as an interface. func validateFieldSchema(val any, field FieldDescription) (any, error) { switch field.Kind { - case FieldKind_DocID, FieldKind_STRING, FieldKind_BLOB: + case FieldKind_DocID, FieldKind_STRING, FieldKind_BLOB, FieldKind_JSON: return getString(val) case FieldKind_STRING_ARRAY: diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index d3e2e35c36..ad4f7bb855 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -415,6 +415,7 @@ func astTypeToKind(t ast.Type) (client.FieldKind, error) { typeDateTime string = "DateTime" typeString string = "String" typeBlob string = "Blob" + typeJSON string = "JSON" ) switch astTypeVal := t.(type) { @@ -465,6 +466,8 @@ func astTypeToKind(t ast.Type) (client.FieldKind, error) { return client.FieldKind_STRING, nil case typeBlob: return client.FieldKind_BLOB, nil + case typeJSON: + return client.FieldKind_JSON, nil default: return client.FieldKind_FOREIGN_OBJECT, nil } diff --git a/request/graphql/schema/descriptions.go b/request/graphql/schema/descriptions.go index 147c494c74..01b1d8b0cb 100644 --- a/request/graphql/schema/descriptions.go +++ b/request/graphql/schema/descriptions.go @@ -34,6 +34,7 @@ var ( &gql.List{}: client.FieldKind_FOREIGN_OBJECT_ARRAY, // Custom scalars schemaTypes.BlobScalarType: client.FieldKind_BLOB, + schemaTypes.JSONScalarType: client.FieldKind_JSON, // More custom ones to come // - JSON // - Counters @@ -55,6 +56,7 @@ var ( client.FieldKind_STRING_ARRAY: gql.NewList(gql.NewNonNull(gql.String)), client.FieldKind_NILLABLE_STRING_ARRAY: gql.NewList(gql.String), client.FieldKind_BLOB: schemaTypes.BlobScalarType, + client.FieldKind_JSON: schemaTypes.JSONScalarType, } // This map is fine to use @@ -74,6 +76,7 @@ var ( client.FieldKind_STRING_ARRAY: client.LWW_REGISTER, client.FieldKind_NILLABLE_STRING_ARRAY: client.LWW_REGISTER, client.FieldKind_BLOB: client.LWW_REGISTER, + client.FieldKind_JSON: client.LWW_REGISTER, client.FieldKind_FOREIGN_OBJECT: client.LWW_REGISTER, client.FieldKind_FOREIGN_OBJECT_ARRAY: client.NONE_CRDT, } diff --git a/request/graphql/schema/types/scalars.go b/request/graphql/schema/types/scalars.go index a0e9dca369..1d944a0f73 100644 --- a/request/graphql/schema/types/scalars.go +++ b/request/graphql/schema/types/scalars.go @@ -16,6 +16,7 @@ import ( "github.com/sourcenetwork/graphql-go" "github.com/sourcenetwork/graphql-go/language/ast" + "github.com/valyala/fastjson" ) // BlobPattern is a regex for validating blob hex strings @@ -63,3 +64,55 @@ var BlobScalarType = graphql.NewScalar(graphql.ScalarConfig{ } }, }) + +// coerceJSON converts the given value into a valid json string. +// If the value cannot be converted nil is returned. +func coerceJSON(value any) any { + switch value := value.(type) { + case []byte: + err := fastjson.ValidateBytes(value) + if err != nil { + // ignore this error because the value + // cannot be converted to a json string + return nil + } + return string(value) + + case *[]byte: + return coerceJSON(*value) + + case string: + err := fastjson.Validate(value) + if err != nil { + // ignore this error because the value + // cannot be converted to a json string + return nil + } + return value + + case *string: + return coerceJSON(*value) + + default: + return nil + } +} + +var JSONScalarType = graphql.NewScalar(graphql.ScalarConfig{ + Name: "JSON", + Description: "The `JSON` scalar type represents a JSON string.", + // Serialize converts the value to a json string + Serialize: coerceJSON, + // ParseValue converts the value to a json string + ParseValue: coerceJSON, + // ParseLiteral converts the ast value to a json string + ParseLiteral: func(valueAST ast.Value) any { + switch valueAST := valueAST.(type) { + case *ast.StringValue: + return coerceJSON(valueAST.Value) + default: + // return nil if the value cannot be parsed + return nil + } + }, +}) diff --git a/request/graphql/schema/types/scalars_test.go b/request/graphql/schema/types/scalars_test.go index 5126f2e6a2..6be3fa23fa 100644 --- a/request/graphql/schema/types/scalars_test.go +++ b/request/graphql/schema/types/scalars_test.go @@ -86,3 +86,86 @@ func TestBlobScalarTypeParseLiteral(t *testing.T) { assert.Equal(t, c.expect, result) } } + +func TestJSONScalarTypeParseAndSerialize(t *testing.T) { + validString := `"hello"` + validBytes := []byte(`"hello"`) + + boolString := "true" + boolBytes := []byte("true") + + intString := "0" + intBytes := []byte("0") + + floatString := "3.14" + floatBytes := []byte("3.14") + + objectString := `{"name": "Bob"}` + objectBytes := []byte(`{"name": "Bob"}`) + + invalidString := "invalid" + invalidBytes := []byte("invalid") + + cases := []struct { + input any + expect any + }{ + {validString, `"hello"`}, + {&validString, `"hello"`}, + {validBytes, `"hello"`}, + {&validBytes, `"hello"`}, + {boolString, "true"}, + {&boolString, "true"}, + {boolBytes, "true"}, + {&boolBytes, "true"}, + {[]byte("true"), "true"}, + {[]byte("false"), "false"}, + {intString, "0"}, + {&intString, "0"}, + {intBytes, "0"}, + {&intBytes, "0"}, + {floatString, "3.14"}, + {&floatString, "3.14"}, + {floatBytes, "3.14"}, + {&floatBytes, "3.14"}, + {invalidString, nil}, + {&invalidString, nil}, + {invalidBytes, nil}, + {&invalidBytes, nil}, + {objectString, `{"name": "Bob"}`}, + {&objectString, `{"name": "Bob"}`}, + {objectBytes, `{"name": "Bob"}`}, + {&objectBytes, `{"name": "Bob"}`}, + {nil, nil}, + {0, nil}, + {false, nil}, + } + for _, c := range cases { + parsed := JSONScalarType.ParseValue(c.input) + assert.Equal(t, c.expect, parsed) + + serialized := JSONScalarType.Serialize(c.input) + assert.Equal(t, c.expect, serialized) + } +} + +func TestJSONScalarTypeParseLiteral(t *testing.T) { + cases := []struct { + input ast.Value + expect any + }{ + {&ast.StringValue{Value: "0"}, "0"}, + {&ast.StringValue{Value: "invalid"}, nil}, + {&ast.IntValue{}, nil}, + {&ast.BooleanValue{}, nil}, + {&ast.NullValue{}, nil}, + {&ast.EnumValue{}, nil}, + {&ast.FloatValue{}, nil}, + {&ast.ListValue{}, nil}, + {&ast.ObjectValue{}, nil}, + } + for _, c := range cases { + result := JSONScalarType.ParseLiteral(c.input) + assert.Equal(t, c.expect, result) + } +} diff --git a/tests/integration/schema/updates/add/field/kind/invalid_test.go b/tests/integration/schema/updates/add/field/kind/invalid_test.go index 98f026ecc2..b9c6dbbf31 100644 --- a/tests/integration/schema/updates/add/field/kind/invalid_test.go +++ b/tests/integration/schema/updates/add/field/kind/invalid_test.go @@ -64,30 +64,6 @@ func TestSchemaUpdatesAddFieldKind9(t *testing.T) { testUtils.ExecuteTestCase(t, test) } -func TestSchemaUpdatesAddFieldKind14(t *testing.T) { - test := testUtils.TestCase{ - Description: "Test schema update, add field with kind deprecated (14)", - Actions: []any{ - testUtils.SchemaUpdate{ - Schema: ` - type Users { - name: String - } - `, - }, - testUtils.SchemaPatch{ - Patch: ` - [ - { "op": "add", "path": "/Users/Fields/-", "value": {"Name": "foo", "Kind": 14} } - ] - `, - ExpectedError: "no type found for given name. Type: 14", - }, - }, - } - testUtils.ExecuteTestCase(t, test) -} - func TestSchemaUpdatesAddFieldKind15(t *testing.T) { test := testUtils.TestCase{ Description: "Test schema update, add field with kind deprecated (15)", diff --git a/tests/integration/schema/updates/add/field/kind/json_test.go b/tests/integration/schema/updates/add/field/kind/json_test.go new file mode 100644 index 0000000000..37e2886a58 --- /dev/null +++ b/tests/integration/schema/updates/add/field/kind/json_test.go @@ -0,0 +1,137 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package kind + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestSchemaUpdatesAddFieldKindJSON(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test schema update, add field with kind json (14)", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Fields/-", "value": {"Name": "foo", "Kind": 14} } + ] + `, + }, + testUtils.Request{ + Request: `query { + Users { + name + foo + } + }`, + Results: []map[string]any{}, + }, + }, + } + testUtils.ExecuteTestCase(t, test) +} + +func TestSchemaUpdatesAddFieldKindJSONWithCreate(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test schema update, add field with kind json (14) with create", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Fields/-", "value": {"Name": "foo", "Kind": 14} } + ] + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "John", + "foo": "{}" + }`, + }, + testUtils.Request{ + Request: `query { + Users { + name + foo + } + }`, + Results: []map[string]any{ + { + "name": "John", + "foo": "{}", + }, + }, + }, + }, + } + testUtils.ExecuteTestCase(t, test) +} + +func TestSchemaUpdatesAddFieldKindJSONSubstitutionWithCreate(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test schema update, add field with kind json substitution with create", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Fields/-", "value": {"Name": "foo", "Kind": "JSON"} } + ] + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "John", + "foo": "{}" + }`, + }, + testUtils.Request{ + Request: `query { + Users { + name + foo + } + }`, + Results: []map[string]any{ + { + "name": "John", + "foo": "{}", + }, + }, + }, + }, + } + testUtils.ExecuteTestCase(t, test) +}