feat: Move relation field properties onto collection (#2529)

## Relevant issue(s)

Resolves #2451 #1911 #2408

## Description

Moves relation field properties onto collection description from schema
description.

This allows for one-sided relations to be defined via `PatchSchema`.
There is no way atm to create them using an SDL, but we could add a
directive or something at somepoint if we want.

As the adding of fields via `PatchCollection` has not yet been enabled,
this does prevent users from adding secondary relation fields to an
existing schema. I'll create a ticket before merging this to allow this,
it is my strong preference to not do that in this already large and
fiddly PR though.

As the internal codebase relies on the setting of secondary fields via
`client.Document`, `client.Document` now requires a collection, not just
the schema, when being constructed. We will likely want to find a way to
avoid that in the future.

This PR also moves some validation from the graphql package into `db`,
not all of it has been moved, but that is a long term wish of mine. The
`db` package validation can/should be improved further making rule reuse
across `PatchCollection`, `PatchSchema` and `CreateCollection` (SDL),
however I would rather not spend too much effort on that in this PR.
This includes moving it out of the `collection.go` file.

It might resolve #2380
but I'd rather not bring that into scope. If I'm waiting around for
reviews I might verify that here though.

This should conclude the transfer of local properties off of the schema
object :)

acp/README.md

@@ -212,7 +212,7 @@ Result:
     "Name": "Users",
     "ID": 1,
     "RootID": 1,
-    "SchemaVersionID": "bafkreibthhctfd3rykinfa6ivvkhegp7sbhk5yvujdkhase7ilj5dz5gqi",
+    "SchemaVersionID": "bafkreihhd6bqrjhl5zidwztgxzeseveplv3cj3fwtn3unjkdx7j2vr2vrq",
     "Sources": [],
     "Fields": [
       {

cli/collection_create.go

@@ -69,14 +69,14 @@ Example: create from stdin:
 			}
 
 			if client.IsJSONArray(docData) {
-				docs, err := client.NewDocsFromJSON(docData, col.Schema())
+				docs, err := client.NewDocsFromJSON(docData, col.Definition())
 				if err != nil {
 					return err
 				}
 				return col.CreateMany(cmd.Context(), docs)
 			}
 
-			doc, err := client.NewDocFromJSON(docData, col.Schema())
+			doc, err := client.NewDocFromJSON(docData, col.Definition())
 			if err != nil {
 				return err
 			}

client/collection_description.go

@@ -60,7 +60,11 @@ type CollectionDescription struct {
 	// - [CollectionSource]
 	Sources []any
 
-	// Fields contains the fields within this Collection.
+	// Fields contains the fields local to the node within this Collection.
+	//
+	// Most fields defined here will also be present on the [SchemaDescription]. A notable
+	// exception to this are the fields of the (optional) secondary side of a relation
+	// which are local only, and will not be present on the [SchemaDescription].
 	Fields []CollectionFieldDescription
 
 	// Indexes contains the secondary indexes that this Collection has.
@@ -136,16 +140,15 @@ func (col CollectionDescription) GetFieldByRelation(
 	relationName string,
 	otherCollectionName string,
 	otherFieldName string,
-	schema *SchemaDescription,
-) (SchemaFieldDescription, bool) {
-	for _, field := range schema.Fields {
-		if field.RelationName == relationName &&
+) (CollectionFieldDescription, bool) {
+	for _, field := range col.Fields {
+		if field.RelationName.Value() == relationName &&
 			!(col.Name.Value() == otherCollectionName && otherFieldName == field.Name) &&
-			field.Kind != FieldKind_DocID {
+			field.Kind.Value() != FieldKind_DocID {
 			return field, true
 		}
 	}
-	return SchemaFieldDescription{}, false
+	return CollectionFieldDescription{}, false
 }
 
 // QuerySources returns all the Sources of type [QuerySource]

client/collection_field_description.go

@@ -10,7 +10,12 @@
 
 package client
 
-import "fmt"
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/sourcenetwork/immutable"
+)
 
 // FieldID is a unique identifier for a field in a schema.
 type FieldID uint32
@@ -22,8 +27,52 @@ type CollectionFieldDescription struct {
 
 	// ID contains the local, internal ID of this field.
 	ID FieldID
+
+	// Kind contains the local field kind if this is a local-only field (e.g. the secondary
+	// side of a relation).
+	//
+	// If the field is globaly defined (on the Schema), this will be [None].
+	Kind immutable.Option[FieldKind]
+
+	// RelationName contains the name of this relation, if this field is part of a relationship.
+	//
+	// Otherwise will be [None].
+	RelationName immutable.Option[string]
 }
 
 func (f FieldID) String() string {
 	return fmt.Sprint(uint32(f))
 }
+
+// collectionFieldDescription is a private type used to facilitate the unmarshalling
+// of json to a [CollectionFieldDescription].
+type collectionFieldDescription struct {
+	Name         string
+	ID           FieldID
+	RelationName immutable.Option[string]
+
+	// Properties below this line are unmarshalled using custom logic in [UnmarshalJSON]
+	Kind json.RawMessage
+}
+
+func (f *CollectionFieldDescription) UnmarshalJSON(bytes []byte) error {
+	var descMap collectionFieldDescription
+	err := json.Unmarshal(bytes, &descMap)
+	if err != nil {
+		return err
+	}
+
+	f.Name = descMap.Name
+	f.ID = descMap.ID
+	f.RelationName = descMap.RelationName
+	kind, err := parseFieldKind(descMap.Kind)
+	if err != nil {
+		return err
+	}
+
+	if kind != FieldKind_None {
+		f.Kind = immutable.Some(kind)
+	}
+
+	return nil
+}

client/definitions.go

@@ -25,35 +25,79 @@ type CollectionDefinition struct {
 // GetFieldByName returns the field for the given field name. If such a field is found it
 // will return it and true, if it is not found it will return false.
 func (def CollectionDefinition) GetFieldByName(fieldName string) (FieldDefinition, bool) {
-	collectionField, ok := def.Description.GetFieldByName(fieldName)
-	if ok {
-		schemaField, ok := def.Schema.GetFieldByName(fieldName)
-		if ok {
-			return NewFieldDefinition(
-				collectionField,
-				schemaField,
-			), true
-		}
+	collectionField, existsOnCollection := def.Description.GetFieldByName(fieldName)
+	schemaField, existsOnSchema := def.Schema.GetFieldByName(fieldName)
+
+	if existsOnCollection && existsOnSchema {
+		return NewFieldDefinition(
+			collectionField,
+			schemaField,
+		), true
+	} else if existsOnCollection && !existsOnSchema {
+		// If the field exists only on the collection, it is a local only field, for example the
+		// secondary side of a relation.
+		return NewLocalFieldDefinition(
+			collectionField,
+		), true
+	} else if !existsOnCollection && existsOnSchema {
+		// If the field only exist on the schema it is likely that this is a schema-only object
+		// definition, for example for an embedded object.
+		return NewSchemaOnlyFieldDefinition(
+			schemaField,
+		), true
 	}
+
 	return FieldDefinition{}, false
 }
 
 // GetFields returns the combined local and global field elements on this [CollectionDefinition]
 // as a single set.
 func (def CollectionDefinition) GetFields() []FieldDefinition {
 	fields := []FieldDefinition{}
+	localFieldNames := map[string]struct{}{}
+
 	for _, localField := range def.Description.Fields {
 		globalField, ok := def.Schema.GetFieldByName(localField.Name)
 		if ok {
 			fields = append(
 				fields,
 				NewFieldDefinition(localField, globalField),
 			)
+		} else {
+			// This must be a local only field, for example the secondary side of a relation.
+			fields = append(
+				fields,
+				NewLocalFieldDefinition(localField),
+			)
+		}
+		localFieldNames[localField.Name] = struct{}{}
+	}
+
+	for _, schemaField := range def.Schema.Fields {
+		if _, ok := localFieldNames[schemaField.Name]; ok {
+			continue
 		}
+		// This must be a global only field, for example on an embedded object.
+		fields = append(
+			fields,
+			NewSchemaOnlyFieldDefinition(schemaField),
+		)
 	}
+
 	return fields
 }
 
+// GetName gets the name of this definition.
+//
+// If the collection description has a name (e.g. it is an active collection) it will return that,
+// otherwise it will return the schema name.
+func (def CollectionDefinition) GetName() string {
+	if def.Description.Name.HasValue() {
+		return def.Description.Name.Value()
+	}
+	return def.Schema.Name
+}
+
 // FieldDefinition describes the combined local and global set of properties that constitutes
 // a field on a collection.
 //
@@ -94,13 +138,39 @@ type FieldDefinition struct {
 // NewFieldDefinition returns a new [FieldDefinition], combining the given local and global elements
 // into a single object.
 func NewFieldDefinition(local CollectionFieldDescription, global SchemaFieldDescription) FieldDefinition {
+	var kind FieldKind
+	if local.Kind.HasValue() {
+		kind = local.Kind.Value()
+	} else {
+		kind = global.Kind
+	}
+
 	return FieldDefinition{
 		Name:              global.Name,
 		ID:                local.ID,
-		Kind:              global.Kind,
-		RelationName:      global.RelationName,
+		Kind:              kind,
+		RelationName:      local.RelationName.Value(),
 		Typ:               global.Typ,
-		IsPrimaryRelation: global.IsPrimaryRelation,
+		IsPrimaryRelation: kind.IsObject() && !kind.IsArray(),
+	}
+}
+
+// NewLocalFieldDefinition returns a new [FieldDefinition] from the given local [CollectionFieldDescription].
+func NewLocalFieldDefinition(local CollectionFieldDescription) FieldDefinition {
+	return FieldDefinition{
+		Name:         local.Name,
+		ID:           local.ID,
+		Kind:         local.Kind.Value(),
+		RelationName: local.RelationName.Value(),
+	}
+}
+
+// NewSchemaOnlyFieldDefinition returns a new [FieldDefinition] from the given global [SchemaFieldDescription].
+func NewSchemaOnlyFieldDefinition(global SchemaFieldDescription) FieldDefinition {
+	return FieldDefinition{
+		Name: global.Name,
+		Kind: global.Kind,
+		Typ:  global.Typ,
 	}
 }
 

client/document.go

@@ -66,28 +66,28 @@ type Document struct {
 	// marks if document has unsaved changes
 	isDirty bool
 
-	schemaDescription SchemaDescription
+	collectionDefinition CollectionDefinition
 }
 
-func newEmptyDoc(sd SchemaDescription) *Document {
+func newEmptyDoc(collectionDefinition CollectionDefinition) *Document {
 	return &Document{
-		fields:            make(map[string]Field),
-		values:            make(map[Field]*FieldValue),
-		schemaDescription: sd,
+		fields:               make(map[string]Field),
+		values:               make(map[Field]*FieldValue),
+		collectionDefinition: collectionDefinition,
 	}
 }
 
 // NewDocWithID creates a new Document with a specified key.
-func NewDocWithID(docID DocID, sd SchemaDescription) *Document {
-	doc := newEmptyDoc(sd)
+func NewDocWithID(docID DocID, collectionDefinition CollectionDefinition) *Document {
+	doc := newEmptyDoc(collectionDefinition)
 	doc.id = docID
 	return doc
 }
 
 // NewDocFromMap creates a new Document from a data map.
-func NewDocFromMap(data map[string]any, sd SchemaDescription) (*Document, error) {
+func NewDocFromMap(data map[string]any, collectionDefinition CollectionDefinition) (*Document, error) {
 	var err error
-	doc := newEmptyDoc(sd)
+	doc := newEmptyDoc(collectionDefinition)
 
 	// check if document contains special _docID field
 	k, hasDocID := data[request.DocIDFieldName]
@@ -126,8 +126,8 @@ func IsJSONArray(obj []byte) bool {
 }
 
 // NewFromJSON creates a new instance of a Document from a raw JSON object byte array.
-func NewDocFromJSON(obj []byte, sd SchemaDescription) (*Document, error) {
-	doc := newEmptyDoc(sd)
+func NewDocFromJSON(obj []byte, collectionDefinition CollectionDefinition) (*Document, error) {
+	doc := newEmptyDoc(collectionDefinition)
 	err := doc.SetWithJSON(obj)
 	if err != nil {
 		return nil, err
@@ -141,7 +141,7 @@ func NewDocFromJSON(obj []byte, sd SchemaDescription) (*Document, error) {
 
 // ManyFromJSON creates a new slice of Documents from a raw JSON array byte array.
 // It will return an error if the given byte array is not a valid JSON array.
-func NewDocsFromJSON(obj []byte, sd SchemaDescription) ([]*Document, error) {
+func NewDocsFromJSON(obj []byte, collectionDefinition CollectionDefinition) ([]*Document, error) {
 	v, err := fastjson.ParseBytes(obj)
 	if err != nil {
 		return nil, err
@@ -157,7 +157,7 @@ func NewDocsFromJSON(obj []byte, sd SchemaDescription) ([]*Document, error) {
 		if err != nil {
 			return nil, err
 		}
-		doc := newEmptyDoc(sd)
+		doc := newEmptyDoc(collectionDefinition)
 		err = doc.setWithFastJSONObject(o)
 		if err != nil {
 			return nil, err
@@ -176,7 +176,7 @@ func NewDocsFromJSON(obj []byte, sd SchemaDescription) ([]*Document, error) {
 // and ensures it matches the supplied field description.
 // It will do any minor parsing, like dates, and return
 // the typed value again as an interface.
-func validateFieldSchema(val any, field SchemaFieldDescription) (NormalValue, error) {
+func validateFieldSchema(val any, field FieldDefinition) (NormalValue, error) {
 	if field.Kind.IsNillable() {
 		if val == nil {
 			return NewNormalNil(field.Kind)
@@ -187,7 +187,7 @@ func validateFieldSchema(val any, field SchemaFieldDescription) (NormalValue, er
 	}
 
 	if field.Kind.IsObjectArray() {
-		return nil, NewErrFieldOrAliasToFieldNotExist(field.Name)
+		return nil, NewErrFieldNotExist(field.Name)
 	}
 
 	if field.Kind.IsObject() {
@@ -588,15 +588,15 @@ func (doc *Document) setWithFastJSONObject(obj *fastjson.Object) error {
 
 // Set the value of a field.
 func (doc *Document) Set(field string, value any) error {
-	fd, exists := doc.schemaDescription.GetFieldByName(field)
+	fd, exists := doc.collectionDefinition.GetFieldByName(field)
 	if !exists {
 		return NewErrFieldNotExist(field)
 	}
-	if fd.IsRelation() && !fd.Kind.IsObjectArray() {
+	if fd.Kind.IsObject() && !fd.Kind.IsObjectArray() {
 		if !strings.HasSuffix(field, request.RelatedObjectID) {
 			field = field + request.RelatedObjectID
 		}
-		fd, exists = doc.schemaDescription.GetFieldByName(field)
+		fd, exists = doc.collectionDefinition.GetFieldByName(field)
 		if !exists {
 			return NewErrFieldNotExist(field)
 		}