Merge branch 'develop' into nasdf/refactor/http-config

client/descriptions.go

@@ -80,7 +80,9 @@ func (col CollectionDescription) GetFieldByRelation(
 	schema *SchemaDescription,
 ) (FieldDescription, bool) {
 	for _, field := range schema.Fields {
-		if field.RelationName == relationName && !(col.Name.Value() == otherCollectionName && otherFieldName == field.Name) {
+		if field.RelationName == relationName &&
+			!(col.Name.Value() == otherCollectionName && otherFieldName == field.Name) &&
+			field.Kind != FieldKind_DocID {
 			return field, true
 		}
 	}
@@ -250,18 +252,6 @@ var FieldKindStringToEnumMapping = map[string]FieldKind{
 // RelationType describes the type of relation between two types.
 type RelationType uint8
 
-// Note: These values are serialized and persisted in the database, avoid modifying existing values
-const (
-	Relation_Type_ONE         RelationType = 1   // 0b0000 0001
-	Relation_Type_MANY        RelationType = 2   // 0b0000 0010
-	Relation_Type_ONEONE      RelationType = 4   // 0b0000 0100
-	Relation_Type_ONEMANY     RelationType = 8   // 0b0000 1000
-	Relation_Type_MANYMANY    RelationType = 16  // 0b0001 0000
-	_                         RelationType = 32  // 0b0010 0000
-	Relation_Type_INTERNAL_ID RelationType = 64  // 0b0100 0000
-	Relation_Type_Primary     RelationType = 128 // 0b1000 0000 Primary reference entity on relation
-)
-
 // FieldID is a unique identifier for a field in a schema.
 type FieldID uint32
 
@@ -302,14 +292,7 @@ type FieldDescription struct {
 	// It is currently immutable.
 	Typ CType
 
-	// RelationType contains the relationship type if this field is a relation field. Otherwise this
-	// will be empty.
-	RelationType RelationType
-}
-
-// IsInternal returns true if this field is internally generated.
-func (f FieldDescription) IsInternal() bool {
-	return (f.Name == request.DocIDFieldName) || f.RelationType&Relation_Type_INTERNAL_ID != 0
+	IsPrimaryRelation bool
 }
 
 // IsObject returns true if this field is an object type.
@@ -323,14 +306,9 @@ func (f FieldDescription) IsObjectArray() bool {
 	return (f.Kind == FieldKind_FOREIGN_OBJECT_ARRAY)
 }
 
-// IsPrimaryRelation returns true if this field is a relation, and is the primary side.
-func (f FieldDescription) IsPrimaryRelation() bool {
-	return f.RelationType > 0 && f.RelationType&Relation_Type_Primary != 0
-}
-
 // IsRelation returns true if this field is a relation.
 func (f FieldDescription) IsRelation() bool {
-	return f.RelationType > 0
+	return f.RelationName != ""
 }
 
 // IsArray returns true if this field is an array type which includes inline arrays as well

client/document.go

@@ -12,6 +12,7 @@ package client
 
 import (
 	"encoding/json"
+	"errors"
 	"regexp"
 	"strings"
 	"sync"
@@ -468,6 +469,16 @@ func (doc *Document) GetValue(field string) (*FieldValue, error) {
 	}
 }
 
+// TryGetValue returns the value for a given field, if it exists.
+// If the field does not exist then return nil and an error.
+func (doc *Document) TryGetValue(field string) (*FieldValue, error) {
+	val, err := doc.GetValue(field)
+	if err != nil && errors.Is(err, ErrFieldNotExist) {
+		return nil, nil
+	}
+	return val, err
+}
+
 // GetValueWithField gets the Value type from a given Field type
 func (doc *Document) GetValueWithField(f Field) (*FieldValue, error) {
 	doc.mu.RLock()

db/collection.go

@@ -169,13 +169,12 @@ func (db *db) updateSchema(
 	}
 
 	for _, field := range schema.Fields {
-		if field.RelationType.IsSet(client.Relation_Type_ONE) {
+		if field.Kind == client.FieldKind_FOREIGN_OBJECT {
 			idFieldName := field.Name + "_id"
 			if _, ok := schema.GetField(idFieldName); !ok {
 				schema.Fields = append(schema.Fields, client.FieldDescription{
 					Name:         idFieldName,
 					Kind:         client.FieldKind_DocID,
-					RelationType: client.Relation_Type_INTERNAL_ID,
 					RelationName: field.RelationName,
 				})
 			}
@@ -316,40 +315,11 @@ func validateUpdateSchemaFields(
 				return false, NewErrSchemaNotFound(proposedField.Name, proposedField.Schema)
 			}
 
-			if proposedField.Kind == client.FieldKind_FOREIGN_OBJECT {
-				if !proposedField.RelationType.IsSet(client.Relation_Type_ONE) ||
-					!(proposedField.RelationType.IsSet(client.Relation_Type_ONEONE) ||
-						proposedField.RelationType.IsSet(client.Relation_Type_ONEMANY)) {
-					return false, NewErrRelationalFieldInvalidRelationType(
-						proposedField.Name,
-						fmt.Sprintf(
-							"%v and %v or %v, with optionally %v",
-							client.Relation_Type_ONE,
-							client.Relation_Type_ONEONE,
-							client.Relation_Type_ONEMANY,
-							client.Relation_Type_Primary,
-						),
-						proposedField.RelationType,
-					)
-				}
-			}
-
-			if proposedField.Kind == client.FieldKind_FOREIGN_OBJECT_ARRAY {
-				if !proposedField.RelationType.IsSet(client.Relation_Type_MANY) ||
-					!proposedField.RelationType.IsSet(client.Relation_Type_ONEMANY) {
-					return false, NewErrRelationalFieldInvalidRelationType(
-						proposedField.Name,
-						client.Relation_Type_MANY|client.Relation_Type_ONEMANY,
-						proposedField.RelationType,
-					)
-				}
-			}
-
 			if proposedField.RelationName == "" {
 				return false, NewErrRelationalFieldMissingRelationName(proposedField.Name)
 			}
 
-			if proposedField.RelationType.IsSet(client.Relation_Type_Primary) {
+			if proposedField.IsPrimaryRelation {
 				if proposedField.Kind == client.FieldKind_FOREIGN_OBJECT_ARRAY {
 					return false, NewErrPrimarySideOnMany(proposedField.Name)
 				}
@@ -363,14 +333,6 @@ func validateUpdateSchemaFields(
 						return false, NewErrRelationalFieldIDInvalidType(idField.Name, client.FieldKind_DocID, idField.Kind)
 					}
 
-					if idField.RelationType != client.Relation_Type_INTERNAL_ID {
-						return false, NewErrRelationalFieldInvalidRelationType(
-							idField.Name,
-							client.Relation_Type_INTERNAL_ID,
-							idField.RelationType,
-						)
-					}
-
 					if idField.RelationName == "" {
 						return false, NewErrRelationalFieldMissingRelationName(idField.Name)
 					}
@@ -381,7 +343,7 @@ func validateUpdateSchemaFields(
 			var relatedField client.FieldDescription
 			for _, field := range relatedDesc.Fields {
 				if field.RelationName == proposedField.RelationName &&
-					!field.RelationType.IsSet(client.Relation_Type_INTERNAL_ID) &&
+					field.Kind != client.FieldKind_DocID &&
 					!(relatedDesc.Name == proposedDesc.Name && field.Name == proposedField.Name) {
 					relatedFieldFound = true
 					relatedField = field
@@ -393,43 +355,13 @@ func validateUpdateSchemaFields(
 				return false, client.NewErrRelationOneSided(proposedField.Name, proposedField.Schema)
 			}
 
-			if !(proposedField.RelationType.IsSet(client.Relation_Type_Primary) ||
-				relatedField.RelationType.IsSet(client.Relation_Type_Primary)) {
+			if !(proposedField.IsPrimaryRelation || relatedField.IsPrimaryRelation) {
 				return false, NewErrPrimarySideNotDefined(proposedField.RelationName)
 			}
 
-			if proposedField.RelationType.IsSet(client.Relation_Type_Primary) &&
-				relatedField.RelationType.IsSet(client.Relation_Type_Primary) {
+			if proposedField.IsPrimaryRelation && relatedField.IsPrimaryRelation {
 				return false, NewErrBothSidesPrimary(proposedField.RelationName)
 			}
-
-			if proposedField.RelationType.IsSet(client.Relation_Type_ONEONE) &&
-				relatedField.Kind != client.FieldKind_FOREIGN_OBJECT {
-				return false, NewErrRelatedFieldKindMismatch(
-					proposedField.RelationName,
-					client.FieldKind_FOREIGN_OBJECT,
-					relatedField.Kind,
-				)
-			}
-
-			if proposedField.RelationType.IsSet(client.Relation_Type_ONEMANY) &&
-				proposedField.Kind == client.FieldKind_FOREIGN_OBJECT &&
-				relatedField.Kind != client.FieldKind_FOREIGN_OBJECT_ARRAY {
-				return false, NewErrRelatedFieldKindMismatch(
-					proposedField.RelationName,
-					client.FieldKind_FOREIGN_OBJECT_ARRAY,
-					relatedField.Kind,
-				)
-			}
-
-			if proposedField.RelationType.IsSet(client.Relation_Type_ONEONE) &&
-				!relatedField.RelationType.IsSet(client.Relation_Type_ONEONE) {
-				return false, NewErrRelatedFieldRelationTypeMismatch(
-					proposedField.RelationName,
-					client.Relation_Type_ONEONE,
-					relatedField.RelationType,
-				)
-			}
 		}
 
 		if _, isDuplicate := newFieldNames[proposedField.Name]; isDuplicate {
@@ -1099,7 +1031,7 @@ func (c *collection) validateOneToOneLinkDoesntAlreadyExist(
 	fieldDescription client.FieldDescription,
 	value any,
 ) error {
-	if !fieldDescription.RelationType.IsSet(client.Relation_Type_INTERNAL_ID) {
+	if fieldDescription.Kind != client.FieldKind_DocID {
 		return nil
 	}
 
@@ -1111,7 +1043,23 @@ func (c *collection) validateOneToOneLinkDoesntAlreadyExist(
 	if !ok {
 		return client.NewErrFieldNotExist(strings.TrimSuffix(fieldDescription.Name, request.RelatedObjectID))
 	}
-	if !objFieldDescription.RelationType.IsSet(client.Relation_Type_ONEONE) {
+	if objFieldDescription.Kind != client.FieldKind_FOREIGN_OBJECT {
+		return nil
+	}
+
+	otherCol, err := c.db.getCollectionByName(ctx, txn, objFieldDescription.Schema)
+	if err != nil {
+		return err
+	}
+	otherSchema := otherCol.Schema()
+	otherObjFieldDescription, _ := otherCol.Description().GetFieldByRelation(
+		fieldDescription.RelationName,
+		c.Name().Value(),
+		objFieldDescription.Name,
+		&otherSchema,
+	)
+	if otherObjFieldDescription.Kind != client.FieldKind_FOREIGN_OBJECT {
+		// If the other field is not an object field then this is not a one to one relation and we can continue
 		return nil
 	}
 

db/collection_update.go

@@ -292,14 +292,14 @@ func (c *collection) updateWithFilter(
 
 // isSecondaryIDField returns true if the given field description represents a secondary relation field ID.
 func (c *collection) isSecondaryIDField(fieldDesc client.FieldDescription) (client.FieldDescription, bool) {
-	if fieldDesc.RelationType != client.Relation_Type_INTERNAL_ID {
+	if fieldDesc.RelationName == "" || fieldDesc.Kind != client.FieldKind_DocID {
 		return client.FieldDescription{}, false
 	}
 
 	relationFieldDescription, valid := c.Schema().GetField(
 		strings.TrimSuffix(fieldDesc.Name, request.RelatedObjectID),
 	)
-	return relationFieldDescription, valid && !relationFieldDescription.IsPrimaryRelation()
+	return relationFieldDescription, valid && !relationFieldDescription.IsPrimaryRelation
 }
 
 // patchPrimaryDoc patches the (primary) document linked to from the document of the given DocID via the

db/errors.go

@@ -253,15 +253,6 @@ func NewErrRelationalFieldMissingSchema(name string, kind client.FieldKind) erro
 	)
 }
 
-func NewErrRelationalFieldInvalidRelationType(name string, expected any, actual client.RelationType) error {
-	return errors.New(
-		errRelationalFieldInvalidRelationType,
-		errors.NewKV("Field", name),
-		errors.NewKV("Expected", expected),
-		errors.NewKV("Actual", actual),
-	)
-}
-
 func NewErrRelationalFieldMissingIDField(name string, expectedName string) error {
 	return errors.New(
 		errRelationalFieldMissingIDField,
@@ -307,19 +298,6 @@ func NewErrRelatedFieldKindMismatch(relationName string, expected client.FieldKi
 	)
 }
 
-func NewErrRelatedFieldRelationTypeMismatch(
-	relationName string,
-	expected client.RelationType,
-	actual client.RelationType,
-) error {
-	return errors.New(
-		errRelatedFieldRelationTypeMismatch,
-		errors.NewKV("RelationName", relationName),
-		errors.NewKV("Expected", expected),
-		errors.NewKV("Actual", actual),
-	)
-}
-
 func NewErrRelationalFieldIDInvalidType(name string, expected, actual client.FieldKind) error {
 	return errors.New(
 		errRelationalFieldIDInvalidType,

db/fetcher/errors.go

@@ -27,7 +27,7 @@ const (
 	errFailedToGetDagNode           string = "failed to get DAG Node"
 	errMissingMapper                string = "missing document mapper"
 	errInvalidInOperatorValue       string = "invalid _in/_nin value"
-	errInvalidIndexFilterCondition  string = "invalid index filter condition"
+	errInvalidFilterOperator        string = "invalid filter operator is provided"
 )
 
 var (
@@ -44,7 +44,7 @@ var (
 	ErrMissingMapper                = errors.New(errMissingMapper)
 	ErrSingleSpanOnly               = errors.New("spans must contain only a single entry")
 	ErrInvalidInOperatorValue       = errors.New(errInvalidInOperatorValue)
-	ErrInvalidIndexFilterCondition  = errors.New(errInvalidIndexFilterCondition)
+	ErrInvalidFilterOperator        = errors.New(errInvalidFilterOperator)
 )
 
 // NewErrFieldIdNotFound returns an error indicating that the given FieldId was not found.
@@ -97,3 +97,8 @@ func NewErrVFetcherFailedToGetDagLink(inner error) error {
 func NewErrFailedToGetDagNode(inner error) error {
 	return errors.Wrap(errFailedToGetDagNode, inner)
 }
+
+// NewErrInvalidFilterOperator returns an error indicating that the given filter operator is invalid.
+func NewErrInvalidFilterOperator(operator string) error {
+	return errors.New(errInvalidFilterOperator, errors.NewKV("Operator", operator))
+}

db/fetcher/indexer.go

@@ -125,23 +125,28 @@ func (f *IndexFetcher) FetchNext(ctx context.Context) (EncodedDocument, ExecInfo
 			return nil, f.execInfo, nil
 		}
 
+		// This CBOR-specific value will be gone soon once we implement
+		// our own encryption package
+		hasNilField := false
+		const cborNil = 0xf6
 		for i, indexedField := range f.indexedFields {
 			property := &encProperty{
 				Desc: indexedField,
 				Raw:  res.key.FieldValues[i],
 			}
+			if len(res.key.FieldValues[i]) == 1 && res.key.FieldValues[i][0] == cborNil {
+				hasNilField = true
+			}
 
 			f.doc.properties[indexedField] = property
 		}
 
-		if f.indexDesc.Unique {
+		if f.indexDesc.Unique && !hasNilField {
 			f.doc.id = res.value
 		} else {
 			f.doc.id = res.key.FieldValues[len(res.key.FieldValues)-1]
 		}
 
-		f.execInfo.FieldsFetched++
-
 		if f.docFetcher != nil && len(f.docFields) > 0 {
 			targetKey := base.MakeDataStoreKeyWithCollectionAndDocID(f.col.Description(), string(f.doc.id))
 			spans := core.NewSpans(core.NewSpan(targetKey, targetKey.PrefixEnd()))