From d07bfd1fe4f38ce70d803a747824f87df055c3c6 Mon Sep 17 00:00:00 2001 From: AndrewSisley Date: Fri, 19 Jan 2024 10:05:57 -0500 Subject: [PATCH] refactor: Make CollectionDescription.Name Option (#2223) ## Relevant issue(s) Partially resolves #2198 ## Description Makes CollectionDescription.Name Option. Changes secondary indexes to be indexed by Collection ID. Moves view queries into Sources property. --- cli/collection.go | 2 +- client/collection.go | 4 +- client/descriptions.go | 96 +++++++++++++++++-- client/errors.go | 46 +++++---- core/key.go | 27 ++++-- core/key_test.go | 31 +++--- db/backup.go | 6 +- db/collection.go | 49 ++++++++-- db/collection_index.go | 35 ++++--- db/collection_update.go | 4 +- db/description/collection.go | 52 ++++++---- db/index_test.go | 50 +++++----- db/indexed_docs_test.go | 26 ++--- db/view.go | 5 +- .../i2198-sec-index-key-change.md | 3 + http/client_collection.go | 68 ++++++++++--- net/peer_replicator.go | 2 +- planner/datasource.go | 2 +- planner/scan.go | 2 +- planner/select.go | 5 +- planner/type_join.go | 4 +- planner/view.go | 4 +- request/graphql/schema/collection.go | 6 +- request/graphql/schema/descriptions_test.go | 27 +++--- request/graphql/schema/generate.go | 17 ++-- tests/clients/cli/wrapper_collection.go | 88 ++++++++++++++--- tests/gen/cli/gendocs.go | 4 +- tests/gen/gen_auto.go | 8 +- tests/gen/gen_auto_config.go | 2 +- tests/gen/gen_auto_test.go | 19 ++-- tests/gen/schema_parser.go | 2 +- tests/integration/utils2.go | 12 +-- tests/predefined/gen_predefined.go | 12 +-- 33 files changed, 498 insertions(+), 222 deletions(-) create mode 100644 docs/data_format_changes/i2198-sec-index-key-change.md diff --git a/cli/collection.go b/cli/collection.go index 8af1839b47..996af66f9a 100644 --- a/cli/collection.go +++ b/cli/collection.go @@ -81,7 +81,7 @@ func MakeCollectionCommand(cfg *config.Config) *cobra.Command { fetchedCols := cols cols = nil for _, c := range fetchedCols { - if c.Name() == name { + if c.Name().Value() == name { cols = append(cols, c) break } diff --git a/client/collection.go b/client/collection.go index 3a42871c62..9ce1e135d6 100644 --- a/client/collection.go +++ b/client/collection.go @@ -13,6 +13,8 @@ package client import ( "context" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/datastore" ) @@ -32,7 +34,7 @@ type CollectionDefinition struct { // Many functions on this object will interact with the underlying datastores. type Collection interface { // Name returns the name of this collection. - Name() string + Name() immutable.Option[string] // ID returns the ID of this Collection. ID() uint32 diff --git a/client/descriptions.go b/client/descriptions.go index 7a4ec0ba7e..cb312e8fac 100644 --- a/client/descriptions.go +++ b/client/descriptions.go @@ -11,8 +11,11 @@ package client import ( + "encoding/json" "fmt" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/client/request" ) @@ -22,7 +25,7 @@ type CollectionDescription struct { // // It is conceptually local to the node hosting the DefraDB instance, but currently there // is no means to update the local value so that it differs from the (global) schema name. - Name string + Name immutable.Option[string] // ID is the local identifier of this collection. // @@ -32,12 +35,11 @@ type CollectionDescription struct { // The ID of the schema version that this collection is at. SchemaVersionID string - // BaseQuery contains the base query of this view, if this collection is a view. + // Sources is the set of sources from which this collection draws data. // - // The query will be saved, and then may be accessed by other actors on demand. Actor defined - // aggregates, filters and other logic (such as LensVM transforms) will execute on top of this - // base query before the result is returned to the actor. - BaseQuery *request.Select + // Currently supported source types are: + // - [QuerySource] + Sources []any // Indexes contains the secondary indexes that this Collection has. Indexes []IndexDescription @@ -78,13 +80,37 @@ func (col CollectionDescription) GetFieldByRelation( schema *SchemaDescription, ) (FieldDescription, bool) { for _, field := range schema.Fields { - if field.RelationName == relationName && !(col.Name == otherCollectionName && otherFieldName == field.Name) { + if field.RelationName == relationName && !(col.Name.Value() == otherCollectionName && otherFieldName == field.Name) { return field, true } } return FieldDescription{}, false } +// QuerySources returns all the Sources of type [QuerySource] +func (col CollectionDescription) QuerySources() []*QuerySource { + return sourcesOfType[*QuerySource](col) +} + +func sourcesOfType[ResultType any](col CollectionDescription) []ResultType { + result := []ResultType{} + for _, source := range col.Sources { + if typedSource, isOfType := source.(ResultType); isOfType { + result = append(result, typedSource) + } + } + return result +} + +// QuerySource represents a collection data source from a query. +// +// The query will be executed when data from this source is requested, and the query results +// yielded to the consumer. +type QuerySource struct { + // Query contains the base query of this data source. + Query request.Select +} + // SchemaDescription describes a Schema and its associated metadata. type SchemaDescription struct { // Root is the version agnostic identifier for this schema. @@ -322,3 +348,59 @@ func (f FieldDescription) IsArray() bool { func (m RelationType) IsSet(target RelationType) bool { return m&target > 0 } + +// collectionDescription is a private type used to facilitate the unmarshalling +// of json to a [CollectionDescription]. +type collectionDescription struct { + // These properties are unmarshalled using the default json unmarshaller + Name immutable.Option[string] + ID uint32 + SchemaVersionID string + Indexes []IndexDescription + + // Properties below this line are unmarshalled using custom logic in [UnmarshalJSON] + Sources []map[string]json.RawMessage +} + +func (c *CollectionDescription) UnmarshalJSON(bytes []byte) error { + var descMap collectionDescription + err := json.Unmarshal(bytes, &descMap) + if err != nil { + return err + } + + c.Name = descMap.Name + c.ID = descMap.ID + c.SchemaVersionID = descMap.SchemaVersionID + c.Indexes = descMap.Indexes + c.Sources = make([]any, len(descMap.Sources)) + + for i, source := range descMap.Sources { + sourceJson, err := json.Marshal(source) + if err != nil { + return err + } + + var sourceValue any + // We detect which concrete type each `Source` object is by detecting + // non-nillable fields, if the key is present it must be of that type. + // They must be non-nillable as nil values may have their keys omitted from + // the json. This also relies on the fields being unique. We may wish to change + // this later to custom-serialize with a `_type` property. + if _, ok := source["Query"]; ok { + // This must be a QuerySource, as only the `QuerySource` type has a `Query` field + var querySource QuerySource + err := json.Unmarshal(sourceJson, &querySource) + if err != nil { + return err + } + sourceValue = &querySource + } else { + return ErrFailedToUnmarshalCollection + } + + c.Sources[i] = sourceValue + } + + return nil +} diff --git a/client/errors.go b/client/errors.go index 78daf3531b..60ccac9669 100644 --- a/client/errors.go +++ b/client/errors.go @@ -17,17 +17,19 @@ import ( ) const ( - errFieldNotExist string = "The given field does not exist" - errUnexpectedType string = "unexpected type" - errParsingFailed string = "failed to parse argument" - errUninitializeProperty string = "invalid state, required property is uninitialized" - errMaxTxnRetries string = "reached maximum transaction reties" - errRelationOneSided string = "relation must be defined on both schemas" - errCollectionNotFound string = "collection not found" - errFieldOrAliasToFieldNotExist string = "The given field or alias to field does not exist" - errUnknownCRDT string = "unknown crdt" - errCRDTKindMismatch string = "CRDT type %s can't be assigned to field kind %s" - errInvalidCRDTType string = "CRDT type not supported" + errFieldNotExist string = "The given field does not exist" + errUnexpectedType string = "unexpected type" + errParsingFailed string = "failed to parse argument" + errUninitializeProperty string = "invalid state, required property is uninitialized" + errMaxTxnRetries string = "reached maximum transaction reties" + errRelationOneSided string = "relation must be defined on both schemas" + errCollectionNotFound string = "collection not found" + errFieldOrAliasToFieldNotExist string = "The given field or alias to field does not exist" + errUnknownCRDT string = "unknown crdt" + errCRDTKindMismatch string = "CRDT type %s can't be assigned to field kind %s" + errInvalidCRDTType string = "CRDT type not supported" + errFailedToUnmarshalCollection string = "failed to unmarshal collection json" + errOperationNotPermittedOnNamelessCols string = "operation not permitted on nameless collection" ) // Errors returnable from this package. @@ -35,16 +37,18 @@ const ( // This list is incomplete and undefined errors may also be returned. // Errors returned from this package may be tested against these errors with errors.Is. var ( - ErrFieldNotExist = errors.New(errFieldNotExist) - ErrUnexpectedType = errors.New(errUnexpectedType) - ErrFieldNotObject = errors.New("trying to access field on a non object type") - ErrValueTypeMismatch = errors.New("value does not match indicated type") - ErrDocumentNotFound = errors.New("no document for the given ID exists") - ErrInvalidUpdateTarget = errors.New("the target document to update is of invalid type") - ErrInvalidUpdater = errors.New("the updater of a document is of invalid type") - ErrInvalidDeleteTarget = errors.New("the target document to delete is of invalid type") - ErrMalformedDocID = errors.New("malformed document ID, missing either version or cid") - ErrInvalidDocIDVersion = errors.New("invalid document ID version") + ErrFieldNotExist = errors.New(errFieldNotExist) + ErrUnexpectedType = errors.New(errUnexpectedType) + ErrFailedToUnmarshalCollection = errors.New(errFailedToUnmarshalCollection) + ErrOperationNotPermittedOnNamelessCols = errors.New(errOperationNotPermittedOnNamelessCols) + ErrFieldNotObject = errors.New("trying to access field on a non object type") + ErrValueTypeMismatch = errors.New("value does not match indicated type") + ErrDocumentNotFound = errors.New("no document for the given ID exists") + ErrInvalidUpdateTarget = errors.New("the target document to update is of invalid type") + ErrInvalidUpdater = errors.New("the updater of a document is of invalid type") + ErrInvalidDeleteTarget = errors.New("the target document to delete is of invalid type") + ErrMalformedDocID = errors.New("malformed document ID, missing either version or cid") + ErrInvalidDocIDVersion = errors.New("invalid document ID version") ) // NewErrFieldNotExist returns an error indicating that the given field does not exist. diff --git a/core/key.go b/core/key.go index 0c038b11dd..cb67cc45d6 100644 --- a/core/key.go +++ b/core/key.go @@ -17,6 +17,7 @@ import ( "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" + "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/errors" @@ -132,8 +133,8 @@ var _ Key = (*CollectionSchemaVersionKey)(nil) // CollectionIndexKey to a stored description of an index type CollectionIndexKey struct { - // CollectionName is the name of the collection that the index is on - CollectionName string + // CollectionID is the id of the collection that the index is on + CollectionID immutable.Option[uint32] // IndexName is the name of the index IndexName string } @@ -291,14 +292,14 @@ func NewCollectionSchemaVersionKeyFromString(key string) (CollectionSchemaVersio } // NewCollectionIndexKey creates a new CollectionIndexKey from a collection name and index name. -func NewCollectionIndexKey(colID, indexName string) CollectionIndexKey { - return CollectionIndexKey{CollectionName: colID, IndexName: indexName} +func NewCollectionIndexKey(colID immutable.Option[uint32], indexName string) CollectionIndexKey { + return CollectionIndexKey{CollectionID: colID, IndexName: indexName} } // NewCollectionIndexKeyFromString creates a new CollectionIndexKey from a string. // It expects the input string is in the following format: // -// /collection/index/[CollectionName]/[IndexName] +// /collection/index/[CollectionID]/[IndexName] // // Where [IndexName] might be omitted. Anything else will return an error. func NewCollectionIndexKeyFromString(key string) (CollectionIndexKey, error) { @@ -306,7 +307,13 @@ func NewCollectionIndexKeyFromString(key string) (CollectionIndexKey, error) { if len(keyArr) < 4 || len(keyArr) > 5 || keyArr[1] != "collection" || keyArr[2] != "index" { return CollectionIndexKey{}, ErrInvalidKey } - result := CollectionIndexKey{CollectionName: keyArr[3]} + + colID, err := strconv.Atoi(keyArr[3]) + if err != nil { + return CollectionIndexKey{}, err + } + + result := CollectionIndexKey{CollectionID: immutable.Some(uint32(colID))} if len(keyArr) == 5 { result.IndexName = keyArr[4] } @@ -315,13 +322,13 @@ func NewCollectionIndexKeyFromString(key string) (CollectionIndexKey, error) { // ToString returns the string representation of the key // It is in the following format: -// /collection/index/[CollectionName]/[IndexName] -// if [CollectionName] is empty, the rest is ignored +// /collection/index/[CollectionID]/[IndexName] +// if [CollectionID] is empty, the rest is ignored func (k CollectionIndexKey) ToString() string { result := COLLECTION_INDEX - if k.CollectionName != "" { - result = result + "/" + k.CollectionName + if k.CollectionID.HasValue() { + result = result + "/" + fmt.Sprint(k.CollectionID.Value()) if k.IndexName != "" { result = result + "/" + k.IndexName } diff --git a/core/key_test.go b/core/key_test.go index 4984c5b14f..52a22a5856 100644 --- a/core/key_test.go +++ b/core/key_test.go @@ -14,6 +14,7 @@ import ( "testing" ds "github.com/ipfs/go-datastore" + "github.com/sourcenetwork/immutable" "github.com/stretchr/testify/assert" ) @@ -110,23 +111,23 @@ func TestNewDataStoreKey_GivenAStringWithExtraSuffix(t *testing.T) { } func TestNewIndexKey_IfEmptyParam_ReturnPrefix(t *testing.T) { - key := NewCollectionIndexKey("", "") + key := NewCollectionIndexKey(immutable.None[uint32](), "") assert.Equal(t, "/collection/index", key.ToString()) } func TestNewIndexKey_IfParamsAreGiven_ReturnFullKey(t *testing.T) { - key := NewCollectionIndexKey("col", "idx") - assert.Equal(t, "/collection/index/col/idx", key.ToString()) + key := NewCollectionIndexKey(immutable.Some[uint32](1), "idx") + assert.Equal(t, "/collection/index/1/idx", key.ToString()) } func TestNewIndexKey_InNoCollectionName_ReturnJustPrefix(t *testing.T) { - key := NewCollectionIndexKey("", "idx") + key := NewCollectionIndexKey(immutable.None[uint32](), "idx") assert.Equal(t, "/collection/index", key.ToString()) } func TestNewIndexKey_InNoIndexName_ReturnWithoutIndexName(t *testing.T) { - key := NewCollectionIndexKey("col", "") - assert.Equal(t, "/collection/index/col", key.ToString()) + key := NewCollectionIndexKey(immutable.Some[uint32](1), "") + assert.Equal(t, "/collection/index/1", key.ToString()) } func TestNewIndexKeyFromString_IfInvalidString_ReturnError(t *testing.T) { @@ -144,17 +145,17 @@ func TestNewIndexKeyFromString_IfInvalidString_ReturnError(t *testing.T) { } func TestNewIndexKeyFromString_IfOnlyCollectionName_ReturnKey(t *testing.T) { - key, err := NewCollectionIndexKeyFromString("/collection/index/col") + key, err := NewCollectionIndexKeyFromString("/collection/index/1") assert.NoError(t, err) - assert.Equal(t, key.CollectionName, "col") - assert.Equal(t, key.IndexName, "") + assert.Equal(t, immutable.Some[uint32](1), key.CollectionID) + assert.Equal(t, "", key.IndexName) } func TestNewIndexKeyFromString_IfFullKeyString_ReturnKey(t *testing.T) { - key, err := NewCollectionIndexKeyFromString("/collection/index/col/idx") + key, err := NewCollectionIndexKeyFromString("/collection/index/1/idx") assert.NoError(t, err) - assert.Equal(t, key.CollectionName, "col") - assert.Equal(t, key.IndexName, "idx") + assert.Equal(t, immutable.Some[uint32](1), key.CollectionID) + assert.Equal(t, "idx", key.IndexName) } func toFieldValues(values ...string) [][]byte { @@ -312,10 +313,10 @@ func TestIndexDatastoreKey_EqualTrue(t *testing.T) { func TestCollectionIndexKey_Bytes(t *testing.T) { key := CollectionIndexKey{ - CollectionName: "col", - IndexName: "idx", + CollectionID: immutable.Some[uint32](1), + IndexName: "idx", } - assert.Equal(t, []byte(COLLECTION_INDEX+"/col/idx"), key.Bytes()) + assert.Equal(t, []byte(COLLECTION_INDEX+"/1/idx"), key.Bytes()) } func TestIndexDatastoreKey_EqualFalse(t *testing.T) { diff --git a/db/backup.go b/db/backup.go index d3a1138686..5573d77894 100644 --- a/db/backup.go +++ b/db/backup.go @@ -137,7 +137,7 @@ func (db *db) basicExport(ctx context.Context, txn datastore.Txn, config *client } colNameCache := map[string]struct{}{} for _, col := range cols { - colNameCache[col.Name()] = struct{}{} + colNameCache[col.Name().Value()] = struct{}{} } tempFile := config.Filepath + ".temp" @@ -181,8 +181,8 @@ func (db *db) basicExport(ctx context.Context, txn datastore.Txn, config *client // set collection err = writeString( f, - fmt.Sprintf("\"%s\":[", col.Name()), - fmt.Sprintf(" \"%s\": [\n", col.Name()), + fmt.Sprintf("\"%s\":[", col.Name().Value()), + fmt.Sprintf(" \"%s\": [\n", col.Name().Value()), config.Pretty, ) if err != nil { diff --git a/db/collection.go b/db/collection.go index f066c1d9fe..50a8acb4f6 100644 --- a/db/collection.go +++ b/db/collection.go @@ -96,12 +96,14 @@ func (db *db) createCollection( schema := def.Schema desc := def.Description - exists, err := description.HasCollectionByName(ctx, txn, desc.Name) - if err != nil { - return nil, err - } - if exists { - return nil, ErrCollectionAlreadyExists + if desc.Name.HasValue() { + exists, err := description.HasCollectionByName(ctx, txn, desc.Name.Value()) + if err != nil { + return nil, err + } + if exists { + return nil, ErrCollectionAlreadyExists + } } colSeq, err := db.getSequence(ctx, txn, core.COLLECTION) @@ -132,7 +134,7 @@ func (db *db) createCollection( } } - return db.getCollectionByName(ctx, txn, desc.Name) + return db.getCollectionByID(ctx, txn, desc.ID) } // updateSchema updates the persisted schema description matching the name of the given @@ -201,6 +203,13 @@ func (db *db) updateSchema( } for _, col := range cols { + if !col.Name.HasValue() { + // Nameless collections cannot be made default as they cannot be queried without a name. + // Note: The `setAsDefaultVersion` block will need a re-write when collections become immutable + // and the schema version stuff gets tracked by [CollectionDescription.Sources] instead. + continue + } + col.SchemaVersionID = schema.VersionID col, err = description.SaveCollection(ctx, txn, col) @@ -208,7 +217,7 @@ func (db *db) updateSchema( return err } - err = db.setDefaultSchemaVersionExplicit(ctx, txn, col.Name, schema.VersionID) + err = db.setDefaultSchemaVersionExplicit(ctx, txn, col.Name.Value(), schema.VersionID) if err != nil { return err } @@ -538,6 +547,26 @@ func (db *db) getCollectionsByVersionID( return collections, nil } +func (db *db) getCollectionByID(ctx context.Context, txn datastore.Txn, id uint32) (client.Collection, error) { + col, err := description.GetCollectionByID(ctx, txn, id) + if err != nil { + return nil, err + } + + schema, err := description.GetSchemaVersion(ctx, txn, col.SchemaVersionID) + if err != nil { + return nil, err + } + + collection := db.newCollection(col, schema) + err = collection.loadIndexes(ctx, txn) + if err != nil { + return nil, err + } + + return collection, nil +} + // getCollectionByName returns an existing collection within the database. func (db *db) getCollectionByName(ctx context.Context, txn datastore.Txn, name string) (client.Collection, error) { if name == "" { @@ -740,7 +769,7 @@ func (c *collection) Description() client.CollectionDescription { } // Name returns the collection name. -func (c *collection) Name() string { +func (c *collection) Name() immutable.Option[string] { return c.Description().Name } @@ -989,7 +1018,7 @@ func (c *collection) save( if isSecondaryRelationID { primaryId := val.Value().(string) - err = c.patchPrimaryDoc(ctx, txn, c.Name(), relationFieldDescription, primaryKey.DocID, primaryId) + err = c.patchPrimaryDoc(ctx, txn, c.Name().Value(), relationFieldDescription, primaryKey.DocID, primaryId) if err != nil { return cid.Undef, err } diff --git a/db/collection_index.go b/db/collection_index.go index 4367d8ebdf..d4f2e4388d 100644 --- a/db/collection_index.go +++ b/db/collection_index.go @@ -18,10 +18,13 @@ import ( "strconv" "strings" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/core" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/db/base" + "github.com/sourcenetwork/defradb/db/description" "github.com/sourcenetwork/defradb/db/fetcher" "github.com/sourcenetwork/defradb/request/graphql/schema" ) @@ -59,7 +62,7 @@ func (db *db) getAllIndexes( ctx context.Context, txn datastore.Txn, ) (map[client.CollectionName][]client.IndexDescription, error) { - prefix := core.NewCollectionIndexKey("", "") + prefix := core.NewCollectionIndexKey(immutable.None[uint32](), "") keys, indexDescriptions, err := datastore.DeserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) @@ -75,8 +78,14 @@ func (db *db) getAllIndexes( if err != nil { return nil, NewErrInvalidStoredIndexKey(indexKey.ToString()) } - indexes[indexKey.CollectionName] = append( - indexes[indexKey.CollectionName], + + col, err := description.GetCollectionByID(ctx, txn, indexKey.CollectionID.Value()) + if err != nil { + return nil, err + } + + indexes[col.Name.Value()] = append( + indexes[col.Name.Value()], indexDescriptions[i], ) } @@ -87,9 +96,9 @@ func (db *db) getAllIndexes( func (db *db) fetchCollectionIndexDescriptions( ctx context.Context, txn datastore.Txn, - colName string, + colID uint32, ) ([]client.IndexDescription, error) { - prefix := core.NewCollectionIndexKey(colName, "") + prefix := core.NewCollectionIndexKey(immutable.Some(colID), "") _, indexDescriptions, err := datastore.DeserializePrefix[client.IndexDescription](ctx, prefix.ToString(), txn.Systemstore()) if err != nil { @@ -338,7 +347,7 @@ func (c *collection) dropIndex(ctx context.Context, txn datastore.Txn, indexName break } } - key := core.NewCollectionIndexKey(c.Name(), indexName) + key := core.NewCollectionIndexKey(immutable.Some(c.ID()), indexName) err = txn.Systemstore().Delete(ctx, key.ToDS()) if err != nil { return err @@ -348,7 +357,7 @@ func (c *collection) dropIndex(ctx context.Context, txn datastore.Txn, indexName } func (c *collection) dropAllIndexes(ctx context.Context, txn datastore.Txn) error { - prefix := core.NewCollectionIndexKey(c.Name(), "") + prefix := core.NewCollectionIndexKey(immutable.Some(c.ID()), "") keys, err := datastore.FetchKeysForPrefix(ctx, prefix.ToString(), txn.Systemstore()) if err != nil { @@ -366,7 +375,7 @@ func (c *collection) dropAllIndexes(ctx context.Context, txn datastore.Txn) erro } func (c *collection) loadIndexes(ctx context.Context, txn datastore.Txn) error { - indexDescriptions, err := c.db.fetchCollectionIndexDescriptions(ctx, txn, c.Name()) + indexDescriptions, err := c.db.fetchCollectionIndexDescriptions(ctx, txn, c.ID()) if err != nil { return err } @@ -428,7 +437,7 @@ func (c *collection) generateIndexNameIfNeededAndCreateKey( nameIncrement := 1 for { desc.Name = generateIndexName(c, desc.Fields, nameIncrement) - indexKey = core.NewCollectionIndexKey(c.Name(), desc.Name) + indexKey = core.NewCollectionIndexKey(immutable.Some(c.ID()), desc.Name) exists, err := txn.Systemstore().Has(ctx, indexKey.ToDS()) if err != nil { return core.CollectionIndexKey{}, err @@ -439,7 +448,7 @@ func (c *collection) generateIndexNameIfNeededAndCreateKey( nameIncrement++ } } else { - indexKey = core.NewCollectionIndexKey(c.Name(), desc.Name) + indexKey = core.NewCollectionIndexKey(immutable.Some(c.ID()), desc.Name) exists, err := txn.Systemstore().Has(ctx, indexKey.ToDS()) if err != nil { return core.CollectionIndexKey{}, err @@ -477,7 +486,11 @@ func generateIndexName(col client.Collection, fields []client.IndexedFieldDescri // at the moment we support only single field indexes that can be stored only in // ascending order. This will change once we introduce composite indexes. direction := "ASC" - sb.WriteString(col.Name()) + if col.Name().HasValue() { + sb.WriteString(col.Name().Value()) + } else { + sb.WriteString(fmt.Sprint(col.ID())) + } sb.WriteByte('_') // we can safely assume that there is at least one field in the slice // because we validate it before calling this function diff --git a/db/collection_update.go b/db/collection_update.go index 4c1895602b..f4fc1eef6e 100644 --- a/db/collection_update.go +++ b/db/collection_update.go @@ -402,7 +402,7 @@ func (c *collection) makeSelectionPlan( return nil, ErrInvalidFilter } - f, err = c.db.parser.NewFilterFromString(c.Name(), fval) + f, err = c.db.parser.NewFilterFromString(c.Name().Value(), fval) if err != nil { return nil, err } @@ -432,7 +432,7 @@ func (c *collection) makeSelectionPlan( func (c *collection) makeSelectLocal(filter immutable.Option[request.Filter]) (*request.Select, error) { slct := &request.Select{ Field: request.Field{ - Name: c.Name(), + Name: c.Name().Value(), }, Filter: filter, Fields: make([]request.Selection, 0), diff --git a/db/description/collection.go b/db/description/collection.go index a334ec6384..3daeaf31de 100644 --- a/db/description/collection.go +++ b/db/description/collection.go @@ -39,15 +39,17 @@ func SaveCollection( return client.CollectionDescription{}, err } - idBuf, err := json.Marshal(desc.ID) - if err != nil { - return client.CollectionDescription{}, err - } + if desc.Name.HasValue() { + idBuf, err := json.Marshal(desc.ID) + if err != nil { + return client.CollectionDescription{}, err + } - nameKey := core.NewCollectionNameKey(desc.Name) - err = txn.Systemstore().Put(ctx, nameKey.ToDS(), idBuf) - if err != nil { - return client.CollectionDescription{}, err + nameKey := core.NewCollectionNameKey(desc.Name.Value()) + err = txn.Systemstore().Put(ctx, nameKey.ToDS(), idBuf) + if err != nil { + return client.CollectionDescription{}, err + } } // The need for this key is temporary, we should replace it with the global collection ID @@ -61,6 +63,26 @@ func SaveCollection( return desc, nil } +func GetCollectionByID( + ctx context.Context, + txn datastore.Txn, + id uint32, +) (client.CollectionDescription, error) { + key := core.NewCollectionKey(id) + buf, err := txn.Systemstore().Get(ctx, key.ToDS()) + if err != nil { + return client.CollectionDescription{}, err + } + + var col client.CollectionDescription + err = json.Unmarshal(buf, &col) + if err != nil { + return client.CollectionDescription{}, err + } + + return col, nil +} + // GetCollectionByName returns the collection with the given name. // // If no collection of that name is found, it will return an error. @@ -81,19 +103,7 @@ func GetCollectionByName( return client.CollectionDescription{}, err } - key := core.NewCollectionKey(id) - buf, err := txn.Systemstore().Get(ctx, key.ToDS()) - if err != nil { - return client.CollectionDescription{}, err - } - - var col client.CollectionDescription - err = json.Unmarshal(buf, &col) - if err != nil { - return client.CollectionDescription{}, err - } - - return col, nil + return GetCollectionByID(ctx, txn, id) } // GetCollectionsBySchemaVersionID returns all collections that use the given diff --git a/db/index_test.go b/db/index_test.go index 911228e649..f1323f3812 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -20,6 +20,7 @@ import ( ds "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" + "github.com/sourcenetwork/immutable" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -59,6 +60,10 @@ type indexTestFixture struct { } func (f *indexTestFixture) addUsersCollection() client.Collection { + if f.users != nil { + return f.users + } + _, err := f.db.AddSchema( f.ctx, fmt.Sprintf( @@ -136,7 +141,7 @@ func newIndexTestFixture(t *testing.T) *indexTestFixture { func (f *indexTestFixture) createCollectionIndex( desc client.IndexDescription, ) (client.IndexDescription, error) { - return f.createCollectionIndexFor(f.users.Name(), desc) + return f.createCollectionIndexFor(f.users.Name().Value(), desc) } func getUsersIndexDescOnName() client.IndexDescription { @@ -176,7 +181,7 @@ func getProductsIndexDescOnCategory() client.IndexDescription { } func (f *indexTestFixture) createUserCollectionIndexOnName() client.IndexDescription { - newDesc, err := f.createCollectionIndexFor(f.users.Name(), getUsersIndexDescOnName()) + newDesc, err := f.createCollectionIndexFor(f.users.Name().Value(), getUsersIndexDescOnName()) require.NoError(f.t, err) return newDesc } @@ -188,13 +193,13 @@ func makeUnique(indexDesc client.IndexDescription) client.IndexDescription { func (f *indexTestFixture) createUserCollectionUniqueIndexOnName() client.IndexDescription { indexDesc := makeUnique(getUsersIndexDescOnName()) - newDesc, err := f.createCollectionIndexFor(f.users.Name(), indexDesc) + newDesc, err := f.createCollectionIndexFor(f.users.Name().Value(), indexDesc) require.NoError(f.t, err) return newDesc } func (f *indexTestFixture) createUserCollectionIndexOnAge() client.IndexDescription { - newDesc, err := f.createCollectionIndexFor(f.users.Name(), getUsersIndexDescOnAge()) + newDesc, err := f.createCollectionIndexFor(f.users.Name().Value(), getUsersIndexDescOnAge()) require.NoError(f.t, err) return newDesc } @@ -204,7 +209,7 @@ func (f *indexTestFixture) dropIndex(colName, indexName string) error { } func (f *indexTestFixture) countIndexPrefixes(colName, indexName string) int { - prefix := core.NewCollectionIndexKey(usersColName, indexName) + prefix := core.NewCollectionIndexKey(immutable.Some(f.users.ID()), indexName) q, err := f.txn.Systemstore().Query(f.ctx, query.Query{ Prefix: prefix.ToString(), }) @@ -247,8 +252,8 @@ func (f *indexTestFixture) getAllIndexes() (map[client.CollectionName][]client.I return f.db.getAllIndexes(f.ctx, f.txn) } -func (f *indexTestFixture) getCollectionIndexes(colName string) ([]client.IndexDescription, error) { - return f.db.fetchCollectionIndexDescriptions(f.ctx, f.txn, colName) +func (f *indexTestFixture) getCollectionIndexes(colID uint32) ([]client.IndexDescription, error) { + return f.db.fetchCollectionIndexDescriptions(f.ctx, f.txn, colID) } func TestCreateIndex_IfFieldsIsEmpty_ReturnError(t *testing.T) { @@ -392,7 +397,7 @@ func TestCreateIndex_ShouldSaveToSystemStorage(t *testing.T) { _, err := f.createCollectionIndex(desc) assert.NoError(t, err) - key := core.NewCollectionIndexKey(f.users.Name(), name) + key := core.NewCollectionIndexKey(immutable.Some(f.users.ID()), name) data, err := f.txn.Systemstore().Get(f.ctx, key.ToDS()) assert.NoError(t, err) var deserialized client.IndexDescription @@ -441,7 +446,7 @@ func TestCreateIndex_WithMultipleCollectionsAndIndexes_AssignIncrementedIDPerCol } createIndexAndAssert := func(col client.Collection, fieldName string, expectedID uint32) { - desc, err := f.createCollectionIndexFor(col.Name(), makeIndex(fieldName)) + desc, err := f.createCollectionIndexFor(col.Name().Value(), makeIndex(fieldName)) require.NoError(t, err) assert.Equal(t, expectedID, desc.ID) seqKey := core.NewSequenceKey(fmt.Sprintf("%s/%d", core.COLLECTION_INDEX, col.ID())) @@ -524,7 +529,7 @@ func TestCreateIndex_IfAttemptToIndexOnUnsupportedType_ReturnError(t *testing.T) f.txn, err = f.db.NewTxn(f.ctx, false) require.NoError(f.t, err) - _, err = f.createCollectionIndexFor(collection.Name(), indexDesc) + _, err = f.createCollectionIndexFor(collection.Name().Value(), indexDesc) require.ErrorIs(f.t, err, NewErrUnsupportedIndexFieldType(unsupportedKind)) } @@ -562,7 +567,7 @@ func TestGetIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { f := newIndexTestFixture(t) defer f.db.Close() - indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") + indexKey := core.NewCollectionIndexKey(immutable.Some(f.users.ID()), "users_name_index") err := f.txn.Systemstore().Put(f.ctx, indexKey.ToDS(), []byte("invalid")) assert.NoError(t, err) @@ -574,7 +579,7 @@ func TestGetIndexes_IfInvalidIndexKeyIsStored_ReturnError(t *testing.T) { f := newIndexTestFixture(t) defer f.db.Close() - indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") + indexKey := core.NewCollectionIndexKey(immutable.Some(f.users.ID()), "users_name_index") key := ds.NewKey(indexKey.ToString() + "/invalid") desc := client.IndexDescription{ Name: "some_index_name", @@ -663,7 +668,7 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) assert.NoError(t, err) - f.getProductsCollectionDesc() + products := f.getProductsCollectionDesc() productsIndexDesc := client.IndexDescription{ Name: "products_description_index", Fields: []client.IndexedFieldDescription{{Name: productsPriceFieldName}}, @@ -675,13 +680,13 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) _, err = f.createCollectionIndexFor(productsColName, productsIndexDesc) assert.NoError(t, err) - userIndexes, err := f.getCollectionIndexes(usersColName) + userIndexes, err := f.getCollectionIndexes(f.users.ID()) assert.NoError(t, err) require.Equal(t, 1, len(userIndexes)) usersIndexDesc.ID = 1 assert.Equal(t, usersIndexDesc, userIndexes[0]) - productIndexes, err := f.getCollectionIndexes(productsColName) + productIndexes, err := f.getCollectionIndexes(products.ID()) assert.NoError(t, err) require.Equal(t, 1, len(productIndexes)) productsIndexDesc.ID = 1 @@ -700,7 +705,7 @@ func TestGetCollectionIndexes_IfSystemStoreFails_ReturnError(t *testing.T) { mockedTxn.EXPECT().Systemstore().Unset() mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore) - _, err := f.getCollectionIndexes(usersColName) + _, err := f.getCollectionIndexes(f.users.ID()) assert.ErrorIs(t, err, testErr) } @@ -716,7 +721,7 @@ func TestGetCollectionIndexes_IfSystemStoreFails_ShouldCloseIterator(t *testing. mockedTxn.EXPECT().Systemstore().Unset() mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore) - _, _ = f.getCollectionIndexes(usersColName) + _, _ = f.getCollectionIndexes(f.users.ID()) } func TestGetCollectionIndexes_IfSystemStoreQueryIteratorFails_ReturnError(t *testing.T) { @@ -732,7 +737,7 @@ func TestGetCollectionIndexes_IfSystemStoreQueryIteratorFails_ReturnError(t *tes mockedTxn.EXPECT().Systemstore().Unset() mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore) - _, err := f.getCollectionIndexes(usersColName) + _, err := f.getCollectionIndexes(f.users.ID()) assert.ErrorIs(t, err, testErr) } @@ -740,11 +745,11 @@ func TestGetCollectionIndexes_IfInvalidIndexIsStored_ReturnError(t *testing.T) { f := newIndexTestFixture(t) defer f.db.Close() - indexKey := core.NewCollectionIndexKey(usersColName, "users_name_index") + indexKey := core.NewCollectionIndexKey(immutable.Some(f.users.ID()), "users_name_index") err := f.txn.Systemstore().Put(f.ctx, indexKey.ToDS(), []byte("invalid")) assert.NoError(t, err) - _, err = f.getCollectionIndexes(usersColName) + _, err = f.getCollectionIndexes(f.users.ID()) assert.ErrorIs(t, err, datastore.NewErrInvalidStoredValue(nil)) } @@ -866,7 +871,6 @@ func TestCollectionGetIndexes_IfFailsToCreateTxn_ShouldNotCache(t *testing.T) { func TestCollectionGetIndexes_IfStoredIndexWithUnsupportedType_ReturnError(t *testing.T) { f := newIndexTestFixtureBare(t) - f.addUsersCollection() const unsupportedKind = client.FieldKind_BOOL_ARRAY _, err := f.db.AddSchema( @@ -1011,7 +1015,7 @@ func TestCollectionGetIndexes_ShouldReturnIndexesInOrderedByName(t *testing.T) { }, } - _, err := f.createCollectionIndexFor(collection.Name(), indexDesc) + _, err := f.createCollectionIndexFor(collection.Name().Value(), indexDesc) require.NoError(t, err) } @@ -1032,7 +1036,7 @@ func TestDropIndex_ShouldDeleteIndex(t *testing.T) { err := f.dropIndex(usersColName, desc.Name) assert.NoError(t, err) - indexKey := core.NewCollectionIndexKey(usersColName, desc.Name) + indexKey := core.NewCollectionIndexKey(immutable.Some(f.users.ID()), desc.Name) _, err = f.txn.Systemstore().Get(f.ctx, indexKey.ToDS()) assert.Error(t, err) } diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index b7c7abbf9d..8e075b22f8 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -19,6 +19,7 @@ import ( ipfsDatastore "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" + "github.com/sourcenetwork/immutable" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -134,7 +135,7 @@ func (b *indexKeyBuilder) Build() core.IndexDataStoreKey { require.NoError(b.f.t, err) var collection client.Collection for _, col := range cols { - if col.Name() == b.colName { + if col.Name().Value() == b.colName { collection = col break } @@ -211,12 +212,15 @@ func (*indexTestFixture) resetSystemStoreStubs(systemStoreOn *mocks.DSReaderWrit } func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_Expecter) { + if f.users == nil { + f.users = f.addUsersCollection() + } desc := getUsersIndexDescOnName() desc.ID = 1 indexOnNameDescData, err := json.Marshal(desc) require.NoError(f.t, err) - colIndexKey := core.NewCollectionIndexKey(usersColName, "") + colIndexKey := core.NewCollectionIndexKey(immutable.Some(f.users.ID()), "") matchPrefixFunc := func(q query.Query) bool { return q.Prefix == colIndexKey.ToDS().String() } @@ -230,7 +234,7 @@ func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_E systemStoreOn.Query(mock.Anything, mock.Anything).Maybe(). Return(mocks.NewQueryResultsWithValues(f.t), nil) - colIndexOnNameKey := core.NewCollectionIndexKey(usersColName, testUsersColIndexName) + colIndexOnNameKey := core.NewCollectionIndexKey(immutable.Some(f.users.ID()), testUsersColIndexName) systemStoreOn.Get(mock.Anything, colIndexOnNameKey.ToDS()).Maybe().Return(indexOnNameDescData, nil) if f.users != nil { @@ -357,9 +361,9 @@ func TestNonUnique_IfMultipleCollectionsWithIndexes_StoreIndexWithCollectionID(t users := f.addUsersCollection() products := f.getProductsCollectionDesc() - _, err := f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnName()) + _, err := f.createCollectionIndexFor(users.Name().Value(), getUsersIndexDescOnName()) require.NoError(f.t, err) - _, err = f.createCollectionIndexFor(products.Name(), getProductsIndexDescOnCategory()) + _, err = f.createCollectionIndexFor(products.Name().Value(), getProductsIndexDescOnCategory()) require.NoError(f.t, err) f.commitTxn() @@ -633,11 +637,11 @@ func TestNonUniqueCreate_IfDatastoreFailsToStoreIndex_ReturnError(t *testing.T) func TestNonUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f := newIndexTestFixtureBare(t) users := f.addUsersCollection() - _, err := f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnName()) + _, err := f.createCollectionIndexFor(users.Name().Value(), getUsersIndexDescOnName()) require.NoError(f.t, err) - _, err = f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnAge()) + _, err = f.createCollectionIndexFor(users.Name().Value(), getUsersIndexDescOnAge()) require.NoError(f.t, err) - _, err = f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnWeight()) + _, err = f.createCollectionIndexFor(users.Name().Value(), getUsersIndexDescOnWeight()) require.NoError(f.t, err) f.commitTxn() @@ -645,7 +649,7 @@ func TestNonUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f.saveDocToCollection(f.newUserDoc("Islam", 23, users), users) products := f.getProductsCollectionDesc() - _, err = f.createCollectionIndexFor(products.Name(), getProductsIndexDescOnCategory()) + _, err = f.createCollectionIndexFor(products.Name().Value(), getProductsIndexDescOnCategory()) require.NoError(f.t, err) f.commitTxn() @@ -1054,9 +1058,9 @@ func TestUnique_IfIndexedFieldIsNil_StoreItAsNil(t *testing.T) { func TestUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f := newIndexTestFixtureBare(t) users := f.addUsersCollection() - _, err := f.createCollectionIndexFor(users.Name(), makeUnique(getUsersIndexDescOnName())) + _, err := f.createCollectionIndexFor(users.Name().Value(), makeUnique(getUsersIndexDescOnName())) require.NoError(f.t, err) - _, err = f.createCollectionIndexFor(users.Name(), makeUnique(getUsersIndexDescOnAge())) + _, err = f.createCollectionIndexFor(users.Name().Value(), makeUnique(getUsersIndexDescOnAge())) require.NoError(f.t, err) f.commitTxn() diff --git a/db/view.go b/db/view.go index 2b4666df22..2a61ff63af 100644 --- a/db/view.go +++ b/db/view.go @@ -57,12 +57,13 @@ func (db *db) addView( } for i := range newDefinitions { - newDefinitions[i].Description.BaseQuery = baseQuery + source := client.QuerySource{Query: *baseQuery} + newDefinitions[i].Description.Sources = append(newDefinitions[i].Description.Sources, &source) } returnDescriptions := make([]client.CollectionDefinition, len(newDefinitions)) for i, definition := range newDefinitions { - if definition.Description.Name == "" { + if !definition.Description.Name.HasValue() { schema, err := description.CreateSchemaVersion(ctx, txn, definition.Schema) if err != nil { return nil, err diff --git a/docs/data_format_changes/i2198-sec-index-key-change.md b/docs/data_format_changes/i2198-sec-index-key-change.md new file mode 100644 index 0000000000..8e372aa6ac --- /dev/null +++ b/docs/data_format_changes/i2198-sec-index-key-change.md @@ -0,0 +1,3 @@ +# Index secondary indexes by collection id + +Secondary indexes are now indexed by collection ID instead of collection name. \ No newline at end of file diff --git a/http/client_collection.go b/http/client_collection.go index 95a81df84f..b44f5045fc 100644 --- a/http/client_collection.go +++ b/http/client_collection.go @@ -20,6 +20,7 @@ import ( "net/url" "strings" + "github.com/sourcenetwork/immutable" sse "github.com/vito/go-sse/sse" "github.com/sourcenetwork/defradb/client" @@ -39,7 +40,7 @@ func (c *Collection) Description() client.CollectionDescription { return c.def.Description } -func (c *Collection) Name() string { +func (c *Collection) Name() immutable.Option[string] { return c.Description().Name } @@ -60,7 +61,11 @@ func (c *Collection) Definition() client.CollectionDefinition { } func (c *Collection) Create(ctx context.Context, doc *client.Document) error { - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name) + if !c.Description().Name.HasValue() { + return client.ErrOperationNotPermittedOnNamelessCols + } + + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value()) body, err := doc.String() if err != nil { @@ -79,7 +84,10 @@ func (c *Collection) Create(ctx context.Context, doc *client.Document) error { } func (c *Collection) CreateMany(ctx context.Context, docs []*client.Document) error { - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name) + if !c.Description().Name.HasValue() { + return client.ErrOperationNotPermittedOnNamelessCols + } + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value()) var docMapList []json.RawMessage for _, doc := range docs { @@ -108,7 +116,11 @@ func (c *Collection) CreateMany(ctx context.Context, docs []*client.Document) er } func (c *Collection) Update(ctx context.Context, doc *client.Document) error { - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, doc.ID().String()) + if !c.Description().Name.HasValue() { + return client.ErrOperationNotPermittedOnNamelessCols + } + + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value(), doc.ID().String()) body, err := doc.ToJSONPatch() if err != nil { @@ -138,7 +150,11 @@ func (c *Collection) Save(ctx context.Context, doc *client.Document) error { } func (c *Collection) Delete(ctx context.Context, docID client.DocID) (bool, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, docID.String()) + if !c.Description().Name.HasValue() { + return false, client.ErrOperationNotPermittedOnNamelessCols + } + + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value(), docID.String()) req, err := http.NewRequestWithContext(ctx, http.MethodDelete, methodURL.String(), nil) if err != nil { @@ -176,7 +192,11 @@ func (c *Collection) updateWith( ctx context.Context, request CollectionUpdateRequest, ) (*client.UpdateResult, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name) + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value()) body, err := json.Marshal(request) if err != nil { @@ -247,7 +267,11 @@ func (c *Collection) deleteWith( ctx context.Context, request CollectionDeleteRequest, ) (*client.DeleteResult, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name) + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value()) body, err := json.Marshal(request) if err != nil { @@ -287,12 +311,16 @@ func (c *Collection) DeleteWithDocIDs(ctx context.Context, docIDs []client.DocID } func (c *Collection) Get(ctx context.Context, docID client.DocID, showDeleted bool) (*client.Document, error) { + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + query := url.Values{} if showDeleted { query.Add("show_deleted", "true") } - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, docID.String()) + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value(), docID.String()) methodURL.RawQuery = query.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, methodURL.String(), nil) @@ -320,7 +348,11 @@ func (c *Collection) WithTxn(tx datastore.Txn) client.Collection { } func (c *Collection) GetAllDocIDs(ctx context.Context) (<-chan client.DocIDResult, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name) + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value()) req, err := http.NewRequestWithContext(ctx, http.MethodGet, methodURL.String(), nil) if err != nil { @@ -372,7 +404,11 @@ func (c *Collection) CreateIndex( ctx context.Context, indexDesc client.IndexDescription, ) (client.IndexDescription, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, "indexes") + if !c.Description().Name.HasValue() { + return client.IndexDescription{}, client.ErrOperationNotPermittedOnNamelessCols + } + + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value(), "indexes") body, err := json.Marshal(&indexDesc) if err != nil { @@ -390,7 +426,11 @@ func (c *Collection) CreateIndex( } func (c *Collection) DropIndex(ctx context.Context, indexName string) error { - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, "indexes", indexName) + if !c.Description().Name.HasValue() { + return client.ErrOperationNotPermittedOnNamelessCols + } + + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value(), "indexes", indexName) req, err := http.NewRequestWithContext(ctx, http.MethodDelete, methodURL.String(), nil) if err != nil { @@ -401,7 +441,11 @@ func (c *Collection) DropIndex(ctx context.Context, indexName string) error { } func (c *Collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { - methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name, "indexes") + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + + methodURL := c.http.baseURL.JoinPath("collections", c.Description().Name.Value(), "indexes") req, err := http.NewRequestWithContext(ctx, http.MethodGet, methodURL.String(), nil) if err != nil { diff --git a/net/peer_replicator.go b/net/peer_replicator.go index 0506e018c4..8756959db8 100644 --- a/net/peer_replicator.go +++ b/net/peer_replicator.go @@ -94,7 +94,7 @@ func (p *Peer) SetReplicator(ctx context.Context, rep client.Replicator) error { for _, col := range added { keysCh, err := col.WithTxn(txn).GetAllDocIDs(ctx) if err != nil { - return NewErrReplicatorDocID(err, col.Name(), rep.Info.ID) + return NewErrReplicatorDocID(err, col.Name().Value(), rep.Info.ID) } p.pushToReplicator(ctx, txn, col, keysCh, rep.Info.ID) } diff --git a/planner/datasource.go b/planner/datasource.go index 6cfb8cf728..cc0bb0a019 100644 --- a/planner/datasource.go +++ b/planner/datasource.go @@ -32,7 +32,7 @@ func (p *Planner) getCollectionScanPlan(mapperSelect *mapper.Select) (planSource } var plan planNode - if col.Description().BaseQuery != nil { + if len(col.Description().QuerySources()) > 0 { var err error plan, err = p.View(mapperSelect, col.Description()) if err != nil { diff --git a/planner/scan.go b/planner/scan.go index 19ae079f5f..f9edb77b54 100644 --- a/planner/scan.go +++ b/planner/scan.go @@ -252,7 +252,7 @@ func (n *scanNode) simpleExplain() (map[string]any, error) { } // Add the collection attributes. - simpleExplainMap[collectionNameLabel] = n.col.Name() + simpleExplainMap[collectionNameLabel] = n.col.Name().Value() simpleExplainMap[collectionIDLabel] = n.col.Description().IDString() // Add the spans attribute. diff --git a/planner/select.go b/planner/select.go index f1d85de9f3..295ad3c475 100644 --- a/planner/select.go +++ b/planner/select.go @@ -373,8 +373,9 @@ func (n *selectNode) initFields(selectReq *mapper.Select) ([]aggregateNode, erro // commit query link fields are always added and need no special treatment here // WARNING: It is important to check collection name is nil and the parent select name // here else we risk falsely identifying user defined fields with the name `links` as a commit links field - } else if n.collection.Description().BaseQuery == nil { - // Views only contain embedded objects and don't require a traditional join here + } else if !(n.collection != nil && len(n.collection.Description().QuerySources()) > 0) { + // Collections sourcing data from queries only contain embedded objects and don't require + // a traditional join here err := n.addTypeIndexJoin(f) if err != nil { return nil, err diff --git a/planner/type_join.go b/planner/type_join.go index fc4e6009cf..28d044938f 100644 --- a/planner/type_join.go +++ b/planner/type_join.go @@ -261,7 +261,7 @@ func (p *Planner) makeTypeJoinOne( subTypeField, subTypeFieldNameFound := subTypeCol.Description().GetFieldByRelation( subTypeFieldDesc.RelationName, - parent.collection.Name(), + parent.collection.Name().Value(), subTypeFieldDesc.Name, &subTypeSchema, ) @@ -396,7 +396,7 @@ func (p *Planner) makeTypeJoinMany( rootField, rootNameFound := subTypeCol.Description().GetFieldByRelation( subTypeFieldDesc.RelationName, - parent.collection.Name(), + parent.collection.Name().Value(), subTypeFieldDesc.Name, &subTypeSchema, ) diff --git a/planner/view.go b/planner/view.go index 48a026f306..f7b015becf 100644 --- a/planner/view.go +++ b/planner/view.go @@ -26,7 +26,9 @@ type viewNode struct { } func (p *Planner) View(query *mapper.Select, desc client.CollectionDescription) (*viewNode, error) { - m, err := mapper.ToSelect(p.ctx, p.db, desc.BaseQuery) + baseQuery := (desc.Sources[0].(*client.QuerySource)).Query + + m, err := mapper.ToSelect(p.ctx, p.db, &baseQuery) if err != nil { return nil, err } diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go index 36f4d61c71..d3e2e35c36 100644 --- a/request/graphql/schema/collection.go +++ b/request/graphql/schema/collection.go @@ -15,6 +15,8 @@ import ( "fmt" "sort" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/request/graphql/schema/types" @@ -151,7 +153,7 @@ func collectionFromAstDefinition( return client.CollectionDefinition{ Description: client.CollectionDescription{ - Name: def.Name.Value, + Name: immutable.Some(def.Name.Value), Indexes: indexDescriptions, }, Schema: client.SchemaDescription{ @@ -513,7 +515,7 @@ func getRelationshipName( func finalizeRelations(relationManager *RelationManager, definitions []client.CollectionDefinition) error { embeddedObjNames := map[string]struct{}{} for _, def := range definitions { - if def.Description.Name == "" { + if !def.Description.Name.HasValue() { embeddedObjNames[def.Schema.Name] = struct{}{} } } diff --git a/request/graphql/schema/descriptions_test.go b/request/graphql/schema/descriptions_test.go index 397436bca2..39a7b5cce2 100644 --- a/request/graphql/schema/descriptions_test.go +++ b/request/graphql/schema/descriptions_test.go @@ -14,6 +14,7 @@ import ( "context" "testing" + "github.com/sourcenetwork/immutable" "github.com/stretchr/testify/assert" "github.com/sourcenetwork/defradb/client" @@ -33,7 +34,7 @@ func TestSingleSimpleType(t *testing.T) { targetDescs: []client.CollectionDefinition{ { Description: client.CollectionDescription{ - Name: "User", + Name: immutable.Some("User"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -82,7 +83,7 @@ func TestSingleSimpleType(t *testing.T) { targetDescs: []client.CollectionDefinition{ { Description: client.CollectionDescription{ - Name: "User", + Name: immutable.Some("User"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -113,7 +114,7 @@ func TestSingleSimpleType(t *testing.T) { }, { Description: client.CollectionDescription{ - Name: "Author", + Name: immutable.Some("Author"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -162,7 +163,7 @@ func TestSingleSimpleType(t *testing.T) { targetDescs: []client.CollectionDefinition{ { Description: client.CollectionDescription{ - Name: "Book", + Name: immutable.Some("Book"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -202,7 +203,7 @@ func TestSingleSimpleType(t *testing.T) { }, { Description: client.CollectionDescription{ - Name: "Author", + Name: immutable.Some("Author"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -260,7 +261,7 @@ func TestSingleSimpleType(t *testing.T) { targetDescs: []client.CollectionDefinition{ { Description: client.CollectionDescription{ - Name: "User", + Name: immutable.Some("User"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -291,7 +292,7 @@ func TestSingleSimpleType(t *testing.T) { }, { Description: client.CollectionDescription{ - Name: "Author", + Name: immutable.Some("Author"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -340,7 +341,7 @@ func TestSingleSimpleType(t *testing.T) { targetDescs: []client.CollectionDefinition{ { Description: client.CollectionDescription{ - Name: "Book", + Name: immutable.Some("Book"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -380,7 +381,7 @@ func TestSingleSimpleType(t *testing.T) { }, { Description: client.CollectionDescription{ - Name: "Author", + Name: immutable.Some("Author"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -438,7 +439,7 @@ func TestSingleSimpleType(t *testing.T) { targetDescs: []client.CollectionDefinition{ { Description: client.CollectionDescription{ - Name: "Book", + Name: immutable.Some("Book"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -478,7 +479,7 @@ func TestSingleSimpleType(t *testing.T) { }, { Description: client.CollectionDescription{ - Name: "Author", + Name: immutable.Some("Author"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -536,7 +537,7 @@ func TestSingleSimpleType(t *testing.T) { targetDescs: []client.CollectionDefinition{ { Description: client.CollectionDescription{ - Name: "Book", + Name: immutable.Some("Book"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ @@ -576,7 +577,7 @@ func TestSingleSimpleType(t *testing.T) { }, { Description: client.CollectionDescription{ - Name: "Author", + Name: immutable.Some("Author"), Indexes: []client.IndexDescription{}, }, Schema: client.SchemaDescription{ diff --git a/request/graphql/schema/generate.go b/request/graphql/schema/generate.go index 1083772d58..32fe562cff 100644 --- a/request/graphql/schema/generate.go +++ b/request/graphql/schema/generate.go @@ -108,7 +108,7 @@ func (g *Generator) generate(ctx context.Context, collections []client.Collectio var isEmbedded bool for _, definition := range collections { - if t.Name() == definition.Schema.Name && definition.Description.Name == "" { + if t.Name() == definition.Schema.Name && !definition.Description.Name.HasValue() { isEmbedded = true break } @@ -194,8 +194,8 @@ func (g *Generator) generate(ctx context.Context, collections []client.Collectio var isReadOnly bool var collectionFound bool for _, definition := range collections { - if t.Name() == definition.Description.Name { - isReadOnly = definition.Description.BaseQuery != nil + if t.Name() == definition.Description.Name.Value() { + isReadOnly = len(definition.Description.QuerySources()) > 0 collectionFound = true break } @@ -416,15 +416,16 @@ func (g *Generator) buildTypes( // TODO remove when Go 1.22 collection := c fieldDescriptions := collection.Schema.Fields - isEmbeddedObject := collection.Description.Name == "" - isViewObject := isEmbeddedObject || collection.Description.BaseQuery != nil + isEmbeddedObject := !collection.Description.Name.HasValue() + isQuerySource := len(collection.Description.QuerySources()) > 0 + isViewObject := isEmbeddedObject || isQuerySource var objectName string if isEmbeddedObject { // If this is an embedded object, take the type name from the Schema objectName = collection.Schema.Name } else { - objectName = collection.Description.Name + objectName = collection.Description.Name.Value() } // check if type exists @@ -529,7 +530,7 @@ func (g *Generator) buildTypes( // for collection create and update mutation operations. func (g *Generator) buildMutationInputTypes(collections []client.CollectionDefinition) error { for _, c := range collections { - if c.Description.Name == "" { + if !c.Description.Name.HasValue() { // If the definition's collection is empty, this must be a collectionless // schema, in which case users cannot mutate documents through it and we // have no need to build mutation input types for it. @@ -541,7 +542,7 @@ func (g *Generator) buildMutationInputTypes(collections []client.CollectionDefin // TODO remove when Go 1.22 collection := c fieldDescriptions := collection.Schema.Fields - mutationInputName := collection.Description.Name + "MutationInputArg" + mutationInputName := collection.Description.Name.Value() + "MutationInputArg" // check if mutation input type exists if _, ok := g.manager.schema.TypeMap()[mutationInputName]; ok { diff --git a/tests/clients/cli/wrapper_collection.go b/tests/clients/cli/wrapper_collection.go index 8295bad8d7..4d94d7b20f 100644 --- a/tests/clients/cli/wrapper_collection.go +++ b/tests/clients/cli/wrapper_collection.go @@ -16,6 +16,8 @@ import ( "fmt" "strings" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/datastore" @@ -34,7 +36,7 @@ func (c *Collection) Description() client.CollectionDescription { return c.def.Description } -func (c *Collection) Name() string { +func (c *Collection) Name() immutable.Option[string] { return c.Description().Name } @@ -55,8 +57,12 @@ func (c *Collection) Definition() client.CollectionDefinition { } func (c *Collection) Create(ctx context.Context, doc *client.Document) error { + if !c.Description().Name.HasValue() { + return client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "create"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) document, err := doc.String() if err != nil { @@ -73,8 +79,12 @@ func (c *Collection) Create(ctx context.Context, doc *client.Document) error { } func (c *Collection) CreateMany(ctx context.Context, docs []*client.Document) error { + if !c.Description().Name.HasValue() { + return client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "create"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) docMapList := make([]map[string]any, len(docs)) for i, doc := range docs { @@ -101,8 +111,12 @@ func (c *Collection) CreateMany(ctx context.Context, docs []*client.Document) er } func (c *Collection) Update(ctx context.Context, doc *client.Document) error { + if !c.Description().Name.HasValue() { + return client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "update"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) args = append(args, "--docID", doc.ID().String()) document, err := doc.ToJSONPatch() @@ -179,8 +193,12 @@ func (c *Collection) UpdateWithFilter( filter any, updater string, ) (*client.UpdateResult, error) { + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "update"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) args = append(args, "--updater", updater) filterJSON, err := json.Marshal(filter) @@ -197,8 +215,12 @@ func (c *Collection) UpdateWithDocID( docID client.DocID, updater string, ) (*client.UpdateResult, error) { + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "update"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) args = append(args, "--docID", docID.String()) args = append(args, "--updater", updater) @@ -210,8 +232,12 @@ func (c *Collection) UpdateWithDocIDs( docIDs []client.DocID, updater string, ) (*client.UpdateResult, error) { + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "update"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) args = append(args, "--updater", updater) strDocIDs := make([]string, len(docIDs)) @@ -252,8 +278,12 @@ func (c *Collection) deleteWith( } func (c *Collection) DeleteWithFilter(ctx context.Context, filter any) (*client.DeleteResult, error) { + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "delete"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) filterJSON, err := json.Marshal(filter) if err != nil { @@ -265,16 +295,24 @@ func (c *Collection) DeleteWithFilter(ctx context.Context, filter any) (*client. } func (c *Collection) DeleteWithDocID(ctx context.Context, docID client.DocID) (*client.DeleteResult, error) { + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "delete"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) args = append(args, "--docID", docID.String()) return c.deleteWith(ctx, args) } func (c *Collection) DeleteWithDocIDs(ctx context.Context, docIDs []client.DocID) (*client.DeleteResult, error) { + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "delete"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) strDocIDs := make([]string, len(docIDs)) for i, v := range docIDs { @@ -286,8 +324,12 @@ func (c *Collection) DeleteWithDocIDs(ctx context.Context, docIDs []client.DocID } func (c *Collection) Get(ctx context.Context, docID client.DocID, showDeleted bool) (*client.Document, error) { + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "get"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) args = append(args, docID.String()) if showDeleted { @@ -315,8 +357,12 @@ func (c *Collection) WithTxn(tx datastore.Txn) client.Collection { } func (c *Collection) GetAllDocIDs(ctx context.Context) (<-chan client.DocIDResult, error) { + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "collection", "docIDs"} - args = append(args, "--name", c.Description().Name) + args = append(args, "--name", c.Description().Name.Value()) stdOut, _, err := c.cmd.executeStream(ctx, args) if err != nil { @@ -354,8 +400,12 @@ func (c *Collection) CreateIndex( ctx context.Context, indexDesc client.IndexDescription, ) (index client.IndexDescription, err error) { + if !c.Description().Name.HasValue() { + return client.IndexDescription{}, client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "index", "create"} - args = append(args, "--collection", c.Description().Name) + args = append(args, "--collection", c.Description().Name.Value()) if indexDesc.Name != "" { args = append(args, "--name", indexDesc.Name) } @@ -380,8 +430,12 @@ func (c *Collection) CreateIndex( } func (c *Collection) DropIndex(ctx context.Context, indexName string) error { + if !c.Description().Name.HasValue() { + return client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "index", "drop"} - args = append(args, "--collection", c.Description().Name) + args = append(args, "--collection", c.Description().Name.Value()) args = append(args, "--name", indexName) _, err := c.cmd.execute(ctx, args) @@ -389,8 +443,12 @@ func (c *Collection) DropIndex(ctx context.Context, indexName string) error { } func (c *Collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, error) { + if !c.Description().Name.HasValue() { + return nil, client.ErrOperationNotPermittedOnNamelessCols + } + args := []string{"client", "index", "list"} - args = append(args, "--collection", c.Description().Name) + args = append(args, "--collection", c.Description().Name.Value()) data, err := c.cmd.execute(ctx, args) if err != nil { diff --git a/tests/gen/cli/gendocs.go b/tests/gen/cli/gendocs.go index 6d388eaf67..226d73bc97 100644 --- a/tests/gen/cli/gendocs.go +++ b/tests/gen/cli/gendocs.go @@ -123,7 +123,7 @@ func saveBatchToCollections( ) error { for colName, colDocs := range colDocsMap { for _, col := range collections { - if col.Description().Name == colName { + if col.Description().Name.Value() == colName { err := col.CreateMany(context.Background(), colDocs) if err != nil { return err @@ -138,7 +138,7 @@ func saveBatchToCollections( func groupDocsByCollection(docs []gen.GeneratedDoc) map[string][]*client.Document { result := make(map[string][]*client.Document) for _, doc := range docs { - result[doc.Col.Description.Name] = append(result[doc.Col.Description.Name], doc.Doc) + result[doc.Col.Description.Name.Value()] = append(result[doc.Col.Description.Name.Value()], doc.Doc) } return result } diff --git a/tests/gen/gen_auto.go b/tests/gen/gen_auto.go index 7ad3bb2d41..2dd3f78d8a 100644 --- a/tests/gen/gen_auto.go +++ b/tests/gen/gen_auto.go @@ -54,7 +54,7 @@ func AutoGenerate(definitions []client.CollectionDefinition, options ...Option) } typeDefs := make(map[string]client.CollectionDefinition) for _, def := range definitions { - typeDefs[def.Description.Name] = def + typeDefs[def.Description.Name.Value()] = def } generator := newRandomDocGenerator(typeDefs, nil) return generator.generateDocs(options...) @@ -212,13 +212,13 @@ func validateDefinitions(definitions []client.CollectionDefinition) error { colNames := make(map[string]struct{}) fieldRefs := []string{} for _, def := range definitions { - if def.Description.Name == "" { + if def.Description.Name.Value() == "" { return NewErrIncompleteColDefinition("description name is empty") } if def.Schema.Name == "" { return NewErrIncompleteColDefinition("schema name is empty") } - if def.Description.Name != def.Schema.Name { + if def.Description.Name.Value() != def.Schema.Name { return NewErrIncompleteColDefinition("description name and schema name do not match") } for _, field := range def.Schema.Fields { @@ -232,7 +232,7 @@ func validateDefinitions(definitions []client.CollectionDefinition) error { fieldRefs = append(fieldRefs, field.Schema) } } - colNames[def.Description.Name] = struct{}{} + colNames[def.Description.Name.Value()] = struct{}{} colIDs[def.Description.ID] = struct{}{} } for _, ref := range fieldRefs { diff --git a/tests/gen/gen_auto_config.go b/tests/gen/gen_auto_config.go index ccebce92d1..1457e67a12 100644 --- a/tests/gen/gen_auto_config.go +++ b/tests/gen/gen_auto_config.go @@ -54,7 +54,7 @@ func (m configsMap) AddForField(typeStr, fieldName string, conf genConfig) { func validateConfig(types map[string]client.CollectionDefinition, configsMap configsMap) error { for typeName, typeConfigs := range configsMap { typeDef := types[typeName] - if typeDef.Description.Name == "" { + if typeDef.Description.Name.Value() == "" { return newNotDefinedTypeErr(typeName) } for fieldName, fieldConfig := range typeConfigs { diff --git a/tests/gen/gen_auto_test.go b/tests/gen/gen_auto_test.go index f22859df0c..25191bbe41 100644 --- a/tests/gen/gen_auto_test.go +++ b/tests/gen/gen_auto_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/sourcenetwork/immutable" "github.com/stretchr/testify/assert" "github.com/sourcenetwork/defradb/client" @@ -71,7 +72,7 @@ func getDocIDsFromDocs(docs []*client.Document) []string { func filterByCollection(docs []GeneratedDoc, name string) []*client.Document { var result []*client.Document for _, doc := range docs { - if doc.Col.Description.Name == name { + if doc.Col.Description.Name.Value() == name { result = append(result, doc.Doc) } } @@ -1200,7 +1201,7 @@ func TestAutoGenerate_IfCollectionDefinitionIsIncomplete_ReturnError(t *testing. return []client.CollectionDefinition{ { Description: client.CollectionDescription{ - Name: "User", + Name: immutable.Some("User"), ID: 0, }, Schema: client.SchemaDescription{ @@ -1221,7 +1222,7 @@ func TestAutoGenerate_IfCollectionDefinitionIsIncomplete_ReturnError(t *testing. }, { Description: client.CollectionDescription{ - Name: "Device", + Name: immutable.Some("Device"), ID: 1, }, Schema: client.SchemaDescription{ @@ -1252,7 +1253,13 @@ func TestAutoGenerate_IfCollectionDefinitionIsIncomplete_ReturnError(t *testing. { name: "description name is empty", changeDefs: func(defs []client.CollectionDefinition) { - defs[0].Description.Name = "" + defs[0].Description.Name = immutable.Some("") + }, + }, + { + name: "description name is none", + changeDefs: func(defs []client.CollectionDefinition) { + defs[0].Description.Name = immutable.None[string]() }, }, { @@ -1312,7 +1319,7 @@ func TestAutoGenerate_IfColDefinitionsAreValid_ShouldGenerate(t *testing.T) { defs := []client.CollectionDefinition{ { Description: client.CollectionDescription{ - Name: "User", + Name: immutable.Some("User"), ID: 0, }, Schema: client.SchemaDescription{ @@ -1341,7 +1348,7 @@ func TestAutoGenerate_IfColDefinitionsAreValid_ShouldGenerate(t *testing.T) { }, { Description: client.CollectionDescription{ - Name: "Device", + Name: immutable.Some("Device"), ID: 1, }, Schema: client.SchemaDescription{ diff --git a/tests/gen/schema_parser.go b/tests/gen/schema_parser.go index 216376c26d..bcce388f22 100644 --- a/tests/gen/schema_parser.go +++ b/tests/gen/schema_parser.go @@ -31,7 +31,7 @@ func parseSDL(gqlSDL string) (map[string]client.CollectionDefinition, error) { } result := make(map[string]client.CollectionDefinition) for _, col := range cols { - result[col.Description.Name] = col + result[col.Description.Name.Value()] = col } return result, nil } diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index dc344d49f7..b2d3b26893 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -344,7 +344,7 @@ func createGenerateDocs(s *state, docs []gen.GeneratedDoc, nodeID immutable.Opti if err != nil { s.t.Fatalf("Failed to generate docs %s", err) } - createDoc(s, CreateDoc{CollectionID: nameToInd[doc.Col.Description.Name], Doc: docJSON, NodeID: nodeID}) + createDoc(s, CreateDoc{CollectionID: nameToInd[doc.Col.Description.Name.Value()], Doc: docJSON, NodeID: nodeID}) } } @@ -352,7 +352,7 @@ func generateDocs(s *state, action GenerateDocs) { collections := getNodeCollections(action.NodeID, s.collections) defs := make([]client.CollectionDefinition, 0, len(collections[0])) for _, col := range collections[0] { - if len(action.ForCollections) == 0 || slice.Contains(action.ForCollections, col.Name()) { + if len(action.ForCollections) == 0 || slice.Contains(action.ForCollections, col.Name().Value()) { defs = append(defs, col.Definition()) } } @@ -739,7 +739,7 @@ func refreshCollections( for i, collectionName := range s.collectionNames { for _, collection := range allCollections { - if collection.Name() == collectionName { + if collection.Name().Value() == collectionName { s.collections[nodeID][i] = collection break } @@ -1155,7 +1155,7 @@ func createDocViaGQL( _docID } }`, - collection.Name(), + collection.Name().Value(), input, ) @@ -1302,7 +1302,7 @@ func updateDocViaGQL( _docID } }`, - collection.Name(), + collection.Name().Value(), doc.ID().String(), input, ) @@ -1883,7 +1883,7 @@ func ParseSDL(gqlSDL string) (map[string]client.CollectionDefinition, error) { } result := make(map[string]client.CollectionDefinition) for _, col := range cols { - result[col.Description.Name] = col + result[col.Description.Name.Value()] = col } return result, nil } diff --git a/tests/predefined/gen_predefined.go b/tests/predefined/gen_predefined.go index 76e143c896..0a203fe2ed 100644 --- a/tests/predefined/gen_predefined.go +++ b/tests/predefined/gen_predefined.go @@ -31,7 +31,7 @@ func parseSDL(gqlSDL string) (map[string]client.CollectionDefinition, error) { } result := make(map[string]client.CollectionDefinition) for _, col := range cols { - result[col.Description.Name] = col + result[col.Description.Name.Value()] = col } return result, nil } @@ -85,7 +85,7 @@ func Create(defs []client.CollectionDefinition, docsList DocsList) ([]gen.Genera resultDocs := make([]gen.GeneratedDoc, 0, len(docsList.Docs)) typeDefs := make(map[string]client.CollectionDefinition) for _, col := range defs { - typeDefs[col.Description.Name] = col + typeDefs[col.Description.Name.Value()] = col } generator := docGenerator{types: typeDefs} for _, doc := range docsList.Docs { @@ -151,7 +151,7 @@ func (this *docGenerator) generatePrimary( result = append(result, subResult...) secondaryDocs, err := this.generateSecondaryDocs( - secDocMapField.(map[string]any), docID, &primType, secType.Description.Name) + secDocMapField.(map[string]any), docID, &primType, secType.Description.Name.Value()) if err != nil { return nil, nil, err } @@ -202,7 +202,7 @@ func (this *docGenerator) generateSecondaryDocs( if !field.IsPrimaryRelation() && (parentTypeName == "" || parentTypeName != field.Schema) { docs, err := this.generateSecondaryDocsForField( - primaryDocMap, primaryType.Description.Name, &field, docID) + primaryDocMap, primaryType.Description.Name.Value(), &field, docID) if err != nil { return nil, err } @@ -231,7 +231,7 @@ func (this *docGenerator) generateSecondaryDocsForField( case []map[string]any: for _, relDoc := range relVal { relDoc[primaryPropName] = primaryDocID - actions, err := this.generateRelatedDocs(relDoc, relTypeDef.Description.Name) + actions, err := this.generateRelatedDocs(relDoc, relTypeDef.Description.Name.Value()) if err != nil { return nil, err } @@ -239,7 +239,7 @@ func (this *docGenerator) generateSecondaryDocsForField( } case map[string]any: relVal[primaryPropName] = primaryDocID - actions, err := this.generateRelatedDocs(relVal, relTypeDef.Description.Name) + actions, err := this.generateRelatedDocs(relVal, relTypeDef.Description.Name.Value()) if err != nil { return nil, err }