From cdd05002eb7f880a11a74d7814345f57282f160a Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Mon, 2 Oct 2023 15:45:06 -0400 Subject: [PATCH] WIP - index tests --- db/index_test.go | 279 +++++++++++++++++----------------------- db/indexed_docs_test.go | 59 ++++----- 2 files changed, 151 insertions(+), 187 deletions(-) diff --git a/db/index_test.go b/db/index_test.go index dce7e65bb4..7392921bba 100644 --- a/db/index_test.go +++ b/db/index_test.go @@ -15,6 +15,7 @@ import ( "encoding/binary" "encoding/json" "fmt" + "strings" "testing" ds "github.com/ipfs/go-datastore" @@ -55,71 +56,62 @@ type indexTestFixture struct { ctx context.Context db *implicitTxnDB txn datastore.Txn - users *collection + users client.Collection t *testing.T } -func getUsersCollectionDesc() client.CollectionDescription { - return client.CollectionDescription{ - Name: usersColName, - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: usersNameFieldName, - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }, - { - Name: usersAgeFieldName, - Kind: client.FieldKind_INT, - Typ: client.LWW_REGISTER, - }, - { - Name: usersWeightFieldName, - Kind: client.FieldKind_FLOAT, - Typ: client.LWW_REGISTER, - }, - }, - }, - } -} +func (f *indexTestFixture) getUsersCollectionDesc() client.Collection { + _, err := f.db.AddSchema( + f.ctx, + fmt.Sprintf( + `type %s { + %s: String + %s: Int + %s: Float + }`, + usersColName, + usersNameFieldName, + usersAgeFieldName, + usersWeightFieldName, + ), + ) + require.NoError(f.t, err) -func getProductsCollectionDesc() client.CollectionDescription { - return client.CollectionDescription{ - Name: productsColName, - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: productsIDFieldName, - Kind: client.FieldKind_INT, - Typ: client.LWW_REGISTER, - }, - { - Name: productsPriceFieldName, - Kind: client.FieldKind_FLOAT, - Typ: client.LWW_REGISTER, - }, - { - Name: productsCategoryFieldName, - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }, - { - Name: productsAvailableFieldName, - Kind: client.FieldKind_BOOL, - Typ: client.LWW_REGISTER, - }, - }, - }, - } + col, err := f.db.GetCollectionByName(f.ctx, usersColName) + require.NoError(f.t, err) + + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) + + return col +} + +func (f *indexTestFixture) getProductsCollectionDesc() client.Collection { + _, err := f.db.AddSchema( + f.ctx, + fmt.Sprintf( + `type %s { + %s: Int + %s: Float + %s: String + %s: Boolean + }`, + productsColName, + productsIDFieldName, + productsPriceFieldName, + productsCategoryFieldName, + productsAvailableFieldName, + ), + ) + require.NoError(f.t, err) + + col, err := f.db.GetCollectionByName(f.ctx, productsColName) + require.NoError(f.t, err) + + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) + + return col } func newIndexTestFixtureBare(t *testing.T) *indexTestFixture { @@ -139,7 +131,7 @@ func newIndexTestFixtureBare(t *testing.T) *indexTestFixture { func newIndexTestFixture(t *testing.T) *indexTestFixture { f := newIndexTestFixtureBare(t) - f.users = f.createCollection(getUsersCollectionDesc()) + f.users = f.getUsersCollectionDesc() return f } @@ -247,18 +239,6 @@ func (f *indexTestFixture) getCollectionIndexes(colName string) ([]client.IndexD return f.db.fetchCollectionIndexDescriptions(f.ctx, f.txn, colName) } -func (f *indexTestFixture) createCollection( - desc client.CollectionDescription, -) *collection { - col, err := f.db.createCollection(f.ctx, f.txn, desc) - assert.NoError(f.t, err) - err = f.txn.Commit(f.ctx) - assert.NoError(f.t, err) - f.txn, err = f.db.NewTxn(f.ctx, false) - assert.NoError(f.t, err) - return col.(*collection) -} - func TestCreateIndex_IfFieldsIsEmpty_ReturnError(t *testing.T) { f := newIndexTestFixture(t) @@ -324,28 +304,6 @@ func TestCreateIndex_IfFieldHasNoDirection_DefaultToAsc(t *testing.T) { assert.Equal(t, client.Ascending, newDesc.Fields[0].Direction) } -func TestCreateIndex_IfNameIsNotSpecified_Generate(t *testing.T) { - f := newIndexTestFixtureBare(t) - colDesc := getUsersCollectionDesc() - const colName = "UsErS" - const fieldName = "NaMe" - colDesc.Name = colName - colDesc.Schema.Name = colName // Which one should we use? - colDesc.Schema.Fields[1].Name = fieldName - f.users = f.createCollection(colDesc) - - desc := client.IndexDescription{ - Name: "", - Fields: []client.IndexedFieldDescription{ - {Name: fieldName, Direction: client.Ascending}, - }, - } - - newDesc, err := f.createCollectionIndex(desc) - assert.NoError(t, err) - assert.Equal(t, colName+"_"+fieldName+"_ASC", newDesc.Name) -} - func TestCreateIndex_IfSingleFieldInDescOrder_ReturnError(t *testing.T) { f := newIndexTestFixture(t) @@ -515,8 +473,8 @@ func TestCreateIndex_IfPropertyDoesntExist_ReturnError(t *testing.T) { func TestCreateIndex_WithMultipleCollectionsAndIndexes_AssignIncrementedIDPerCollection(t *testing.T) { f := newIndexTestFixtureBare(t) - users := f.createCollection(getUsersCollectionDesc()) - products := f.createCollection(getProductsCollectionDesc()) + users := f.getUsersCollectionDesc() + products := f.getProductsCollectionDesc() makeIndex := func(fieldName string) client.IndexDescription { return client.IndexDescription{ @@ -606,24 +564,16 @@ func TestCreateIndex_IfAttemptToIndexOnUnsupportedType_ReturnError(t *testing.T) const unsupportedKind = client.FieldKind_BOOL_ARRAY - desc := client.CollectionDescription{ - Name: "testTypeCol", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "field", - Kind: unsupportedKind, - Typ: client.LWW_REGISTER, - }, - }, - }, - } + _, err := f.db.AddSchema( + f.ctx, + `type testTypeCol { + field: [Boolean!] + }`, + ) + require.NoError(f.t, err) - collection := f.createCollection(desc) + collection, err := f.db.GetCollectionByName(f.ctx, "testTypeCol") + require.NoError(f.t, err) indexDesc := client.IndexDescription{ Fields: []client.IndexedFieldDescription{ @@ -631,7 +581,10 @@ func TestCreateIndex_IfAttemptToIndexOnUnsupportedType_ReturnError(t *testing.T) }, } - _, err := f.createCollectionIndexFor(collection.Name(), indexDesc) + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) + + _, err = f.createCollectionIndexFor(collection.Name(), indexDesc) require.ErrorIs(f.t, err, NewErrUnsupportedIndexFieldType(unsupportedKind)) f.commitTxn() } @@ -652,7 +605,7 @@ func TestCreateIndex_IfFailedToReadIndexUponRetrievingCollectionDesc_ReturnError onSystemStore.Query(mock.Anything, mock.MatchedBy(matchPrefixFunc)).Return(nil, testErr) - descData, err := json.Marshal(getUsersCollectionDesc()) + descData, err := json.Marshal(f.users.Description()) require.NoError(t, err) onSystemStore.Query(mock.Anything, mock.Anything). @@ -676,7 +629,9 @@ func TestGetIndexes_ShouldReturnListOfAllExistingIndexes(t *testing.T) { _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) assert.NoError(t, err) - f.createCollection(getProductsCollectionDesc()) + f.commitTxn() + + f.getProductsCollectionDesc() productsIndexDesc := client.IndexDescription{ Name: "products_description_index", Fields: []client.IndexedFieldDescription{{Name: productsPriceFieldName}}, @@ -829,11 +784,17 @@ func TestGetCollectionIndexes_ShouldReturnListOfCollectionIndexes(t *testing.T) _, err := f.createCollectionIndexFor(usersColName, usersIndexDesc) assert.NoError(t, err) - f.createCollection(getProductsCollectionDesc()) + f.commitTxn() + + f.getProductsCollectionDesc() productsIndexDesc := client.IndexDescription{ Name: "products_description_index", Fields: []client.IndexedFieldDescription{{Name: productsPriceFieldName}}, } + + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) + _, err = f.createCollectionIndexFor(productsColName, productsIndexDesc) assert.NoError(t, err) @@ -1019,27 +980,23 @@ func TestCollectionGetIndexes_IfFailsToCreateTxn_ShouldNotCache(t *testing.T) { func TestCollectionGetIndexes_IfStoredIndexWithUnsupportedType_ReturnError(t *testing.T) { f := newIndexTestFixtureBare(t) + f.getUsersCollectionDesc() const unsupportedKind = client.FieldKind_BOOL_ARRAY + _, err := f.db.AddSchema( + f.ctx, + `type testTypeCol { + name: String + field: [Boolean!] + }`, + ) + require.NoError(f.t, err) - desc := client.CollectionDescription{ - Name: "testTypeCol", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "field", - Kind: unsupportedKind, - Typ: client.LWW_REGISTER, - }, - }, - }, - } + collection, err := f.db.GetCollectionByName(f.ctx, "testTypeCol") + require.NoError(f.t, err) - collection := f.createCollection(desc) + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) indexDesc := client.IndexDescription{ Fields: []client.IndexedFieldDescription{ @@ -1119,17 +1076,6 @@ func TestCollectionGetIndexes_IfIndexIsDropped_ReturnUpdateIndexes(t *testing.T) func TestCollectionGetIndexes_ShouldReturnIndexesInOrderedByName(t *testing.T) { f := newIndexTestFixtureBare(t) - colDesc := client.CollectionDescription{ - Name: "testCollection", - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - }, - }, - } const ( num = 30 fieldNamePrefix = "field_" @@ -1140,17 +1086,34 @@ func TestCollectionGetIndexes_ShouldReturnIndexesInOrderedByName(t *testing.T) { return fmt.Sprintf("%02d", i) } + builder := strings.Builder{} + builder.WriteString("type testCollection {\n") + for i := 1; i <= num; i++ { - colDesc.Schema.Fields = append(colDesc.Schema.Fields, - client.FieldDescription{ - Name: fieldNamePrefix + toSuffix(i), - Kind: client.FieldKind_STRING, - Typ: client.LWW_REGISTER, - }) + _, err := builder.WriteString(fieldNamePrefix) + require.NoError(f.t, err) + + _, err = builder.WriteString(toSuffix(i)) + require.NoError(f.t, err) + + _, err = builder.WriteString(": String\n") + require.NoError(f.t, err) } + _, err := builder.WriteString("}") + require.NoError(f.t, err) - collection := f.createCollection(colDesc) + const unsupportedKind = client.FieldKind_BOOL_ARRAY + _, err = f.db.AddSchema( + f.ctx, + builder.String(), + ) + require.NoError(f.t, err) + collection, err := f.db.GetCollectionByName(f.ctx, "testCollection") + require.NoError(f.t, err) + + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) for i := 1; i <= num; i++ { iStr := toSuffix(i) indexDesc := client.IndexDescription{ @@ -1317,7 +1280,7 @@ func TestDropAllIndexes_ShouldDeleteAllIndexes(t *testing.T) { assert.Equal(t, 2, f.countIndexPrefixes(usersColName, "")) - err = f.users.dropAllIndexes(f.ctx, f.txn) + err = f.users.(*collection).dropAllIndexes(f.ctx, f.txn) assert.NoError(t, err) assert.Equal(t, 0, f.countIndexPrefixes(usersColName, "")) @@ -1329,7 +1292,7 @@ func TestDropAllIndexes_IfStorageFails_ReturnError(t *testing.T) { f.db.Close(f.ctx) - err := f.users.dropAllIndexes(f.ctx, f.txn) + err := f.users.(*collection).dropAllIndexes(f.ctx, f.txn) assert.Error(t, err) } @@ -1384,7 +1347,7 @@ func TestDropAllIndexes_IfSystemStorageFails_ReturnError(t *testing.T) { mockedTxn.EXPECT().Systemstore().Unset() mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore).Maybe() - err := f.users.dropAllIndexes(f.ctx, f.txn) + err := f.users.(*collection).dropAllIndexes(f.ctx, f.txn) assert.ErrorIs(t, err, testErr, testCase.Name) } } @@ -1404,7 +1367,7 @@ func TestDropAllIndexes_ShouldCloseQueryIterator(t *testing.T) { mockedTxn.EXPECT().Systemstore().Unset() mockedTxn.EXPECT().Systemstore().Return(mockedTxn.MockSystemstore).Maybe() - _ = f.users.dropAllIndexes(f.ctx, f.txn) + _ = f.users.(*collection).dropAllIndexes(f.ctx, f.txn) } func TestNewCollectionIndex_IfDescriptionHasNoFields_ReturnError(t *testing.T) { diff --git a/db/indexed_docs_test.go b/db/indexed_docs_test.go index cd3f0b3fea..00a4aa49a3 100644 --- a/db/indexed_docs_test.go +++ b/db/indexed_docs_test.go @@ -234,7 +234,9 @@ func (f *indexTestFixture) stubSystemStore(systemStoreOn *mocks.DSReaderWriter_E systemStoreOn.Get(mock.Anything, colKey.ToDS()).Maybe().Return([]byte(userColVersionID), nil) colVersionIDKey := core.NewCollectionSchemaVersionKey(userColVersionID) - colDesc := getUsersCollectionDesc() + usersCol, err := f.db.GetCollectionByName(f.ctx, usersColName) + require.NoError(f.t, err) + colDesc := usersCol.Description() colDesc.ID = 1 for i := range colDesc.Schema.Fields { colDesc.Schema.Fields[i].ID = client.FieldID(i) @@ -361,8 +363,8 @@ func TestNonUnique_IfIndexIntField_StoreIt(t *testing.T) { func TestNonUnique_IfMultipleCollectionsWithIndexes_StoreIndexWithCollectionID(t *testing.T) { f := newIndexTestFixtureBare(t) - users := f.createCollection(getUsersCollectionDesc()) - products := f.createCollection(getProductsCollectionDesc()) + users := f.getUsersCollectionDesc() + products := f.getProductsCollectionDesc() _, err := f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnName()) require.NoError(f.t, err) @@ -437,24 +439,23 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { } for i, tc := range testCase { - desc := client.CollectionDescription{ - Name: "testTypeCol" + strconv.Itoa(i), - Schema: client.SchemaDescription{ - Fields: []client.FieldDescription{ - { - Name: "_key", - Kind: client.FieldKind_DocKey, - }, - { - Name: "field", - Kind: tc.FieldKind, - Typ: client.LWW_REGISTER, - }, - }, - }, - } + _, err := f.db.AddSchema( + f.ctx, + fmt.Sprintf( + `type %s { + field: %s + }`, + "testTypeCol"+strconv.Itoa(i), + tc.FieldKind.String(), + ), + ) + require.NoError(f.t, err) - collection := f.createCollection(desc) + collection, err := f.db.GetCollectionByName(f.ctx, "testTypeCol"+strconv.Itoa(i)) + require.NoError(f.t, err) + + f.txn, err = f.db.NewTxn(f.ctx, false) + require.NoError(f.t, err) indexDesc := client.IndexDescription{ Fields: []client.IndexedFieldDescription{ @@ -462,7 +463,7 @@ func TestNonUnique_StoringIndexedFieldValueOfDifferentTypes(t *testing.T) { }, } - _, err := f.createCollectionIndexFor(collection.Name(), indexDesc) + _, err = f.createCollectionIndexFor(collection.Name(), indexDesc) require.NoError(f.t, err) f.commitTxn() @@ -596,7 +597,7 @@ func TestNonUniqueCreate_IfUponIndexingExistingDocsFetcherFails_ReturnError(t *t doc := f.newUserDoc("John", 21) f.saveDocToCollection(doc, f.users) - f.users.fetcherFactory = tc.PrepareFetcher + f.users.(*collection).fetcherFactory = tc.PrepareFetcher key := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() _, err := f.users.CreateIndex(f.ctx, getUsersIndexDescOnName()) @@ -614,7 +615,7 @@ func TestNonUniqueCreate_IfDatastoreFailsToStoreIndex_ReturnError(t *testing.T) f.saveDocToCollection(doc, f.users) fieldKeyString := core.DataStoreKey{ - CollectionID: f.users.desc.IDString(), + CollectionID: f.users.Description().IDString(), }.WithDocKey(doc.Key().String()). WithFieldId("1"). WithValueFlag(). @@ -623,7 +624,7 @@ func TestNonUniqueCreate_IfDatastoreFailsToStoreIndex_ReturnError(t *testing.T) invalidKeyString := fieldKeyString + "/doesn't matter/" // Insert an invalid key within the document prefix, this will generate an error within the fetcher. - f.users.db.multistore.Datastore().Put(f.ctx, ipfsDatastore.NewKey(invalidKeyString), []byte("doesn't matter")) + f.db.multistore.Datastore().Put(f.ctx, ipfsDatastore.NewKey(invalidKeyString), []byte("doesn't matter")) _, err := f.users.CreateIndex(f.ctx, getUsersIndexDescOnName()) require.ErrorIs(f.t, err, core.ErrInvalidKey) @@ -631,7 +632,7 @@ func TestNonUniqueCreate_IfDatastoreFailsToStoreIndex_ReturnError(t *testing.T) func TestNonUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f := newIndexTestFixtureBare(t) - users := f.createCollection(getUsersCollectionDesc()) + users := f.getUsersCollectionDesc() _, err := f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnName()) require.NoError(f.t, err) _, err = f.createCollectionIndexFor(users.Name(), getUsersIndexDescOnAge()) @@ -643,7 +644,7 @@ func TestNonUniqueDrop_ShouldDeleteStoredIndexedFields(t *testing.T) { f.saveDocToCollection(f.newUserDoc("John", 21), users) f.saveDocToCollection(f.newUserDoc("Islam", 23), users) - products := f.createCollection(getProductsCollectionDesc()) + products := f.getProductsCollectionDesc() _, err = f.createCollectionIndexFor(products.Name(), getProductsIndexDescOnCategory()) require.NoError(f.t, err) f.commitTxn() @@ -885,7 +886,7 @@ func TestNonUniqueUpdate_IfFetcherFails_ReturnError(t *testing.T) { doc := f.newUserDoc("John", 21) f.saveDocToCollection(doc, f.users) - f.users.fetcherFactory = tc.PrepareFetcher + f.users.(*collection).fetcherFactory = tc.PrepareFetcher oldKey := newIndexKeyBuilder(f).Col(usersColName).Field(usersNameFieldName).Doc(doc).Build() err := doc.Set(usersNameFieldName, "Islam") @@ -931,7 +932,7 @@ func TestNonUniqueUpdate_ShouldPassToFetcherOnlyRelevantFields(t *testing.T) { f.createUserCollectionIndexOnName() f.createUserCollectionIndexOnAge() - f.users.fetcherFactory = func() fetcher.Fetcher { + f.users.(*collection).fetcherFactory = func() fetcher.Fetcher { f := fetcherMocks.NewStubbedFetcher(t) f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Unset() f.EXPECT().Init(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). @@ -999,7 +1000,7 @@ func TestNonUniqueUpdate_IfDatastoreFails_ReturnError(t *testing.T) { schemaVersionID: f.users.Schema().VersionID, } - f.users.fetcherFactory = func() fetcher.Fetcher { + f.users.(*collection).fetcherFactory = func() fetcher.Fetcher { df := fetcherMocks.NewStubbedFetcher(t) df.EXPECT().FetchNext(mock.Anything).Unset() df.EXPECT().FetchNext(mock.Anything).Return(&encodedDoc, fetcher.ExecInfo{}, nil)