From 44f3b6c5cb542fff9bdc60550bdaf284cfeb9293 Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Fri, 15 Sep 2023 13:36:47 -0400 Subject: [PATCH] Add SetDefaultSchemaVersion Will allow users to change their default database schema versions on demand, allowing rapid switching between (application) api/database versions. It also allows them to define and apply schema updates eagerly, and then make the switch at a later date. --- client/db.go | 9 + client/mocks/db.go | 43 ++++ db/collection.go | 26 +- db/txn_db.go | 19 ++ http/client.go | 11 + http/handler_store.go | 16 ++ http/server.go | 1 + http/wrapper.go | 4 + .../migrations/query/with_set_default_test.go | 232 ++++++++++++++++++ .../schema/with_update_set_default_test.go | 142 +++++++++++ tests/integration/test_case.go | 12 + tests/integration/utils2.go | 18 ++ 12 files changed, 532 insertions(+), 1 deletion(-) create mode 100644 tests/integration/schema/migrations/query/with_set_default_test.go create mode 100644 tests/integration/schema/with_update_set_default_test.go diff --git a/client/db.go b/client/db.go index ba4dd0b89d..18f4df680c 100644 --- a/client/db.go +++ b/client/db.go @@ -111,6 +111,15 @@ type Store interface { // [FieldKindStringToEnumMapping]. PatchSchema(context.Context, string) error + // SetDefaultSchemaVersion sets the default schema version to the ID provided. It will be applied to all + // collections using the schema. + // + // This will affect all operations interacting with the schema where a schema version is not explicitly + // provided. This includes GQL queries and Collection operations. + // + // It will return an error if the provided schema version ID does not exist. + SetDefaultSchemaVersion(context.Context, string) error + // SetMigration sets the migration for the given source-destination schema version IDs. Is equivilent to // calling `LensRegistry().SetMigration(ctx, cfg)`. // diff --git a/client/mocks/db.go b/client/mocks/db.go index cb0af26193..afaa1503bf 100644 --- a/client/mocks/db.go +++ b/client/mocks/db.go @@ -1163,6 +1163,49 @@ func (_c *DB_Root_Call) RunAndReturn(run func() datastore.RootStore) *DB_Root_Ca return _c } +// SetDefaultSchemaVersion provides a mock function with given fields: _a0, _a1 +func (_m *DB) SetDefaultSchemaVersion(_a0 context.Context, _a1 string) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DB_SetDefaultSchemaVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetDefaultSchemaVersion' +type DB_SetDefaultSchemaVersion_Call struct { + *mock.Call +} + +// SetDefaultSchemaVersion is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 string +func (_e *DB_Expecter) SetDefaultSchemaVersion(_a0 interface{}, _a1 interface{}) *DB_SetDefaultSchemaVersion_Call { + return &DB_SetDefaultSchemaVersion_Call{Call: _e.mock.On("SetDefaultSchemaVersion", _a0, _a1)} +} + +func (_c *DB_SetDefaultSchemaVersion_Call) Run(run func(_a0 context.Context, _a1 string)) *DB_SetDefaultSchemaVersion_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *DB_SetDefaultSchemaVersion_Call) Return(_a0 error) *DB_SetDefaultSchemaVersion_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *DB_SetDefaultSchemaVersion_Call) RunAndReturn(run func(context.Context, string) error) *DB_SetDefaultSchemaVersion_Call { + _c.Call.Return(run) + return _c +} + // SetMigration provides a mock function with given fields: _a0, _a1 func (_m *DB) SetMigration(_a0 context.Context, _a1 client.LensConfig) error { ret := _m.Called(_a0, _a1) diff --git a/db/collection.go b/db/collection.go index 82ff083842..e8ed363a7b 100644 --- a/db/collection.go +++ b/db/collection.go @@ -300,7 +300,7 @@ func (db *db) updateCollection( return nil, err } - err = db.setDefaultSchemaVersion(ctx, txn, desc.Name, desc.Schema.SchemaID, schemaVersionID) + err = db.setDefaultSchemaVersionExplicit(ctx, txn, desc.Name, desc.Schema.SchemaID, schemaVersionID) if err != nil { return nil, err } @@ -585,6 +585,30 @@ func validateUpdateCollectionIndexes( } func (db *db) setDefaultSchemaVersion( + ctx context.Context, + txn datastore.Txn, + schemaVersionID string, +) error { + col, err := db.getCollectionByVersionID(ctx, txn, schemaVersionID) + if err != nil { + return err + } + + desc := col.Description() + err = db.setDefaultSchemaVersionExplicit(ctx, txn, desc.Name, desc.Schema.SchemaID, schemaVersionID) + if err != nil { + return err + } + + cols, err := db.getCollectionDescriptions(ctx, txn) + if err != nil { + return err + } + + return db.parser.SetSchema(ctx, txn, cols) +} + +func (db *db) setDefaultSchemaVersionExplicit( ctx context.Context, txn datastore.Txn, collectionName string, diff --git a/db/txn_db.go b/db/txn_db.go index b307d96e35..03afaeb34a 100644 --- a/db/txn_db.go +++ b/db/txn_db.go @@ -280,6 +280,25 @@ func (db *explicitTxnDB) PatchSchema(ctx context.Context, patchString string) er return db.patchSchema(ctx, db.txn, patchString) } +func (db *implicitTxnDB) SetDefaultSchemaVersion(ctx context.Context, schemaVersionID string) error { + txn, err := db.NewTxn(ctx, false) + if err != nil { + return err + } + defer txn.Discard(ctx) + + err = db.setDefaultSchemaVersion(ctx, txn, schemaVersionID) + if err != nil { + return err + } + + return txn.Commit(ctx) +} + +func (db *explicitTxnDB) SetDefaultSchemaVersion(ctx context.Context, schemaVersionID string) error { + return db.setDefaultSchemaVersion(ctx, db.txn, schemaVersionID) +} + func (db *implicitTxnDB) SetMigration(ctx context.Context, cfg client.LensConfig) error { txn, err := db.NewTxn(ctx, false) if err != nil { diff --git a/http/client.go b/http/client.go index 16a8924a65..d0a7dba37f 100644 --- a/http/client.go +++ b/http/client.go @@ -223,6 +223,17 @@ func (c *Client) PatchSchema(ctx context.Context, patch string) error { return err } +func (c *Client) SetDefaultSchemaVersion(ctx context.Context, schemaVersionID string) error { + methodURL := c.http.baseURL.JoinPath("schema", "default") + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, methodURL.String(), strings.NewReader(schemaVersionID)) + if err != nil { + return err + } + _, err = c.http.request(req) + return err +} + func (c *Client) SetMigration(ctx context.Context, config client.LensConfig) error { return c.LensRegistry().SetMigration(ctx, config) } diff --git a/http/handler_store.go b/http/handler_store.go index d0cbdf42d2..40a8762d5d 100644 --- a/http/handler_store.go +++ b/http/handler_store.go @@ -164,6 +164,22 @@ func (s *storeHandler) PatchSchema(rw http.ResponseWriter, req *http.Request) { rw.WriteHeader(http.StatusOK) } +func (s *storeHandler) SetDefaultSchemaVersion(rw http.ResponseWriter, req *http.Request) { + store := req.Context().Value(storeContextKey).(client.Store) + + schemaVersionID, err := io.ReadAll(req.Body) + if err != nil { + responseJSON(rw, http.StatusBadRequest, errorResponse{err}) + return + } + err = store.SetDefaultSchemaVersion(req.Context(), string(schemaVersionID)) + if err != nil { + responseJSON(rw, http.StatusBadRequest, errorResponse{err}) + return + } + rw.WriteHeader(http.StatusOK) +} + func (s *storeHandler) GetCollection(rw http.ResponseWriter, req *http.Request) { store := req.Context().Value(storeContextKey).(client.Store) diff --git a/http/server.go b/http/server.go index afee4b9217..f1116d7b3b 100644 --- a/http/server.go +++ b/http/server.go @@ -53,6 +53,7 @@ func NewServer(db client.DB) *Server { api.Route("/schema", func(schema chi.Router) { schema.Post("/", store_handler.AddSchema) schema.Patch("/", store_handler.PatchSchema) + schema.Post("/default", store_handler.SetDefaultSchemaVersion) }) api.Route("/collections", func(collections chi.Router) { collections.Get("/", store_handler.GetCollection) diff --git a/http/wrapper.go b/http/wrapper.go index 558dc79474..31dc37a74f 100644 --- a/http/wrapper.go +++ b/http/wrapper.go @@ -90,6 +90,10 @@ func (w *Wrapper) PatchSchema(ctx context.Context, patch string) error { return w.client.PatchSchema(ctx, patch) } +func (w *Wrapper) SetDefaultSchemaVersion(ctx context.Context, schemaVersionID string) error { + return w.client.SetDefaultSchemaVersion(ctx, schemaVersionID) +} + func (w *Wrapper) SetMigration(ctx context.Context, config client.LensConfig) error { return w.client.SetMigration(ctx, config) } diff --git a/tests/integration/schema/migrations/query/with_set_default_test.go b/tests/integration/schema/migrations/query/with_set_default_test.go new file mode 100644 index 0000000000..3f929da7f9 --- /dev/null +++ b/tests/integration/schema/migrations/query/with_set_default_test.go @@ -0,0 +1,232 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package query + +import ( + "testing" + + "github.com/lens-vm/lens/host-go/config/model" + + "github.com/sourcenetwork/defradb/client" + testUtils "github.com/sourcenetwork/defradb/tests/integration" + "github.com/sourcenetwork/defradb/tests/lenses" +) + +func TestSchemaMigrationQuery_WithSetDefaultToLatest_AppliesForwardMigration(t *testing.T) { + schemaVersionID2 := "bafkreigfqdqnj5dunwgcsf2a6ht6q6m2yv3ys6byw5ifsmi5lfcpeh5t7e" + + test := testUtils.TestCase{ + Description: "Test schema migration", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + verified: Boolean + } + `, + }, + testUtils.CreateDoc{ + Doc: `{ + "name": "John" + }`, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Schema/Fields/-", "value": {"Name": "email", "Kind": "String"} } + ] + `, + }, + testUtils.ConfigureMigration{ + LensConfig: client.LensConfig{ + SourceSchemaVersionID: "bafkreifmgqtwpvepenteuvj27u4ewix6nb7ypvyz6j555wsk5u2n7hrldm", + DestinationSchemaVersionID: schemaVersionID2, + Lens: model.Lens{ + Lenses: []model.LensModule{ + { + Path: lenses.SetDefaultModulePath, + Arguments: map[string]any{ + "dst": "verified", + "value": true, + }, + }, + }, + }, + }, + }, + testUtils.SetDefaultSchemaVersion{ + SchemaVersionID: schemaVersionID2, + }, + testUtils.Request{ + Request: `query { + Users { + name + verified + } + }`, + Results: []map[string]any{ + { + "name": "John", + "verified": true, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestSchemaMigrationQuery_WithSetDefaultToOriginal_AppliesInverseMigration(t *testing.T) { + schemaVersionID1 := "bafkreifmgqtwpvepenteuvj27u4ewix6nb7ypvyz6j555wsk5u2n7hrldm" + schemaVersionID2 := "bafkreigfqdqnj5dunwgcsf2a6ht6q6m2yv3ys6byw5ifsmi5lfcpeh5t7e" + + test := testUtils.TestCase{ + Description: "Test schema migration", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + verified: Boolean + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Schema/Fields/-", "value": {"Name": "email", "Kind": "String"} } + ] + `, + }, + testUtils.SetDefaultSchemaVersion{ + SchemaVersionID: schemaVersionID2, + }, + // Create John using the new schema version + testUtils.CreateDoc{ + Doc: `{ + "name": "John", + "verified": true + }`, + }, + testUtils.ConfigureMigration{ + LensConfig: client.LensConfig{ + SourceSchemaVersionID: schemaVersionID1, + DestinationSchemaVersionID: schemaVersionID2, + Lens: model.Lens{ + Lenses: []model.LensModule{ + { + Path: lenses.SetDefaultModulePath, + Arguments: map[string]any{ + "dst": "verified", + "value": true, + }, + }, + }, + }, + }, + }, + // Set the schema version back to the original + testUtils.SetDefaultSchemaVersion{ + SchemaVersionID: schemaVersionID1, + }, + testUtils.Request{ + Request: `query { + Users { + name + verified + } + }`, + Results: []map[string]any{ + { + "name": "John", + // The inverse lens migration has been applied, clearing the verified field + "verified": nil, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestSchemaMigrationQuery_WithSetDefaultToOriginalVersionThatDocWasCreatedAt_ClearsMigrations(t *testing.T) { + schemaVersionID1 := "bafkreifmgqtwpvepenteuvj27u4ewix6nb7ypvyz6j555wsk5u2n7hrldm" + schemaVersionID2 := "bafkreigfqdqnj5dunwgcsf2a6ht6q6m2yv3ys6byw5ifsmi5lfcpeh5t7e" + + test := testUtils.TestCase{ + Description: "Test schema migration", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + verified: Boolean + } + `, + }, + // Create John using the original schema version + testUtils.CreateDoc{ + Doc: `{ + "name": "John", + "verified": false + }`, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Schema/Fields/-", "value": {"Name": "email", "Kind": "String"} } + ] + `, + }, + testUtils.ConfigureMigration{ + LensConfig: client.LensConfig{ + SourceSchemaVersionID: schemaVersionID1, + DestinationSchemaVersionID: schemaVersionID2, + Lens: model.Lens{ + Lenses: []model.LensModule{ + { + Path: lenses.SetDefaultModulePath, + Arguments: map[string]any{ + "dst": "verified", + "value": true, + }, + }, + }, + }, + }, + }, + // Set the schema version back to the original + testUtils.SetDefaultSchemaVersion{ + SchemaVersionID: schemaVersionID1, + }, + testUtils.Request{ + Request: `query { + Users { + name + verified + } + }`, + Results: []map[string]any{ + { + "name": "John", + // The inverse lens migration has not been applied, the document is returned as it was defined + "verified": false, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/schema/with_update_set_default_test.go b/tests/integration/schema/with_update_set_default_test.go new file mode 100644 index 0000000000..ae92e556f9 --- /dev/null +++ b/tests/integration/schema/with_update_set_default_test.go @@ -0,0 +1,142 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package schema + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestSchema_WithUpdateAndSetDefaultVersionToEmptyString_Errors(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test schema update, set default version to empty string", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Schema/Fields/-", "value": {"Name": "email", "Kind": 11} } + ] + `, + }, + testUtils.SetDefaultSchemaVersion{ + SchemaVersionID: "", + ExpectedError: "schema version ID can't be empty", + }, + }, + } + testUtils.ExecuteTestCase(t, test) +} + +func TestSchema_WithUpdateAndSetDefaultVersionToUnknownVersion_Errors(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test schema update, set default version to invalid string", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Schema/Fields/-", "value": {"Name": "email", "Kind": 11} } + ] + `, + }, + testUtils.SetDefaultSchemaVersion{ + SchemaVersionID: "does not exist", + ExpectedError: "datastore: key not found", + }, + }, + } + testUtils.ExecuteTestCase(t, test) +} + +func TestSchema_WithUpdateAndSetDefaultVersionToOriginal_NewFieldIsNotQueriable(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test schema update, set default version to original schema version", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Schema/Fields/-", "value": {"Name": "email", "Kind": 11} } + ] + `, + }, + testUtils.SetDefaultSchemaVersion{ + SchemaVersionID: "bafkreihn4qameldz3j7rfundmd4ldhxnaircuulk6h2vcwnpcgxl4oqffq", + }, + testUtils.Request{ + Request: `query { + Users { + name + email + } + }`, + // As the email field did not exist at this schema version, it will return a gql error + ExpectedError: `Cannot query field "email" on type "Users".`, + }, + }, + } + testUtils.ExecuteTestCase(t, test) +} + +func TestSchema_WithUpdateAndSetDefaultVersionToNew_AllowsQueryingOfNewField(t *testing.T) { + test := testUtils.TestCase{ + Description: "Test schema update, set default version to new schema version", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + } + `, + }, + testUtils.SchemaPatch{ + Patch: ` + [ + { "op": "add", "path": "/Users/Schema/Fields/-", "value": {"Name": "email", "Kind": 11} } + ] + `, + }, + testUtils.SetDefaultSchemaVersion{ + SchemaVersionID: "bafkreidejaxpsevyijnr4nah4e2l263emwhdaj57fwwv34eu5rea4ff54e", + }, + testUtils.Request{ + Request: `query { + Users { + name + email + } + }`, + Results: []map[string]any{}, + }, + }, + } + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index e17adfdeaa..c8580350b3 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -85,6 +85,18 @@ type SchemaPatch struct { ExpectedError string } +// SetDefaultSchemaVersion is an action that will set the default schema version to the +// given value. +type SetDefaultSchemaVersion struct { + // NodeID may hold the ID (index) of a node to set the default schema version on. + // + // If a value is not provided the default will be set on all nodes. + NodeID immutable.Option[int] + + SchemaVersionID string + ExpectedError string +} + // CreateDoc will attempt to create the given document in the given collection // using the set [MutationType]. type CreateDoc struct { diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 622478f513..717cfc0127 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -418,6 +418,9 @@ func executeTestCase( case SchemaPatch: patchSchema(s, action) + case SetDefaultSchemaVersion: + setDefaultSchemaVersion(s, action) + case ConfigureMigration: configureMigration(s, action) @@ -1070,6 +1073,21 @@ func patchSchema( refreshIndexes(s) } +func setDefaultSchemaVersion( + s *state, + action SetDefaultSchemaVersion, +) { + for _, node := range getNodes(action.NodeID, s.nodes) { + err := node.DB.SetDefaultSchemaVersion(s.ctx, action.SchemaVersionID) + expectedErrorRaised := AssertError(s.t, s.testCase.Description, err, action.ExpectedError) + + assertExpectedErrorRaised(s.t, s.testCase.Description, action.ExpectedError, expectedErrorRaised) + } + + refreshCollections(s) + refreshIndexes(s) +} + // createDoc creates a document using the chosen [mutationType] and caches it in the // test state object. func createDoc(