diff --git a/.gitignore b/.gitignore index 81c1a16d62..40eac1780c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,12 @@ tests/lenses/rust_wasm32_remove/pkg tests/lenses/rust_wasm32_copy/Cargo.lock tests/lenses/rust_wasm32_copy/target tests/lenses/rust_wasm32_copy/pkg +tests/lenses/rust_wasm32_prepend/Cargo.lock +tests/lenses/rust_wasm32_prepend/target +tests/lenses/rust_wasm32_prepend/pkg +tests/lenses/rust_wasm32_filter/Cargo.lock +tests/lenses/rust_wasm32_filter/target +tests/lenses/rust_wasm32_filter/pkg # Ignore OS X metadata files. .history diff --git a/cli/view_add.go b/cli/view_add.go index 46779fb784..44fa6d348e 100644 --- a/cli/view_add.go +++ b/cli/view_add.go @@ -10,34 +10,72 @@ package cli -import "github.com/spf13/cobra" +import ( + "encoding/json" + "io" + "os" + "strings" + + "github.com/lens-vm/lens/host-go/config/model" + "github.com/sourcenetwork/immutable" + "github.com/spf13/cobra" +) func MakeViewAddCommand() *cobra.Command { + var lensFile string var cmd = &cobra.Command{ - Use: "add [query] [sdl]", + Use: "add [query] [sdl] [transform]", Short: "Add new view", Long: `Add new database view. Example: add from an argument string: - defradb client view add 'Foo { name, ...}' 'type Foo { ... }' + defradb client view add 'Foo { name, ...}' 'type Foo { ... }' '{"lenses": [...' Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, + Args: cobra.RangeArgs(2, 4), RunE: func(cmd *cobra.Command, args []string) error { store := mustGetStoreContext(cmd) - if len(args) != 2 { - return ErrViewAddMissingArgs - } - query := args[0] sdl := args[1] - defs, err := store.AddView(cmd.Context(), query, sdl) + var lensCfgJson string + switch { + case lensFile != "": + data, err := os.ReadFile(lensFile) + if err != nil { + return err + } + lensCfgJson = string(data) + case len(args) == 3 && args[2] == "-": + data, err := io.ReadAll(cmd.InOrStdin()) + if err != nil { + return err + } + lensCfgJson = string(data) + case len(args) == 3: + lensCfgJson = args[2] + } + + var transform immutable.Option[model.Lens] + if lensCfgJson != "" { + decoder := json.NewDecoder(strings.NewReader(lensCfgJson)) + decoder.DisallowUnknownFields() + + var lensCfg model.Lens + if err := decoder.Decode(&lensCfg); err != nil { + return NewErrInvalidLensConfig(err) + } + transform = immutable.Some(lensCfg) + } + + defs, err := store.AddView(cmd.Context(), query, sdl, transform) if err != nil { return err } return writeJSON(cmd, defs) }, } + cmd.Flags().StringVarP(&lensFile, "file", "f", "", "Lens configuration file") return cmd } diff --git a/client/db.go b/client/db.go index 24d0212600..2b34550f7f 100644 --- a/client/db.go +++ b/client/db.go @@ -155,7 +155,16 @@ type Store interface { // // It will return the collection definitions of the types defined in the SDL if successful, otherwise an error // will be returned. This function does not execute the given query. - AddView(ctx context.Context, gqlQuery string, sdl string) ([]CollectionDefinition, error) + // + // Optionally, a lens transform configuration may also be provided - it will execute after the query has run. + // The transform is not limited to just transforming the input documents, it may also yield new ones, or filter out + // those passed in from the underlying query. + AddView( + ctx context.Context, + gqlQuery string, + sdl string, + transform immutable.Option[model.Lens], + ) ([]CollectionDefinition, error) // SetMigration sets the migration for all collections using the given source-destination schema version IDs. // diff --git a/client/descriptions.go b/client/descriptions.go index ada44acc25..dfd9948090 100644 --- a/client/descriptions.go +++ b/client/descriptions.go @@ -135,6 +135,13 @@ func sourcesOfType[ResultType any](col CollectionDescription) []ResultType { type QuerySource struct { // Query contains the base query of this data source. Query request.Select + + // Transform is a optional Lens configuration. If specified, data drawn from the [Query] will have the + // transform applied before being returned. + // + // The transform is not limited to just transforming the input documents, it may also yield new ones, or filter out + // those passed in from the underlying query. + Transform immutable.Option[model.Lens] } // CollectionSource represents a collection data source from another collection instance. diff --git a/db/txn_db.go b/db/txn_db.go index a05f2d895d..a6cb477bc5 100644 --- a/db/txn_db.go +++ b/db/txn_db.go @@ -391,14 +391,19 @@ func (db *explicitTxnDB) SetMigration(ctx context.Context, cfg client.LensConfig return db.setMigration(ctx, db.txn, cfg) } -func (db *implicitTxnDB) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) { +func (db *implicitTxnDB) AddView( + ctx context.Context, + query string, + sdl string, + transform immutable.Option[model.Lens], +) ([]client.CollectionDefinition, error) { txn, err := db.NewTxn(ctx, false) if err != nil { return nil, err } defer txn.Discard(ctx) - defs, err := db.addView(ctx, txn, query, sdl) + defs, err := db.addView(ctx, txn, query, sdl, transform) if err != nil { return nil, err } @@ -411,8 +416,13 @@ func (db *implicitTxnDB) AddView(ctx context.Context, query string, sdl string) return defs, nil } -func (db *explicitTxnDB) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) { - return db.addView(ctx, db.txn, query, sdl) +func (db *explicitTxnDB) AddView( + ctx context.Context, + query string, + sdl string, + transform immutable.Option[model.Lens], +) ([]client.CollectionDefinition, error) { + return db.addView(ctx, db.txn, query, sdl, transform) } // BasicImport imports a json dataset. diff --git a/db/view.go b/db/view.go index 2a61ff63af..ea57f94541 100644 --- a/db/view.go +++ b/db/view.go @@ -15,6 +15,9 @@ import ( "errors" "fmt" + "github.com/lens-vm/lens/host-go/config/model" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/datastore" @@ -26,6 +29,7 @@ func (db *db) addView( txn datastore.Txn, inputQuery string, sdl string, + transform immutable.Option[model.Lens], ) ([]client.CollectionDefinition, error) { // Wrap the given query as part of the GQL query object - this simplifies the syntax for users // and ensures that we can't be given mutations. In the future this line should disappear along @@ -57,7 +61,10 @@ func (db *db) addView( } for i := range newDefinitions { - source := client.QuerySource{Query: *baseQuery} + source := client.QuerySource{ + Query: *baseQuery, + Transform: transform, + } newDefinitions[i].Description.Sources = append(newDefinitions[i].Description.Sources, &source) } @@ -78,6 +85,15 @@ func (db *db) addView( return nil, err } returnDescriptions[i] = col.Definition() + + for _, source := range col.Description().QuerySources() { + if source.Transform.HasValue() { + err = db.LensRegistry().SetMigration(ctx, col.ID(), source.Transform.Value()) + if err != nil { + return nil, err + } + } + } } } diff --git a/http/client.go b/http/client.go index 1c4012d76b..5a1e5c5ebb 100644 --- a/http/client.go +++ b/http/client.go @@ -173,14 +173,20 @@ func (c *Client) SetActiveSchemaVersion(ctx context.Context, schemaVersionID str } type addViewRequest struct { - Query string - SDL string + Query string + SDL string + Transform immutable.Option[model.Lens] } -func (c *Client) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) { +func (c *Client) AddView( + ctx context.Context, + query string, + sdl string, + transform immutable.Option[model.Lens], +) ([]client.CollectionDefinition, error) { methodURL := c.http.baseURL.JoinPath("view") - body, err := json.Marshal(addViewRequest{query, sdl}) + body, err := json.Marshal(addViewRequest{query, sdl, transform}) if err != nil { return nil, err } diff --git a/http/handler_store.go b/http/handler_store.go index 314e01e973..f9a65eda8c 100644 --- a/http/handler_store.go +++ b/http/handler_store.go @@ -117,7 +117,7 @@ func (s *storeHandler) AddView(rw http.ResponseWriter, req *http.Request) { return } - defs, err := store.AddView(req.Context(), message.Query, message.SDL) + defs, err := store.AddView(req.Context(), message.Query, message.SDL, message.Transform) if err != nil { responseJSON(rw, http.StatusBadRequest, errorResponse{err}) return diff --git a/planner/datasource.go b/planner/datasource.go index cc0bb0a019..526621d9d4 100644 --- a/planner/datasource.go +++ b/planner/datasource.go @@ -34,7 +34,7 @@ func (p *Planner) getCollectionScanPlan(mapperSelect *mapper.Select) (planSource var plan planNode if len(col.Description().QuerySources()) > 0 { var err error - plan, err = p.View(mapperSelect, col.Description()) + plan, err = p.View(mapperSelect, col) if err != nil { return planSource{}, err } diff --git a/planner/lens.go b/planner/lens.go new file mode 100644 index 0000000000..eba0edd587 --- /dev/null +++ b/planner/lens.go @@ -0,0 +1,174 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package planner + +import ( + "github.com/sourcenetwork/immutable/enumerable" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/client/request" + "github.com/sourcenetwork/defradb/core" +) + +// viewNode applies a lens transform to data yielded from the source node. +// +// It may return a different number of documents to that yielded by its source, +// and there is no guarentee that those documents will actually exist as documents +// in Defra (they may be created by the transform). +type lensNode struct { + docMapper + documentIterator + + p *Planner + source planNode + collection client.CollectionDescription + + input enumerable.Queue[map[string]any] + output enumerable.Enumerable[map[string]any] +} + +func (p *Planner) Lens(source planNode, docMap *core.DocumentMapping, col client.Collection) *lensNode { + return &lensNode{ + docMapper: docMapper{docMap}, + p: p, + source: source, + collection: col.Description(), + } +} + +func (n *lensNode) Init() error { + n.input = enumerable.NewQueue[map[string]any]() + + pipe, err := n.p.db.LensRegistry().MigrateUp(n.p.ctx, n.input, n.collection.ID) + if err != nil { + return err + } + + n.output = pipe + + return n.source.Init() +} + +func (n *lensNode) Start() error { + return n.source.Start() +} + +func (n *lensNode) Spans(spans core.Spans) { + n.source.Spans(spans) +} + +func (n *lensNode) Next() (bool, error) { + hasNext, err := n.output.Next() + if err != nil { + return false, err + } + + if hasNext { + lensDoc, err := n.output.Value() + if err != nil { + return false, err + } + + nextValue, err := n.toDoc(lensDoc) + if err != nil { + return false, err + } + + n.currentValue = nextValue + return true, nil + } + + sourceHasNext, err := n.source.Next() + if err != nil { + return false, err + } + + if !sourceHasNext { + return false, nil + } + + sourceDoc := n.source.Value() + sourceLensDoc := n.source.Source().DocumentMap().ToMap(sourceDoc) + + err = n.input.Put(sourceLensDoc) + if err != nil { + return false, err + } + + return n.Next() +} + +func (n *lensNode) toDoc(mapDoc map[string]any) (core.Doc, error) { + status := client.Active + properties := make([]any, len(mapDoc)) + + for fieldName, fieldValue := range mapDoc { + if fieldName == request.DocIDFieldName && fieldValue != nil { + properties[core.DocIDFieldIndex] = fieldValue.(string) + continue + } + + if fieldName == request.DeletedFieldName { + if wasDeleted, ok := fieldValue.(bool); ok { + if wasDeleted { + status = client.Deleted + } + } + continue + } + + indexes := n.documentMapping.IndexesByName[fieldName] + if len(indexes) == 0 { + // Note: This can happen if a migration returns a field that + // we do not know about. In which case we have to skip it. + continue + } + // Take the last index of this name, this is in order to be consistent with other + // similar logic, for example when converting a core.Doc to a map before passing it + // into a lens transform. + fieldIndex := indexes[len(indexes)-1] + + if len(properties) <= fieldIndex { + // Because the document is sourced from another mapping, we may still need to grow + // the resultant field set. We cannot use [append] because the index of each field + // must still correspond to it's field ID. + originalProps := properties + properties = make([]any, fieldIndex+1) + copy(properties, originalProps) + } + properties[fieldIndex] = fieldValue + } + + return core.Doc{ + Fields: properties, + SchemaVersionID: n.collection.SchemaVersionID, + Status: status, + }, nil +} + +func (n *lensNode) Source() planNode { + return n.source +} + +func (n *lensNode) Kind() string { + return "lensNode" +} + +func (n *lensNode) Close() error { + if n.source != nil { + err := n.source.Close() + if err != nil { + return err + } + } + + return nil +} diff --git a/planner/operations.go b/planner/operations.go index 75d70dcdaf..59aa00a3c3 100644 --- a/planner/operations.go +++ b/planner/operations.go @@ -33,6 +33,7 @@ var ( _ planNode = (*updateNode)(nil) _ planNode = (*valuesNode)(nil) _ planNode = (*viewNode)(nil) + _ planNode = (*lensNode)(nil) _ MultiNode = (*parallelNode)(nil) _ MultiNode = (*topLevelNode)(nil) diff --git a/planner/planner.go b/planner/planner.go index 3ef8ff28e3..0629076924 100644 --- a/planner/planner.go +++ b/planner/planner.go @@ -240,6 +240,9 @@ func (p *Planner) expandPlan(planNode planNode, parentPlan *selectTopNode) error case *viewNode: return p.expandPlan(n.source, parentPlan) + case *lensNode: + return p.expandPlan(n.source, parentPlan) + default: return nil } diff --git a/planner/view.go b/planner/view.go index f7b015becf..f02de06d27 100644 --- a/planner/view.go +++ b/planner/view.go @@ -23,12 +23,17 @@ type viewNode struct { p *Planner desc client.CollectionDescription source planNode + + // This is cached as a boolean to save rediscovering this in the main Next/Value iteration loop + hasTransform bool } -func (p *Planner) View(query *mapper.Select, desc client.CollectionDescription) (*viewNode, error) { - baseQuery := (desc.Sources[0].(*client.QuerySource)).Query +func (p *Planner) View(query *mapper.Select, col client.Collection) (planNode, error) { + // For now, we assume a single source. This will need to change if/when we support multiple sources + querySource := (col.Description().Sources[0].(*client.QuerySource)) + hasTransform := querySource.Transform.HasValue() - m, err := mapper.ToSelect(p.ctx, p.db, &baseQuery) + m, err := mapper.ToSelect(p.ctx, p.db, &querySource.Query) if err != nil { return nil, err } @@ -38,12 +43,19 @@ func (p *Planner) View(query *mapper.Select, desc client.CollectionDescription) return nil, err } - return &viewNode{ - p: p, - desc: desc, - source: source, - docMapper: docMapper{query.DocumentMapping}, - }, nil + if hasTransform { + source = p.Lens(source, query.DocumentMapping, col) + } + + viewNode := &viewNode{ + p: p, + desc: col.Description(), + source: source, + docMapper: docMapper{query.DocumentMapping}, + hasTransform: hasTransform, + } + + return viewNode, nil } func (n *viewNode) Init() error { @@ -63,14 +75,21 @@ func (n *viewNode) Next() (bool, error) { } func (n *viewNode) Value() core.Doc { - sourceValue := n.source.DocumentMap().ToMap(n.source.Value()) + sourceValue := n.source.Value() + if n.hasTransform { + // If this view has a transform the source document will already have been + // converted to the new document mapping. + return sourceValue + } + + sourceMap := n.source.DocumentMap().ToMap(sourceValue) // We must convert the document from the source mapping (which was constructed using the // view's base query) to a document using the output mapping (which was constructed using // the current query and the output schemas). We do this by source output name, which // will take into account any aliases defined in the base query. doc := n.docMapper.documentMapping.NewDoc() - for fieldName, fieldValue := range sourceValue { + for fieldName, fieldValue := range sourceMap { // If the field does not exist, ignore it an continue. It likely means that // the field was declared in the query but not the SDL, and if it is not in the // SDL it cannot be requested/rendered by the user and would be dropped later anyway. diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index 14ea46a7d1..a1ae5e098c 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -218,11 +218,24 @@ func (w *Wrapper) SetActiveSchemaVersion(ctx context.Context, schemaVersionID st return err } -func (w *Wrapper) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) { +func (w *Wrapper) AddView( + ctx context.Context, + query string, + sdl string, + transform immutable.Option[model.Lens], +) ([]client.CollectionDefinition, error) { args := []string{"client", "view", "add"} args = append(args, query) args = append(args, sdl) + if transform.HasValue() { + lenses, err := json.Marshal(transform.Value()) + if err != nil { + return nil, err + } + args = append(args, string(lenses)) + } + data, err := w.cmd.execute(ctx, args) if err != nil { return nil, err diff --git a/tests/clients/http/wrapper.go b/tests/clients/http/wrapper.go index 01799e6c09..5e8ef858ca 100644 --- a/tests/clients/http/wrapper.go +++ b/tests/clients/http/wrapper.go @@ -110,8 +110,13 @@ func (w *Wrapper) SetActiveSchemaVersion(ctx context.Context, schemaVersionID st return w.client.SetActiveSchemaVersion(ctx, schemaVersionID) } -func (w *Wrapper) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) { - return w.client.AddView(ctx, query, sdl) +func (w *Wrapper) AddView( + ctx context.Context, + query string, + sdl string, + transform immutable.Option[model.Lens], +) ([]client.CollectionDefinition, error) { + return w.client.AddView(ctx, query, sdl, transform) } func (w *Wrapper) SetMigration(ctx context.Context, config client.LensConfig) error { diff --git a/tests/integration/explain.go b/tests/integration/explain.go index a8de0e6441..0b9c4c2dab 100644 --- a/tests/integration/explain.go +++ b/tests/integration/explain.go @@ -56,6 +56,7 @@ var ( "updateNode": {}, "valuesNode": {}, "viewNode": {}, + "lensNode": {}, } ) diff --git a/tests/integration/explain/debug/with_view_transform_test.go b/tests/integration/explain/debug/with_view_transform_test.go new file mode 100644 index 0000000000..386324067a --- /dev/null +++ b/tests/integration/explain/debug/with_view_transform_test.go @@ -0,0 +1,88 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package test_explain_debug + +import ( + "testing" + + "github.com/lens-vm/lens/host-go/config/model" + "github.com/sourcenetwork/immutable" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" + "github.com/sourcenetwork/defradb/tests/lenses" +) + +var transformViewPattern = dataMap{ + "explain": dataMap{ + "selectTopNode": dataMap{ + "selectNode": dataMap{ + "viewNode": dataMap{ + "lensNode": dataMap{ + "selectTopNode": dataMap{ + "selectNode": dataMap{ + "scanNode": dataMap{}, + }, + }, + }, + }, + }, + }, + }, +} + +func TestDebugExplainRequestWithViewWithTransform(t *testing.T) { + test := testUtils.TestCase{ + + Description: "Explain (debug) request with view with transform", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + } + `, + }, + testUtils.CreateView{ + Query: ` + User { + name + } + `, + SDL: ` + type UserView { + fullName: String + } + `, + Transform: immutable.Some(model.Lens{ + Lenses: []model.LensModule{ + { + Path: lenses.CopyModulePath, + Arguments: map[string]any{ + "src": "name", + "dst": "fullName", + }, + }, + }, + }), + }, + testUtils.ExplainRequest{ + Request: `query @explain(type: debug) { + UserView { + fullName + } + }`, + ExpectedPatterns: []dataMap{transformViewPattern}, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 5fea4d4478..a2cf6b16d6 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -175,6 +175,9 @@ type CreateView struct { // The SDL containing all types used by the view output. SDL string + // An optional Lens transform to add to the view. + Transform immutable.Option[model.Lens] + // Any error expected from the action. Optional. // // String can be a partial, and the test will pass if an error is returned that diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index cfef7d870f..a00330b93b 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1112,7 +1112,7 @@ func createView( action CreateView, ) { for _, node := range getNodes(action.NodeID, s.nodes) { - _, err := node.AddView(s.ctx, action.Query, action.SDL) + _, err := node.AddView(s.ctx, action.Query, action.SDL, action.Transform) expectedErrorRaised := AssertError(s.t, s.testCase.Description, err, action.ExpectedError) assertExpectedErrorRaised(s.t, s.testCase.Description, action.ExpectedError, expectedErrorRaised) diff --git a/tests/integration/view/one_to_many/with_transform_test.go b/tests/integration/view/one_to_many/with_transform_test.go new file mode 100644 index 0000000000..05b41516f4 --- /dev/null +++ b/tests/integration/view/one_to_many/with_transform_test.go @@ -0,0 +1,192 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package one_to_many + +import ( + "testing" + + "github.com/lens-vm/lens/host-go/config/model" + "github.com/sourcenetwork/immutable" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" + "github.com/sourcenetwork/defradb/tests/lenses" +) + +func TestView_OneToManyWithTransformOnOuter(t *testing.T) { + test := testUtils.TestCase{ + Description: "One to many view with transform on outer", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Author { + name: String + books: [Book] + } + type Book { + name: String + author: Author + } + `, + }, + testUtils.CreateView{ + Query: ` + Author { + name + books { + name + } + } + `, + SDL: ` + type AuthorView { + fullName: String + books: [BookView] + } + interface BookView { + name: String + } + `, + Transform: immutable.Some(model.Lens{ + // This transform will copy the value from `name` into the `fullName` field, + // like an overly-complicated alias + Lenses: []model.LensModule{ + { + Path: lenses.CopyModulePath, + Arguments: map[string]any{ + "src": "name", + "dst": "fullName", + }, + }, + }, + }), + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Ferdowsi" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "name": "Shahnameh", + "author": "bae-db3c6923-c6a4-5386-8301-b20a5454bf1d" + }`, + }, + testUtils.Request{ + Request: ` + query { + AuthorView { + fullName + books { + name + } + } + } + `, + Results: []map[string]any{ + { + "fullName": "Ferdowsi", + "books": []any{ + map[string]any{ + "name": "Shahnameh", + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestView_OneToManyWithTransformAddingInnerDocs(t *testing.T) { + test := testUtils.TestCase{ + Description: "One to many view with transform adding inner docs", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Author { + name: String + } + `, + }, + testUtils.CreateView{ + Query: ` + Author { + name + } + `, + SDL: ` + type AuthorView { + name: String + books: [BookView] + } + interface BookView { + name: String + } + `, + Transform: immutable.Some(model.Lens{ + Lenses: []model.LensModule{ + { + Path: lenses.SetDefaultModulePath, + Arguments: map[string]any{ + "dst": "books", + "value": []map[string]any{ + { + "name": "The Tragedy of Sohrab and Rostam", + }, + { + "name": "The Legend of Seyavash", + }, + }, + }, + }, + }, + }), + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Ferdowsi" + }`, + }, + testUtils.Request{ + Request: ` + query { + AuthorView { + name + books { + name + } + } + } + `, + Results: []map[string]any{ + { + "name": "Ferdowsi", + "books": []any{ + map[string]any{ + "name": "The Tragedy of Sohrab and Rostam", + }, + map[string]any{ + "name": "The Legend of Seyavash", + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/view/one_to_one/with_transform_test.go b/tests/integration/view/one_to_one/with_transform_test.go new file mode 100644 index 0000000000..cc638596e0 --- /dev/null +++ b/tests/integration/view/one_to_one/with_transform_test.go @@ -0,0 +1,108 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package one_to_one + +import ( + "testing" + + "github.com/lens-vm/lens/host-go/config/model" + "github.com/sourcenetwork/immutable" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" + "github.com/sourcenetwork/defradb/tests/lenses" +) + +func TestView_OneToOneWithTransformOnOuter(t *testing.T) { + test := testUtils.TestCase{ + Description: "One to one view with transform on outer", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Author { + name: String + book: Book + } + type Book { + name: String + author: Author + } + `, + }, + testUtils.CreateView{ + Query: ` + Author { + name + book { + name + } + } + `, + SDL: ` + type AuthorView { + fullName: String + book: BookView + } + interface BookView { + name: String + } + `, + Transform: immutable.Some(model.Lens{ + // This transform will copy the value from `name` into the `fullName` field, + // like an overly-complicated alias + Lenses: []model.LensModule{ + { + Path: lenses.CopyModulePath, + Arguments: map[string]any{ + "src": "name", + "dst": "fullName", + }, + }, + }, + }), + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Ferdowsi" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "name": "Shahnameh", + "author": "bae-db3c6923-c6a4-5386-8301-b20a5454bf1d" + }`, + }, + testUtils.Request{ + Request: ` + query { + AuthorView { + fullName + book { + name + } + } + } + `, + Results: []map[string]any{ + { + "fullName": "Ferdowsi", + "book": map[string]any{ + "name": "Shahnameh", + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/view/simple/with_transform_test.go b/tests/integration/view/simple/with_transform_test.go new file mode 100644 index 0000000000..fc148357e9 --- /dev/null +++ b/tests/integration/view/simple/with_transform_test.go @@ -0,0 +1,323 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package simple + +import ( + "testing" + + "github.com/lens-vm/lens/host-go/config/model" + "github.com/sourcenetwork/immutable" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" + "github.com/sourcenetwork/defradb/tests/lenses" +) + +func TestView_SimpleWithTransform(t *testing.T) { + test := testUtils.TestCase{ + Description: "Simple view with transform", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + } + `, + }, + testUtils.CreateView{ + Query: ` + User { + name + } + `, + SDL: ` + type UserView { + fullName: String + } + `, + Transform: immutable.Some(model.Lens{ + // This transform will copy the value from `name` into the `fullName` field, + // like an overly-complicated alias + Lenses: []model.LensModule{ + { + Path: lenses.CopyModulePath, + Arguments: map[string]any{ + "src": "name", + "dst": "fullName", + }, + }, + }, + }), + }, + testUtils.CreateDoc{ + // Set the `name` field only + Doc: `{ + "name": "John" + }`, + }, + testUtils.CreateDoc{ + // Set the `name` field only + Doc: `{ + "name": "Fred" + }`, + }, + testUtils.Request{ + Request: ` + query { + UserView { + fullName + } + } + `, + Results: []map[string]any{ + { + "fullName": "Fred", + }, + { + "fullName": "John", + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestView_SimpleWithMultipleTransforms(t *testing.T) { + test := testUtils.TestCase{ + Description: "Simple view with multiple transforms", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + } + `, + }, + testUtils.CreateView{ + Query: ` + User { + name + } + `, + SDL: ` + type UserView { + fullName: String + age: Int + } + `, + Transform: immutable.Some(model.Lens{ + // This transform will copy the value from `name` into the `fullName` field, + // like an overly-complicated alias. It will then set `age` to 23. + // + // It is important that this test tests the returning of more fields than it is + // provided with, given the production code. + Lenses: []model.LensModule{ + { + Path: lenses.CopyModulePath, + Arguments: map[string]any{ + "src": "name", + "dst": "fullName", + }, + }, + { + Path: lenses.SetDefaultModulePath, + Arguments: map[string]any{ + "dst": "age", + "value": 23, + }, + }, + }, + }), + }, + testUtils.CreateDoc{ + Doc: `{ + "name": "John" + }`, + }, + testUtils.CreateDoc{ + Doc: `{ + "name": "Fred" + }`, + }, + testUtils.Request{ + Request: ` + query { + UserView { + fullName + age + } + } + `, + Results: []map[string]any{ + { + "fullName": "Fred", + "age": 23, + }, + { + "fullName": "John", + "age": 23, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestView_SimpleWithTransformReturningMoreDocsThanInput(t *testing.T) { + test := testUtils.TestCase{ + Description: "Simple view with transform returning more docs than input", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + } + `, + }, + testUtils.CreateView{ + Query: ` + User { + name + } + `, + SDL: ` + type UserView { + name: String + } + `, + Transform: immutable.Some(model.Lens{ + Lenses: []model.LensModule{ + { + Path: lenses.PrependModulePath, + Arguments: map[string]any{ + "values": []map[string]any{ + { + "name": "Fred", + }, + { + "name": "Shahzad", + }, + }, + }, + }, + }, + }), + }, + testUtils.CreateDoc{ + Doc: `{ + "name": "John" + }`, + }, + testUtils.Request{ + Request: ` + query { + UserView { + name + } + } + `, + Results: []map[string]any{ + { + "name": "Fred", + }, + { + "name": "Shahzad", + }, + { + "name": "John", + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestView_SimpleWithTransformReturningFewerDocsThanInput(t *testing.T) { + test := testUtils.TestCase{ + Description: "Simple view with transform returning fewer docs than input", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + valid: Boolean + } + `, + }, + testUtils.CreateView{ + Query: ` + User { + name + valid + } + `, + SDL: ` + type UserView { + name: String + } + `, + Transform: immutable.Some(model.Lens{ + Lenses: []model.LensModule{ + { + Path: lenses.FilterModulePath, + Arguments: map[string]any{ + "src": "valid", + "value": true, + }, + }, + }, + }), + }, + testUtils.CreateDoc{ + Doc: `{ + "name": "John", + "valid": true + }`, + }, + testUtils.CreateDoc{ + Doc: `{ + "name": "Fred", + "valid": false + }`, + }, + testUtils.CreateDoc{ + Doc: `{ + "name": "Shahzad", + "valid": true + }`, + }, + testUtils.Request{ + Request: ` + query { + UserView { + name + } + } + `, + Results: []map[string]any{ + { + "name": "Shahzad", + }, + { + "name": "John", + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/lenses/Makefile b/tests/lenses/Makefile index 7370a04b80..5ebd3a0217 100644 --- a/tests/lenses/Makefile +++ b/tests/lenses/Makefile @@ -2,3 +2,5 @@ build: cargo build --target wasm32-unknown-unknown --manifest-path "./rust_wasm32_set_default/Cargo.toml" cargo build --target wasm32-unknown-unknown --manifest-path "./rust_wasm32_remove/Cargo.toml" cargo build --target wasm32-unknown-unknown --manifest-path "./rust_wasm32_copy/Cargo.toml" + cargo build --target wasm32-unknown-unknown --manifest-path "./rust_wasm32_prepend/Cargo.toml" + cargo build --target wasm32-unknown-unknown --manifest-path "./rust_wasm32_filter/Cargo.toml" diff --git a/tests/lenses/rust_wasm32_filter/Cargo.toml b/tests/lenses/rust_wasm32_filter/Cargo.toml new file mode 100644 index 0000000000..c3f815599f --- /dev/null +++ b/tests/lenses/rust_wasm32_filter/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "rust-wasm32-filter" +version = "0.1.0" +edition = "2018" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0.87" +lens_sdk = "^0.5" diff --git a/tests/lenses/rust_wasm32_filter/src/lib.rs b/tests/lenses/rust_wasm32_filter/src/lib.rs new file mode 100644 index 0000000000..5736d867f9 --- /dev/null +++ b/tests/lenses/rust_wasm32_filter/src/lib.rs @@ -0,0 +1,106 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; +use std::sync::RwLock; +use std::error::Error; +use std::{fmt, error}; +use serde::Deserialize; +use lens_sdk::StreamOption; +use lens_sdk::option::StreamOption::{Some, None, EndOfStream}; + +#[link(wasm_import_module = "lens")] +extern "C" { + fn next() -> *mut u8; +} + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +enum ModuleError { + ParametersNotSetError, + PropertyNotFoundError{requested: String}, +} + +impl error::Error for ModuleError { } + +impl fmt::Display for ModuleError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &*self { + ModuleError::ParametersNotSetError => f.write_str("Parameters have not been set."), + ModuleError::PropertyNotFoundError { requested } => + write!(f, "The requested property was not found. Requested: {}", requested), + } + } +} + +#[derive(Deserialize, Clone)] +pub struct Parameters { + pub src: String, + pub value: serde_json::Value, +} + +static PARAMETERS: RwLock> = RwLock::new(None); + +#[no_mangle] +pub extern fn alloc(size: usize) -> *mut u8 { + lens_sdk::alloc(size) +} + +#[no_mangle] +pub extern fn set_param(ptr: *mut u8) -> *mut u8 { + match try_set_param(ptr) { + Ok(_) => lens_sdk::nil_ptr(), + Err(e) => lens_sdk::to_mem(lens_sdk::ERROR_TYPE_ID, &e.to_string().as_bytes()) + } +} + +fn try_set_param(ptr: *mut u8) -> Result<(), Box> { + let parameter = lens_sdk::try_from_mem::(ptr)? + .ok_or(ModuleError::ParametersNotSetError)?; + + let mut dst = PARAMETERS.write()?; + *dst = Some(parameter); + Ok(()) +} + +#[no_mangle] +pub extern fn transform() -> *mut u8 { + match try_transform() { + Ok(o) => match o { + Some(result_json) => lens_sdk::to_mem(lens_sdk::JSON_TYPE_ID, &result_json), + None => lens_sdk::nil_ptr(), + EndOfStream => lens_sdk::to_mem(lens_sdk::EOS_TYPE_ID, &[]), + }, + Err(e) => lens_sdk::to_mem(lens_sdk::ERROR_TYPE_ID, &e.to_string().as_bytes()) + } +} + +fn try_transform() -> Result>, Box> { + let ptr = unsafe { next() }; + let mut input = match lens_sdk::try_from_mem::>(ptr)? { + Some(v) => v, + // Implementations of `transform` are free to handle nil however they like. In this + // implementation we chose to return nil given a nil input. + None => return Ok(None), + EndOfStream => return Ok(EndOfStream) + }; + + let params = PARAMETERS.read()? + .clone() + .ok_or(ModuleError::ParametersNotSetError)? + .clone(); + + let value = input.get_mut(¶ms.src) + .ok_or(ModuleError::PropertyNotFoundError{requested: params.src.clone()})? + .clone(); + + if value != params.value { + return try_transform(); + } + + let result = input.clone(); + + let result_json = serde_json::to_vec(&result)?; + lens_sdk::free_transport_buffer(ptr)?; + Ok(Some(result_json)) +} diff --git a/tests/lenses/rust_wasm32_prepend/Cargo.toml b/tests/lenses/rust_wasm32_prepend/Cargo.toml new file mode 100644 index 0000000000..7038a68b9c --- /dev/null +++ b/tests/lenses/rust_wasm32_prepend/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "rust-wasm32-prepend" +version = "0.1.0" +edition = "2018" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0.87" +lens_sdk = "^0.5" diff --git a/tests/lenses/rust_wasm32_prepend/src/lib.rs b/tests/lenses/rust_wasm32_prepend/src/lib.rs new file mode 100644 index 0000000000..30dcab58f8 --- /dev/null +++ b/tests/lenses/rust_wasm32_prepend/src/lib.rs @@ -0,0 +1,111 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; +use std::sync::RwLock; +use std::error::Error; +use std::{fmt, error}; +use serde::Deserialize; +use lens_sdk::StreamOption; +use lens_sdk::option::StreamOption::{Some, None, EndOfStream}; + +#[link(wasm_import_module = "lens")] +extern "C" { + fn next() -> *mut u8; +} + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +enum ModuleError { + ParametersNotSetError, +} + +impl error::Error for ModuleError { } + +impl fmt::Display for ModuleError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &*self { + ModuleError::ParametersNotSetError => f.write_str("Parameters have not been set."), + } + } +} + +#[derive(Deserialize, Clone)] +pub struct Parameters { + pub values: Vec>, +} + +static PARAMETERS: RwLock> = RwLock::new(None); +static PARAM_INDEX: RwLock = RwLock::new(0); + +#[no_mangle] +pub extern fn alloc(size: usize) -> *mut u8 { + lens_sdk::alloc(size) +} + +#[no_mangle] +pub extern fn set_param(ptr: *mut u8) -> *mut u8 { + match try_set_param(ptr) { + Ok(_) => lens_sdk::nil_ptr(), + Err(e) => lens_sdk::to_mem(lens_sdk::ERROR_TYPE_ID, &e.to_string().as_bytes()) + } +} + +fn try_set_param(ptr: *mut u8) -> Result<(), Box> { + let parameter = lens_sdk::try_from_mem::(ptr)? + .ok_or(ModuleError::ParametersNotSetError)? + .clone(); + + let mut dst = PARAMETERS.write()?; + *dst = Some(parameter); + Ok(()) +} + +#[no_mangle] +pub extern fn transform() -> *mut u8 { + match try_transform() { + Ok(o) => match o { + Some(result_json) => lens_sdk::to_mem(lens_sdk::JSON_TYPE_ID, &result_json), + None => lens_sdk::nil_ptr(), + EndOfStream => lens_sdk::to_mem(lens_sdk::EOS_TYPE_ID, &[]), + }, + Err(e) => lens_sdk::to_mem(lens_sdk::ERROR_TYPE_ID, &e.to_string().as_bytes()) + } +} + +fn try_transform() -> Result>, Box> { + let params = PARAMETERS.read()? + .clone() + .ok_or(ModuleError::ParametersNotSetError)? + .clone(); + + let param_index = PARAM_INDEX.read()? + .clone(); + + if param_index < params.values.len() { + let result = ¶ms.values[param_index]; + let result_json = serde_json::to_vec(&result)?; + + let mut dst = PARAM_INDEX.write()?; + *dst = param_index+1; + return Ok(Some(result_json)) + } + + // Note: The following is a very unperformant, but simple way of yielding the input documents, + // as this module is only used for testing, this is preferred. + + let ptr = unsafe { next() }; + let input = match lens_sdk::try_from_mem::>(ptr)? { + Some(v) => v, + // Implementations of `transform` are free to handle nil however they like. In this + // implementation we chose to return nil given a nil input. + None => return Ok(None), + EndOfStream => return Ok(EndOfStream) + }; + + let result = input.clone(); + + let result_json = serde_json::to_vec(&result)?; + lens_sdk::free_transport_buffer(ptr)?; + Ok(Some(result_json)) +} diff --git a/tests/lenses/utils.go b/tests/lenses/utils.go index 132c7d33c4..0fab59d3a6 100644 --- a/tests/lenses/utils.go +++ b/tests/lenses/utils.go @@ -44,6 +44,25 @@ var CopyModulePath string = getPathRelativeToProjectRoot( "/tests/lenses/rust_wasm32_copy/target/wasm32-unknown-unknown/debug/rust_wasm32_copy.wasm", ) +// PrependModulePath is the path to the `Prepend` lens module compiled to wasm. +// +// The module has one parameter: +// - `values` is an array of `map[string]string`s, the module will yield these documents before +// any documents fed to it (from Defra). +var PrependModulePath string = getPathRelativeToProjectRoot( + "/tests/lenses/rust_wasm32_prepend/target/wasm32-unknown-unknown/debug/rust_wasm32_prepend.wasm", +) + +// FilterModulePath is the path to the `Filter` lens module compiled to wasm. +// +// The module has two parameters: +// - `src` is a string and is the name of the property you wish to evaluate +// - `value` can be any valid json value and will be compared to the document value at the `src` location +// only documents with values that match this given value will be returned. +var FilterModulePath string = getPathRelativeToProjectRoot( + "/tests/lenses/rust_wasm32_filter/target/wasm32-unknown-unknown/debug/rust_wasm32_filter.wasm", +) + func getPathRelativeToProjectRoot(relativePath string) string { _, filename, _, _ := runtime.Caller(0) root := path.Dir(path.Dir(path.Dir(filename)))