Merge branch 'develop' into nasdf/refactor/config

.gitignore

@@ -17,6 +17,12 @@ tests/lenses/rust_wasm32_remove/pkg
 tests/lenses/rust_wasm32_copy/Cargo.lock
 tests/lenses/rust_wasm32_copy/target
 tests/lenses/rust_wasm32_copy/pkg
+tests/lenses/rust_wasm32_prepend/Cargo.lock
+tests/lenses/rust_wasm32_prepend/target
+tests/lenses/rust_wasm32_prepend/pkg
+tests/lenses/rust_wasm32_filter/Cargo.lock
+tests/lenses/rust_wasm32_filter/target
+tests/lenses/rust_wasm32_filter/pkg
 
 # Ignore OS X metadata files.
 .history

cli/view_add.go

@@ -10,34 +10,72 @@
 
 package cli
 
-import "github.com/spf13/cobra"
+import (
+	"encoding/json"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/lens-vm/lens/host-go/config/model"
+	"github.com/sourcenetwork/immutable"
+	"github.com/spf13/cobra"
+)
 
 func MakeViewAddCommand() *cobra.Command {
+	var lensFile string
 	var cmd = &cobra.Command{
-		Use:   "add [query] [sdl]",
+		Use:   "add [query] [sdl] [transform]",
 		Short: "Add new view",
 		Long: `Add new database view.
 
 Example: add from an argument string:
-  defradb client view add 'Foo { name, ...}' 'type Foo { ... }'
+  defradb client view add 'Foo { name, ...}' 'type Foo { ... }' '{"lenses": [...'
 
 Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`,
+		Args: cobra.RangeArgs(2, 4),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			store := mustGetContextStore(cmd)
 
-			if len(args) != 2 {
-				return ErrViewAddMissingArgs
-			}
-
 			query := args[0]
 			sdl := args[1]
 
-			defs, err := store.AddView(cmd.Context(), query, sdl)
+			var lensCfgJson string
+			switch {
+			case lensFile != "":
+				data, err := os.ReadFile(lensFile)
+				if err != nil {
+					return err
+				}
+				lensCfgJson = string(data)
+			case len(args) == 3 && args[2] == "-":
+				data, err := io.ReadAll(cmd.InOrStdin())
+				if err != nil {
+					return err
+				}
+				lensCfgJson = string(data)
+			case len(args) == 3:
+				lensCfgJson = args[2]
+			}
+
+			var transform immutable.Option[model.Lens]
+			if lensCfgJson != "" {
+				decoder := json.NewDecoder(strings.NewReader(lensCfgJson))
+				decoder.DisallowUnknownFields()
+
+				var lensCfg model.Lens
+				if err := decoder.Decode(&lensCfg); err != nil {
+					return NewErrInvalidLensConfig(err)
+				}
+				transform = immutable.Some(lensCfg)
+			}
+
+			defs, err := store.AddView(cmd.Context(), query, sdl, transform)
 			if err != nil {
 				return err
 			}
 			return writeJSON(cmd, defs)
 		},
 	}
+	cmd.Flags().StringVarP(&lensFile, "file", "f", "", "Lens configuration file")
 	return cmd
 }

client/db.go

@@ -155,7 +155,16 @@ type Store interface {
 	//
 	// It will return the collection definitions of the types defined in the SDL if successful, otherwise an error
 	// will be returned.  This function does not execute the given query.
-	AddView(ctx context.Context, gqlQuery string, sdl string) ([]CollectionDefinition, error)
+	//
+	// Optionally, a lens transform configuration may also be provided - it will execute after the query has run.
+	// The transform is not limited to just transforming the input documents, it may also yield new ones, or filter out
+	// those passed in from the underlying query.
+	AddView(
+		ctx context.Context,
+		gqlQuery string,
+		sdl string,
+		transform immutable.Option[model.Lens],
+	) ([]CollectionDefinition, error)
 
 	// SetMigration sets the migration for all collections using the given source-destination schema version IDs.
 	//

client/descriptions.go

@@ -135,6 +135,13 @@ func sourcesOfType[ResultType any](col CollectionDescription) []ResultType {
 type QuerySource struct {
 	// Query contains the base query of this data source.
 	Query request.Select
+
+	// Transform is a optional Lens configuration.  If specified, data drawn from the [Query] will have the
+	// transform applied before being returned.
+	//
+	// The transform is not limited to just transforming the input documents, it may also yield new ones, or filter out
+	// those passed in from the underlying query.
+	Transform immutable.Option[model.Lens]
 }
 
 // CollectionSource represents a collection data source from another collection instance.

db/txn_db.go

@@ -391,14 +391,19 @@ func (db *explicitTxnDB) SetMigration(ctx context.Context, cfg client.LensConfig
 	return db.setMigration(ctx, db.txn, cfg)
 }
 
-func (db *implicitTxnDB) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) {
+func (db *implicitTxnDB) AddView(
+	ctx context.Context,
+	query string,
+	sdl string,
+	transform immutable.Option[model.Lens],
+) ([]client.CollectionDefinition, error) {
 	txn, err := db.NewTxn(ctx, false)
 	if err != nil {
 		return nil, err
 	}
 	defer txn.Discard(ctx)
 
-	defs, err := db.addView(ctx, txn, query, sdl)
+	defs, err := db.addView(ctx, txn, query, sdl, transform)
 	if err != nil {
 		return nil, err
 	}
@@ -411,8 +416,13 @@ func (db *implicitTxnDB) AddView(ctx context.Context, query string, sdl string)
 	return defs, nil
 }
 
-func (db *explicitTxnDB) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) {
-	return db.addView(ctx, db.txn, query, sdl)
+func (db *explicitTxnDB) AddView(
+	ctx context.Context,
+	query string,
+	sdl string,
+	transform immutable.Option[model.Lens],
+) ([]client.CollectionDefinition, error) {
+	return db.addView(ctx, db.txn, query, sdl, transform)
 }
 
 // BasicImport imports a json dataset.

db/view.go

@@ -15,6 +15,9 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/lens-vm/lens/host-go/config/model"
+	"github.com/sourcenetwork/immutable"
+
 	"github.com/sourcenetwork/defradb/client"
 	"github.com/sourcenetwork/defradb/client/request"
 	"github.com/sourcenetwork/defradb/datastore"
@@ -26,6 +29,7 @@ func (db *db) addView(
 	txn datastore.Txn,
 	inputQuery string,
 	sdl string,
+	transform immutable.Option[model.Lens],
 ) ([]client.CollectionDefinition, error) {
 	// Wrap the given query as part of the GQL query object - this simplifies the syntax for users
 	// and ensures that we can't be given mutations.  In the future this line should disappear along
@@ -57,7 +61,10 @@ func (db *db) addView(
 	}
 
 	for i := range newDefinitions {
-		source := client.QuerySource{Query: *baseQuery}
+		source := client.QuerySource{
+			Query:     *baseQuery,
+			Transform: transform,
+		}
 		newDefinitions[i].Description.Sources = append(newDefinitions[i].Description.Sources, &source)
 	}
 
@@ -78,6 +85,15 @@ func (db *db) addView(
 				return nil, err
 			}
 			returnDescriptions[i] = col.Definition()
+
+			for _, source := range col.Description().QuerySources() {
+				if source.Transform.HasValue() {
+					err = db.LensRegistry().SetMigration(ctx, col.ID(), source.Transform.Value())
+					if err != nil {
+						return nil, err
+					}
+				}
+			}
 		}
 	}
 

go.mod

@@ -43,12 +43,11 @@ require (
 	github.com/ugorji/go/codec v1.2.12
 	github.com/valyala/fastjson v1.6.4
 	github.com/vito/go-sse v1.0.0
-	go.opentelemetry.io/otel/metric v1.22.0
-	go.opentelemetry.io/otel/sdk/metric v1.22.0
+	go.opentelemetry.io/otel/metric v1.23.1
+	go.opentelemetry.io/otel/sdk/metric v1.23.1
 	go.uber.org/zap v1.26.0
-	golang.org/x/crypto v0.18.0
 	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc
-	golang.org/x/net v0.20.0
+	golang.org/x/net v0.21.0
 	google.golang.org/grpc v1.61.0
 	google.golang.org/protobuf v1.32.0
 )
@@ -175,16 +174,17 @@ require (
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/otel v1.22.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.22.0 // indirect
-	go.opentelemetry.io/otel/trace v1.22.0 // indirect
+	go.opentelemetry.io/otel v1.23.1 // indirect
+	go.opentelemetry.io/otel/sdk v1.23.1 // indirect
+	go.opentelemetry.io/otel/trace v1.23.1 // indirect
 	go.uber.org/dig v1.17.1 // indirect
 	go.uber.org/fx v1.20.1 // indirect
 	go.uber.org/mock v0.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.16.1 // indirect
 	gonum.org/v1/gonum v0.14.0 // indirect

go.sum

@@ -584,16 +584,16 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
-go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
-go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
-go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
-go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
-go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
-go.opentelemetry.io/otel/sdk/metric v1.22.0 h1:ARrRetm1HCVxq0cbnaZQlfwODYJHo3gFL8Z3tSmHBcI=
-go.opentelemetry.io/otel/sdk/metric v1.22.0/go.mod h1:KjQGeMIDlBNEOo6HvjhxIec1p/69/kULDcp4gr0oLQQ=
-go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
-go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
+go.opentelemetry.io/otel v1.23.1 h1:Za4UzOqJYS+MUczKI320AtqZHZb7EqxO00jAHE0jmQY=
+go.opentelemetry.io/otel v1.23.1/go.mod h1:Td0134eafDLcTS4y+zQ26GE8u3dEuRBiBCTUIRHaikA=
+go.opentelemetry.io/otel/metric v1.23.1 h1:PQJmqJ9u2QaJLBOELl1cxIdPcpbwzbkjfEyelTl2rlo=
+go.opentelemetry.io/otel/metric v1.23.1/go.mod h1:mpG2QPlAfnK8yNhNJAxDZruU9Y1/HubbC+KyH8FaCWI=
+go.opentelemetry.io/otel/sdk v1.23.1 h1:O7JmZw0h76if63LQdsBMKQDWNb5oEcOThG9IrxscV+E=
+go.opentelemetry.io/otel/sdk v1.23.1/go.mod h1:LzdEVR5am1uKOOwfBWFef2DCi1nu3SA8XQxx2IerWFk=
+go.opentelemetry.io/otel/sdk/metric v1.23.1 h1:T9/8WsYg+ZqIpMWwdISVVrlGb/N0Jr1OHjR/alpKwzg=
+go.opentelemetry.io/otel/sdk/metric v1.23.1/go.mod h1:8WX6WnNtHCgUruJ4TJ+UssQjMtpxkpX0zveQC8JG/E0=
+go.opentelemetry.io/otel/trace v1.23.1 h1:4LrmmEd8AU2rFvU1zegmvqW7+kWarxtNOPyeL6HmYY8=
+go.opentelemetry.io/otel/trace v1.23.1/go.mod h1:4IpnpJFwr1mo/6HL8XIPJaE9y0+u1KcVmuW7dwFSVrI=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
@@ -627,8 +627,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
-golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM=
 golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
@@ -664,8 +664,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
-golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -707,8 +707,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

http/client.go

@@ -173,14 +173,20 @@ func (c *Client) SetActiveSchemaVersion(ctx context.Context, schemaVersionID str
 }
 
 type addViewRequest struct {
-	Query string
-	SDL   string
+	Query     string
+	SDL       string
+	Transform immutable.Option[model.Lens]
 }
 
-func (c *Client) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) {
+func (c *Client) AddView(
+	ctx context.Context,
+	query string,
+	sdl string,
+	transform immutable.Option[model.Lens],
+) ([]client.CollectionDefinition, error) {
 	methodURL := c.http.baseURL.JoinPath("view")
 
-	body, err := json.Marshal(addViewRequest{query, sdl})
+	body, err := json.Marshal(addViewRequest{query, sdl, transform})
 	if err != nil {
 		return nil, err
 	}

http/handler_store.go

@@ -117,7 +117,7 @@ func (s *storeHandler) AddView(rw http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	defs, err := store.AddView(req.Context(), message.Query, message.SDL)
+	defs, err := store.AddView(req.Context(), message.Query, message.SDL, message.Transform)
 	if err != nil {
 		responseJSON(rw, http.StatusBadRequest, errorResponse{err})
 		return

planner/datasource.go

@@ -34,7 +34,7 @@ func (p *Planner) getCollectionScanPlan(mapperSelect *mapper.Select) (planSource
 	var plan planNode
 	if len(col.Description().QuerySources()) > 0 {
 		var err error
-		plan, err = p.View(mapperSelect, col.Description())
+		plan, err = p.View(mapperSelect, col)
 		if err != nil {
 			return planSource{}, err
 		}