Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add support for views with Lens transforms #2311

Merged
merged 5 commits into from
Feb 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ tests/lenses/rust_wasm32_remove/pkg
tests/lenses/rust_wasm32_copy/Cargo.lock
tests/lenses/rust_wasm32_copy/target
tests/lenses/rust_wasm32_copy/pkg
tests/lenses/rust_wasm32_prepend/Cargo.lock
tests/lenses/rust_wasm32_prepend/target
tests/lenses/rust_wasm32_prepend/pkg
tests/lenses/rust_wasm32_filter/Cargo.lock
tests/lenses/rust_wasm32_filter/target
tests/lenses/rust_wasm32_filter/pkg

# Ignore OS X metadata files.
.history
Expand Down
54 changes: 46 additions & 8 deletions cli/view_add.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,34 +10,72 @@

package cli

import "github.com/spf13/cobra"
import (
"encoding/json"
"io"
"os"
"strings"

"github.com/lens-vm/lens/host-go/config/model"
"github.com/sourcenetwork/immutable"
"github.com/spf13/cobra"
)

func MakeViewAddCommand() *cobra.Command {
var lensFile string
var cmd = &cobra.Command{
Use: "add [query] [sdl]",
Use: "add [query] [sdl] [transform]",
Short: "Add new view",
Long: `Add new database view.

Example: add from an argument string:
defradb client view add 'Foo { name, ...}' 'type Foo { ... }'
defradb client view add 'Foo { name, ...}' 'type Foo { ... }' '{"lenses": [...'

Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`,
Args: cobra.RangeArgs(2, 4),
RunE: func(cmd *cobra.Command, args []string) error {
store := mustGetStoreContext(cmd)

if len(args) != 2 {
return ErrViewAddMissingArgs
}

query := args[0]
sdl := args[1]

defs, err := store.AddView(cmd.Context(), query, sdl)
var lensCfgJson string
switch {
case lensFile != "":
data, err := os.ReadFile(lensFile)
if err != nil {
return err
}
lensCfgJson = string(data)
case len(args) == 3 && args[2] == "-":
data, err := io.ReadAll(cmd.InOrStdin())
if err != nil {
return err
}
lensCfgJson = string(data)
case len(args) == 3:
lensCfgJson = args[2]
}

var transform immutable.Option[model.Lens]
if lensCfgJson != "" {
decoder := json.NewDecoder(strings.NewReader(lensCfgJson))
decoder.DisallowUnknownFields()

var lensCfg model.Lens
if err := decoder.Decode(&lensCfg); err != nil {
return NewErrInvalidLensConfig(err)
}
transform = immutable.Some(lensCfg)
}

defs, err := store.AddView(cmd.Context(), query, sdl, transform)
if err != nil {
return err
}
return writeJSON(cmd, defs)
},
}
cmd.Flags().StringVarP(&lensFile, "file", "f", "", "Lens configuration file")
return cmd
}
11 changes: 10 additions & 1 deletion client/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,16 @@ type Store interface {
//
// It will return the collection definitions of the types defined in the SDL if successful, otherwise an error
// will be returned. This function does not execute the given query.
AddView(ctx context.Context, gqlQuery string, sdl string) ([]CollectionDefinition, error)
//
// Optionally, a lens transform configuration may also be provided - it will execute after the query has run.
// The transform is not limited to just transforming the input documents, it may also yield new ones, or filter out
// those passed in from the underlying query.
AddView(
ctx context.Context,
gqlQuery string,
sdl string,
transform immutable.Option[model.Lens],
) ([]CollectionDefinition, error)

// SetMigration sets the migration for all collections using the given source-destination schema version IDs.
//
Expand Down
7 changes: 7 additions & 0 deletions client/descriptions.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,13 @@ func sourcesOfType[ResultType any](col CollectionDescription) []ResultType {
type QuerySource struct {
// Query contains the base query of this data source.
Query request.Select

// Transform is a optional Lens configuration. If specified, data drawn from the [Query] will have the
// transform applied before being returned.
//
// The transform is not limited to just transforming the input documents, it may also yield new ones, or filter out
// those passed in from the underlying query.
Transform immutable.Option[model.Lens]
}

// CollectionSource represents a collection data source from another collection instance.
Expand Down
18 changes: 14 additions & 4 deletions db/txn_db.go
Original file line number Diff line number Diff line change
Expand Up @@ -391,14 +391,19 @@ func (db *explicitTxnDB) SetMigration(ctx context.Context, cfg client.LensConfig
return db.setMigration(ctx, db.txn, cfg)
}

func (db *implicitTxnDB) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) {
func (db *implicitTxnDB) AddView(
ctx context.Context,
query string,
sdl string,
transform immutable.Option[model.Lens],
) ([]client.CollectionDefinition, error) {
txn, err := db.NewTxn(ctx, false)
if err != nil {
return nil, err
}
defer txn.Discard(ctx)

defs, err := db.addView(ctx, txn, query, sdl)
defs, err := db.addView(ctx, txn, query, sdl, transform)
if err != nil {
return nil, err
}
Expand All @@ -411,8 +416,13 @@ func (db *implicitTxnDB) AddView(ctx context.Context, query string, sdl string)
return defs, nil
}

func (db *explicitTxnDB) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) {
return db.addView(ctx, db.txn, query, sdl)
func (db *explicitTxnDB) AddView(
ctx context.Context,
query string,
sdl string,
transform immutable.Option[model.Lens],
) ([]client.CollectionDefinition, error) {
return db.addView(ctx, db.txn, query, sdl, transform)
}

// BasicImport imports a json dataset.
Expand Down
18 changes: 17 additions & 1 deletion db/view.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ import (
"errors"
"fmt"

"github.com/lens-vm/lens/host-go/config/model"
"github.com/sourcenetwork/immutable"

"github.com/sourcenetwork/defradb/client"
"github.com/sourcenetwork/defradb/client/request"
"github.com/sourcenetwork/defradb/datastore"
Expand All @@ -26,6 +29,7 @@ func (db *db) addView(
txn datastore.Txn,
inputQuery string,
sdl string,
transform immutable.Option[model.Lens],
) ([]client.CollectionDefinition, error) {
// Wrap the given query as part of the GQL query object - this simplifies the syntax for users
// and ensures that we can't be given mutations. In the future this line should disappear along
Expand Down Expand Up @@ -57,7 +61,10 @@ func (db *db) addView(
}

for i := range newDefinitions {
source := client.QuerySource{Query: *baseQuery}
source := client.QuerySource{
Query: *baseQuery,
Transform: transform,
}
newDefinitions[i].Description.Sources = append(newDefinitions[i].Description.Sources, &source)
}

Expand All @@ -78,6 +85,15 @@ func (db *db) addView(
return nil, err
}
returnDescriptions[i] = col.Definition()

for _, source := range col.Description().QuerySources() {
if source.Transform.HasValue() {
err = db.LensRegistry().SetMigration(ctx, col.ID(), source.Transform.Value())
if err != nil {
return nil, err
}
}
}
}
}

Expand Down
14 changes: 10 additions & 4 deletions http/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,14 +173,20 @@ func (c *Client) SetActiveSchemaVersion(ctx context.Context, schemaVersionID str
}

type addViewRequest struct {
Query string
SDL string
Query string
SDL string
Transform immutable.Option[model.Lens]
}

func (c *Client) AddView(ctx context.Context, query string, sdl string) ([]client.CollectionDefinition, error) {
func (c *Client) AddView(
ctx context.Context,
query string,
sdl string,
transform immutable.Option[model.Lens],
) ([]client.CollectionDefinition, error) {
methodURL := c.http.baseURL.JoinPath("view")

body, err := json.Marshal(addViewRequest{query, sdl})
body, err := json.Marshal(addViewRequest{query, sdl, transform})
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion http/handler_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func (s *storeHandler) AddView(rw http.ResponseWriter, req *http.Request) {
return
}

defs, err := store.AddView(req.Context(), message.Query, message.SDL)
defs, err := store.AddView(req.Context(), message.Query, message.SDL, message.Transform)
if err != nil {
responseJSON(rw, http.StatusBadRequest, errorResponse{err})
return
Expand Down
2 changes: 1 addition & 1 deletion planner/datasource.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func (p *Planner) getCollectionScanPlan(mapperSelect *mapper.Select) (planSource
var plan planNode
if len(col.Description().QuerySources()) > 0 {
var err error
plan, err = p.View(mapperSelect, col.Description())
plan, err = p.View(mapperSelect, col)
if err != nil {
return planSource{}, err
}
Expand Down
Loading
Loading