Model Collection SchemaVersions and migrations on Collections

cli/cli.go

@@ -48,7 +48,7 @@ func NewDefraCommand(cfg *config.Config) *cobra.Command {
 	schema_migrate := MakeSchemaMigrationCommand()
 	schema_migrate.AddCommand(
 		MakeSchemaMigrationSetCommand(),
-		MakeSchemaMigrationGetCommand(),
+		MakeSchemaMigrationSetRegistryCommand(),
 		MakeSchemaMigrationReloadCommand(),
 		MakeSchemaMigrationUpCommand(),
 		MakeSchemaMigrationDownCommand(),
@@ -58,7 +58,7 @@ func NewDefraCommand(cfg *config.Config) *cobra.Command {
 	schema.AddCommand(
 		MakeSchemaAddCommand(),
 		MakeSchemaPatchCommand(),
-		MakeSchemaSetDefaultCommand(),
+		MakeSchemaSetActiveCommand(),
 		MakeSchemaDescribeCommand(),
 		schema_migrate,
 	)

cli/collection_describe.go

@@ -17,6 +17,7 @@ import (
 )
 
 func MakeCollectionDescribeCommand() *cobra.Command {
+	var getInactive bool
 	var cmd = &cobra.Command{
 		Use:   "describe",
 		Short: "View collection description.",
@@ -42,7 +43,7 @@ Example: view collection by version id
 				return writeJSON(cmd, col.Definition())
 			}
 			// if no collection specified list all collections
-			cols, err := store.GetAllCollections(cmd.Context())
+			cols, err := store.GetAllCollections(cmd.Context(), getInactive)
 			if err != nil {
 				return err
 			}
@@ -53,5 +54,6 @@ Example: view collection by version id
 			return writeJSON(cmd, colDesc)
 		},
 	}
+	cmd.Flags().BoolVar(&getInactive, "get-inactive", false, "Get inactive collections as well as active")
 	return cmd
 }

cli/schema_migration_down.go

@@ -23,21 +23,21 @@ import (
 
 func MakeSchemaMigrationDownCommand() *cobra.Command {
 	var file string
-	var schemaVersionID string
+	var collectionID uint32
 	var cmd = &cobra.Command{
-		Use:   "down --version <version> <documents>",
-		Short: "Reverses the migration from the specified schema version.",
-		Long: `Reverses the migration from the specified schema version.
+		Use:   "down --collection <collectionID> <documents>",
+		Short: "Reverses the migration to the specified collection version.",
+		Long: `Reverses the migration to the specified collection version.
 Documents is a list of documents to reverse the migration from.
 
 Example: migrate from string
-  defradb client schema migration down --version bae123 '[{"name": "Bob"}]'
+  defradb client schema migration down --collection 2 '[{"name": "Bob"}]'
 
 Example: migrate from file
-  defradb client schema migration down --version bae123 -f documents.json
+  defradb client schema migration down --collection 2 -f documents.json
 
 Example: migrate from stdin
-  cat documents.json | defradb client schema migration down --version bae123 -
+  cat documents.json | defradb client schema migration down --collection 2 -
 		`,
 		Args: cobra.RangeArgs(0, 1),
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -71,7 +71,8 @@ Example: migrate from stdin
 			if tx, ok := cmd.Context().Value(txContextKey).(datastore.Txn); ok {
 				lens = lens.WithTxn(tx)
 			}
-			out, err := lens.MigrateDown(cmd.Context(), enumerable.New(src), schemaVersionID)
+
+			out, err := lens.MigrateDown(cmd.Context(), enumerable.New(src), collectionID)
 			if err != nil {
 				return err
 			}
@@ -86,6 +87,6 @@ Example: migrate from stdin
 		},
 	}
 	cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)")
-	cmd.Flags().StringVar(&schemaVersionID, "version", "", "Schema version id")
+	cmd.Flags().Uint32Var(&collectionID, "collection", 0, "Collection id")
 	return cmd
 }

cli/schema_migration_set.go

@@ -27,7 +27,7 @@ func MakeSchemaMigrationSetCommand() *cobra.Command {
 	var cmd = &cobra.Command{
 		Use:   "set [src] [dst] [cfg]",
 		Short: "Set a schema migration within DefraDB",
-		Long: `Set a migration between two schema versions within the local DefraDB node.
+		Long: `Set a migration between collections of the given schema versions within the local DefraDB node.
 
 Example: set from an argument string:
   defradb client schema migration set bae123 bae456 '{"lenses": [...'
@@ -80,7 +80,7 @@ Learn more about the DefraDB GraphQL Schema Language on https://docs.source.netw
 				Lens:                       lensCfg,
 			}
 
-			return store.LensRegistry().SetMigration(cmd.Context(), migrationCfg)
+			return store.SetMigration(cmd.Context(), migrationCfg)
 		},
 	}
 	cmd.Flags().StringVarP(&lensFile, "file", "f", "", "Lens configuration file")

cli/schema_migration_set_registry.go

@@ -0,0 +1,54 @@
+// Copyright 2023 Democratized Data Foundation
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package cli
+
+import (
+	"encoding/json"
+	"strconv"
+	"strings"
+
+	"github.com/lens-vm/lens/host-go/config/model"
+	"github.com/spf13/cobra"
+)
+
+func MakeSchemaMigrationSetRegistryCommand() *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   "set-registry [collectionID] [cfg]",
+		Short: "Set a schema migration within the DefraDB LensRegistry",
+		Long: `Set a migration to a collection within the LensRegistry of the local DefraDB node.
+Does not persist the migration after restart.
+
+Example: set from an argument string:
+  defradb client schema migration set-registry 2 '{"lenses": [...'
+
+Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`,
+		Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			store := mustGetStoreContext(cmd)
+
+			decoder := json.NewDecoder(strings.NewReader(args[1]))
+			decoder.DisallowUnknownFields()
+
+			var lensCfg model.Lens
+			if err := decoder.Decode(&lensCfg); err != nil {
+				return NewErrInvalidLensConfig(err)
+			}
+
+			collectionID, err := strconv.ParseUint(args[0], 10, 64)
+			if err != nil {
+				return err
+			}
+
+			return store.LensRegistry().SetMigration(cmd.Context(), uint32(collectionID), lensCfg)
+		},
+	}
+	return cmd
+}

cli/schema_migration_up.go

@@ -23,21 +23,21 @@ import (
 
 func MakeSchemaMigrationUpCommand() *cobra.Command {
 	var file string
-	var schemaVersionID string
+	var collectionID uint32
 	var cmd = &cobra.Command{
-		Use:   "up --version <version> <documents>",
-		Short: "Applies the migration to the specified schema version.",
-		Long: `Applies the migration to the specified schema version.
+		Use:   "up --collection <collectionID> <documents>",
+		Short: "Applies the migration to the specified collection version.",
+		Long: `Applies the migration to the specified collection version.
 Documents is a list of documents to apply the migration to.		
 
 Example: migrate from string
-  defradb client schema migration up --version bae123 '[{"name": "Bob"}]'
+  defradb client schema migration up --collection 2 '[{"name": "Bob"}]'
 
 Example: migrate from file
-  defradb client schema migration up --version bae123 -f documents.json
+  defradb client schema migration up --collection 2 -f documents.json
 
 Example: migrate from stdin
-  cat documents.json | defradb client schema migration up --version bae123 -
+  cat documents.json | defradb client schema migration up --collection 2 -
 		`,
 		Args: cobra.RangeArgs(0, 1),
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -71,7 +71,8 @@ Example: migrate from stdin
 			if tx, ok := cmd.Context().Value(txContextKey).(datastore.Txn); ok {
 				lens = lens.WithTxn(tx)
 			}
-			out, err := lens.MigrateUp(cmd.Context(), enumerable.New(src), schemaVersionID)
+
+			out, err := lens.MigrateUp(cmd.Context(), enumerable.New(src), collectionID)
 			if err != nil {
 				return err
 			}
@@ -86,6 +87,6 @@ Example: migrate from stdin
 		},
 	}
 	cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)")
-	cmd.Flags().StringVar(&schemaVersionID, "version", "", "Schema version id")
+	cmd.Flags().Uint32Var(&collectionID, "collection", 0, "Collection id")
 	return cmd
 }

cli/schema_patch.go

@@ -11,16 +11,21 @@
 package cli
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
+	"strings"
 
+	"github.com/lens-vm/lens/host-go/config/model"
+	"github.com/sourcenetwork/immutable"
 	"github.com/spf13/cobra"
 )
 
 func MakeSchemaPatchCommand() *cobra.Command {
 	var patchFile string
-	var setDefault bool
+	var lensFile string
+	var setActive bool
 	var cmd = &cobra.Command{
 		Use:   "patch [schema]",
 		Short: "Patch an existing schema type",
@@ -29,7 +34,7 @@ func MakeSchemaPatchCommand() *cobra.Command {
 Uses JSON Patch to modify schema types.
 
 Example: patch from an argument string:
-  defradb client schema patch '[{ "op": "add", "path": "...", "value": {...} }]'
+  defradb client schema patch '[{ "op": "add", "path": "...", "value": {...} }]' '{"lenses": [...'
 
 Example: patch from file:
   defradb client schema patch -f patch.json
@@ -55,16 +60,42 @@ To learn more about the DefraDB GraphQL Schema Language, refer to https://docs.s
 					return err
 				}
 				patch = string(data)
-			case len(args) > 0:
+			case len(args) >= 1:
 				patch = args[0]
 			default:
 				return fmt.Errorf("patch cannot be empty")
 			}
 
-			return store.PatchSchema(cmd.Context(), patch, setDefault)
+			var lensCfgJson string
+			switch {
+			case lensFile != "":
+				data, err := os.ReadFile(lensFile)
+				if err != nil {
+					return err
+				}
+				patch = string(data)
+			case len(args) == 2:
+				lensCfgJson = args[1]
+			}
+
+			decoder := json.NewDecoder(strings.NewReader(lensCfgJson))
+			decoder.DisallowUnknownFields()
+
+			var migration immutable.Option[model.Lens]
+			if lensCfgJson != "" {
+				var lensCfg model.Lens
+				if err := decoder.Decode(&lensCfg); err != nil {
+					return NewErrInvalidLensConfig(err)
+				}
+				migration = immutable.Some(lensCfg)
+			}
+
+			return store.PatchSchema(cmd.Context(), patch, migration, setActive)
 		},
 	}
-	cmd.Flags().BoolVar(&setDefault, "set-default", false, "Set default schema version")
-	cmd.Flags().StringVarP(&patchFile, "file", "f", "", "File to load a patch from")
+	cmd.Flags().BoolVar(&setActive, "set-active", false,
+		"Set the active schema version for all collections using the root schem")
+	cmd.Flags().StringVarP(&patchFile, "patch-file", "p", "", "File to load a patch from")
+	cmd.Flags().StringVarP(&lensFile, "lens-file", "t", "", "File to load a lens config from")
 	return cmd
 }

cli/schema_set_default.go → cli/schema_set_active.go

@@ -14,15 +14,16 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func MakeSchemaSetDefaultCommand() *cobra.Command {
+func MakeSchemaSetActiveCommand() *cobra.Command {
 	var cmd = &cobra.Command{
-		Use:   "set-default [versionID]",
-		Short: "Set the default schema version",
-		Long:  `Set the default schema version`,
-		Args:  cobra.ExactArgs(1),
+		Use:   "set-active [versionID]",
+		Short: "Set the active collection version",
+		Long: `Activates all collection versions with the given schema version, and deactivates all
+those without it (if they share the same schema root).`,
+		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			store := mustGetStoreContext(cmd)
-			return store.SetDefaultSchemaVersion(cmd.Context(), args[0])
+			return store.SetActiveSchemaVersion(cmd.Context(), args[0])
 		},
 	}
 	return cmd