feat: Model Col. SchemaVersions and migrations on Cols (sourcenetwork…

…#2286)

## Relevant issue(s)

Resolves sourcenetwork#2198

## Description

Models Collection SchemaVersions and migrations on Collections, instead
of in the Lens Registry.

Primary motivators for the change:
1. Boost the DevEx RE Lens and schema versions. It seemed pretty painful
before to find transforms, yet alone figure out where that lay in the
version history, or add them to a collection.
2. Unify the Collection history/schema history with the View system, so
that they both roughly work in the same way. Reducing the cognitive and
maintenance overhead of having two different systems for the same thing
for both us and the users.

Noteworthy stuff:
1. The branching of schema is not new, it is just more visible with this
change. There is nothing we can really do to block it in a distributed
system. I do now see it as supported, excluding Lens, which still needs
a relatively minor tweak in order to handle the migration-pathing.
2. The model change in client/descriptions.go
3. The removal of stuff from the LensRegistry. This is currently kept to
a minimum, however I would very much like to completely remove
LensRegistry in the near future.
4. The CLI and the Http clients used to rely on the fact that
db.SetMigration and registry.SetMigration did the same thing. This is no
longer the case.
5. The CLI and the Http clients followed a different, flatter structure
than the Go client, flattening the lens registry into the
`lens`/`schema` pathways. This was inconsistent, and only worked when
(4) was as it was. This has been changed out of necessity and the 3
clients now have matching structures.
6. The CLI still paths everything under `schema`, and the http client
everything under `lens`, which is strange, but out of scope. I will open
a ticket to make them the same.
7. Some of the schema versions in the existing schema tests have
changed, I think there was a bug in the framework where it was not
caring about their accuracy. It now cares.
8. Users can now specify their transforms when patching a schema.
9. Out of the 4 collection getting funcs on `client.Store`, only
`GetAllCollections` has the ability to get inactive collections. I
currently very much dislike that we have 4 functions for the same thing
in the Go client, especially given that this is not reflected in the
Http and CLI clients. In another ticket I would very much like us to
collapse these four functions into one, and have that one function
support the fetching of inactive collections.

cli/cli.go

@@ -48,7 +48,7 @@ func NewDefraCommand(cfg *config.Config) *cobra.Command {
 	schema_migrate := MakeSchemaMigrationCommand()
 	schema_migrate.AddCommand(
 		MakeSchemaMigrationSetCommand(),
-		MakeSchemaMigrationGetCommand(),
+		MakeSchemaMigrationSetRegistryCommand(),
 		MakeSchemaMigrationReloadCommand(),
 		MakeSchemaMigrationUpCommand(),
 		MakeSchemaMigrationDownCommand(),
@@ -58,7 +58,7 @@ func NewDefraCommand(cfg *config.Config) *cobra.Command {
 	schema.AddCommand(
 		MakeSchemaAddCommand(),
 		MakeSchemaPatchCommand(),
-		MakeSchemaSetDefaultCommand(),
+		MakeSchemaSetActiveCommand(),
 		MakeSchemaDescribeCommand(),
 		schema_migrate,
 	)

cli/collection_describe.go

@@ -17,6 +17,7 @@ import (
 )
 
 func MakeCollectionDescribeCommand() *cobra.Command {
+	var getInactive bool
 	var cmd = &cobra.Command{
 		Use:   "describe",
 		Short: "View collection description.",
@@ -42,7 +43,7 @@ Example: view collection by version id
 				return writeJSON(cmd, col.Definition())
 			}
 			// if no collection specified list all collections
-			cols, err := store.GetAllCollections(cmd.Context())
+			cols, err := store.GetAllCollections(cmd.Context(), getInactive)
 			if err != nil {
 				return err
 			}
@@ -53,5 +54,6 @@ Example: view collection by version id
 			return writeJSON(cmd, colDesc)
 		},
 	}
+	cmd.Flags().BoolVar(&getInactive, "get-inactive", false, "Get inactive collections as well as active")
 	return cmd
 }

cli/schema_migration_down.go

@@ -23,21 +23,21 @@ import (
 
 func MakeSchemaMigrationDownCommand() *cobra.Command {
 	var file string
-	var schemaVersionID string
+	var collectionID uint32
 	var cmd = &cobra.Command{
-		Use:   "down --version <version> <documents>",
-		Short: "Reverses the migration from the specified schema version.",
-		Long: `Reverses the migration from the specified schema version.
+		Use:   "down --collection <collectionID> <documents>",
+		Short: "Reverses the migration to the specified collection version.",
+		Long: `Reverses the migration to the specified collection version.
 Documents is a list of documents to reverse the migration from.
 
 Example: migrate from string
-  defradb client schema migration down --version bae123 '[{"name": "Bob"}]'
+  defradb client schema migration down --collection 2 '[{"name": "Bob"}]'
 
 Example: migrate from file
-  defradb client schema migration down --version bae123 -f documents.json
+  defradb client schema migration down --collection 2 -f documents.json
 
 Example: migrate from stdin
-  cat documents.json | defradb client schema migration down --version bae123 -
+  cat documents.json | defradb client schema migration down --collection 2 -
 		`,
 		Args: cobra.RangeArgs(0, 1),
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -71,7 +71,8 @@ Example: migrate from stdin
 			if tx, ok := cmd.Context().Value(txContextKey).(datastore.Txn); ok {
 				lens = lens.WithTxn(tx)
 			}
-			out, err := lens.MigrateDown(cmd.Context(), enumerable.New(src), schemaVersionID)
+
+			out, err := lens.MigrateDown(cmd.Context(), enumerable.New(src), collectionID)
 			if err != nil {
 				return err
 			}
@@ -86,6 +87,6 @@ Example: migrate from stdin
 		},
 	}
 	cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)")
-	cmd.Flags().StringVar(&schemaVersionID, "version", "", "Schema version id")
+	cmd.Flags().Uint32Var(&collectionID, "collection", 0, "Collection id")
 	return cmd
 }

cli/schema_migration_set.go

@@ -27,7 +27,8 @@ func MakeSchemaMigrationSetCommand() *cobra.Command {
 	var cmd = &cobra.Command{
 		Use:   "set [src] [dst] [cfg]",
 		Short: "Set a schema migration within DefraDB",
-		Long: `Set a migration between two schema versions within the local DefraDB node.
+		Long: `Set a migration from a source schema version to a destination schema version for
+all collections that are on the given source schema version within the local DefraDB node.
 
 Example: set from an argument string:
   defradb client schema migration set bae123 bae456 '{"lenses": [...'
@@ -80,7 +81,7 @@ Learn more about the DefraDB GraphQL Schema Language on https://docs.source.netw
 				Lens:                       lensCfg,
 			}
 
-			return store.LensRegistry().SetMigration(cmd.Context(), migrationCfg)
+			return store.SetMigration(cmd.Context(), migrationCfg)
 		},
 	}
 	cmd.Flags().StringVarP(&lensFile, "file", "f", "", "Lens configuration file")

cli/schema_migration_set_registry.go

@@ -0,0 +1,54 @@
+// Copyright 2023 Democratized Data Foundation
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package cli
+
+import (
+	"encoding/json"
+	"strconv"
+	"strings"
+
+	"github.com/lens-vm/lens/host-go/config/model"
+	"github.com/spf13/cobra"
+)
+
+func MakeSchemaMigrationSetRegistryCommand() *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   "set-registry [collectionID] [cfg]",
+		Short: "Set a schema migration within the DefraDB LensRegistry",
+		Long: `Set a migration to a collection within the LensRegistry of the local DefraDB node.
+Does not persist the migration after restart.
+
+Example: set from an argument string:
+  defradb client schema migration set-registry 2 '{"lenses": [...'
+
+Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`,
+		Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			store := mustGetStoreContext(cmd)
+
+			decoder := json.NewDecoder(strings.NewReader(args[1]))
+			decoder.DisallowUnknownFields()
+
+			var lensCfg model.Lens
+			if err := decoder.Decode(&lensCfg); err != nil {
+				return NewErrInvalidLensConfig(err)
+			}
+
+			collectionID, err := strconv.ParseUint(args[0], 10, 64)
+			if err != nil {
+				return err
+			}
+
+			return store.LensRegistry().SetMigration(cmd.Context(), uint32(collectionID), lensCfg)
+		},
+	}
+	return cmd
+}

cli/schema_migration_up.go

@@ -23,21 +23,21 @@ import (
 
 func MakeSchemaMigrationUpCommand() *cobra.Command {
 	var file string
-	var schemaVersionID string
+	var collectionID uint32
 	var cmd = &cobra.Command{
-		Use:   "up --version <version> <documents>",
-		Short: "Applies the migration to the specified schema version.",
-		Long: `Applies the migration to the specified schema version.
+		Use:   "up --collection <collectionID> <documents>",
+		Short: "Applies the migration to the specified collection version.",
+		Long: `Applies the migration to the specified collection version.
 Documents is a list of documents to apply the migration to.		
 
 Example: migrate from string
-  defradb client schema migration up --version bae123 '[{"name": "Bob"}]'
+  defradb client schema migration up --collection 2 '[{"name": "Bob"}]'
 
 Example: migrate from file
-  defradb client schema migration up --version bae123 -f documents.json
+  defradb client schema migration up --collection 2 -f documents.json
 
 Example: migrate from stdin
-  cat documents.json | defradb client schema migration up --version bae123 -
+  cat documents.json | defradb client schema migration up --collection 2 -
 		`,
 		Args: cobra.RangeArgs(0, 1),
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -71,7 +71,8 @@ Example: migrate from stdin
 			if tx, ok := cmd.Context().Value(txContextKey).(datastore.Txn); ok {
 				lens = lens.WithTxn(tx)
 			}
-			out, err := lens.MigrateUp(cmd.Context(), enumerable.New(src), schemaVersionID)
+
+			out, err := lens.MigrateUp(cmd.Context(), enumerable.New(src), collectionID)
 			if err != nil {
 				return err
 			}
@@ -86,6 +87,6 @@ Example: migrate from stdin
 		},
 	}
 	cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)")
-	cmd.Flags().StringVar(&schemaVersionID, "version", "", "Schema version id")
+	cmd.Flags().Uint32Var(&collectionID, "collection", 0, "Collection id")
 	return cmd
 }

cli/schema_patch.go

@@ -11,25 +11,30 @@
 package cli
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
+	"strings"
 
+	"github.com/lens-vm/lens/host-go/config/model"
+	"github.com/sourcenetwork/immutable"
 	"github.com/spf13/cobra"
 )
 
 func MakeSchemaPatchCommand() *cobra.Command {
 	var patchFile string
-	var setDefault bool
+	var lensFile string
+	var setActive bool
 	var cmd = &cobra.Command{
-		Use:   "patch [schema]",
+		Use:   "patch [schema] [migration]",
 		Short: "Patch an existing schema type",
 		Long: `Patch an existing schema.
 
 Uses JSON Patch to modify schema types.
 
 Example: patch from an argument string:
-  defradb client schema patch '[{ "op": "add", "path": "...", "value": {...} }]'
+  defradb client schema patch '[{ "op": "add", "path": "...", "value": {...} }]' '{"lenses": [...'
 
 Example: patch from file:
   defradb client schema patch -f patch.json
@@ -55,16 +60,42 @@ To learn more about the DefraDB GraphQL Schema Language, refer to https://docs.s
 					return err
 				}
 				patch = string(data)
-			case len(args) > 0:
+			case len(args) >= 1:
 				patch = args[0]
 			default:
 				return fmt.Errorf("patch cannot be empty")
 			}
 
-			return store.PatchSchema(cmd.Context(), patch, setDefault)
+			var lensCfgJson string
+			switch {
+			case lensFile != "":
+				data, err := os.ReadFile(lensFile)
+				if err != nil {
+					return err
+				}
+				patch = string(data)
+			case len(args) == 2:
+				lensCfgJson = args[1]
+			}
+
+			decoder := json.NewDecoder(strings.NewReader(lensCfgJson))
+			decoder.DisallowUnknownFields()
+
+			var migration immutable.Option[model.Lens]
+			if lensCfgJson != "" {
+				var lensCfg model.Lens
+				if err := decoder.Decode(&lensCfg); err != nil {
+					return NewErrInvalidLensConfig(err)
+				}
+				migration = immutable.Some(lensCfg)
+			}
+
+			return store.PatchSchema(cmd.Context(), patch, migration, setActive)
 		},
 	}
-	cmd.Flags().BoolVar(&setDefault, "set-default", false, "Set default schema version")
-	cmd.Flags().StringVarP(&patchFile, "file", "f", "", "File to load a patch from")
+	cmd.Flags().BoolVar(&setActive, "set-active", false,
+		"Set the active schema version for all collections using the root schem")
+	cmd.Flags().StringVarP(&patchFile, "patch-file", "p", "", "File to load a patch from")
+	cmd.Flags().StringVarP(&lensFile, "lens-file", "t", "", "File to load a lens config from")
 	return cmd
 }

cli/schema_set_default.go → cli/schema_set_active.go

@@ -14,15 +14,16 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func MakeSchemaSetDefaultCommand() *cobra.Command {
+func MakeSchemaSetActiveCommand() *cobra.Command {
 	var cmd = &cobra.Command{
-		Use:   "set-default [versionID]",
-		Short: "Set the default schema version",
-		Long:  `Set the default schema version`,
-		Args:  cobra.ExactArgs(1),
+		Use:   "set-active [versionID]",
+		Short: "Set the active collection version",
+		Long: `Activates all collection versions with the given schema version, and deactivates all
+those without it (if they share the same schema root).`,
+		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			store := mustGetStoreContext(cmd)
-			return store.SetDefaultSchemaVersion(cmd.Context(), args[0])
+			return store.SetActiveSchemaVersion(cmd.Context(), args[0])
 		},
 	}
 	return cmd