WIP - Add cached views

client/collection_description.go

@@ -79,6 +79,9 @@ type CollectionDescription struct {
 	// parsing is done, to avoid storing an invalid policyID or policy resource
 	// that may not even exist on acp.
 	Policy immutable.Option[PolicyDescription]
+
+	// todo: Make sure 	PatchCollection errors if trying to make this false for normal collections (temp.)
+	IsMaterialized bool
 }
 
 // QuerySource represents a collection data source from a query.
@@ -179,6 +182,7 @@ type collectionDescription struct {
 	ID              uint32
 	RootID          uint32
 	SchemaVersionID string
+	IsMaterialized  bool
 	Policy          immutable.Option[PolicyDescription]
 	Indexes         []IndexDescription
 	Fields          []CollectionFieldDescription
@@ -198,6 +202,7 @@ func (c *CollectionDescription) UnmarshalJSON(bytes []byte) error {
 	c.ID = descMap.ID
 	c.RootID = descMap.RootID
 	c.SchemaVersionID = descMap.SchemaVersionID
+	c.IsMaterialized = descMap.IsMaterialized
 	c.Indexes = descMap.Indexes
 	c.Fields = descMap.Fields
 	c.Sources = make([]any, len(descMap.Sources))

client/db.go

@@ -196,6 +196,8 @@ type Store interface {
 		transform immutable.Option[model.Lens],
 	) ([]CollectionDefinition, error)
 
+	RefreshViews(context.Context, CollectionFetchOptions) error
+
 	// SetMigration sets the migration for all collections using the given source-destination schema version IDs.
 	//
 	// There may only be one migration per collection version.  If another migration was registered it will be
@@ -304,3 +306,11 @@ type SchemaFetchOptions struct {
 	// If provided, only the schema with this id will be returned.
 	ID immutable.Option[string]
 }
+
+/*
+// ViewFetchOptions represents a set of options used for fetching views.
+type ViewFetchOptions struct {
+	// If provided, only views with this name will be returned.
+	Name immutable.Option[string]
+}
+*/

http/client.go

@@ -219,6 +219,10 @@ func (c *Client) AddView(
 	return descriptions, nil
 }
 
+func (c *Client) RefreshViews(ctx context.Context, opts client.CollectionFetchOptions) error {
+	panic("todo")
+}
+
 func (c *Client) SetMigration(ctx context.Context, config client.LensConfig) error {
 	methodURL := c.http.baseURL.JoinPath("lens")
 

internal/core/key.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/sourcenetwork/defradb/client"
 	"github.com/sourcenetwork/defradb/errors"
+	"github.com/sourcenetwork/defradb/internal/encoding"
 )
 
 var (
@@ -43,12 +44,14 @@ const (
 )
 
 const (
-	COLLECTION                     = "collection"
-	COLLECTION_ID                  = "/collection/id"
-	COLLECTION_NAME                = "/collection/name"
-	COLLECTION_SCHEMA_VERSION      = "/collection/version"
-	COLLECTION_ROOT                = "/collection/root"
-	COLLECTION_INDEX               = "/collection/index"
+	COLLECTION                = "collection"
+	COLLECTION_ID             = "/collection/id"
+	COLLECTION_NAME           = "/collection/name"
+	COLLECTION_SCHEMA_VERSION = "/collection/version"
+	COLLECTION_ROOT           = "/collection/root"
+	COLLECTION_INDEX          = "/collection/index"
+	// todo- doc temp-ness
+	COLLECTION_VIEW_ITEMS          = "/collection/vi"
 	SCHEMA_VERSION                 = "/schema/version/v"
 	SCHEMA_VERSION_ROOT            = "/schema/version/r"
 	COLLECTION_SEQ                 = "/seq/collection"
@@ -77,6 +80,15 @@ type DataStoreKey struct {
 
 var _ Key = (*DataStoreKey)(nil)
 
+// todo: note this may be removed/merged into DataStoreKey
+type ViewCacheKey struct {
+	CollectionRootID uint32
+	// todo - note this is temp type:
+	ItemID uint
+}
+
+var _ Key = (*ViewCacheKey)(nil)
+
 // IndexedField contains information necessary for storing a single
 // value of a field in an index.
 type IndexedField struct {
@@ -527,6 +539,57 @@ func (k DataStoreKey) ToPrimaryDataStoreKey() PrimaryDataStoreKey {
 	}
 }
 
+func NewViewCacheColPrefix(rootID uint32) ViewCacheKey {
+	return ViewCacheKey{
+		CollectionRootID: rootID,
+	}
+}
+
+func NewViewCacheKey(rootID uint32, itemID uint) ViewCacheKey {
+	return ViewCacheKey{
+		CollectionRootID: rootID,
+		ItemID:           itemID,
+	}
+}
+
+func (k ViewCacheKey) ToString() string {
+	return string(k.Bytes())
+}
+
+func (k ViewCacheKey) Bytes() []byte {
+	result := []byte(COLLECTION_VIEW_ITEMS)
+
+	if k.CollectionRootID != 0 {
+		result = append(result, '/')
+		result = encoding.EncodeUvarintAscending(result, uint64(k.CollectionRootID))
+		//result = []byte(string(result) + "/" + strconv.Itoa(int(k.CollectionRootID)))
+	}
+
+	if k.ItemID != 0 {
+		result = append(result, '/')
+		result = encoding.EncodeUvarintAscending(result, uint64(k.ItemID))
+	}
+
+	return result
+}
+
+func (k ViewCacheKey) ToDS() ds.Key {
+	return ds.NewKey(k.ToString())
+}
+
+func (k ViewCacheKey) PrettyPrint() string {
+	result := COLLECTION_VIEW_ITEMS
+
+	if k.CollectionRootID != 0 {
+		result = result + "/" + strconv.Itoa(int(k.CollectionRootID))
+	}
+	if k.ItemID != 0 {
+		result = result + "/" + strconv.Itoa(int(k.ItemID))
+	}
+
+	return result
+}
+
 // NewIndexDataStoreKey creates a new IndexDataStoreKey from a collection ID, index ID and fields.
 // It also validates values of the fields.
 func NewIndexDataStoreKey(collectionID, indexID uint32, fields []IndexedField) IndexDataStoreKey {

internal/db/definition_validation.go

@@ -164,6 +164,8 @@ var globalValidators = []definitionValidator{
 	validateTypeAndKindCompatible,
 	validateFieldNotDuplicated,
 	validateSelfReferences,
+	validateCollectionMaterialized,
+	validateMaterializedHasNoACP,
 }
 
 var createValidators = append(
@@ -978,3 +980,35 @@ func validateSchemaNameNotEmpty(
 
 	return nil
 }
+
+func validateCollectionMaterialized(
+	ctx context.Context,
+	db *db,
+	newState *definitionState,
+	oldState *definitionState,
+) error {
+	for _, col := range newState.collections {
+		if len(col.QuerySources()) == 0 && !col.IsMaterialized {
+			return NewErrColNotMaterialized(col.Name.Value())
+		}
+
+	}
+
+	return nil
+}
+
+func validateMaterializedHasNoACP(
+	ctx context.Context,
+	db *db,
+	newState *definitionState,
+	oldState *definitionState,
+) error {
+	for _, col := range newState.collections {
+		if col.IsMaterialized && len(col.QuerySources()) != 0 && col.Policy.HasValue() {
+			return NewErrMaterializedViewAndACPNotSupported(col.Name.Value())
+		}
+
+	}
+
+	return nil
+}

internal/db/errors.go

@@ -102,6 +102,8 @@ const (
 	errReplicatorNotFound                       string = "replicator not found"
 	errCanNotEncryptBuiltinField                string = "can not encrypt build-in field"
 	errSelfReferenceWithoutSelf                 string = "must specify 'Self' kind for self referencing relations"
+	errColNotMaterialized                       string = "non-materialized collections (only views) are not supported"
+	errMaterializedViewAndACPNotSupported       string = "materialized views do not support ACP"
 )
 
 var (
@@ -143,6 +145,8 @@ var (
 	ErrReplicatorNotFound                       = errors.New(errReplicatorNotFound)
 	ErrCanNotEncryptBuiltinField                = errors.New(errCanNotEncryptBuiltinField)
 	ErrSelfReferenceWithoutSelf                 = errors.New(errSelfReferenceWithoutSelf)
+	ErrColNotMaterialized                       = errors.New(errColNotMaterialized)
+	ErrMaterializedViewAndACPNotSupported       = errors.New(errMaterializedViewAndACPNotSupported)
 )
 
 // NewErrFailedToGetHeads returns a new error indicating that the heads of a document
@@ -659,3 +663,17 @@ func NewErrSelfReferenceWithoutSelf(fieldName string) error {
 		errors.NewKV("Field", fieldName),
 	)
 }
+
+func NewErrColNotMaterialized(collection string) error {
+	return errors.New(
+		errColNotMaterialized,
+		errors.NewKV("Collection", collection),
+	)
+}
+
+func NewErrMaterializedViewAndACPNotSupported(collection string) error {
+	return errors.New(
+		errMaterializedViewAndACPNotSupported,
+		errors.NewKV("Collection", collection),
+	)
+}

internal/db/lens.go

@@ -52,6 +52,7 @@ func (db *db) setMigration(ctx context.Context, cfg client.LensConfig) error {
 			ID:              uint32(colID),
 			RootID:          client.OrphanRootID,
 			SchemaVersionID: cfg.SourceSchemaVersionID,
+			IsMaterialized:  true,
 		}
 
 		col, err := description.SaveCollection(ctx, txn, desc)
@@ -96,6 +97,7 @@ func (db *db) setMigration(ctx context.Context, cfg client.LensConfig) error {
 				ID:              uint32(colID),
 				RootID:          sourceCol.RootID,
 				SchemaVersionID: cfg.DestinationSchemaVersionID,
+				IsMaterialized:  true,
 				Sources: []any{
 					&client.CollectionSource{
 						SourceCollectionID: sourceCol.ID,

internal/db/store.go

@@ -237,6 +237,23 @@ func (db *db) AddView(
 	return defs, nil
 }
 
+func (db *db) RefreshViews(ctx context.Context, opts client.CollectionFetchOptions) error {
+	ctx, txn, err := ensureContextTxn(ctx, db, false)
+	if err != nil {
+		return err
+	}
+	defer txn.Discard(ctx)
+
+	db.refreshViews(ctx, opts)
+
+	err = txn.Commit(ctx)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // BasicImport imports a json dataset.
 // filepath must be accessible to the node.
 func (db *db) BasicImport(ctx context.Context, filepath string) error {