refactor: Change from protobuf to cbor for IPLD (#2604)

## Relevant issue(s)

Resolves #2603 

## Description

This PR changes the encoding of the blocks in the blockstore to use cbor
instead of protobof. As part of this, we move away from
encoding/decoding individual deltas (no more Marshalling and
Unmarshalling deltas) and instead, encode/decode at the block level. To
do this, we use the new `github.com/ipld/go-ipld-prime` package that
uses IPLD schemas to map the encoded blocks to the appropriate fields
and CRDT types. The new `core/block` package handles that part of the
change.

The `net` package is also affected by that change with the `DAGSyncer`
being replaced by the new `ipld/linking.LinkSystem`. I expect further
usage of the `LinkSystem` in a couple subsequent PRs.

client/db.go

@@ -13,7 +13,7 @@ package client
 import (
 	"context"
 
-	blockstore "github.com/ipfs/boxo/blockstore"
+	ds "github.com/ipfs/go-datastore"
 	"github.com/lens-vm/lens/host-go/config/model"
 	"github.com/sourcenetwork/immutable"
 
@@ -48,13 +48,18 @@ type DB interface {
 	// Blockstore returns the blockstore, within which all blocks (commits) managed by DefraDB are held.
 	//
 	// It sits within the rootstore returned by [Root].
-	Blockstore() blockstore.Blockstore
+	Blockstore() datastore.DAGStore
 
 	// Peerstore returns the peerstore where known host information is stored.
 	//
 	// It sits within the rootstore returned by [Root].
 	Peerstore() datastore.DSBatching
 
+	// Headstore returns the headstore where the current heads of the database are stored.
+	//
+	// It is read-only and sits within the rootstore returned by [Root].
+	Headstore() ds.Read
+
 	// Close closes the database instance and releases any resources held.
 	//
 	// The behaviour of other functions in this package after this function has been called is undefined

datastore/blockstore.go

@@ -13,13 +13,14 @@ package datastore
 import (
 	"context"
 
-	blockstore "github.com/ipfs/boxo/blockstore"
+	"github.com/ipfs/boxo/blockstore"
 	dshelp "github.com/ipfs/boxo/datastore/dshelp"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
 	dsq "github.com/ipfs/go-datastore/query"
 	ipld "github.com/ipfs/go-ipld-format"
+	"github.com/ipld/go-ipld-prime/storage/bsadapter"
 
 	"github.com/sourcenetwork/defradb/errors"
 )
@@ -44,18 +45,33 @@ import (
 
 // NewBlockstore returns a default Blockstore implementation
 // using the provided datastore.Batching backend.
-func NewBlockstore(store DSReaderWriter) blockstore.Blockstore {
+func newBlockstore(store DSReaderWriter) *bstore {
 	return &bstore{
 		store: store,
 	}
 }
 
+func newIPLDStore(store blockstore.Blockstore) *bsadapter.Adapter {
+	return &bsadapter.Adapter{Wrapped: store}
+}
+
 type bstore struct {
 	store DSReaderWriter
 
 	rehash bool
 }
 
+var _ blockstore.Blockstore = (*bstore)(nil)
+var _ DAGStore = (*bstore)(nil)
+
+// AsIPLDStorage returns an IPLDStorage instance.
+//
+// It wraps the blockstore in an IPLD Blockstore adapter for use with
+// the IPLD LinkSystem.
+func (bs *bstore) AsIPLDStorage() IPLDStorage {
+	return newIPLDStore(bs)
+}
+
 // HashOnRead enables or disables rehashing of blocks on read.
 func (bs *bstore) HashOnRead(enabled bool) {
 	bs.rehash = enabled

datastore/multi.go

@@ -46,7 +46,7 @@ func MultiStoreFrom(rootstore ds.Datastore) MultiStore {
 		head:   prefix(rootRW, headStoreKey),
 		peer:   namespace.Wrap(rootstore, peerStoreKey),
 		system: prefix(rootRW, systemStoreKey),
-		dag:    NewDAGStore(prefix(rootRW, blockStoreKey)),
+		dag:    newBlockstore(prefix(rootRW, blockStoreKey)),
 	}
 
 	return ms

datastore/store.go

@@ -11,8 +11,9 @@
 package datastore
 
 import (
-	blockstore "github.com/ipfs/boxo/blockstore"
+	"github.com/ipfs/boxo/blockstore"
 	ds "github.com/ipfs/go-datastore"
+	"github.com/ipld/go-ipld-prime/storage"
 
 	"github.com/sourcenetwork/corelog"
 
@@ -72,6 +73,13 @@ type DSReaderWriter interface {
 // DAGStore proxies the ipld.DAGService under the /core namespace for future-proofing
 type DAGStore interface {
 	blockstore.Blockstore
+	AsIPLDStorage() IPLDStorage
+}
+
+// IPLDStorage provides the methods needed for an IPLD LinkSystem.
+type IPLDStorage interface {
+	storage.ReadableStorage
+	storage.WritableStorage
 }
 
 // DSBatching wraps the Batching interface from go-datastore

docs/data_format_changes/i2603-ipld-protobuf-to-cbor.md

@@ -0,0 +1,3 @@
+# Change encoding from protobuf to cbor and use the new IPLD schema
+
+The DAG blocks are now encoded using CBOR instead of protobuf and we use the new `github.com/ipld/go-ipld-prime` package to handle block encoding and decoding. It makes use of the new IPLD schema to define the block structure.

events/db_update.go

@@ -12,7 +12,7 @@ package events
 
 import (
 	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
+
 	"github.com/sourcenetwork/immutable"
 )
 
@@ -36,10 +36,10 @@ type Update struct {
 	// SchemaRoot is the root identifier of the schema that defined the shape of the document that was updated.
 	SchemaRoot string
 
-	// Block is the contents of this composite commit, it contains the Cids of the field level commits that
+	// Block is the encoded contents of this composite commit, it contains the Cids of the field level commits that
 	// also formed this update.
-	Block ipld.Node
+	Block []byte
 
-	// Priority is used to determine the order in which concurrent updates are applied.
-	Priority uint64
+	// IsCreate is true if this update is the creation of a new document.
+	IsCreate bool
 }