fix(backend): refactor db schema and fix site reference materials

backend/cmd/mintter-site/main.go

@@ -9,7 +9,7 @@ import (
 	"os"
 
 	"mintter/backend/cmd/mintter-site/sites"
-	"mintter/backend/daemon"
+	"mintter/backend/daemon/storage"
 
 	"github.com/burdiyan/go/mainutil"
 	"github.com/peterbourgon/ff/v3"
@@ -59,7 +59,7 @@ Flags:
 			return err
 		}
 
-		dir, err := daemon.InitRepo(cfg.Base.DataDir, nil)
+		dir, err := storage.InitRepo(cfg.Base.DataDir, nil)
 		if err != nil {
 			return err
 		}

backend/cmd/mintter-site/sites/sites.go

@@ -288,43 +288,24 @@ func (ws *Website) PublishBlobs(ctx context.Context, in *groups.PublishBlobsRequ
 		return nil, fmt.Errorf("error getting groupID on the site, is the site initialized?: %w", err)
 	}
 
+	groupOwner, err := storage.GetKV(conn, keySiteOwner)
+	if err != nil || groupOwner == "" {
+		return nil, fmt.Errorf("error getting group owner on the site, is the site initialized?: %w", err)
+	}
+
 	var role groups.Role
-	{
-		edb, err := hypersql.LookupEnsure(conn, storage.LookupResource, groupID)
+	if groupOwner == callerAccount.String() {
+		role = groups.Role_OWNER
+	} else {
+		r, err := hypersql.GetGroupRole(conn, groupID, "hm://a/"+callerAccount.String())
 		if err != nil {
-			return nil, fmt.Errorf("couldn't get group (%s) resource: %w", groupID, err)
-		}
-
-		owner, err := storage.GetKV(conn, keySiteOwner)
-		if err != nil || owner == "" {
-			return nil, fmt.Errorf("error getting owner on the site, is the site initialized?: %w", err)
-		}
-
-		if owner == callerAccount.String() {
-			role = groups.Role_OWNER
-		} else {
-			groupOwner, err := hypersql.ResourceGetOwner(conn, edb)
-			if err != nil {
-				return nil, fmt.Errorf("couldn't get the owner of the group %s: %w", groupID, err)
-			}
-
-			pkdb, err := hypersql.LookupEnsure(conn, storage.LookupPublicKey, callerAccount)
-			if err != nil {
-				return nil, fmt.Errorf("couldn't get member entity for account [%s]: %w", callerAccount.String(), err)
-			}
-
-			// See if the caller is in the owner's group
-			r, err := hypersql.GroupGetRole(conn, edb, groupOwner, pkdb)
-			if err != nil {
-				return nil, fmt.Errorf("couldn't get role of member %s in group %s: %w", callerAccount.String(), groupID, err)
-			}
-
-			role = groups.Role(r)
+			return nil, err
 		}
+		role = groups.Role(r)
 	}
 
 	if role != groups.Role_OWNER && role != groups.Role_EDITOR {
-		return nil, status.Errorf(codes.PermissionDenied, "Caller [%s] does not have enough permissions to publish to this site.", callerAccount.String())
+		return nil, status.Errorf(codes.PermissionDenied, "Caller %q does not have enough permissions to publish to this site.", callerAccount.String())
 	}
 
 	blobs, err := ws.blobs.Await(ctx)

backend/cmd/mintter-site/sites/sites_test.go

@@ -6,6 +6,7 @@ import (
 	"mintter/backend/config"
 	"mintter/backend/core/coretest"
 	"mintter/backend/daemon"
+	"mintter/backend/daemon/storage"
 	accounts "mintter/backend/genproto/accounts/v1alpha"
 	groups "mintter/backend/genproto/groups/v1alpha"
 	"mintter/backend/ipfs"
@@ -121,7 +122,7 @@ func makeTestSite(t *testing.T, name string) *App {
 	user := coretest.NewTester(name)
 
 	cfg := testConfig(t)
-	dir, err := daemon.InitRepo(cfg.Base.DataDir, user.Device.Wrapped())
+	dir, err := storage.InitRepo(cfg.Base.DataDir, user.Device.Wrapped())
 	require.NoError(t, err)
 
 	app, err := Load(ctx, "http://127.0.0.1:"+strconv.Itoa(cfg.HTTP.Port), cfg, dir)

backend/cmd/mintterd/main.go

@@ -12,6 +12,7 @@ import (
 
 	"mintter/backend/config"
 	"mintter/backend/daemon"
+	"mintter/backend/daemon/storage"
 	"mintter/backend/logging"
 
 	"github.com/burdiyan/go/mainutil"
@@ -67,7 +68,7 @@ func main() {
 			defer sentry.Flush(2 * time.Second)
 		}
 
-		dir, err := daemon.InitRepo(cfg.Base.DataDir, nil)
+		dir, err := storage.InitRepo(cfg.Base.DataDir, nil)
 		if err != nil {
 			return err
 		}

backend/cmd/mkdb/main.go

@@ -10,6 +10,7 @@ import (
 	"mintter/backend/config"
 	"mintter/backend/core/coretest"
 	"mintter/backend/daemon"
+	"mintter/backend/daemon/storage"
 	"os"
 
 	"github.com/burdiyan/go/mainutil"
@@ -38,7 +39,7 @@ func run() error {
 		return err
 	}
 
-	dir, err := daemon.InitRepo(cfg.Base.DataDir, alice.Device.Wrapped())
+	dir, err := storage.InitRepo(cfg.Base.DataDir, alice.Device.Wrapped())
 	if err != nil {
 		return err
 	}

backend/core/principal.go

@@ -3,6 +3,7 @@ package core
 import (
 	"encoding/binary"
 	"fmt"
+	"unsafe"
 
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -65,6 +66,12 @@ func (p Principal) String() string {
 	return s
 }
 
+// UnsafeString casts raw bytes to a without copying.
+// Useful for using as map key.
+func (p Principal) UnsafeString() string {
+	return unsafe.String(&p[0], len(p))
+}
+
 // Verify implements Verifier.
 func (p Principal) Verify(data []byte, sig Signature) error {
 	code, key := p.Explode()

backend/core/principal_test.go

@@ -0,0 +1,14 @@
+package core
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPrincipalUnsafeString(t *testing.T) {
+	// Hardcoded key generated offline.
+	me, err := DecodePrincipal("z6Mkv1LjkRosErBhmqrkmb5sDxXNs6EzBDSD8ktywpYLLGuC")
+	require.NoError(t, err)
+	require.Equal(t, string(me), me.UnsafeString())
+}

backend/daemon/api/documents/v1alpha/content_graph.go

@@ -22,14 +22,17 @@ func (srv *Server) ListCitations(ctx context.Context, in *documents.ListCitation
 
 	targetEntity := in.DocumentId
 
-	var backlinks []hypersql.BacklinksForEntityResult
+	var backlinks []hypersql.BacklinksForDocumentResult
 	if err := srv.blobs.Query(ctx, func(conn *sqlite.Conn) error {
 		edb, err := hypersql.EntitiesLookupID(conn, targetEntity)
 		if err != nil {
 			return err
 		}
+		if edb.ResourcesID == 0 {
+			return status.Error(codes.NotFound, "document not found")
+		}
 
-		list, err := hypersql.BacklinksForEntity(conn, edb.EntitiesID)
+		list, err := hypersql.BacklinksForDocument(conn, edb.ResourcesID)
 		backlinks = list
 		return err
 	}); err != nil {
@@ -41,17 +44,17 @@ func (srv *Server) ListCitations(ctx context.Context, in *documents.ListCitation
 	}
 
 	for i, link := range backlinks {
-		var ld hyper.LinkData
-		if err := json.Unmarshal(link.BlobAttrsExtra, &ld); err != nil {
+		var ld hyper.DocLinkMeta
+		if err := json.Unmarshal(link.ResourceLinksMeta, &ld); err != nil {
 			return nil, fmt.Errorf("failed to decode link data: %w", err)
 		}
 
 		src := cid.NewCidV1(uint64(link.BlobsCodec), link.BlobsMultihash)
 
 		resp.Links[i] = &documents.Link{
 			Source: &documents.LinkNode{
-				DocumentId: link.EntitiesEID,
-				BlockId:    link.BlobAttrsAnchor,
+				DocumentId: link.ResourcesIRI,
+				BlockId:    ld.Anchor,
 				Version:    src.String(),
 			},
 			Target: &documents.LinkNode{

backend/daemon/api/documents/v1alpha/content_graph_test.go

@@ -47,9 +47,7 @@ func TestBacklinks(t *testing.T) {
 					Type:   "link",
 					Starts: []int32{0},
 					Ends:   []int32{5},
-					Attributes: map[string]string{
-						"url": pub.Document.Id + "?v=" + pub.Version + "#b1",
-					},
+					Ref:    pub.Document.Id + "?v=" + pub.Version + "#b1",
 				},
 			},
 		}}},
@@ -63,9 +61,7 @@ func TestBacklinks(t *testing.T) {
 					Type:   "link",
 					Starts: []int32{0},
 					Ends:   []int32{5},
-					Attributes: map[string]string{
-						"url": pub.Document.Id + "?v=" + pub.Version,
-					},
+					Ref:    pub.Document.Id + "?v=" + pub.Version,
 				},
 			},
 		}}},
@@ -122,6 +118,7 @@ func TestBacklinks(t *testing.T) {
 		},
 	}
 
+	require.Equal(t, len(want.Links), len(cits.Links), "number of links must match")
 	diff := cmp.Diff(makeSet(want.Links), makeSet(cits.Links), testutil.ExportedFieldsFilter())
 	if diff != "" {
 		t.Fatal(diff)

backend/daemon/api/entities/v1alpha/entities.go

@@ -83,11 +83,11 @@ func getChange(conn *sqlite.Conn, c cid.Cid, id int64) (*entities.Change, error)
 	out = &entities.Change{
 		Id:         c.String(),
 		Author:     core.Principal(info.PublicKeysPrincipal).String(),
-		CreateTime: timestamppb.New(hlc.Unpack(info.ChangesHLCTime).Time()),
+		CreateTime: timestamppb.New(hlc.Unpack(info.StructuralBlobsTs).Time()),
 		IsTrusted:  info.IsTrusted > 0,
 	}
 
-	deps, err := hypersql.ChangesGetDeps(conn, info.ChangesBlob)
+	deps, err := hypersql.ChangesGetDeps(conn, info.StructuralBlobsID)
 	if err != nil {
 		return nil, err
 	}
@@ -116,11 +116,11 @@ func (api *Server) GetEntityTimeline(ctx context.Context, in *entities.GetEntity
 		if err != nil {
 			return err
 		}
-		if eid.EntitiesID == 0 {
+		if eid.ResourcesID == 0 {
 			return errutil.NotFound("no such entity %s", in.Id)
 		}
 
-		changes, err := hypersql.ChangesInfoForEntity(conn, eid.EntitiesID)
+		changes, err := hypersql.ChangesInfoForEntity(conn, eid.ResourcesID)
 		if err != nil {
 			return err
 		}
@@ -137,13 +137,13 @@ func (api *Server) GetEntityTimeline(ctx context.Context, in *entities.GetEntity
 			chpb := &entities.Change{
 				Id:         cs,
 				Author:     core.Principal(ch.PublicKeysPrincipal).String(),
-				CreateTime: timestamppb.New(hlc.Unpack(ch.ChangesHLCTime).Time()),
+				CreateTime: timestamppb.New(hlc.Unpack(ch.StructuralBlobsTs).Time()),
 				IsTrusted:  ch.IsTrusted > 0,
 			}
 			heads[cs] = struct{}{}
 			out.ChangesByTime = append(out.ChangesByTime, cs)
 
-			deps, err := hypersql.ChangesGetDeps(conn, ch.ChangesBlob)
+			deps, err := hypersql.ChangesGetDeps(conn, ch.StructuralBlobsID)
 			if err != nil {
 				return err
 			}