refactor(backend): optimize database indices and queries for blob met…

…adata
seed-hypermedia · Mar 20, 2024 · c8e6bc7 · c8e6bc7
1 parent d2dd71f
commit c8e6bc7
Show file tree

Hide file tree

Showing 12 changed files with 56 additions and 155 deletions.
diff --git a/backend/daemon/api/entities/v1alpha/entities.go b/backend/daemon/api/entities/v1alpha/entities.go
@@ -272,7 +272,7 @@ var qGetEntityTimeline = dqb.Str(`
 		public_keys.principal AS author,
 		group_concat(change_deps.parent, ' ') AS deps
 	FROM structural_blobs
-	JOIN blobs ON blobs.id = structural_blobs.id
+	JOIN blobs INDEXED BY blobs_metadata ON blobs.id = structural_blobs.id
 	JOIN public_keys ON public_keys.id = structural_blobs.author
 	LEFT JOIN change_deps ON change_deps.child = structural_blobs.id
 	LEFT JOIN drafts ON (drafts.resource, drafts.blob) = (structural_blobs.resource, structural_blobs.id)

diff --git a/backend/daemon/api/groups/v1alpha/db.go b/backend/daemon/api/groups/v1alpha/db.go
@@ -176,7 +176,7 @@ var QCollectBlobs = dqb.Str(`
 		blobs.codec,
 		blobs.multihash
 	FROM selected
-	JOIN blobs ON blobs.id = selected.id AND selected.kind = 1
+	JOIN blobs INDEXED BY blobs_metadata ON blobs.id = selected.id AND selected.kind = 1
 	ORDER BY blobs.id;
 `)
 

diff --git a/backend/daemon/api/groups/v1alpha/groups.go b/backend/daemon/api/groups/v1alpha/groups.go
@@ -847,7 +847,7 @@ var qListDocumentGroups = dqb.Str(`
 		structural_blobs.ts AS ts
 	FROM resource_links
 	JOIN structural_blobs ON structural_blobs.id = resource_links.source
-	JOIN blobs ON blobs.id = structural_blobs.id
+	JOIN blobs INDEXED BY blobs_metadata ON blobs.id = structural_blobs.id
 	JOIN resources ON resources.id = structural_blobs.resource
 	WHERE resource_links.type = 'group/content'
 	AND resource_links.target = :document

diff --git a/backend/daemon/storage/migrations.go b/backend/daemon/storage/migrations.go
@@ -386,6 +386,34 @@ var migrations = []migration{
 			DELETE FROM kv WHERE key = 'last_reindex_time';
 		`))
 	}},
+	{Version: "2024-03-19.01", Run: func(_ *Dir, conn *sqlite.Conn) error {
+		return sqlitex.ExecScript(conn, sqlfmt(`
+			DROP VIEW IF EXISTS key_delegations_view;
+			CREATE VIEW IF NOT EXISTS key_delegations_view AS
+			SELECT
+				kd.id AS blob,
+				blobs.codec AS blob_codec,
+				blobs.multihash AS blob_multihash,
+				iss.principal AS issuer,
+				del.principal AS delegate
+			FROM key_delegations kd
+			JOIN blobs INDEXED BY blobs_metadata ON blobs.id = kd.id
+			JOIN public_keys iss ON iss.id = kd.issuer
+			JOIN public_keys del ON del.id = kd.delegate;
+
+			DROP VIEW IF EXISTS drafts_view;
+			CREATE VIEW IF NOT EXISTS drafts_view AS
+			SELECT
+				drafts.resource AS resource_id,
+				drafts.blob AS blob_id,
+				resources.iri AS resource,
+				blobs.codec AS codec,
+				blobs.multihash AS multihash
+			FROM drafts
+			JOIN resources ON resources.id = drafts.resource
+			JOIN blobs INDEXED BY blobs_metadata ON blobs.id = drafts.blob;
+		`))
+	}},
 }
 
 const (

diff --git a/backend/daemon/storage/schema.gensum b/backend/daemon/storage/schema.gensum
@@ -1,2 +1,2 @@
-srcs: 73de8491dac69809249c20522cd0a451
+srcs: c0bb41a9dd9db2130a02aa83b977269f
 outs: 0efc08608cead13c5b8756e4beee2ecd
diff --git a/backend/daemon/storage/schema.sql b/backend/daemon/storage/schema.sql
@@ -99,7 +99,7 @@ SELECT
     iss.principal AS issuer,
     del.principal AS delegate
 FROM key_delegations kd
-JOIN blobs ON blobs.id = kd.id
+JOIN blobs INDEXED BY blobs_metadata ON blobs.id = kd.id
 JOIN public_keys iss ON iss.id = kd.issuer
 JOIN public_keys del ON del.id = kd.delegate;
 
@@ -176,7 +176,7 @@ SELECT
     blobs.multihash AS multihash
 FROM drafts
 JOIN resources ON resources.id = drafts.resource
-JOIN blobs ON blobs.id = drafts.blob;
+JOIN blobs INDEXED BY blobs_metadata ON blobs.id = drafts.blob;
 
 -- View for dependency links between changes.
 CREATE VIEW change_deps AS

diff --git a/backend/hyper/hypersql/queries.gen.go b/backend/hyper/hypersql/queries.gen.go
diff --git a/backend/hyper/hypersql/queries.gensum b/backend/hyper/hypersql/queries.gensum
@@ -1,2 +1,2 @@
-srcs: 800af070fac742f44f90f3142c9ffcdd
-outs: ed8e2acc2115513302d723aab1d251e1
+srcs: 2e1bd3b9f4e7baaf51f43553662842dc
+outs: 63faeb2bdfe750e843383133579a0550
diff --git a/backend/hyper/hypersql/queries.go b/backend/hyper/hypersql/queries.go
@@ -14,7 +14,7 @@ func generateQueries() error {
 	code, err := sgen.CodegenQueries("hypersql",
 		qb.MakeQuery(s.Schema, "BlobsHave", sgen.QueryKindSingle,
 			"SELECT", qb.Results(qb.ResultExpr("1", "have", sgen.TypeInt)), '\n',
-			"FROM", s.Blobs, '\n',
+			"FROM", s.Blobs, "INDEXED BY blobs_metadata_by_hash", '\n',
 			"WHERE", s.BlobsMultihash, "=", qb.VarCol(s.BlobsMultihash), '\n',
 			"AND", s.BlobsSize, ">=", "0",
 		),
@@ -35,7 +35,7 @@ func generateQueries() error {
 				s.BlobsID,
 				s.BlobsSize,
 			), '\n',
-			"FROM", s.Blobs, '\n',
+			"FROM", s.Blobs, "INDEXED BY blobs_metadata_by_hash", '\n',
 			"WHERE", s.BlobsMultihash, "=", qb.VarCol(s.BlobsMultihash),
 		),
 		qb.MakeQuery(s.Schema, "BlobsInsert", sgen.QueryKindSingle,
@@ -66,7 +66,7 @@ func generateQueries() error {
 				s.BlobsMultihash,
 				s.BlobsCodec,
 			), '\n',
-			"FROM", s.Blobs, '\n',
+			"FROM", s.Blobs, "INDEXED BY blobs_metadata", '\n',
 			"LEFT JOIN", s.Drafts, "ON", s.DraftsBlob, "=", s.BlobsID, '\n',
 			"WHERE", s.BlobsSize, ">=", "0", '\n',
 			"AND", s.DraftsBlob, "IS NULL", '\n',
@@ -173,7 +173,7 @@ func generateQueries() error {
 				s.KeyDelegationsIssuer,
 			), '\n',
 			"FROM", s.KeyDelegations, '\n',
-			"JOIN", s.Blobs, "ON", s.BlobsID, "=", s.KeyDelegationsID, '\n',
+			"JOIN", s.Blobs, "INDEXED BY blobs_metadata_by_hash ON", s.BlobsID, "=", s.KeyDelegationsID, '\n',
 			"WHERE", s.BlobsMultihash, "=", qb.VarCol(s.BlobsMultihash), '\n',
 			"LIMIT 1",
 		),
@@ -242,23 +242,6 @@ func generateQueries() error {
 			"WHERE", s.ResourcesIRI, "=", qb.VarCol(s.ResourcesIRI), '\n',
 			"ORDER BY", s.StructuralBlobsTs,
 		),
-		qb.MakeQuery(s.Schema, "ChangesListPublicNoData", sgen.QueryKindMany,
-			"SELECT", qb.Results(
-				s.StructuralBlobsViewBlobID,
-				s.StructuralBlobsViewCodec,
-				s.StructuralBlobsViewResourceID,
-				s.StructuralBlobsViewTs,
-				s.StructuralBlobsViewMultihash,
-				s.StructuralBlobsViewSize,
-				s.StructuralBlobsViewResource,
-				s.DraftsBlob,
-			), '\n',
-			"FROM", s.StructuralBlobsView, '\n',
-			"LEFT JOIN", s.Drafts, "ON", s.DraftsResource, "=", s.StructuralBlobsViewResourceID, '\n',
-			"WHERE", s.DraftsBlob, "IS NULL", '\n',
-			"AND", s.StructuralBlobsViewBlobType, "= 'Change'",
-			"ORDER BY", qb.Enumeration(s.StructuralBlobsViewResource, s.StructuralBlobsViewTs),
-		),
 		qb.MakeQuery(s.Schema, "ChangesResolveHeads", sgen.QueryKindSingle,
 			"WITH RECURSIVE changeset (change) AS", qb.SubQuery(
 				"SELECT value",
@@ -328,24 +311,9 @@ WHERE blob NOT IN deps`,
 				s.BlobsMultihash,
 			), '\n',
 			"FROM", s.ChangeDeps, '\n',
-			"JOIN", s.Blobs, "ON", s.BlobsID, "=", s.ChangeDepsParent, '\n',
+			"JOIN", s.Blobs, "INDEXED BY blobs_metadata ON", s.BlobsID, "=", s.ChangeDepsParent, '\n',
 			"WHERE", s.ChangeDepsChild, "=", qb.VarCol(s.ChangeDepsChild),
 		),
-		qb.MakeQuery(s.Schema, "ChangesInfoForEntity", sgen.QueryKindMany,
-			"SELECT", qb.Results(
-				s.BlobsCodec,
-				s.BlobsMultihash,
-				s.StructuralBlobsID,
-				s.StructuralBlobsTs,
-				s.PublicKeysPrincipal,
-				qb.ResultExpr(s.C_TrustedAccountsID+" > 0", "is_trusted", sgen.TypeInt),
-			), '\n',
-			"FROM", s.StructuralBlobs, '\n',
-			"JOIN", s.Blobs, "ON", s.BlobsID, "=", s.StructuralBlobsID, '\n',
-			"JOIN", s.PublicKeys, "ON", s.PublicKeysID, "=", s.StructuralBlobsAuthor, '\n',
-			"LEFT JOIN", s.TrustedAccounts, "ON", s.TrustedAccountsID, "=", s.StructuralBlobsAuthor, '\n',
-			"WHERE", s.StructuralBlobsResource, "=", qb.VarCol(s.StructuralBlobsResource),
-		),
 
 		qb.MakeQuery(s.Schema, "BacklinksForDocument", sgen.QueryKindMany,
 			"SELECT", qb.Results(
@@ -361,7 +329,7 @@ WHERE blob NOT IN deps`,
 			"FROM", s.ResourceLinks, '\n',
 			"JOIN", s.StructuralBlobs, "ON", s.StructuralBlobsID, "=", s.ResourceLinksSource, '\n',
 			"JOIN", s.Resources, "ON", s.ResourcesID, "=", s.StructuralBlobsResource, '\n',
-			"JOIN", s.Blobs, "ON", s.BlobsID, "=", s.StructuralBlobsID, '\n',
+			"JOIN", s.Blobs, "INDEXED BY blobs_metadata ON", s.BlobsID, "=", s.StructuralBlobsID, '\n',
 			"WHERE", s.ResourceLinksType, "GLOB 'doc/*'", '\n',
 			"AND", s.ResourceLinksTarget, "=", qb.VarCol(s.ResourceLinksTarget),
 		),

diff --git a/backend/mttnet/list_blobs.go b/backend/mttnet/list_blobs.go
@@ -75,7 +75,7 @@ var qListBlobs = dqb.Str(`
 		blobs.id,
 		blobs.codec,
 		blobs.multihash
-	FROM blobs
+	FROM blobs INDEXED BY blobs_metadata
 	LEFT OUTER JOIN drafts ON drafts.blob = blobs.id
 	WHERE blobs.size >= 0
 	AND drafts.blob IS NULL

diff --git a/backend/mttnet/providing.go b/backend/mttnet/providing.go
@@ -6,7 +6,6 @@ import (
 	"mintter/backend/hyper"
 	"mintter/backend/hyper/hypersql"
 	"mintter/backend/logging"
-	"mintter/backend/pkg/dqb"
 	"time"
 
 	"crawshaw.io/sqlite/sqlitex"
@@ -15,16 +14,6 @@ import (
 	"go.uber.org/zap"
 )
 
-var qAllPublicBlobs = dqb.Str(`
-	SELECT
-		blobs.codec,
-		blobs.multihash
-	FROM blobs
-	LEFT OUTER JOIN drafts ON drafts.blob = blobs.id
-	WHERE blobs.size >= 0
-	AND drafts.blob IS NULL;
-`)
-
 var randSrc = rand.NewSource(time.Now().UnixNano())
 
 func makeProvidingStrategy(db *sqlitex.Pool, logLevel string) provider.KeyChanFunc {