Add a hook to fix multihash mismatch errors (#423)

* Add storage read opner error hook Allow the calling application to specify custom error handling logic when storage read opener error occurs. The fix is designed to fix the 'ad multihash mismatch error'. * Check for storageReadOpenerErrorHook when link system created --------- Co-authored-by: gammazero <[email protected]>
ipni · Feb 19, 2024 · 991632f · 991632f
1 parent a8b0416
commit 991632f
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 1 deletion.
diff --git a/engine/linksystem.go b/engine/linksystem.go
@@ -25,7 +25,8 @@ var (
 // Creates the main engine linksystem.
 func (e *Engine) mkLinkSystem() ipld.LinkSystem {
 	lsys := cidlink.DefaultLinkSystem()
-	lsys.StorageReadOpener = func(lctx ipld.LinkContext, lnk ipld.Link) (io.Reader, error) {
+
+	storageReadOpener := func(lctx ipld.LinkContext, lnk ipld.Link) (io.Reader, error) {
 		// If link corresponds to schema.NoEntries return error immediately.
 		if lnk == schema.NoEntries {
 			return nil, errNoEntries
@@ -149,6 +150,21 @@ func (e *Engine) mkLinkSystem() ipld.LinkSystem {
 
 		return bytes.NewBuffer(val), nil
 	}
+
+	// If error hook provided, call error hook function on storageReadOpener
+	// error. Otherwise, only call storageReadOpener.
+	if e.options.storageReadOpenerErrorHook != nil {
+		lsys.StorageReadOpener = func(lctx ipld.LinkContext, lnk ipld.Link) (io.Reader, error) {
+			r, err := storageReadOpener(lctx, lnk)
+			if err != nil {
+				return r, e.options.storageReadOpenerErrorHook(lctx, lnk, err)
+			}
+			return r, nil
+		}
+	} else {
+		lsys.StorageReadOpener = storageReadOpener
+	}
+
 	lsys.StorageWriteOpener = func(lctx ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) {
 		buf := bytes.NewBuffer(nil)
 		return buf, func(lnk ipld.Link) error {

diff --git a/engine/options.go b/engine/options.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/ipfs/go-datastore"
 	dssync "github.com/ipfs/go-datastore/sync"
+	"github.com/ipld/go-ipld-prime"
 	_ "github.com/ipni/go-libipni/maurl"
 	"github.com/ipni/index-provider/engine/chunker"
 	"github.com/ipni/index-provider/engine/policy"
@@ -96,6 +97,8 @@ type (
 		chunker     chunker.NewChunkerFunc
 
 		syncPolicy *policy.Policy
+
+		storageReadOpenerErrorHook func(lctx ipld.LinkContext, lnk ipld.Link, err error) error
 	}
 )
 
@@ -431,3 +434,13 @@ func WithExtraGossipData(extraData []byte) Option {
 		return nil
 	}
 }
+
+// WithStorageReadOpenerErrorHook allows the calling applicaiton to invoke a custom piece logic whenever a storage read opener error occurs.
+// For example the calling application can delete corrupted / create a new advertisement if the datastore was corrupted for some reason.
+// The calling application can return ipld.ErrNotFound{} to indicate IPNI that this advertisement should be skipped without halting processing of the rest of the chain.
+func WithStorageReadOpenerErrorHook(hook func(ipld.LinkContext, ipld.Link, error) error) Option {
+	return func(o *options) error {
+		o.storageReadOpenerErrorHook = hook
+		return nil
+	}
+}