fix requests requesting nodes the other peer doesnt have

lib/replicator.js

@@ -1,6 +1,7 @@
 const b4a = require('b4a')
 const safetyCatch = require('safety-catch')
 const RandomIterator = require('random-array-iterator')
+const flatTree = require('flat-tree')
 const ReceiverQueue = require('./receiver-queue')
 const RemoteBitfield = require('./remote-bitfield')
 const { REQUEST_CANCELLED, REQUEST_TIMEOUT, INVALID_CAPABILITY, SNAPSHOT_NOT_AVAILABLE } = require('hypercore-errors')
@@ -583,7 +584,7 @@ class Peer {
         proof = await this._getProof(msg)
       } catch (err) {
         safetyCatch(err)
-        if (isCriticalError(err)) throw err
+        if (msg.fork === this.core.tree.fork && isCriticalError(err)) throw err
       }
     }
 
@@ -830,6 +831,7 @@ class Peer {
 
       if (this.remoteBitfield.get(index) === false) continue
       if (this.core.bitfield.get(index) === true) continue
+      if (!this._hasTreeParent(index)) continue
 
       // Check if this block is currently inflight - if so pick another
       const b = this.replicator._blocks.get(index)
@@ -860,13 +862,48 @@ class Peer {
     return this.remoteBitfield.get(b.index)
   }
 
+  _hasTreeParent (index) {
+    if (this.remoteLength >= this.core.tree.length) return true
+
+    const ite = flatTree.iterator(index * 2)
+
+    let span = 2
+    let length = 0
+
+    while (true) {
+      ite.parent()
+
+      const left = (ite.index - ite.factor / 2 + 1) / 2
+      length = left + span
+
+      // if larger than local AND larger than remote - they share the root so its ok
+      if (length > this.core.tree.length) {
+        if (length > this.remoteLength) return true
+        break
+      }
+
+      // its less than local but larger than remote so skip it
+      if (length > this.remoteLength) break
+
+      span *= 2
+      const first = this.core.bitfield.findFirst(true, left)
+      if (first > -1 && first < length) return true
+    }
+
+    // TODO: push to async queue and check against our local merkle tree if we actually can request this block
+    return false
+  }
+
   _requestBlock (b) {
     const { length, fork } = this.core.tree
 
     if (this.remoteBitfield.get(b.index) === false || fork !== this.remoteFork) {
       this._maybeWant(b.index)
       return false
     }
+    if (!this._hasTreeParent(b.index)) {
+      return false
+    }
 
     const req = this._makeRequest(b.index >= length, b.priority)
     if (req === null) return false
@@ -916,6 +953,10 @@ class Peer {
         continue
       }
 
+      if (!this._hasTreeParent(index)) {
+        continue
+      }
+
       const b = this.replicator._blocks.add(index, PRIORITY.NORMAL)
 
       if (b.inflight.length > 0) {

test/replicate.js

@@ -1377,6 +1377,38 @@ test('manifests gossip eagerly sync', async function (t) {
   })
 })
 
+test('remote has larger tree', async function (t) {
+  const a = await create()
+  const b = await create(a.key)
+  const c = await create(a.key)
+
+  await a.append(['a', 'b', 'c', 'd', 'e'])
+
+  {
+    const [s1, s2] = replicate(a, b, t)
+    await b.get(2)
+    await b.get(3)
+    await b.get(4)
+    s1.destroy()
+    s2.destroy()
+  }
+
+  await a.append('f')
+
+  {
+    const [s1, s2] = replicate(a, c, t)
+    await eventFlush()
+    s1.destroy()
+    s2.destroy()
+  }
+
+  replicate(b, c, t)
+  c.get(4) // unresolvable but should not crash anything
+  await eventFlush()
+  t.ok(!!(await c.get(2)), 'got block #2')
+  t.ok(!!(await c.get(3)), 'got block #3')
+})
+
 async function waitForRequestBlock (core, opts) {
   while (true) {
     const reqBlock = core.replicator._inflight._requests.find(req => req && req.block)