Merge branch 'main' of github.com:holepunchto/hypercore into pause-on…

…-full-storage

lib/core.js

@@ -225,7 +225,7 @@ module.exports = class Core {
     return false
   }
 
-  async copyFrom (src, signature, { length = src.tree.length } = {}) {
+  async copyFrom (src, signature, { length = src.tree.length, additional = [] } = {}) {
     await this._mutex.lock()
 
     try {
@@ -235,16 +235,19 @@ module.exports = class Core {
       throw err
     }
 
+    const initialLength = this.tree.length
+
     try {
       const updates = []
 
       let pos = 0
+      const copyLength = Math.min(src.tree.length, length)
 
-      while (pos < length) {
+      while (pos < copyLength) {
         const segmentStart = maximumSegmentStart(pos, src.bitfield, this.bitfield)
         if (segmentStart >= length || segmentStart < 0) break
 
-        const segmentEnd = Math.min(length, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
+        const segmentEnd = Math.min(src.tree.length, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
 
         const segment = []
 
@@ -268,7 +271,7 @@ module.exports = class Core {
         })
       }
 
-      for (let i = 0; i < length * 2; i++) {
+      for (let i = 0; i < copyLength * 2; i++) {
         const node = await src.tree.get(i, false)
         if (node === null) continue
 
@@ -277,35 +280,67 @@ module.exports = class Core {
 
       await this.tree.flush()
 
-      if (length > this.tree.length) {
-        this.tree.fork = src.tree.fork
-        this.tree.roots = [...src.tree.roots]
-        this.tree.length = src.tree.length
-        this.tree.byteLength = src.tree.byteLength
-
-        if (length < this.tree.length) {
-          const batch = await src.tree.truncate(length)
-          this.tree.roots = [...batch.roots]
-          this.tree.length = batch.length
-          this.tree.byteLength = batch.byteLength
+      let batch = this.tree.batch()
+
+      // add additional blocks
+      if (length > src.tree.length) {
+        const missing = length - src.tree.length
+
+        if (additional.length < missing) {
+          throw INVALID_OPERATION('Insufficient additional nodes were passed')
         }
 
+        const source = src.tree.batch()
+
+        batch.roots = [...source.roots]
+        batch.length = source.length
+        batch.byteLength = source.byteLength
+
+        const blocks = additional.length === missing ? additional : additional.slice(0, missing)
+
+        await this.blocks.putBatch(source.length, blocks, source.byteLength)
+        this.bitfield.setRange(source.length, missing, true)
+
+        updates.push({
+          drop: false,
+          start: source.length,
+          length: missing
+        })
+
+        for (const block of blocks) await batch.append(block)
+      } else {
+        const source = length < src.tree.length ? await src.tree.truncate(length) : src.tree.batch()
+
+        this.tree.roots = [...source.roots]
+        this.tree.length = source.length
+        this.tree.byteLength = source.byteLength
+
+        // update batch
+        batch = this.tree.batch()
+      }
+
+      // verify if upgraded
+      if (batch.length > initialLength) {
         try {
-          const batch = this.tree.batch()
           batch.signature = signature
+
           this._verifyBatchUpgrade(batch, this.header.manifest)
           this.tree.signature = signature
         } catch (err) {
           this.tree.signature = null
           // TODO: how to handle signature failure?
           throw err
         }
-
-        this.header.tree.length = this.tree.length
-        this.header.tree.rootHash = this.tree.hash()
-        this.header.tree.signature = this.tree.signature
       }
 
+      await batch.commit()
+
+      this.tree.fork = src.tree.fork
+
+      this.header.tree.length = this.tree.length
+      this.header.tree.rootHash = this.tree.hash()
+      this.header.tree.signature = this.tree.signature
+
       this.header.userData = src.header.userData.slice(0)
       this.header.hints.contiguousLength = Math.min(src.header.hints.contiguousLength, this.header.tree.length)
 

lib/messages.js

@@ -87,6 +87,11 @@ const manifestv0 = {
     hashes.preencode(state, m.hash)
     state.end++ // type
 
+    if (m.prologue && m.signers.length === 0) {
+      c.fixed32.preencode(state, m.prologue.hash)
+      return
+    }
+
     if (m.quorum === 1 && m.signers.length === 1 && !m.allowPatch) {
       signer.preencode(state, m.signers[0])
     } else {
@@ -98,6 +103,12 @@ const manifestv0 = {
   encode (state, m) {
     hashes.encode(state, m.hash)
 
+    if (m.prologue && m.signers.length === 0) {
+      c.uint.encode(state, 0)
+      c.fixed32.encode(state, m.prologue.hash)
+      return
+    }
+
     if (m.quorum === 1 && m.signers.length === 1 && !m.allowPatch) {
       c.uint.encode(state, 1)
       signer.encode(state, m.signers[0])
@@ -112,9 +123,22 @@ const manifestv0 = {
     const hash = hashes.decode(state)
     const type = c.uint.decode(state)
 
-    if (type === 0) throw new Error('Type 0 is deprecated')
     if (type > 2) throw new Error('Unknown type: ' + type)
 
+    if (type === 0) {
+      return {
+        version: 0,
+        hash,
+        allowPatch: false,
+        quorum: 0,
+        signers: [],
+        prologue: {
+          hash: c.fixed32.decode(state),
+          length: 0
+        }
+      }
+    }
+
     if (type === 1) {
       return {
         version: 0,

lib/verifier.js

@@ -47,13 +47,14 @@ class CompatSigner extends Signer {
 module.exports = class Verifier {
   constructor (manifest, { compat = false, crypto = defaultCrypto, legacy = false } = {}) {
     this.compat = compat || manifest === null
-    this.manifest = manifest
-    this.version = this.compat ? 0 : typeof this.manifest.version === 'number' ? this.manifest.version : 1
+    this.version = this.compat ? 0 : typeof manifest.version === 'number' ? manifest.version : 1
     this.hash = manifest.hash || 'blake2b'
-    this.allowPatch = !this.compat && !!this.manifest.allowPatch
-    this.quorum = this.compat ? 1 : (this.manifest.quorum || 0)
-    this.signers = this.manifest.signers.map((s, index) => this.compat ? new CompatSigner(crypto, index, s, legacy) : new Signer(crypto, index, s))
-    this.prologue = this.compat ? null : (this.manifest.prologue || null)
+    this.allowPatch = !this.compat && !!manifest.allowPatch
+    this.quorum = this.compat ? 1 : defaultQuorum(manifest)
+    this.signers = manifest.signers
+      ? manifest.signers.map((s, index) => this.compat ? new CompatSigner(crypto, index, s, legacy) : new Signer(crypto, index, s))
+      : []
+    this.prologue = this.compat ? null : (manifest.prologue || null)
   }
 
   _verifyCompat (batch, signature) {
@@ -169,15 +170,15 @@ module.exports = class Verifier {
       version: typeof inp.version === 'number' ? inp.version : 1,
       hash: 'blake2b',
       allowPatch: !!inp.allowPatch,
-      quorum: inp.quorum || 0,
-      signers: inp.signers.map(parseSigner),
+      quorum: defaultQuorum(inp),
+      signers: inp.signers ? inp.signers.map(parseSigner) : [],
       prologue: null
     }
 
     if (inp.hash && inp.hash !== 'blake2b') throw BAD_ARGUMENT('Only Blake2b hashes are supported')
 
     if (inp.prologue) {
-      if (!(b4a.isBuffer(inp.prologue.hash) && inp.prologue.hash.byteLength === 32) || !inp.prologue.length) {
+      if (!(b4a.isBuffer(inp.prologue.hash) && inp.prologue.hash.byteLength === 32) || !(inp.prologue.length >= 0)) {
         throw BAD_ARGUMENT('Invalid prologue')
       }
       manifest.prologue = inp.prologue
@@ -206,6 +207,12 @@ function toMap (nodes) {
   return m
 }
 
+function defaultQuorum (man) {
+  if (typeof man.quorum === 'number') return man.quorum
+  if (!man.signers || !man.signers.length) return 0
+  return (man.signers.length >> 1) + 1
+}
+
 function generateUpgrade (patch, start, length) {
   const upgrade = { start, length, nodes: null, additionalNodes: [] }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore",
-  "version": "10.31.12",
+  "version": "10.32.3",
   "description": "Hypercore is a secure, distributed append-only log",
   "main": "index.js",
   "scripts": {

test/core.js

@@ -403,6 +403,161 @@ test('core - partial clone', async function (t) {
     b4a.from('0'),
     b4a.from('1')
   ])
+
+  await t.exception(copy.blocks.get(2))
+})
+
+test('core - clone with additional', async function (t) {
+  const { core } = await create()
+  const { core: copy } = await create({ keyPair: core.header.keyPair })
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.keyPair.publicKey } })
+
+  await core.append([b4a.from('a'), b4a.from('b')])
+  await copy.copyFrom(core, core.tree.signature)
+
+  t.is(copy.header.keyPair.publicKey, core.header.keyPair.publicKey)
+  t.is(copy.header.keyPair.publicKey, clone.header.keyPair.publicKey)
+
+  // copy should be independent
+  await core.append([b4a.from('c')])
+
+  await clone.copyFrom(copy, core.tree.signature, { length: 3, additional: [b4a.from('c')] })
+
+  t.is(clone.header.tree.length, 3)
+  t.alike(clone.header.tree.signature, core.header.tree.signature)
+
+  t.is(clone.tree.length, core.tree.length)
+  t.is(clone.tree.byteLength, core.tree.byteLength)
+  t.alike(clone.roots, core.roots)
+
+  t.alike(await clone.blocks.get(0), b4a.from('a'))
+  t.alike(await clone.blocks.get(1), b4a.from('b'))
+  t.alike(await clone.blocks.get(2), b4a.from('c'))
+})
+
+test('core - clone with additional, larger tree', async function (t) {
+  const { core } = await create()
+  const { core: copy } = await create({ keyPair: core.header.keyPair })
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.keyPair.publicKey } })
+
+  await core.append([b4a.from('a'), b4a.from('b')])
+  await copy.copyFrom(core, core.tree.signature)
+
+  t.is(copy.header.keyPair.publicKey, core.header.keyPair.publicKey)
+  t.is(copy.header.keyPair.publicKey, clone.header.keyPair.publicKey)
+
+  const additional = [
+    b4a.from('c'),
+    b4a.from('d'),
+    b4a.from('e'),
+    b4a.from('f'),
+    b4a.from('g'),
+    b4a.from('h'),
+    b4a.from('i'),
+    b4a.from('j')
+  ]
+
+  await core.append(additional)
+
+  // copy should be independent
+  await clone.copyFrom(copy, core.tree.signature, { length: core.tree.length, additional })
+
+  t.is(clone.header.tree.length, core.header.tree.length)
+  t.alike(clone.header.tree.signature, core.header.tree.signature)
+
+  t.is(clone.tree.length, core.tree.length)
+  t.is(clone.tree.byteLength, core.tree.byteLength)
+  t.alike(clone.roots, core.roots)
+
+  t.alike(await clone.blocks.get(0), b4a.from('a'))
+  t.alike(await clone.blocks.get(1), b4a.from('b'))
+  t.alike(await clone.blocks.get(2), b4a.from('c'))
+  t.alike(await clone.blocks.get(3), b4a.from('d'))
+  t.alike(await clone.blocks.get(4), b4a.from('e'))
+  t.alike(await clone.blocks.get(5), b4a.from('f'))
+  t.alike(await clone.blocks.get(6), b4a.from('g'))
+  t.alike(await clone.blocks.get(7), b4a.from('h'))
+  t.alike(await clone.blocks.get(8), b4a.from('i'))
+  t.alike(await clone.blocks.get(9), b4a.from('j'))
+})
+
+test('core - clone with too many additional', async function (t) {
+  const { core } = await create()
+  const { core: copy } = await create({ keyPair: core.header.keyPair })
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.keyPair.publicKey } })
+
+  await core.append([b4a.from('a'), b4a.from('b')])
+  await copy.copyFrom(core, core.tree.signature)
+
+  t.is(copy.header.keyPair.publicKey, core.header.keyPair.publicKey)
+  t.is(copy.header.keyPair.publicKey, clone.header.keyPair.publicKey)
+
+  // copy should be independent
+  await core.append([b4a.from('c')])
+
+  await clone.copyFrom(copy, core.tree.signature, {
+    length: 3,
+    additional: [
+      b4a.from('c'),
+      b4a.from('d')
+    ]
+  })
+
+  t.is(clone.header.tree.length, 3)
+  t.alike(clone.header.tree.signature, core.header.tree.signature)
+
+  t.is(clone.tree.length, core.tree.length)
+  t.is(clone.tree.byteLength, core.tree.byteLength)
+  t.alike(clone.roots, core.roots)
+
+  t.alike(await clone.blocks.get(0), b4a.from('a'))
+  t.alike(await clone.blocks.get(1), b4a.from('b'))
+  t.alike(await clone.blocks.get(2), b4a.from('c'))
+
+  await t.exception(clone.blocks.get(3))
+})
+
+test('core - clone fills in with additional', async function (t) {
+  const { core } = await create()
+  const { core: copy } = await create({ keyPair: core.header.keyPair })
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.keyPair.publicKey } })
+
+  t.is(copy.header.keyPair.publicKey, core.header.keyPair.publicKey)
+  t.is(copy.header.keyPair.publicKey, clone.header.keyPair.publicKey)
+
+  await clone.copyFrom(core, core.tree.signature)
+
+  // copy should be independent
+  await core.append([b4a.from('a')])
+  await copy.copyFrom(core, core.tree.signature)
+
+  // upgrade clone
+  {
+    const p = await core.tree.proof({ upgrade: { start: 0, length: 1 } })
+    t.ok(await clone.verify(p))
+  }
+
+  await core.append([b4a.from('b')])
+
+  // verify state
+  t.is(copy.tree.length, 1)
+  t.is(clone.tree.length, 1)
+
+  await t.exception(clone.blocks.get(0))
+  await t.exception(copy.blocks.get(1))
+
+  // copy should both fill in and upgrade
+  await clone.copyFrom(copy, core.tree.signature, { length: 2, additional: [b4a.from('b')] })
+
+  t.is(clone.header.tree.length, 2)
+  t.alike(clone.header.tree.signature, core.header.tree.signature)
+
+  t.is(clone.tree.length, core.tree.length)
+  t.is(clone.tree.byteLength, core.tree.byteLength)
+  t.alike(clone.roots, core.roots)
+
+  t.alike(await clone.blocks.get(0), b4a.from('a'))
+  t.alike(await clone.blocks.get(1), b4a.from('b'))
 })
 
 async function create (opts) {