Skip to content

Commit

Permalink
Manifest (#430 + #431)
Browse files Browse the repository at this point in the history 
* Initial manifest support (#426)

* add auth section to upgrade

* exchange auth definitions through a manifest

* fix a ton of stuff

* skip custom keypair tests for now

* allow explicit compat mode

* no need to special case keypair

* actually fix preload test

* make manifest explicit

* also set compat mode when replicating manifest

* very explicit manifest exchange

* verifyBatch -> verifyBatchUpgraded

* tweak namespacing and add explicit default ns

* add preliminary functions for parsing manifest

* typo on imports

* add encodings and entropy/namespacing

* add compat signer and more tests

* lib/core uses manifest createAuth function

* default to compat mode

* make signers classes and update tests

* correctly set compat on sessions/clones/upgrade

* fix bad merge

* need to specify compat for clone

* replace auth with manifest

* update clone and ensure that key is different

* update tests to manifest api

* load new auth if secret key is provided

* remove custom auth test

* support passing a sign function

* remove auth tests

* manifest renamed to multipleSigners

* properly set compat and keyPair

* pass batch to patched verify

* port tests from hypercore-crypto-multisig

* include file for multisig helpers

* rename to verifiers and add explicit compat verifier

* split out signing and verification

* remove sign option and just pass keyPair or signature

* only set keyPair after openCapabilities

* clarify code around creating a manifest

* conflict test now works

* fix core copyFrom and core clone tests

* fix clone and clone tests

* standardise opts across append and truncate

* tidy up

* add multisig lib file

* expose helper

* fix batch flush api

* fix noManifest logic

* fix some typos/missing apis

* set signature on tree only after verification

* rename static helpers

* rename defaultAuth to verifier

* rename defaultAuth to verifier

* remove static createVerifier method

* review by @mafintosh

* move manifestHash to lib/manifest

---------

Co-authored-by: Mathias Buus <[email protected]>

* last part of the constructor should be openSession

* simplify key handler

* fix keypair handler

* define writable condition in one place

* fix typo in manifestHash

* pass manifest directly to createVerifier

* no need to dbl copy manifest now

* tweak manifest construction

* consistent errors

* add manifest getter

* tweak manifest checks on load

* review by @chm-diederichs

* set keypair in constructor if possible

* simplify core append signature

* tweak batch flush

* simplify compat option

* update batch test

* move compat default to lib/core

* move compat check to a single func

* no manifesthash in caps

---------

Co-authored-by: Christophe Diederichs <[email protected]>
Co-authored-by: Christophe Diederichs <[email protected]>

Manifest tweaks (#431)

* more tests for now truncate/append options

* move more manifest functinality into manifest.js

* move isCompat to manifest and simplify clone options

* enable all tests

* pass batches to verifiers and move namespace support to batch

* fix storage test
  • Loading branch information
@mafintosh
mafintosh committed Sep 7, 2023
1 parent c572826 commit 52201f2
Show file tree
Hide file tree
Showing 17 changed files with 1,648 additions and 535 deletions.
96 changes: 55 additions & 41 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ const BlockEncryption = require('./lib/block-encryption')
const Info = require('./lib/info')
const Download = require('./lib/download')
const Batch = require('./lib/batch')
const { manifestHash, defaultSignerManifest, createVerifier, createManifest, isCompat } = require('./lib/manifest')
const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
const {
BAD_ARGUMENT,
Expand Down Expand Up @@ -69,30 +70,31 @@ module.exports = class Hypercore extends EventEmitter {

this.id = null
this.key = key || null
this.keyPair = null
this.keyPair = opts.keyPair || null
this.readable = true
this.writable = false
this.opened = false
this.closed = false
this.snapshotted = !!opts.snapshot
this.sparse = opts.sparse !== false
this.sessions = opts._sessions || [this]
this.auth = opts.auth || null
this.autoClose = !!opts.autoClose
this.onwait = opts.onwait || null
this.wait = opts.wait !== false
this.timeout = opts.timeout || 0
this._clone = opts.clone || null
this._readonly = opts.writable === false

this.closing = null
this.opening = this._openSession(key, storage, opts)
this.opening.catch(noop)
this.opening = null

this._manifest = opts.manifest || null
this._clone = opts.clone || null
this._readonly = opts.writable === false
this._preappend = preappend.bind(this)
this._snapshot = null
this._batch = opts._batch || null
this._findingPeers = 0

this.opening = this._openSession(key, storage, opts)
this.opening.catch(noop)
}

[inspect] (depth, opts) {
Expand Down Expand Up @@ -139,6 +141,14 @@ module.exports = class Hypercore extends EventEmitter {
indent + ')'
}

static key (manifest, { compat } = {}) {
return compat ? manifest.signer.publicKey : manifestHash(manifest)
}

static discoveryKey (key) {
return hypercoreCrypto.discoveryKey(key)
}

static getProtocolMuxer (stream) {
return stream.noiseStream.userData
}
Expand Down Expand Up @@ -245,20 +255,19 @@ module.exports = class Hypercore extends EventEmitter {
}

setKeyPair (keyPair) {
this.auth = Core.createAuth(this.crypto, { keyPair })
this.writable = !this._readonly && !!this.auth && !!this.auth.sign
this.keyPair = keyPair
this.writable = this._isWritable()
}

_passCapabilities (o) {
if (!this.auth) this.auth = o.auth

if (!this.keyPair) this.keyPair = o.keyPair
this.crypto = o.crypto
this.id = o.id
this.key = o.key
this.core = o.core
this.replicator = o.replicator
this.encryption = o.encryption
this.writable = !this._readonly && !!this.auth && !!this.auth.sign
this.writable = this._isWritable()
this.autoClose = o.autoClose

if (this.snapshotted && this.core && !this._snapshot) this._updateSnapshot()
Expand Down Expand Up @@ -288,14 +297,6 @@ module.exports = class Hypercore extends EventEmitter {
if (!isFirst) await opts._opening
if (opts.preload) opts = { ...opts, ...(await this._retryPreload(opts.preload)) }

const keyPair = opts.keyPair

if (opts.auth) {
this.auth = opts.auth
} else if (keyPair && keyPair.secretKey) {
this.setKeyPair(keyPair)
}

if (isFirst) {
await this._openCapabilities(key, storage, opts)
// Only the root session should pass capabilities to other sessions.
Expand All @@ -306,15 +307,14 @@ module.exports = class Hypercore extends EventEmitter {

// copy state over
if (this._clone) {
const { from, upgrade } = this._clone
const { from, signature } = this._clone
await from.opening
await this.core.copyFrom(from.core, upgrade)
await this.core.copyFrom(from.core, signature)
this._clone = null
}
}

if (!this.auth) this.auth = this.core.defaultAuth
this.writable = !this._readonly && !!this.auth && !!this.auth.sign
this.writable = this._isWritable()

if (opts.valueEncoding) {
this.valueEncoding = c.from(opts.valueEncoding)
Expand Down Expand Up @@ -353,7 +353,7 @@ module.exports = class Hypercore extends EventEmitter {
this.storage = Hypercore.defaultStorage(opts.storage || storage, { unlocked, writable: !unlocked })

this.core = await Core.open(this.storage, {
compat: opts.compat !== false, // default to true for now
compat: opts.compat,
force: opts.force,
createIfMissing: opts.createIfMissing,
readonly: unlocked,
Expand All @@ -362,7 +362,7 @@ module.exports = class Hypercore extends EventEmitter {
keyPair: opts.keyPair,
crypto: this.crypto,
legacy: opts.legacy,
auth: opts.auth,
manifest: opts.manifest,
onupdate: this._oncoreupdate.bind(this),
onconflict: this._oncoreconflict.bind(this)
})
Expand Down Expand Up @@ -417,6 +417,10 @@ module.exports = class Hypercore extends EventEmitter {
return prev.length !== next.length || prev.fork !== next.fork
}

_isWritable () {
return !this._readonly && !!(this.keyPair && this.keyPair.secretKey)
}

close (err) {
if (this.closing) return this.closing
this.closing = this._close(err || null)
Expand Down Expand Up @@ -465,23 +469,22 @@ module.exports = class Hypercore extends EventEmitter {
this.emit('close', true)
}

clone (storage, opts = {}) {
clone (keyPair, storage, opts = {}) {
// TODO: current limitation is no forking
if ((opts.fork && opts.fork !== 0) || this.fork !== 0) {
throw BAD_ARGUMENT('Cannot clone a fork')
}

const key = opts.key === undefined ? opts.keyPair ? null : this.key : opts.key
const keyPair = (opts.auth || opts.keyPair === undefined) ? null : opts.keyPair
const manifest = opts.manifest || defaultSignerManifest(keyPair.publicKey)
const key = opts.key || (opts.compat !== false ? manifest.signer.publicKey : manifestHash(manifest))

let auth = this.core.defaultAuth
if (opts.auth) {
auth = opts.auth
} else if (keyPair && keyPair.secretKey) {
auth = Core.createAuth(this.crypto, { keyPair, manifest: { signer: { publicKey: keyPair.publicKey } } })
if (b4a.equals(key, this.key)) {
throw BAD_ARGUMENT('Clone cannot share verification information')
}

const upgrade = opts.upgrade === undefined ? null : opts.upgrade
const signature = opts.signature === undefined
? createVerifier(createManifest(manifest), { compat: isCompat(key, manifest) }).sign(this.core.tree.batch(), keyPair)
: opts.signature

const sparse = opts.sparse === false ? false : this.sparse
const wait = opts.wait === false ? false : this.wait
Expand All @@ -497,11 +500,11 @@ module.exports = class Hypercore extends EventEmitter {
wait,
onwait,
timeout,
auth,
manifest,
overwrite: true,
clone: {
from: this,
upgrade
signature
}
})
}
Expand Down Expand Up @@ -542,6 +545,10 @@ module.exports = class Hypercore extends EventEmitter {
return this.replicator === null ? null : this.replicator.discoveryKey
}

get manifest () {
return this._manifest || (this.core === null ? null : this.core.header.manifest)
}

get length () {
if (this._snapshot) return this._snapshot.length
if (this.core === null) return 0
Expand Down Expand Up @@ -934,16 +941,22 @@ module.exports = class Hypercore extends EventEmitter {
if (this.opened === false) await this.opening
if (this.writable === false) throw SESSION_NOT_WRITABLE()

const { fork = this.core.tree.fork + 1, force = false } = typeof opts === 'number' ? { fork: opts } : opts
const {
fork = this.core.tree.fork + 1,
force = false,
keyPair = this.keyPair,
signature = null
} = typeof opts === 'number' ? { fork: opts } : opts

if (this._batch && !force) throw BATCH_UNFLUSHED()

await this.core.truncate(newLength, fork, this.auth)
await this.core.truncate(newLength, fork, { keyPair, signature })

// TODO: Should propagate from an event triggered by the oplog
this.replicator.updateAll()
}

async append (blocks, opts) {
async append (blocks, opts = {}) {
if (this._batch && this !== this._batch.session) throw BATCH_UNFLUSHED()

if (this.opened === false) await this.opening
Expand All @@ -961,7 +974,8 @@ module.exports = class Hypercore extends EventEmitter {
}
}

return this.core.append(buffers, (opts && opts.auth) || this.auth, { preappend })
const { keyPair = this.keyPair, signature = null } = opts
return this.core.append(buffers, { keyPair, signature, preappend })
}

async treeHash (length) {
Expand Down
15 changes: 9 additions & 6 deletions lib/batch.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ module.exports = class HypercoreBatch extends EventEmitter {
return this.session.core
}

get auth () {
return this.session.auth
get manifest () {
return this.session.manifest
}

async ready () {
Expand Down Expand Up @@ -230,12 +230,14 @@ module.exports = class HypercoreBatch extends EventEmitter {
return info
}

async flush (length = this._appends.length, auth) {
async flush (opts = {}) {
if (this.opened === false) await this.opening
if (this.closing) throw SESSION_CLOSED()

const { length = this._appends.length, keyPair = this.session.keyPair, signature = null } = opts

while (this._flushing) await this._flushing
this._flushing = this._flush(length, auth)
this._flushing = this._flush(length, keyPair, signature)

try {
await this._flushing
Expand All @@ -246,11 +248,12 @@ module.exports = class HypercoreBatch extends EventEmitter {
if (this.autoClose) await this.close()
}

async _flush (length, auth) { // TODO: make this safe to interact with a parallel truncate...
async _flush (length, keyPair, signature) { // TODO: make this safe to interact with a parallel truncate...
if (this._appends.length === 0) return

const flushingLength = Math.min(length, this._appends.length)
const info = await this.session.append(flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends, { auth })
const blocks = flushingLength < this._appends.length ? this._appends.slice(0, flushingLength) : this._appends
const info = await this.session.append(blocks, { keyPair, signature })
const delta = info.byteLength - this._sessionByteLength

this._sessionLength = info.length
Expand Down
22 changes: 8 additions & 14 deletions lib/caps.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const c = require('compact-encoding')

const [TREE, REPLICATE_INITIATOR, REPLICATE_RESPONDER, MANIFEST, DEFAULT_NAMESPACE] = crypto.namespace('hypercore', 5)

exports.MANIFEST = MANIFEST
exports.DEFAULT_NAMESPACE = DEFAULT_NAMESPACE

exports.replicate = function (isInitiator, key, handshakeHash) {
Expand All @@ -16,27 +17,20 @@ exports.replicate = function (isInitiator, key, handshakeHash) {
return out
}

exports.manifestHash = function (manifest) {
const state = { start: 0, end: 32 + manifest.byteLength, buffer: null }
state.buffer = b4a.allocUnsafe(state.end)
c.raw.encode(state, MANIFEST)
c.raw.encode(state, manifest)
const out = b4a.allocUnsafe(32)
sodium.crypto_generichash(out, state.buffer)
return out
}

exports.treeSignable = function (hash, length, fork) {
const state = { start: 0, end: 80, buffer: b4a.allocUnsafe(80) }
exports.treeSignable = function (namespace, hash, length, fork) {
const state = { start: 0, end: 112, buffer: b4a.allocUnsafe(112) }
c.raw.encode(state, TREE)
c.raw.encode(state, namespace)
c.raw.encode(state, hash)
c.uint64.encode(state, length)
c.uint64.encode(state, fork)
return state.buffer
}

exports.treeSignableLegacy = function (hash, length, fork) {
const state = { start: 0, end: 48, buffer: b4a.allocUnsafe(48) }
exports.treeSignableCompat = function (hash, length, fork, noHeader) {
const end = noHeader ? 48 : 80
const state = { start: 0, end, buffer: b4a.allocUnsafe(end) }
if (!noHeader) c.raw.encode(state, TREE) // ultra legacy mode, kill in future major
c.raw.encode(state, hash)
c.uint64.encode(state, length)
c.uint64.encode(state, fork)
Expand Down
Loading

0 comments on commit 52201f2

Please sign in to comment.