support nested keys (#29)

* support nested keys

* bump deps and fix suffix

* fix bug and regen

* no runtime resolves

* final fixes

* standard

builder/codegen.js

@@ -72,7 +72,7 @@ module.exports = function generateCode (hyperdb, { directory = '.' } = {}) {
   str += '\n'
   str += `const { IndexEncoder, c } = require('${pkg.name}/runtime')\n`
   str += '\n'
-  str += 'const { version, resolveStruct } = require(\'./messages.js\')\n'
+  str += 'const { version, getEncoding, setVersion } = require(\'./messages.js\')\n'
   str += '\n'
 
   for (const ns of hyperdb.namespaces.values()) {
@@ -162,15 +162,15 @@ function generateCommonPrefix (type) {
   if (len === 0) {
     str += '  return []\n'
   } else if (len === 1) {
-    str += `  const a = ${getKeyPath(type.fullKey[0], 'record')}\n`
+    str += `  const a = ${getKeyPath(type.fullKey[0], 'record', true)}\n`
     str += '  return a === undefined ? [] : [a]\n'
   } else {
     str += '  const arr = []\n'
     str += '\n'
 
     for (let i = 0; i < len; i++) {
       const key = type.fullKey[i]
-      str += `  const a${i} = ${getKeyPath(key, 'record')}\n`
+      str += `  const a${i} = ${getKeyPath(key, 'record', true)}\n`
       str += `  if (a${i} === undefined) return arr\n`
       str += `  arr.push(a${i})\n`
       str += '\n'
@@ -189,6 +189,10 @@ function generateCollectionDefinition (collection) {
 
   let str = generateCommonPrefix(collection)
 
+  str += `// ${s(collection.fqn)} value encoding\n`
+  str += `const ${id}_enc = getEncoding(${s(collection.valueEncoding)})\n`
+  str += '\n'
+
   if (collection.trigger) {
     str += `// ${s(collection.fqn)} has the following schema defined trigger\n`
     str += `const ${id}_trigger = ${gen('helpers' + collection.getNamespace().id, collection.trigger)}\n`
@@ -198,34 +202,22 @@ function generateCollectionDefinition (collection) {
   str += `// ${s(collection.fqn)} reconstruction function\n`
   str += `function ${id}_reconstruct (version, keyBuf, valueBuf) {\n`
   if (collection.key.length) str += `  const key = ${id}_key.decode(keyBuf)\n`
-  str += `  const value = c.decode(resolveStruct(${s(collection.valueEncoding)}, version), valueBuf)\n`
-  if (collection.key.length === 0) {
-    str += '  return value\n'
-  } else {
-    str += '  // TODO: This should be fully code generated\n'
-    str += '  return {\n'
-    for (let i = 0; i < collection.key.length; i++) {
-      const key = collection.key[i]
-      str += `    ${gen.property(key)}: key[${i}],\n`
-    }
-    str += '    ...value\n'
-    str += '  }\n'
+  str += '  setVersion(version)\n'
+  str += `  const record = c.decode(${id}_enc, valueBuf)\n`
+
+  for (let i = 0; i < collection.key.length; i++) {
+    const key = collection.key[i]
+    str += `  ${getKeyPath(key, 'record', false)} = key[${i}]\n`
   }
+
+  str += '  return record\n'
   str += '}\n'
 
   str += `// ${s(collection.fqn)} key reconstruction function\n`
   str += `function ${id}_reconstruct_key (keyBuf) {\n`
   if (collection.key.length) str += `  const key = ${id}_key.decode(keyBuf)\n`
-  if (collection.key.length === 0) {
-    str += '  return {}\n'
-  } else {
-    str += '  return {\n'
-    for (let i = 0; i < collection.key.length; i++) {
-      const key = collection.key[i]
-      str += `    ${gen.property(key)}: key[${i}]${i < collection.key.length - 1 ? ',' : ''}\n`
-    }
-    str += '  }\n'
-  }
+
+  str += generateKeyReconstruct('  ', collection.key, 'key') + '\n'
   str += '}\n'
 
   str += '\n'
@@ -283,9 +275,12 @@ function generateEncodeKeyRange (index, sep) {
 }
 
 function generateEncodeCollectionValue (collection, sep) {
+  const id = getId(collection)
+
   let str = ''
   str += '  encodeValue (version, record) {\n'
-  str += `    return c.encode(resolveStruct(${s(collection.valueEncoding)}, version), record)\n`
+  str += '    setVersion(version)\n'
+  str += `    return c.encode(${id}_enc, record)\n`
   str += `  }${sep}\n`
   return str
 }
@@ -340,6 +335,52 @@ function toProps (name, keys) {
   return keys.map(c => c === null ? name : c.split('.').reduce(gen, name))
 }
 
+function generateKeyReconstruct (indent, keys, key) {
+  if (keys.length === 0) return indent + 'return {}\n'
+
+  const grouped = new Map()
+
+  for (let index = 0; index < keys.length; index++) {
+    const k = keys[index].split('.')
+    let map = grouped
+
+    for (let i = 0; i < k.length; i++) {
+      let info = map.get(k[i])
+
+      if (!info) {
+        info = { key: null, index: -1, map: null }
+        map.set(k[i], info)
+      }
+
+      if (i === k.length - 1) {
+        info.key = keys[index]
+        info.index = index
+      } else {
+        map = info.map = new Map()
+      }
+    }
+  }
+
+  return indent + 'return ' + generate(indent, grouped)
+
+  function generate (indent, map) {
+    let s = '{\n'
+    const all = [...map]
+    for (let i = 0; i < all.length; i++) {
+      const [k, v] = all[i]
+      s += indent + `  ${gen.property(k)}: `
+      if (v.index !== -1) {
+        s += `${key}[${v.index}]`
+      } else {
+        s += generate(indent + '  ', v.map)
+      }
+      s += (i === all.length - 1) ? '\n' : ',\n'
+    }
+    s += indent + '}'
+    return s
+  }
+}
+
 function generateIndexKeyEncoding (type) {
   let str = 'new IndexEncoder([\n'
   for (let i = 0; i < type.keyEncoding.length; i++) {
@@ -364,8 +405,8 @@ function getIndexId (index) {
   return 'index' + index.id
 }
 
-function getKeyPath (key, name) {
+function getKeyPath (key, name, optional) {
   if (key === null) return name
-  const r = (a, b, i) => (i === 0) ? gen(a, b) : gen.optional(a, b)
-  return key.split('.').reduce(r, 'record')
+  const r = (a, b, i) => (i === 0 || !optional) ? gen(a, b) : gen.optional(a, b)
+  return key.split('.').reduce(r, name)
 }

builder/index.js

@@ -80,26 +80,55 @@ class Collection extends DBType {
     this.trigger = (typeof description.trigger === 'function') ? description.trigger.toString() : (description.trigger || null)
 
     this.keyEncoding = []
-    this.valueEncoding = this.fqn + '/value'
+
     if (this.key.length) {
       for (const component of this.key) {
-        const field = this.schema.fieldsByName.get(component)
+        const field = resolvePathToType(component, this.schema)
         if (!field) throw new Error('Field not found: ' + component)
         const resolvedType = this.builder.schema.resolve(field.type.fqn, { aliases: false })
         this.keyEncoding.push(resolvedType.name)
       }
     }
 
     // Register a value encoding type (the portion of the record that will not be in the primary key)
-    const primaryKeySet = new Set(this.key)
+    this.valueEncoding = this._deriveValueSchema().fqn
+  }
+
+  _deriveValueSchema (schema = this.schema, prefix = '', primaryKeySet = new Set(this.key)) {
+    const fields = []
+    const type = '/hyperdb#' + this.id
+
+    if (!schema.isStruct) return { external: false, fqn: schema.name }
+
+    let external = false
+
+    for (const f of schema.fields) {
+      const name = prefix ? prefix + '.' + f.name : f.name
+      const cpy = f.toJSON()
+
+      if (primaryKeySet.has(name)) {
+        external = cpy.external = true
+      } else if (this._deriveValueSchema(f.type, name, primaryKeySet).external) {
+        external = true
+      }
+
+      fields.push(cpy)
+    }
+
+    if (!external) {
+      return { external: false, fqn: getFQN(schema.namespace, schema.name) }
+    }
+
     this.builder.schema.register({
-      ...this.schema.toJSON(),
+      ...schema.toJSON(),
       derived: true,
       flagsPosition: -1,
-      namespace: this.namespace,
-      name: this.description.name + '/value',
-      fields: this.schema.fields.filter(f => !primaryKeySet.has(f.name)).map(f => f.toJSON())
+      namespace: schema.namespace,
+      name: schema.name + type,
+      fields
     })
+
+    return { external: true, fqn: getFQN(schema.namespace, schema.name + type) }
   }
 
   toJSON () {
@@ -352,3 +381,15 @@ function getFQN (namespace, name) {
   if (namespace === null) return name
   return '@' + namespace + '/' + name
 }
+
+function resolvePathToType (name, schema) {
+  const parts = name.split('.')
+
+  let field = schema.fieldsByName.get(parts[0])
+
+  for (let i = 1; i < parts.length && field; i++) {
+    field = field.type.fieldsByName.get(parts[i])
+  }
+
+  return field
+}

package.json

@@ -29,7 +29,7 @@
     "generate-object-property": "^2.0.0",
     "generate-string": "^1.0.1",
     "hyperbee": "^2.20.4",
-    "hyperschema": "^1.0.0",
+    "hyperschema": "^1.3.0",
     "index-encoder": "^3.2.0",
     "rocksdb-native": "^3.0.0",
     "streamx": "^2.20.0"

test/basic.js

@@ -358,3 +358,47 @@ test('update does not break existing snaps', async function ({ create }, t) {
 
   await db.close()
 })
+
+test('nested keys', async function ({ create, bee }, t) {
+  const db = await create({ fixture: 4 })
+
+  await db.insert('@db/nested-members', { member: { id: 'maf', age: 50 }, fun: true })
+  await db.insert('@db/nested-members', { member: { id: 'andrew', age: 40 }, fun: false })
+
+  await db.flush()
+
+  const all = await db.find('@db/nested-members').toArray()
+  t.is(all.length, 2)
+
+  const one = await db.get('@db/nested-members', { member: { id: 'maf' } })
+  t.alike(one, { member: { id: 'maf', age: 50 }, fun: true })
+
+  await db.delete('@db/nested-members', { member: { id: 'andrew' } })
+  await db.flush()
+
+  if (bee) {
+    t.comment('only test changes feed on bee engine')
+
+    const ops = []
+    for await (const op of db.changes()) ops.push(op)
+
+    t.alike(ops, [{
+      type: 'insert',
+      seq: 1,
+      collection: '@db/nested-members',
+      value: { member: { id: 'maf', age: 50 }, fun: true }
+    }, {
+      type: 'insert',
+      seq: 2,
+      collection: '@db/nested-members',
+      value: { member: { id: 'andrew', age: 40 }, fun: false }
+    }, {
+      type: 'delete',
+      seq: 3,
+      collection: '@db/nested-members',
+      value: { member: { id: 'andrew' } }
+    }])
+  }
+
+  await db.close()
+})

test/fixtures/builders/4.js

@@ -0,0 +1,54 @@
+const HyperDB = require('../../../builder')
+const Hyperschema = require('hyperschema')
+const path = require('path')
+
+const SCHEMA_DIR = path.join(__dirname, '../generated/4/hyperschema')
+const DB_DIR = path.join(__dirname, '../generated/4/hyperdb')
+
+const schema = Hyperschema.from(SCHEMA_DIR)
+
+const dbSchema = schema.namespace('db')
+
+dbSchema.register({
+  name: 'member',
+  fields: [
+    {
+      name: 'id',
+      type: 'string',
+      required: true
+    },
+    {
+      name: 'age',
+      type: 'uint',
+      required: true
+    }
+  ]
+})
+
+dbSchema.register({
+  name: 'nested',
+  fields: [
+    {
+      name: 'member',
+      type: '@db/member',
+      required: true
+    },
+    {
+      name: 'fun',
+      type: 'bool'
+    }
+  ]
+})
+
+Hyperschema.toDisk(schema)
+
+const db = HyperDB.from(SCHEMA_DIR, DB_DIR)
+const testDb = db.namespace('db')
+
+testDb.collections.register({
+  name: 'nested-members',
+  schema: '@db/nested',
+  key: ['member.id']
+})
+
+HyperDB.toDisk(db)

test/fixtures/generate.js

@@ -1,3 +1,4 @@
 require('./builders/1.js')
 require('./builders/2.js')
 require('./builders/3.js')
+require('./builders/4.js')

test/fixtures/generated/1/hyperdb/index.js

@@ -3,7 +3,7 @@
 
 const { IndexEncoder, c } = require('hyperdb/runtime')
 
-const { version, resolveStruct } = require('./messages.js')
+const { version, getEncoding, setVersion } = require('./messages.js')
 
 // '@db/members' collection key
 const collection0_key = new IndexEncoder([
@@ -15,15 +15,16 @@ function collection0_indexify (record) {
   return a === undefined ? [] : [a]
 }
 
+// '@db/members' value encoding
+const collection0_enc = getEncoding('@db/member/hyperdb#0')
+
 // '@db/members' reconstruction function
 function collection0_reconstruct (version, keyBuf, valueBuf) {
   const key = collection0_key.decode(keyBuf)
-  const value = c.decode(resolveStruct('@db/members/value', version), valueBuf)
-  // TODO: This should be fully code generated
-  return {
-    id: key[0],
-    ...value
-  }
+  setVersion(version)
+  const record = c.decode(collection0_enc, valueBuf)
+  record.id = key[0]
+  return record
 }
 // '@db/members' key reconstruction function
 function collection0_reconstruct_key (keyBuf) {
@@ -50,7 +51,8 @@ const collection0 = {
     })
   },
   encodeValue (version, record) {
-    return c.encode(resolveStruct('@db/members/value', version), record)
+    setVersion(version)
+    return c.encode(collection0_enc, record)
   },
   trigger: null,
   reconstruct: collection0_reconstruct,