Skip to content

Commit

Permalink
Add index key maps (#4)
Browse files Browse the repository at this point in the history 
* Pass offset through constructor

* Add index key maps

* tweak codegen

---------

Co-authored-by: Mathias Buus <[email protected]>
  • Loading branch information
@andrewosh @mafintosh
andrewosh and mafintosh authored Sep 14, 2024
1 parent 3a01c0a commit 593298d
Show file tree
Hide file tree
Showing 4 changed files with 158 additions and 34 deletions.
44 changes: 36 additions & 8 deletions builder/codegen.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ const type = contract.resolveIndex('@keet/devices-by-name')
*/

const gen = require('generate-object-property')
const genFunc = require('generate-function')
const s = require('generate-string')

const IndexTypeMap = new Map([
Expand Down Expand Up @@ -122,13 +123,28 @@ module.exports = function generateCode (hyperdb) {
return str
}

function toArrayFunction (fn) {
const i = fn.indexOf('(')
const isArrow = !fn.slice(0, i).includes('function')
fn = fn.slice(i)
if (isArrow) return fn
return fn.replace('{', '=> {')
}

function generateCommonPrefix (id, type) {
let str = ''

str += `// ${s(type.fqn)} collection key\n`
str += `const ${id}_key = ${generateIndexKeyEncoding(type)}\n`
str += '\n'

if (type.isMapped) {
const fn = genFunc()
fn(type.map)
str += `const ${id}_map = ${toArrayFunction(fn.toString())}\n`
str += '\n'
}

str += `function ${id}_indexify (record) {\n`
str += ' const arr = []\n'
str += '\n'
Expand Down Expand Up @@ -227,23 +243,35 @@ function generateEncodeCollectionKey (id, collection) {
}

function generateEncodeIndexKeys (id, index) {
const accessors = index.fullKey.map(c => {
return c.split('.').reduce(gen, 'record')
})
let str = ''
str += 'function encodeKeys (record) {\n'
str += ` const key = [${accessors.join(', ')}]\n`
str += ` return [${id + '_key'}.encode(key)]\n`
str += 'function encodeKeys (record, context) {\n'
if (index.isMapped) {
const accessors = index.fullKey.map(c => {
return c.split('.').reduce(gen, 'structKey')
})
str += ` const mapped = ${id}_map(record, context)\n`
str += ' const keys = new Array(mapped.length)\n'
str += ' for (let i = 0; i < keys.length; i++) {\n'
str += ' const structKey = mapped[i]\n'
str += ` keys[i] = ${id + '_key'}.encode([${accessors.join(', ')}])\n`
str += ' }\n'
str += ' return keys\n'
} else {
const accessors = index.fullKey.map(c => {
return c.split('.').reduce(gen, 'record')
})
str += ` return [${id + '_key'}.encode([${accessors.join(', ')}])]\n`
}
str += ' }'
return str
}

function generateIndexKeyEncoding (type) {
let str = 'new IndexEncoder([\n'
for (let i = 0; i < type.fullKey.length; i++) {
for (let i = 0; i < type.keyEncoding.length; i++) {
const component = type.keyEncoding[i]
str += ' ' + IndexTypeMap.get(component)
if (i !== type.fullKey.length - 1) str += ',\n'
if (i !== type.keyEncoding.length - 1) str += ',\n'
else str += '\n'
}
str += `], { prefix: ${type.id} })`
Expand Down
82 changes: 74 additions & 8 deletions builder/example.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,6 @@ example.register({
name: 'field2',
type: '@example/struct1',
required: true
},
{
name: 'name',
type: 'string'
}
]
})
Expand All @@ -62,14 +58,38 @@ example.register({
type: 'uint',
required: true
},
{
name: 'id3',
type: 'uint',
required: true
},
{
name: 'struct1',
type: '@example/struct2',
required: true
},
{
name: 'text',
name: 'name',
type: 'string'
},
{
name: 'age',
type: 'uint'
},
{
name: 'tags',
type: 'string',
array: true
}
]
})

example.register({
name: 'collection-info',
fields: [
{
name: 'count',
type: 'uint'
}
]
})
Expand All @@ -79,17 +99,63 @@ Hyperschema.toDisk(schema)
const db = HyperDB.from(SCHEMA_DIR, DB_DIR)
const exampleDb = db.namespace('example')

exampleDb.collections.register({
name: 'collection1-info',
schema: '@example/collection-info',
derived: true
})

exampleDb.collections.register({
name: 'collection1',
schema: '@example/record1',
key: ['id1', 'id2']
key: ['id1', 'id2'],
trigger: async (db, key, record, context) => {
const info = (await db.get('@example/collection1-info')) || { count: 0 }
const existing = await db.get('@example/collection1', key)
if (existing && record) return
await db.insert('@example/collection1-info', { count: record ? info.count + 1 : info.count - 1 })
}
})

exampleDb.indexes.register({
name: 'collection1-by-struct',
name: 'collection1-by-struct-mapped',
collection: '@example/collection1',
key: {
type: {
fields: [
{
name: 'name',
type: 'string'
},
{
name: 'age',
type: 'uint'
}
]
},
map: (record, context) => [
{ name: record.name, age: record.age }
]
}
})

exampleDb.indexes.register({
name: 'collection1-by-id3',
collection: '@example/collection1',
key: ['struct1.field1', 'struct1.field2'],
key: ['id3'],
unique: true
})

exampleDb.indexes.register({
name: 'collection1-by-struct',
collection: '@example/collection1',
key: ['name', 'age']
})

exampleDb.indexes.register({
name: 'collection1-by-tags',
collection: '@example/collection1',
key: ['name', 'tags']
})

HyperDB.toDisk(db)
63 changes: 46 additions & 17 deletions builder/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class DBType {
this.key = description.key
this.fullKey = []

this.isMapped = false
this.isIndex = false
this.isCollection = false

Expand Down Expand Up @@ -64,20 +65,23 @@ class Collection extends DBType {
constructor (builder, namespace, description) {
super(builder, namespace, description)
this.isCollection = true
this.derived = !!description.derived
this.indexes = []

this.schema = this.builder.schema.resolve(description.schema)
if (!this.schema) throw new Error('Schema not found: ' + description.schema)

this.key = description.key
this.key = description.key || []
this.fullKey = this.key

this.keyEncoding = []
this.valueEncoding = this.fqn + '/value'
for (const component of this.key) {
const field = this.schema.fieldsByName.get(component)
const resolvedType = this.builder.schema.resolve(field.type.fqn, { aliases: false })
this.keyEncoding.push(resolvedType.name)
if (this.key.length) {
for (const component of this.key) {
const field = this.schema.fieldsByName.get(component)
const resolvedType = this.builder.schema.resolve(field.type.fqn, { aliases: false })
this.keyEncoding.push(resolvedType.name)
}
}

// Register a value encoding type (the portion of the record that will not be in the primary key)
Expand All @@ -98,6 +102,7 @@ class Collection extends DBType {
type: COLLECTION_TYPE,
indexes: this.indexes.map(i => i.fqn),
schema: this.schema.fqn,
derived: this.derived,
key: this.key
}
}
Expand All @@ -108,24 +113,44 @@ class Index extends DBType {
super(builder, namespace, description)
this.isIndex = true
this.unique = !!description.unique
this.isMapped = !Array.isArray(description.key)

this.collection = this.builder.typesByName.get(description.collection)
this.keyEncoding = []

this.key = description.key
this.fullKey = [...this.key]

if (!this.collection || !this.collection.isCollection) {
throw new Error('Invalid index target: ' + description.collection)
}
this.collection.indexes.push(this)

this.key = description.key
this.fullKey = null

this.map = null
if (this.isMapped) {
this.map = (typeof this.key.map === 'function') ? this.key.map.toString() : this.key.map
}

// Key encoding will be an IndexEncoder of the secondary index's key fields
// If the key is not unique, then the primary key should also be included
for (const component of this.key) {
const resolvedType = this._resolveKey(this.collection.schema, component)
this.keyEncoding.push(resolvedType.name)
// If an Array is provided, the keys are intepreted as fields from the source collection
// This can be overridden by providing { type, map } options to the key field
if (Array.isArray(this.key)) {
this.fullKey = [...this.key]
for (const component of this.key) {
const resolvedType = this._resolveKey(this.collection.schema, component)
this.keyEncoding.push(resolvedType.name)
}
} else {
this.fullKey = []
for (const field of this.key.type.fields) {
const resolvedType = this.builder.schema.resolve(field.type, { aliases: false })
this.keyEncoding.push(resolvedType.name)
this.fullKey.push(field.name)
}
}
if (!this.unique) {

// If the key is not unique, then the primary key should also be included
if (!this.unique && !this.isMapped) {
for (let i = 0; i < this.collection.keyEncoding.length; i++) {
this.keyEncoding.push(this.collection.keyEncoding[i])
this.fullKey.push(this.collection.key[i])
Expand All @@ -139,7 +164,12 @@ class Index extends DBType {
type: INDEX_TYPE,
collection: this.description.collection,
unique: this.unique,
key: this.key
key: Array.isArray(this.key)
? this.key
: {
type: this.key.type,
map: (typeof this.key.map === 'function') ? this.key.map.toString() : this.key.map
}
}
}
}
Expand Down Expand Up @@ -174,7 +204,6 @@ class BuilderNamespace {

this.collections = new BuilderCollections(this)
this.indexes = new BuilderIndexes(this)
this.schema = this.builder.schema.namespace(this.name)

this.descriptions = []
}
Expand All @@ -188,10 +217,10 @@ class BuilderNamespace {
}

class Builder {
constructor (schema, dbJson, { dbDir = null, schemaDir = null } = {}) {
constructor (schema, dbJson, { offset, dbDir = null, schemaDir = null } = {}) {
this.schema = schema
this.version = dbJson ? dbJson.version : 0
this.offset = dbJson ? dbJson.offset : 0
this.offset = dbJson ? dbJson.offset : (offset || 0)
this.dbDir = dbDir
this.schemaDir = schemaDir

Expand Down
3 changes: 2 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@
"dependencies": {
"b4a": "^1.6.6",
"compact-encoding": "^2.15.0",
"hyperschema": "^0.0.14",
"generate-function": "^2.3.1",
"generate-object-property": "^2.0.0",
"generate-string": "^1.0.1",
"hyperschema": "^0.0.14",
"rocksdb-native": "^2.3.1",
"streamx": "^2.20.0"
},
Expand Down

0 comments on commit 593298d

Please sign in to comment.