From eaa12e34b9513b8f23039f0adb7ef9879ff4c2f1 Mon Sep 17 00:00:00 2001 From: Devin Rousso Date: Mon, 13 Mar 2023 18:31:09 -0600 Subject: [PATCH] create a concrete wrapper for `level` (#259) the behavior adjustments made inside `BlockstoreLevel` (e.g. `get` not throwing if the key is not found) should be used everywhere so that we have a consistent pattern for using `level` --- README.md | 2 +- src/store/blockstore-level.ts | 91 ++-------------- src/store/create-level.ts | 12 --- src/store/data-store-level.ts | 2 +- src/store/index-level.ts | 54 +++------- src/store/level-wrapper.ts | 173 ++++++++++++++++++++++++++++++ src/store/message-store-level.ts | 2 +- tests/store/index-level.spec.ts | 13 ++- tests/store/message-store.spec.ts | 4 +- 9 files changed, 212 insertions(+), 141 deletions(-) delete mode 100644 src/store/create-level.ts create mode 100644 src/store/level-wrapper.ts diff --git a/README.md b/README.md index eabe24163..14f5a447a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-94.93%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.92%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-93.27%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-94.93%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-95.07%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.99%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-93.53%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-95.07%25-brightgreen.svg?style=flat) ## Introduction diff --git a/src/store/blockstore-level.ts b/src/store/blockstore-level.ts index fbce2aac4..92701168b 100644 --- a/src/store/blockstore-level.ts +++ b/src/store/blockstore-level.ts @@ -1,11 +1,8 @@ import type { AwaitIterable, Batch, KeyQuery, Pair, Query } from 'interface-store'; import type { Blockstore, Options } from 'interface-blockstore'; -import type { LevelDatabase, LevelDatabaseOptions } from './create-level.js'; -import { abortOr } from '../utils/abort.js'; import { CID } from 'multiformats'; -import { createLevelDatabase } from './create-level.js'; -import { sleep } from '../utils/time.js'; +import { createLevelDatabase, LevelWrapper } from './level-wrapper.js'; // `level` works in Node.js 12+ and Electron 5+ on Linux, Mac OS, Windows and // FreeBSD, including any future Node.js and Electron release thanks to Node-API, including ARM @@ -19,94 +16,36 @@ import { sleep } from '../utils/time.js'; export class BlockstoreLevel implements Blockstore { config: BlockstoreLevelConfig; - db: LevelDatabase; + db: LevelWrapper; - /** - * @param location - must be a directory path (relative or absolute) where LevelDB will store its - * files, or in browsers, the name of - * the {@link https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase IDBDatabase} - * to be opened. - */ - constructor(config: BlockstoreLevelConfig) { + constructor(config: BlockstoreLevelConfig, db?: LevelWrapper) { this.config = { createLevelDatabase, ...config }; + + this.db = db ?? new LevelWrapper({ ...this.config, valueEncoding: 'binary' }); } async open(): Promise { - await this.createLevelDatabase(); - - while (this.db.status === 'opening' || this.db.status === 'closing') { - await sleep(200); - } - - if (this.db.status === 'open') { - return; - } - - // db.open() is automatically called by the database constructor. We're calling it explicitly - // in order to explicitly catch an error that would otherwise not surface - // until another method like db.get() is called. Once open() has then been called, - // any read & write operations will again be queued internally - // until opening has finished. return this.db.open(); } - /** - * releases all file handles and locks held by the underlying db. - */ async close(): Promise { - if (!this.db) { - return; - } - - while (this.db.status === 'opening' || this.db.status === 'closing') { - await sleep(200); - } - - if (this.db.status === 'closed') { - return; - } - return this.db.close(); } async partition(name: string): Promise { - await this.createLevelDatabase(); - - return new BlockstoreLevel({ - location : '', - createLevelDatabase : async (_location: string, options?: LevelDatabaseOptions): Promise> => { - return this.db.sublevel(name, options); - } - }); + const db = await this.db.partition(name); + return new BlockstoreLevel({ ...this.config, location: '' }, db); } async put(key: CID, val: Uint8Array, options?: Options): Promise { - options?.signal?.throwIfAborted(); - - await abortOr(options?.signal, this.createLevelDatabase()); - - return abortOr(options?.signal, this.db.put(key.toString(), val)); + return this.db.put(key.toString(), val, options); } async get(key: CID, options?: Options): Promise { - options?.signal?.throwIfAborted(); - - await abortOr(options?.signal, this.createLevelDatabase()); - - try { - const val = await abortOr(options?.signal, this.db.get(key.toString())); - return val; - } catch (e) { - // level throws an error if the key is not present. Return undefined in this case - if (e.code === 'LEVEL_NOT_FOUND') { - return undefined; - } else { - throw e; - } - } + return this.db.get(key.toString(), options); } async has(key: CID, options?: Options): Promise { @@ -114,11 +53,7 @@ export class BlockstoreLevel implements Blockstore { } async delete(key: CID, options?: Options): Promise { - options?.signal?.throwIfAborted(); - - await abortOr(options?.signal, this.createLevelDatabase()); - - return abortOr(options?.signal, this.db.del(key.toString())); + return this.db.delete(key.toString(), options); } async * putMany(source: AwaitIterable>, options?: Options): @@ -145,14 +80,10 @@ export class BlockstoreLevel implements Blockstore { } } - private async createLevelDatabase(): Promise { - this.db ??= await this.config.createLevelDatabase(this.config.location, { keyEncoding: 'utf8', valueEncoding: 'binary' }); - } - /** * deletes all entries */ - clear(): Promise { + async clear(): Promise { return this.db.clear(); } diff --git a/src/store/create-level.ts b/src/store/create-level.ts deleted file mode 100644 index dfabce24a..000000000 --- a/src/store/create-level.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { AbstractDatabaseOptions, AbstractLevel } from 'abstract-level'; - -export type LevelDatabase = AbstractLevel; - -export type LevelDatabaseOptions = AbstractDatabaseOptions; - -export async function createLevelDatabase(location: string, options?: LevelDatabaseOptions): Promise> { - // Only import `'level'` when it's actually necessary (i.e. only when the default `createLevelDatabase` is used). - // Overriding `createLevelDatabase` will prevent this from happening. - const { Level } = await import('level'); - return new Level(location, options); -} diff --git a/src/store/data-store-level.ts b/src/store/data-store-level.ts index 3b9861ce1..eca0cbcae 100644 --- a/src/store/data-store-level.ts +++ b/src/store/data-store-level.ts @@ -3,7 +3,7 @@ import type { PutResult } from './data-store.js'; import { BlockstoreLevel } from './blockstore-level.js'; import { CID } from 'multiformats/cid'; -import { createLevelDatabase } from './create-level.js'; +import { createLevelDatabase } from './level-wrapper.js'; import { DataStore } from './data-store.js'; import { exporter } from 'ipfs-unixfs-exporter'; import { importer } from 'ipfs-unixfs-importer'; diff --git a/src/store/index-level.ts b/src/store/index-level.ts index c097b1818..2eda08d36 100644 --- a/src/store/index-level.ts +++ b/src/store/index-level.ts @@ -1,10 +1,8 @@ -import type { LevelDatabase } from './create-level.js'; -import type { AbstractBatchDelOperation, AbstractBatchOperation, AbstractIteratorOptions } from 'abstract-level'; import type { Filter, RangeFilter } from '../core/types.js'; +import type { LevelWrapperBatchOperation, LevelWrapperIteratorOptions } from './level-wrapper.js'; -import { abortOr } from '../utils/abort.js'; -import { createLevelDatabase } from './create-level.js'; import { flatten } from '../utils/object.js'; +import { createLevelDatabase, LevelWrapper } from './level-wrapper.js'; export type Entry = { _id: string, @@ -21,38 +19,30 @@ export interface IndexLevelOptions { export class IndexLevel { config: IndexLevelConfig; - db: LevelDatabase; + db: LevelWrapper; constructor(config?: IndexLevelConfig) { this.config = { createLevelDatabase, ...config }; + + this.db = new LevelWrapper({ ...this.config, valueEncoding: 'utf8' }); } async open(): Promise { - await this.createLevelDatabase(); - return this.db.open(); } async close(): Promise { - if (!this.db) { - return; - } - return this.db.close(); } async put(entry: Entry, options?: IndexLevelOptions): Promise { - options?.signal?.throwIfAborted(); - - await abortOr(options?.signal, this.createLevelDatabase()); - entry = flatten(entry) as Entry; const { _id } = entry; - const ops: AbstractBatchOperation, string, string>[] = [ ]; + const ops: LevelWrapperBatchOperation[] = [ ]; const prefixes: string[] = [ ]; for (const property in entry) { if (property === '_id') { @@ -67,14 +57,10 @@ export class IndexLevel { } ops.push({ type: 'put', key: `__${_id}__prefixes`, value: JSON.stringify(prefixes) }); - await abortOr(options?.signal, this.db.batch(ops)); + return this.db.batch(ops, options); } async query(filter: Filter, options?: IndexLevelOptions): Promise> { - options?.signal?.throwIfAborted(); - - await abortOr(options?.signal, this.createLevelDatabase()); - const requiredProperties = new Set(); const missingPropertiesForID: { [id: string]: Set } = { }; const promises: Promise[] = [ ]; @@ -116,32 +102,28 @@ export class IndexLevel { } async delete(id: string, options?: IndexLevelOptions): Promise { - options?.signal?.throwIfAborted(); - - await abortOr(options?.signal, this.createLevelDatabase()); - - const prefixes = await abortOr(options?.signal, this.db.get(`__${id}__prefixes`)); + const prefixes = await this.db.get(`__${id}__prefixes`, options); if (!prefixes) { return; } - const ops: AbstractBatchDelOperation, string>[] = [ ]; + const ops: LevelWrapperBatchOperation[] = [ ]; for (const prefix of JSON.parse(prefixes)) { ops.push({ type: 'del', key: this.join(prefix, id) }); } ops.push({ type: 'del', key: `__${id}__prefixes` }); - await abortOr(options?.signal, this.db.batch(ops)); + return this.db.batch(ops, options); } - clear(): Promise { + async clear(): Promise { return this.db.clear(); } private async findExactMatches(propertyName: string, propertyValue: unknown, options?: IndexLevelOptions): Promise { const propertyKey = this.join(propertyName, this.encodeValue(propertyValue)); - const iteratorOptions: AbstractIteratorOptions = { + const iteratorOptions: LevelWrapperIteratorOptions = { gt: propertyKey }; @@ -151,7 +133,7 @@ export class IndexLevel { private async findRangeMatches(propertyName: string, range: RangeFilter, options?: IndexLevelOptions): Promise { const propertyKey = this.join(propertyName); - const iteratorOptions: AbstractIteratorOptions = { }; + const iteratorOptions: LevelWrapperIteratorOptions = { }; for (const comparator in range) { iteratorOptions[comparator] = this.join(propertyName, this.encodeValue(range[comparator])); } @@ -171,7 +153,7 @@ export class IndexLevel { private async findMatches( propertyName: string, - iteratorOptions: AbstractIteratorOptions, + iteratorOptions: LevelWrapperIteratorOptions, options?: IndexLevelOptions ): Promise { // Since we will stop iterating if we encounter entries that do not start with the `propertyName`, we need to always start from the upper bound. @@ -181,9 +163,7 @@ export class IndexLevel { } const matches = new Map; - for await (const [ key, value ] of this.db.iterator(iteratorOptions)) { - options?.signal?.throwIfAborted(); - + for await (const [ key, value ] of this.db.iterator(iteratorOptions, options)) { if (!key.startsWith(propertyName)) { break; } @@ -206,10 +186,6 @@ export class IndexLevel { private join(...values: unknown[]): string { return values.join(`\x00`); } - - private async createLevelDatabase(): Promise { - this.db ??= await this.config.createLevelDatabase(this.config.location, { keyEncoding: 'utf8', valueEncoding: 'utf8' }); - } } type IndexLevelConfig = { diff --git a/src/store/level-wrapper.ts b/src/store/level-wrapper.ts new file mode 100644 index 000000000..bcebece3a --- /dev/null +++ b/src/store/level-wrapper.ts @@ -0,0 +1,173 @@ +import type { AbstractBatchOperation, AbstractDatabaseOptions, AbstractIteratorOptions, AbstractLevel } from 'abstract-level'; + +import { abortOr } from '../utils/abort.js'; +import { sleep } from '../utils/time.js'; + +export type CreateLevelDatabaseOptions = AbstractDatabaseOptions; + +export type LevelDatabase = AbstractLevel; + +export async function createLevelDatabase(location: string, options?: CreateLevelDatabaseOptions): Promise> { + // Only import `'level'` when it's actually necessary (i.e. only when the default `createLevelDatabase` is used). + // Overriding `createLevelDatabase` will prevent this from happening. + const { Level } = await import('level'); + return new Level(location, { ...options, keyEncoding: 'utf8' }); +} + +export interface LevelWrapperOptions { + signal?: AbortSignal; +} + +export type LevelWrapperBatchOperation = AbstractBatchOperation, string, V>; + +export type LevelWrapperIteratorOptions = AbstractIteratorOptions; + +// `Level` works in Node.js 12+ and Electron 5+ on Linux, Mac OS, Windows and FreeBSD, including any +// future Node.js and Electron release thanks to Node-API, including ARM platforms like Raspberry Pi +// and Android, as well as in Chrome, Firefox, Edge, Safari, iOS Safari and Chrome for Android. +export class LevelWrapper { + config: LevelWrapperConfig; + + db: LevelDatabase; + + /** + * @param config.location - must be a directory path (relative or absolute) where `Level`` will + * store its files, or in browsers, the name of the {@link https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase `IDBDatabase`} + * to be opened. + */ + constructor(config: LevelWrapperConfig, db?: LevelDatabase) { + this.config = { + createLevelDatabase, + ...config + }; + + this.db = db; + } + + async open(): Promise { + await this.createLevelDatabase(); + + while (this.db.status === 'opening' || this.db.status === 'closing') { + await sleep(200); + } + + if (this.db.status === 'open') { + return; + } + + // `db.open()` is automatically called by the database constructor. We're calling it explicitly + // in order to explicitly catch an error that would otherwise not surface until another method + // like `db.get()` is called. Once `db.open()` has then been called, any read & write + // operations will again be queued internally until opening has finished. + return this.db.open(); + } + + async close(): Promise { + if (!this.db) { + return; + } + + while (this.db.status === 'opening' || this.db.status === 'closing') { + await sleep(200); + } + + if (this.db.status === 'closed') { + return; + } + + return this.db.close(); + } + + async partition(name: string): Promise> { + await this.createLevelDatabase(); + + return new LevelWrapper({ ...this.config, location: '' }, this.db.sublevel(name, { + keyEncoding : 'utf8', + valueEncoding : this.config.valueEncoding + })); + } + + async get(key: string, options?: LevelWrapperOptions): Promise { + options?.signal?.throwIfAborted(); + + await abortOr(options?.signal, this.createLevelDatabase()); + + try { + const value = await abortOr(options?.signal, this.db.get(String(key))); + return value; + } catch (error) { + // `Level`` throws an error if the key is not present. Return `undefined` in this case. + if (error.code === 'LEVEL_NOT_FOUND') { + return undefined; + } else { + throw error; + } + } + } + + async * keys(options?: LevelWrapperOptions): AsyncGenerator { + options?.signal?.throwIfAborted(); + + await abortOr(options?.signal, this.createLevelDatabase()); + + for await (const key of this.db.keys()) { + options?.signal?.throwIfAborted(); + + yield key; + } + } + + async * iterator(iteratorOptions: LevelWrapperIteratorOptions, options?: LevelWrapperOptions): AsyncGenerator<[string, V]> { + options?.signal?.throwIfAborted(); + + await abortOr(options?.signal, this.createLevelDatabase()); + + for await (const entry of this.db.iterator(iteratorOptions)) { + options?.signal?.throwIfAborted(); + + yield entry; + } + } + + async put(key: string, value: V, options?: LevelWrapperOptions): Promise { + options?.signal?.throwIfAborted(); + + await abortOr(options?.signal, this.createLevelDatabase()); + + return abortOr(options?.signal, this.db.put(String(key), value)); + } + + async delete(key: string, options?: LevelWrapperOptions): Promise { + options?.signal?.throwIfAborted(); + + await abortOr(options?.signal, this.createLevelDatabase()); + + return abortOr(options?.signal, this.db.del(String(key))); + } + + async clear(): Promise { + await this.createLevelDatabase(); + + return this.db.clear(); + } + + async batch(operations: Array>, options?: LevelWrapperOptions): Promise { + options?.signal?.throwIfAborted(); + + await abortOr(options?.signal, this.createLevelDatabase()); + + return abortOr(options?.signal, this.db.batch(operations)); + } + + private async createLevelDatabase(): Promise { + this.db ??= await this.config.createLevelDatabase(this.config.location, { + keyEncoding : 'utf8', + valueEncoding : this.config.valueEncoding + }); + } +} + +type LevelWrapperConfig = CreateLevelDatabaseOptions & { + location: string, + createLevelDatabase?: typeof createLevelDatabase, +}; \ No newline at end of file diff --git a/src/store/message-store-level.ts b/src/store/message-store-level.ts index 44b8ff6dc..2d57aca81 100644 --- a/src/store/message-store-level.ts +++ b/src/store/message-store-level.ts @@ -7,7 +7,7 @@ import * as cbor from '@ipld/dag-cbor'; import { abortOr } from '../utils/abort.js'; import { BlockstoreLevel } from './blockstore-level.js'; import { CID } from 'multiformats/cid'; -import { createLevelDatabase } from './create-level.js'; +import { createLevelDatabase } from './level-wrapper.js'; import { IndexLevel } from './index-level.js'; import { sha256 } from 'multiformats/hashes/sha2'; diff --git a/tests/store/index-level.spec.ts b/tests/store/index-level.spec.ts index 7e66a70e2..7d5d36f0b 100644 --- a/tests/store/index-level.spec.ts +++ b/tests/store/index-level.spec.ts @@ -31,7 +31,10 @@ describe('Index Level', () => { 'c' : 'd' }); - const keys = await index.db.keys().all(); + const keys = [ ]; + for await (const key of index.db.keys()) { + keys.push(key); + } expect(keys.length).to.equal(4); }); @@ -57,8 +60,8 @@ describe('Index Level', () => { }; await index.put(doc); - await expect(index.db.get(index['join']('empty', '[object Object]', doc._id))).to.be.rejected; - await expect(index.db.get(index['join']('empty.nested', '[object Object]', doc._id))).to.be.rejected; + await expect(index.db.get(index['join']('empty', '[object Object]', doc._id))).to.eventually.be.undefined; + await expect(index.db.get(index['join']('empty.nested', '[object Object]', doc._id))).to.eventually.be.undefined; }); it('removes empty arrays', async () => { @@ -68,8 +71,8 @@ describe('Index Level', () => { }; await index.put(doc); - await expect(index.db.get(index['join']('empty', '', doc._id))).to.be.rejected; - await expect(index.db.get(index['join']('empty.0', '', doc._id))).to.be.rejected; + await expect(index.db.get(index['join']('empty', '', doc._id))).to.eventually.be.undefined; + await expect(index.db.get(index['join']('empty.0', '', doc._id))).to.eventually.be.undefined; }); it('should not put anything if aborted beforehand', async () => { diff --git a/tests/store/message-store.spec.ts b/tests/store/message-store.spec.ts index cf988b86f..f064dcf81 100644 --- a/tests/store/message-store.spec.ts +++ b/tests/store/message-store.spec.ts @@ -5,7 +5,7 @@ import { Message } from '../../src/core/message.js'; import { MessageStoreLevel } from '../../src/store/message-store-level.js'; import { RecordsWriteMessage } from '../../src/interfaces/records/types.js'; import { TestDataGenerator } from '../utils/test-data-generator.js'; -import { createLevelDatabase, LevelDatabase, LevelDatabaseOptions } from '../../src/store/create-level.js'; +import { createLevelDatabase, CreateLevelDatabaseOptions, LevelDatabase } from '../../src/store/level-wrapper.js'; let messageStore: MessageStoreLevel; @@ -131,7 +131,7 @@ describe('MessageStoreLevel Tests', () => { const messageStore = new MessageStoreLevel({ blockstoreLocation : 'TEST-BLOCKSTORE', indexLocation : 'TEST-INDEX', - createLevelDatabase(location, options?: LevelDatabaseOptions): Promise> { + createLevelDatabase(location, options?: CreateLevelDatabaseOptions): Promise> { locations.add(location); return createLevelDatabase(location, options); }