From c5e0a2c2a0729a8a42f3134b0d4334d350374af0 Mon Sep 17 00:00:00 2001 From: oliver-oloughlin Date: Thu, 2 Nov 2023 17:57:17 +0100 Subject: [PATCH 1/3] feat: added setInterval(), deprecated cron(), created AtomicWrapper --- README.md | 38 +- src/atomic_wrapper.ts | 115 ++++ src/collection.ts | 29 +- src/constants.ts | 7 +- src/indexable_collection.ts | 4 +- src/kvdex.ts | 178 +++++- src/large_collection.ts | 7 +- src/types.ts | 45 +- src/utils.ts | 51 +- tests/collection/enqueue.test.ts | 82 +-- tests/collection/listenQueue.test.ts | 86 +-- tests/db/atomic.test.ts | 546 +++++++++--------- tests/db/cron.test.ts | 66 ++- tests/db/deleteAll.test.ts | 5 +- tests/db/enqueue.test.ts | 90 +-- tests/db/listenQueue.test.ts | 74 +-- tests/db/setInterval.test.ts | 43 ++ tests/indexable_collection/enqueue.test.ts | 82 +-- .../indexable_collection/listenQueue.test.ts | 86 +-- tests/large_collection/enqueue.test.ts | 82 +-- tests/large_collection/listenQueue.test.ts | 86 +-- 21 files changed, 1083 insertions(+), 719 deletions(-) create mode 100644 src/atomic_wrapper.ts create mode 100644 tests/db/setInterval.test.ts diff --git a/README.md b/README.md index 48acc22..1e86ec9 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ `kvdex` is a high-level abstraction layer for Deno KV with zero third-party dependencies. It's purpose is to enhance the experience of using Deno's KV store through additional features such as indexing, strongly typed collections, and -cron jobs, while maintaining as much of the native functionality as possible, -like atomic operations and queue listeners. +intervalled queue callbacks, while maintaining as much of the native +functionality as possible, like atomic operations and queue listeners. ## Highlights @@ -13,7 +13,7 @@ like atomic operations and queue listeners. - Extensible model strategy (Zod supported) - Segmented storage for large objects that exceed the native size limit. - Support for pagination and filtering. -- Create repeating cron jobs. +- Set intervals built on queues. - Message queues at database and collection level with topics. - Support for atomic operations. @@ -58,7 +58,7 @@ like atomic operations and queue listeners. - [findUndelivered()](#findundelivered-1) - [enqueue()](#enqueue-1) - [listenQueue()](#listenqueue-1) - - [cron()](#cron) + - [setInterval()](#setinterval) - [atomic()](#atomic) - [Atomic Operations](#atomic-operations) - [Without checking](#without-checking) @@ -73,9 +73,9 @@ like atomic operations and queue listeners. Collections are typed using models. Standard models can be defined using the `model()` function. Alternatively, any object that extends or implements the Model type can be used as a model. Zod is therefore fully compatible, without -being a dependency. The standard model uses TypeScript inference only and does -not validate any data when parsing. It is up to the devloper to choose the -strategy that fits their use case the best. +being a dependency. The standard model uses type casting only, and does not +validate any data when parsing. It is up to the developer to choose the strategy +that fits their use case the best. **_NOTE_:** When using interfaces instead of types, they must extend the KvValue type. @@ -701,28 +701,28 @@ db.listenQueue(async (data) => { }, { topic: "posts" }) ``` -### cron() +### setInterval() -Create a cron job that will run on interval, either indefinitely or until an -exit condition is met. Interval defaults to 1 hour if not set. Like with queue -listeners, multiple cron jobs can be created. +Create an interval with a callback function that can run indefinitely or until +an exit condition is met. Interval defaults to 1 hour if not set. Like with +queue listeners, multiple intervals can be created. ```ts // Will repeat indefinitely with 1 hour interval -db.cron(() => console.log("Hello World!")) +db.setInterval(() => console.log("Hello World!")) -// First job starts with a 10 second delay, after that there is a 5 second delay between jobs -db.cron(() => console.log("I terminate after running 10 times"), { - // Delay before the first job is invoked +// First callback is invoked after a 10 second delay, after that there is a 5 second delay between callbacks +db.setInterval(() => console.log("I terminate after running 10 times"), { + // Delay before the first callback is invoked startDelay: 10_000, - // Fixed interval + // Fixed interval of 5 seconds interval: 5_000, - // If this is set it will override the fixed interval - setInterval: ({ count }) => count * 500 + // ...or set a dynamic interval + interval: ({ count }) => count * 500 - // Count starts at 0, exitOn is run before the current job + // Count starts at 0, exitOn is run before the current callback exitOn: ({ count }) => count === 10, }) ``` diff --git a/src/atomic_wrapper.ts b/src/atomic_wrapper.ts new file mode 100644 index 0000000..4f4dc39 --- /dev/null +++ b/src/atomic_wrapper.ts @@ -0,0 +1,115 @@ +import { ATOMIC_OPERATION_MUTATION_LIMIT } from "./constants.ts" +import { SetOptions } from "./types.ts" + +export class AtomicWrapper implements Deno.AtomicOperation { + private kv: Deno.Kv + private current: Deno.AtomicOperation + private atomics: Deno.AtomicOperation[] + private count: number + private atomicBatchSize: number + + constructor( + kv: Deno.Kv, + atomicBatchSize = ATOMIC_OPERATION_MUTATION_LIMIT / 2, + ) { + this.kv = kv + this.current = kv.atomic() + this.atomics = [] + this.count = 0 + this.atomicBatchSize = atomicBatchSize + } + + set(key: Deno.KvKey, value: unknown, options?: SetOptions) { + this.addMutation((op) => op.set(key, value, options)) + return this + } + + delete(key: Deno.KvKey) { + this.addMutation((op) => op.delete(key)) + return this + } + + mutate(...mutations: Deno.KvMutation[]) { + this.addMutation((op) => op.mutate(...mutations)) + return this + } + + check(...checks: Deno.AtomicCheck[]) { + this.addMutation((op) => op.check(...checks)) + return this + } + + sum(key: Deno.KvKey, n: bigint) { + this.addMutation((op) => op.sum(key, n)) + return this + } + + max(key: Deno.KvKey, n: bigint) { + this.addMutation((op) => op.max(key, n)) + return this + } + + min(key: Deno.KvKey, n: bigint): this { + this.addMutation((op) => op.min(key, n)) + return this + } + + enqueue( + value: unknown, + options?: { + delay?: number | undefined + keysIfUndelivered?: Deno.KvKey[] | undefined + } | undefined, + ) { + this.addMutation((op) => op.enqueue(value, options)) + return this + } + + async commit(): Promise { + // Add curent operation to atomics list + this.atomics.push(this.current) + + // Commit all operations + const settled = await Promise.allSettled( + this.atomics.map((op) => op.commit()), + ) + + // Check status of all commits + const success = settled.every((v) => v.status === "fulfilled") + + // If successful, return commit result + if (success) { + return { + ok: true, + versionstamp: "0", + } + } + + // Return commit error + return { + ok: false, + } + } + + /** PRIVATE METHODS */ + + /** + * Add an atomic mutation within a batched operation. + * + * @param mutation - Atomic mutation. + */ + private addMutation( + mutation: (op: Deno.AtomicOperation) => Deno.AtomicOperation, + ) { + // Add atomic mutation and increment count + this.current = mutation(this.current) + this.count++ + + // Add current operation to atomics list if batch size is reached, reset current and count + if (this.count >= this.atomicBatchSize - 1) { + this.atomics.push(this.current) + this.current = this.kv.atomic() + this.count = 0 + } + } +} diff --git a/src/collection.ts b/src/collection.ts index efbcfd7..24b4c74 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -27,7 +27,6 @@ import type { } from "./types.ts" import { allFulfilled, - atomicDelete, createHandlerId, createListSelector, extendKey, @@ -36,9 +35,11 @@ import { isKvObject, kvGetMany, prepareEnqueue, + selectsAll, } from "./utils.ts" import { Document } from "./document.ts" import { model } from "./model.ts" +import { AtomicWrapper } from "./atomic_wrapper.ts" /** * Create a collection builder function. @@ -62,7 +63,7 @@ export function collection( kv: Deno.Kv, key: KvKey, queueHandlers: Map[]>, - idempotentListener: () => void, + idempotentListener: () => Promise, ) => new Collection>( kv, @@ -84,7 +85,7 @@ export class Collection< const T2 extends CollectionOptions, > { private queueHandlers: Map[]> - private idempotentListener: () => void + private idempotentListener: () => Promise protected kv: Deno.Kv @@ -97,7 +98,7 @@ export class Collection< key: KvKey, model: Model, queueHandlers: Map[]>, - idempotentListener: () => void, + idempotentListener: () => Promise, options?: T2, ) { // Set reference to queue handlers and idempotent listener @@ -442,14 +443,7 @@ export class Collection< */ async deleteMany(options?: ListOptions) { // Perform quick delete if all documents are to be deleted - if ( - !options?.consistency && - !options?.cursor && - !options?.endId && - !options?.startId && - !options?.filter && - !options?.limit - ) { + if (selectsAll(options)) { // Create list iterator and empty keys list const iter = this.kv.list({ prefix: this._keys.baseKey }, options) const keys: Deno.KvKey[] = [] @@ -460,7 +454,9 @@ export class Collection< } // Delete all keys and return - return await atomicDelete(this.kv, keys, options?.batchSize) + const atomic = new AtomicWrapper(this.kv, options?.batchSize) + keys.forEach((key) => atomic.delete(key)) + await atomic.commit() } // Execute delete operation for each document entry @@ -651,7 +647,7 @@ export class Collection< * @param options - Queue listener options. * @returns void. */ - listenQueue( + async listenQueue( handler: QueueMessageHandler, options?: QueueListenerOptions, ) { @@ -664,7 +660,7 @@ export class Collection< this.queueHandlers.set(handlerId, handlers) // Activate idempotent listener - this.idempotentListener() + return await this.idempotentListener() } /** @@ -759,7 +755,8 @@ export class Collection< }) // Filter document and add to documents list - if (!options?.filter || options.filter(doc)) { + const filter = options?.filter + if (!filter || filter(doc)) { docs.push(doc) } } diff --git a/src/constants.ts b/src/constants.ts index 6cb2a83..d754969 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -14,7 +14,7 @@ export const UNDELIVERED_KEY_PREFIX = "__undelivered__" // Fixed limits export const ATOMIC_OPERATION_MUTATION_LIMIT = 1_000 -export const ATOMIC_OPERATION_SAFE_MUTATION_LIMIT = 20 +export const ATOMIC_OPERATION_CONSERVATIVE_MUTATION_LIMIT = 20 export const GET_MANY_KEY_LIMIT = 10 @@ -24,3 +24,8 @@ export const LARGE_COLLECTION_STRING_LIMIT = 25_000 export const DEFAULT_CRON_INTERVAL = 60 * 60 * 1_000 // 1 hour export const DEFAULT_CRON_RETRY = 10 + +// Interval constants +export const DEFAULT_INTERVAL = 60 * 60 * 1_000 // 1 hour + +export const DEFAULT_INTERVAL_RETRY = 10 diff --git a/src/indexable_collection.ts b/src/indexable_collection.ts index b9d0b3e..6a7d889 100644 --- a/src/indexable_collection.ts +++ b/src/indexable_collection.ts @@ -61,7 +61,7 @@ export function indexableCollection< kv: Deno.Kv, key: KvKey, queueHandlers: Map[]>, - idempotentListener: () => void, + idempotentListener: () => Promise, ) => new IndexableCollection( kv, @@ -92,7 +92,7 @@ export class IndexableCollection< key: KvKey, model: Model, queueHandlers: Map[]>, - idempotentListener: () => void, + idempotentListener: () => Promise, options: T2, ) { // Invoke super constructor diff --git a/src/kvdex.ts b/src/kvdex.ts index 6d317ba..2c618a2 100644 --- a/src/kvdex.ts +++ b/src/kvdex.ts @@ -7,6 +7,7 @@ import type { DeleteAllOptions, EnqueueOptions, FindOptions, + IntervalMessage, KvId, KvKey, KvValue, @@ -15,12 +16,12 @@ import type { QueueValue, Schema, SchemaDefinition, + SetIntervalOptions, } from "./types.ts" import { Collection } from "./collection.ts" import { Document } from "./document.ts" import { allFulfilled, - atomicDelete, createHandlerId, extendKey, parseQueueMessage, @@ -30,10 +31,12 @@ import { AtomicBuilder } from "./atomic_builder.ts" import { DEFAULT_CRON_INTERVAL, DEFAULT_CRON_RETRY, + DEFAULT_INTERVAL_RETRY, KVDEX_KEY_PREFIX, UNDELIVERED_KEY_PREFIX, } from "./constants.ts" import { model } from "./model.ts" +import { AtomicWrapper } from "./atomic_wrapper.ts" /** * Create a new database instance. @@ -79,34 +82,32 @@ export function kvdex( schemaDefinition: T, ) { // Set listener activated flag and queue handlers map - let listenerIsActivated = false + let listener: Promise const queueHandlers = new Map[]>() // Create idempotent listener activator const idempotentListener = () => { - // If listener is already activated, cancel - if (listenerIsActivated) { - return - } - - // Set listener activated flag - listenerIsActivated = true + // Create new queue listener if not already created + if (!listener) { + // Add queue listener + listener = kv.listenQueue(async (msg) => { + // Parse queue message + const parsed = parseQueueMessage(msg) + if (!parsed.ok) { + return + } - // Add queue listener - kv.listenQueue(async (msg) => { - // Parse queue message - const parsed = parseQueueMessage(msg) - if (!parsed.ok) { - return - } + // Find correct queue handlers + const { __data__, __handlerId__ } = parsed.msg + const handlers = queueHandlers.get(__handlerId__) - // Find correct queue handlers - const { __data__, __handlerId__ } = parsed.msg - const handlers = queueHandlers.get(__handlerId__) + // Run queue handlers + await allFulfilled(handlers?.map((handler) => handler(__data__)) ?? []) + }) + } - // Run queue handlers - await allFulfilled(handlers?.map((handler) => handler(__data__)) ?? []) - }) + // Return queue listener + return listener } // Create schema @@ -128,13 +129,13 @@ export class KvDex> { private kv: Deno.Kv private schema: T private queueHandlers: Map[]> - private idempotentListener: () => void + private idempotentListener: () => Promise constructor( kv: Deno.Kv, schema: T, queueHandlers: Map[]>, - idempotentListener: () => void, + idempotentListener: () => Promise, ) { this.kv = kv this.schema = schema @@ -196,11 +197,14 @@ export class KvDex> { // Collect all kvdex keys const keys: Deno.KvKey[] = [] for await (const { key } of iter) { + console.log(key) keys.push(key) } // Delete all entries - await atomicDelete(this.kv, keys, options?.atomicBatchSize) + const atomic = new AtomicWrapper(this.kv, options?.atomicBatchSize) + keys.forEach((key) => atomic.delete(key)) + await atomic.commit() } /** @@ -358,6 +362,8 @@ export class KvDex> { * @param job - Work that will be run for each job interval. * @param options - Cron options. * @returns Cron job ID. + * + * @deprecated This method will be removed in a future release. Use kvdex.setInterval() instead, or the Deno.cron() API for cron jobs. */ async cron( job: (msg: CronMessage) => unknown, @@ -447,6 +453,126 @@ export class KvDex> { // Return the cron job id return id } + + /** + * Set an interval for a callback function to be invoked. + * + * Interval defaults to 1 hour if not set. + * + * Will repeat indefinitely if no exit condition is set. + * + * @example + * ```ts + * // Will repeat indeefinitely without delay + * db.setInterval(() => console.log("Hello World!")) + * + * // First callback starts after a 10 second delay, after that there is a 5 second delay between callbacks + * db.setInterval(() => console.log("I terminate after the 10th run"), { + * // 10 second delay before the first job callback invoked + * startDelay: 10_000, + * + * // Fixed interval of 5 seconds + * interval: 5_000, + * + * // ...or set a dynamic interval + * interval: ({ count }) => count * 500 + * + * // Count starts at 0 and is given before the current callback is run + * exit: ({ count }) => count === 10, + * }) + * ``` + * + * @param fn - Callback function. + * @param options - Set interval options. + * @returns The interval id. + */ + async setInterval( + fn: (msg: IntervalMessage) => unknown, + options?: SetIntervalOptions, + ) { + // Create interval handler id + const id = crypto.randomUUID() + + // Create interval enqueuer + const enqueue = async ( + msg: IntervalMessage, + delay: number | undefined, + ) => { + // Try enqueuing until delivered on number of retries is exhausted + for (let i = 0; i <= (options?.retry ?? DEFAULT_INTERVAL_RETRY); i++) { + await this.enqueue(msg, { + idsIfUndelivered: [id], + delay, + topic: id, + }) + + // Check if message was delivered, break loop if successful + const doc = await this.findUndelivered(id) + + if (doc === null) { + break + } + + // Delete undelivered entry before retrying + await this.deleteUndelivered(id) + } + } + + // Add interval listener + this.listenQueue(async (msg) => { + // Check if exit criteria is met, terminate interval if true + let exit = false + try { + exit = await options?.exitOn?.(msg) ?? false + } catch (e) { + console.error( + `An error was caught while running exitOn task for interval {ID = ${id}}`, + e, + ) + } + + if (exit) { + await options?.onExit?.(msg) + return + } + + // Set the next interval + let interval = DEFAULT_CRON_INTERVAL + try { + if (typeof options?.interval === "function") { + interval = await options.interval(msg) + } else { + interval = options?.interval ?? DEFAULT_CRON_INTERVAL + } + } catch (e) { + console.error( + `An error was caught while setting the next callback delay for interval {ID = ${id}}`, + e, + ) + } + + await allFulfilled([ + // Enqueue next callback + enqueue({ + count: msg.count + 1, + previousInterval: interval, + previousTimestamp: new Date(), + }, interval), + + // Invoke callback function + fn(msg), + ]) + }, { topic: id }) + + // Enqueue first cron job + await enqueue({ + count: 0, + previousInterval: options?.startDelay ?? 0, + previousTimestamp: null, + }, options?.startDelay) + + return id + } } /*************************/ @@ -467,7 +593,7 @@ function _createSchema( def: T, kv: Deno.Kv, queueHandlers: Map[]>, - idempotentListener: () => void, + idempotentListener: () => Promise, treeKey?: KvKey, ): Schema { // Get all the definition entries diff --git a/src/large_collection.ts b/src/large_collection.ts index 4237228..155b945 100644 --- a/src/large_collection.ts +++ b/src/large_collection.ts @@ -54,7 +54,7 @@ export function largeCollection( kv: Deno.Kv, key: KvKey, queueHandlers: Map[]>, - idempotentListener: () => void, + idempotentListener: () => Promise, ) => new LargeCollection>( kv, @@ -77,7 +77,7 @@ export class LargeCollection< key: KvKey, model: Model, queueHandlers: Map[]>, - idempotentListener: () => void, + idempotentListener: () => Promise, options?: T2, ) { // Invoke super constructor @@ -217,7 +217,8 @@ export class LargeCollection< } // Filter document and add to documents list - if (!options?.filter || options.filter(doc)) { + const filter = options?.filter + if (!filter || filter(doc)) { docs.push(doc) } } diff --git a/src/types.ts b/src/types.ts index bca14f3..1c8952e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -7,7 +7,7 @@ export type CollectionBuilderFn = ( kv: Deno.Kv, key: KvKey, queueHandlers: Map[]>, - idempotentListener: () => void, + idempotentListener: () => Promise, ) => Collection> export type CheckKeyOf = K extends keyof T ? T[K] : never @@ -81,6 +81,49 @@ export type CronMessage = { enqueueTimestamp: Date } +// Interval types +export type SetIntervalOptions = { + /** + * Static or dynamic interval in milliseconds. + * + * @default 3_600_000 // Defaults to 1 hour + */ + interval?: number | ((msg: IntervalMessage) => number | Promise) + + /** Exit condition used to terminate interval. */ + exitOn?: (msg: IntervalMessage) => boolean | Promise + + /** Task to be run when terminating an interval, executed after ```exitOn()``` returns true. */ + onExit?: (msg: IntervalMessage) => unknown + + /** + * Delay before running the first job. + * + * If not set, will run first job immediately. + */ + startDelay?: number + + /** + * Number of retry attempts upon failed job deliver. + * + * When all retry attempts are spent the cron job will exit. + * + * @default 10 + */ + retry?: number +} + +export type IntervalMessage = { + /** Job number, starts at 0. */ + count: number + + /** Previously set interval. */ + previousInterval: number + + /** Timestamp of previous executed callback, is null for first callback. */ + previousTimestamp: Date | null +} + // Atomic Builder Types export type CollectionSelector< T1 extends Schema, diff --git a/src/utils.ts b/src/utils.ts index 00e9d60..3ae5fa6 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,5 @@ import { - ATOMIC_OPERATION_MUTATION_LIMIT, - ATOMIC_OPERATION_SAFE_MUTATION_LIMIT, + ATOMIC_OPERATION_CONSERVATIVE_MUTATION_LIMIT, GET_MANY_KEY_LIMIT, UNDELIVERED_KEY_PREFIX, } from "./constants.ts" @@ -325,10 +324,10 @@ export async function useAtomics( for ( let i = 0; i < elements.length; - i += ATOMIC_OPERATION_SAFE_MUTATION_LIMIT + i += ATOMIC_OPERATION_CONSERVATIVE_MUTATION_LIMIT ) { slicedElements.push( - elements.slice(i, i + ATOMIC_OPERATION_SAFE_MUTATION_LIMIT), + elements.slice(i, i + ATOMIC_OPERATION_CONSERVATIVE_MUTATION_LIMIT), ) } @@ -488,38 +487,20 @@ export function createListSelector( } /** - * Perform multiple delete operations with optimal efficiency using atomic operations. + * Checks whether the specified list options selects all entries or potentially limits the selection. * - * @param kv - Deno KV instance. - * @param keys - Keys of documents to be deleted. - * @param batchSize - Batch size of deletes in a single atomic operation. + * @param options - List options. + * @returns true if list options selects all entries, false if potentially not. */ -export async function atomicDelete( - kv: Deno.Kv, - keys: Deno.KvKey[], - batchSize = ATOMIC_OPERATION_MUTATION_LIMIT / 2, +export function selectsAll( + options: ListOptions | undefined, ) { - // Initiate atomic operation and check - let atomic = kv.atomic() - let check = 0 - - // Loop over and add delete operation for each key - for (const key of keys) { - atomic.delete(key) - - // If check is at limit, commit atomic operation - if (check >= batchSize - 1) { - await atomic.commit() - - // Reset atomic operation and check - atomic = kv.atomic() - check = 0 - } - - // Increment check - check++ - } - - // Commit final atomic operation - await atomic.commit() + return ( + !options?.consistency && + !options?.cursor && + !options?.endId && + !options?.startId && + !options?.filter && + !options?.limit + ) } diff --git a/tests/collection/enqueue.test.ts b/tests/collection/enqueue.test.ts index bede448..99065f2 100644 --- a/tests/collection/enqueue.test.ts +++ b/tests/collection/enqueue.test.ts @@ -9,59 +9,63 @@ import { createHandlerId } from "../../src/utils.ts" import { assert } from "../deps.ts" import { sleep, useDb, useKv } from "../utils.ts" -Deno.test("collection - enqueue", async (t) => { - await t.step("Should enqueue message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" +Deno.test({ + name: "collection - enqueue", + fn: async (t) => { + await t.step("Should enqueue message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" - const db = kvdex(kv, { - numbers: collection(model()), - }) + const db = kvdex(kv, { + numbers: collection(model()), + }) - const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) + const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) - let assertion = false + let assertion = false - kv.listenQueue((msg) => { - const qMsg = msg as QueueMessage - assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data - }) + kv.listenQueue((msg) => { + const qMsg = msg as QueueMessage + assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data + }) - await db.numbers.enqueue(data, { - idsIfUndelivered: [undeliveredId], - }) + await db.numbers.enqueue(data, { + idsIfUndelivered: [undeliveredId], + }) - await sleep(100) + await sleep(100) - const undelivered = await db.numbers.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) + const undelivered = await db.numbers.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + }) }) - }) - await t.step("Should enqueue message in correct topic", async () => { - await useDb(async (db) => { - const data = "data" - const undeliveredId = "undelivered" - const topic = "topic" + await t.step("Should enqueue message in correct topic", async () => { + await useDb(async (db) => { + const data = "data" + const undeliveredId = "undelivered" + const topic = "topic" - let assertion1 = false - let assertion2 = true + let assertion1 = false + let assertion2 = true - db.users.listenQueue(() => assertion1 = true, { topic }) + db.users.listenQueue(() => assertion1 = true, { topic }) - db.users.listenQueue(() => assertion2 = false) + db.users.listenQueue(() => assertion2 = false) - await db.users.enqueue("data", { - idsIfUndelivered: [undeliveredId], - topic, - }) + await db.users.enqueue("data", { + idsIfUndelivered: [undeliveredId], + topic, + }) - await sleep(100) + await sleep(100) - const undelivered = await db.users.findUndelivered(undeliveredId) - assert(assertion1 || typeof undelivered?.value === typeof data) - assert(assertion2) + const undelivered = await db.users.findUndelivered(undeliveredId) + assert(assertion1 || typeof undelivered?.value === typeof data) + assert(assertion2) + }) }) - }) + }, + sanitizeOps: false, }) diff --git a/tests/collection/listenQueue.test.ts b/tests/collection/listenQueue.test.ts index 23ad447..e8b1b7b 100644 --- a/tests/collection/listenQueue.test.ts +++ b/tests/collection/listenQueue.test.ts @@ -13,59 +13,67 @@ import { createHandlerId, extendKey } from "../../src/utils.ts" import { assert } from "../deps.ts" import { sleep, useKv } from "../utils.ts" -Deno.test("collection - listenQueue", async (t) => { - await t.step("Should receive message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "id" +Deno.test({ + name: "collection - listenQueue", + fn: async (t) => { + await t.step("Should receive message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "id" - const db = kvdex(kv, { - numbers: collection(model()), - }) + const db = kvdex(kv, { + numbers: collection(model()), + }) - const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) + const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) - let assertion = false + let assertion = false - db.numbers.listenQueue((msgData) => { - assertion = msgData === data - }) + db.numbers.listenQueue((msgData) => { + assertion = msgData === data + }) - const msg: QueueMessage = { - __handlerId__: handlerId, - __data__: data, - } + const msg: QueueMessage = { + __handlerId__: handlerId, + __data__: data, + } - await kv.enqueue(msg, { - keysIfUndelivered: [ - extendKey([KVDEX_KEY_PREFIX], UNDELIVERED_KEY_PREFIX, undeliveredId), - ], - }) + await kv.enqueue(msg, { + keysIfUndelivered: [ + extendKey( + [KVDEX_KEY_PREFIX], + UNDELIVERED_KEY_PREFIX, + undeliveredId, + ), + ], + }) - await sleep(100) + await sleep(100) - const undelivered = await db.numbers.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) + const undelivered = await db.numbers.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + }) }) - }) - await t.step("Should not receive db queue message", async () => { - await useKv(async (kv) => { - const db = kvdex(kv, { - numbers: collection(model()), - }) + await t.step("Should not receive db queue message", async () => { + await useKv(async (kv) => { + const db = kvdex(kv, { + numbers: collection(model()), + }) - let assertion = true + let assertion = true - db.numbers.listenQueue(() => { - assertion = false - }) + db.numbers.listenQueue(() => { + assertion = false + }) - await db.enqueue("data") + await db.enqueue("data") - await sleep(100) + await sleep(100) - assert(assertion) + assert(assertion) + }) }) - }) + }, + sanitizeOps: false, }) diff --git a/tests/db/atomic.test.ts b/tests/db/atomic.test.ts index 6d94c79..c689b6c 100644 --- a/tests/db/atomic.test.ts +++ b/tests/db/atomic.test.ts @@ -10,347 +10,351 @@ import { assert } from "../deps.ts" import { mockUser1, mockUser2, mockUserInvalid } from "../mocks.ts" import { sleep, useDb, useKv } from "../utils.ts" -Deno.test("db - atomic", async (t) => { - await t.step("Should add documents to collection", async () => { - await useDb(async (db) => { - const cr = await db - .atomic((schema) => schema.users) - .add(mockUser1) - .add(mockUser2) - .commit() - - assert(cr.ok) - - const count = await db.users.count() - assert(count === 2) - }) - }) - - await t.step( - "Should only set first document with colliding ids", - async () => { +Deno.test({ + name: "db - atomic", + sanitizeOps: false, + fn: async (t) => { + await t.step("Should add documents to collection", async () => { await useDb(async (db) => { - const id = "id" - const cr = await db .atomic((schema) => schema.users) - .set(id, mockUser1) - .set(id, mockUser2) + .add(mockUser1) + .add(mockUser2) .commit() assert(cr.ok) const count = await db.users.count() - assert(count === 1) + assert(count === 2) }) - }, - ) - - await t.step("Should delete document", async () => { - await useDb(async (db) => { - const cr1 = await db.users.add(mockUser1) - assert(cr1.ok) - - const cr2 = await db - .atomic((schema) => schema.users) - .delete(cr1.id) - .commit() - - assert(cr2.ok) - - const count = await db.users.count() - const doc = await db.users.find(cr1.id) - assert(count === 0) - assert(doc === null) }) - }) - - await t.step("Should perform sum operation", async () => { - await useDb(async (db) => { - const initial = 100n - const additional = 10n - const cr1 = await db.u64s.add(new Deno.KvU64(initial)) - assert(cr1.ok) + await t.step( + "Should only set first document with colliding ids", + async () => { + await useDb(async (db) => { + const id = "id" - const cr2 = await db - .atomic((schema) => schema.u64s) - .sum(cr1.id, additional) - .commit() + const cr = await db + .atomic((schema) => schema.users) + .set(id, mockUser1) + .set(id, mockUser2) + .commit() - assert(cr2.ok) + assert(cr.ok) - const doc = await db.u64s.find(cr1.id) - assert(doc?.value.value === initial + additional) - }) - }) + const count = await db.users.count() + assert(count === 1) + }) + }, + ) - await t.step( - "Should perform min operation and set document value to the given value", - async () => { + await t.step("Should delete document", async () => { await useDb(async (db) => { - const initial = 100n - const min = 10n - - const cr1 = await db.u64s.add(new Deno.KvU64(initial)) + const cr1 = await db.users.add(mockUser1) assert(cr1.ok) const cr2 = await db - .atomic((schema) => schema.u64s) - .min(cr1.id, min) + .atomic((schema) => schema.users) + .delete(cr1.id) .commit() assert(cr2.ok) - const doc = await db.u64s.find(cr1.id) - assert(doc?.value.value === min) + const count = await db.users.count() + const doc = await db.users.find(cr1.id) + assert(count === 0) + assert(doc === null) }) - }, - ) + }) - await t.step( - "Should perform min operation and set document value to the existing value", - async () => { + await t.step("Should perform sum operation", async () => { await useDb(async (db) => { const initial = 100n - const min = 200n + const additional = 10n const cr1 = await db.u64s.add(new Deno.KvU64(initial)) assert(cr1.ok) const cr2 = await db .atomic((schema) => schema.u64s) - .min(cr1.id, min) + .sum(cr1.id, additional) .commit() assert(cr2.ok) const doc = await db.u64s.find(cr1.id) - assert(doc?.value.value === initial) + assert(doc?.value.value === initial + additional) }) - }, - ) + }) - await t.step( - "Should perform max operation and set document value to the given value", - async () => { - await useDb(async (db) => { - const initial = 100n - const max = 200n + await t.step( + "Should perform min operation and set document value to the given value", + async () => { + await useDb(async (db) => { + const initial = 100n + const min = 10n - const cr1 = await db.u64s.add(new Deno.KvU64(initial)) - assert(cr1.ok) + const cr1 = await db.u64s.add(new Deno.KvU64(initial)) + assert(cr1.ok) - const cr2 = await db - .atomic((schema) => schema.u64s) - .max(cr1.id, max) - .commit() + const cr2 = await db + .atomic((schema) => schema.u64s) + .min(cr1.id, min) + .commit() - assert(cr2.ok) + assert(cr2.ok) - const doc = await db.u64s.find(cr1.id) - assert(doc?.value.value === max) - }) - }, - ) + const doc = await db.u64s.find(cr1.id) + assert(doc?.value.value === min) + }) + }, + ) - await t.step( - "Should perform max operation and set document value to the existing value", - async () => { - await useDb(async (db) => { - const initial = 100n - const max = 10n + await t.step( + "Should perform min operation and set document value to the existing value", + async () => { + await useDb(async (db) => { + const initial = 100n + const min = 200n - const cr1 = await db.u64s.add(new Deno.KvU64(initial)) - assert(cr1.ok) + const cr1 = await db.u64s.add(new Deno.KvU64(initial)) + assert(cr1.ok) - const cr2 = await db - .atomic((schema) => schema.u64s) - .max(cr1.id, max) - .commit() + const cr2 = await db + .atomic((schema) => schema.u64s) + .min(cr1.id, min) + .commit() - assert(cr2.ok) + assert(cr2.ok) - const doc = await db.u64s.find(cr1.id) - assert(doc?.value.value === initial) - }) - }, - ) - - await t.step("Should perform mutation operations", async () => { - await useDb(async (db) => { - const initial = new Deno.KvU64(100n) - const value = new Deno.KvU64(200n) - const id = "id" - const addition = new Deno.KvU64(100n) - const min1 = new Deno.KvU64(10n) - const min2 = new Deno.KvU64(200n) - const max1 = new Deno.KvU64(200n) - const max2 = new Deno.KvU64(10n) - - const cr1 = await db.u64s.add(initial) - const cr2 = await db.u64s.add(initial) - const cr3 = await db.u64s.add(initial) - const cr4 = await db.u64s.add(initial) - const cr5 = await db.u64s.add(initial) - const cr6 = await db.u64s.add(initial) - - assert(cr1.ok && cr2.ok && cr3.ok && cr4.ok && cr5.ok && cr6.ok) - - await db - .atomic((schema) => schema.u64s) - .mutate( - { - id, - type: "set", - value, - }, - { - id: cr1.id, - type: "sum", - value: addition, - }, - { - id: cr2.id, - type: "min", - value: min1, - }, - { - id: cr3.id, - type: "min", - value: min2, - }, - { - id: cr4.id, - type: "max", - value: max1, - }, - { - id: cr5.id, - type: "max", - value: max2, - }, - { - id: cr6.id, - type: "delete", - }, - ) - .commit() - - const docNew = await db.u64s.find(id) - const doc1 = await db.u64s.find(cr1.id) - const doc2 = await db.u64s.find(cr2.id) - const doc3 = await db.u64s.find(cr3.id) - const doc4 = await db.u64s.find(cr4.id) - const doc5 = await db.u64s.find(cr5.id) - const doc6 = await db.u64s.find(cr6.id) - - assert(docNew?.value.value === value.value) - assert(doc1?.value.value === initial.value + addition.value) - assert(doc2?.value.value === min1.value) - assert(doc3?.value.value === initial.value) - assert(doc4?.value.value === max1.value) - assert(doc5?.value.value === initial.value) - assert(doc6 === null) - }) - }) + const doc = await db.u64s.find(cr1.id) + assert(doc?.value.value === initial) + }) + }, + ) - await t.step("Should enqueue message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" + await t.step( + "Should perform max operation and set document value to the given value", + async () => { + await useDb(async (db) => { + const initial = 100n + const max = 200n - const db = kvdex(kv, { - numbers: collection(model()), - }) + const cr1 = await db.u64s.add(new Deno.KvU64(initial)) + assert(cr1.ok) - const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) + const cr2 = await db + .atomic((schema) => schema.u64s) + .max(cr1.id, max) + .commit() - let assertion = false + assert(cr2.ok) - kv.listenQueue((msg) => { - const qMsg = msg as QueueMessage - assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data - }) + const doc = await db.u64s.find(cr1.id) + assert(doc?.value.value === max) + }) + }, + ) + + await t.step( + "Should perform max operation and set document value to the existing value", + async () => { + await useDb(async (db) => { + const initial = 100n + const max = 10n - await db - .atomic((schema) => schema.numbers) - .enqueue("data", { - idsIfUndelivered: [undeliveredId], + const cr1 = await db.u64s.add(new Deno.KvU64(initial)) + assert(cr1.ok) + + const cr2 = await db + .atomic((schema) => schema.u64s) + .max(cr1.id, max) + .commit() + + assert(cr2.ok) + + const doc = await db.u64s.find(cr1.id) + assert(doc?.value.value === initial) }) - .commit() + }, + ) - await sleep(100) + await t.step("Should perform mutation operations", async () => { + await useDb(async (db) => { + const initial = new Deno.KvU64(100n) + const value = new Deno.KvU64(200n) + const id = "id" + const addition = new Deno.KvU64(100n) + const min1 = new Deno.KvU64(10n) + const min2 = new Deno.KvU64(200n) + const max1 = new Deno.KvU64(200n) + const max2 = new Deno.KvU64(10n) + + const cr1 = await db.u64s.add(initial) + const cr2 = await db.u64s.add(initial) + const cr3 = await db.u64s.add(initial) + const cr4 = await db.u64s.add(initial) + const cr5 = await db.u64s.add(initial) + const cr6 = await db.u64s.add(initial) + + assert(cr1.ok && cr2.ok && cr3.ok && cr4.ok && cr5.ok && cr6.ok) + + await db + .atomic((schema) => schema.u64s) + .mutate( + { + id, + type: "set", + value, + }, + { + id: cr1.id, + type: "sum", + value: addition, + }, + { + id: cr2.id, + type: "min", + value: min1, + }, + { + id: cr3.id, + type: "min", + value: min2, + }, + { + id: cr4.id, + type: "max", + value: max1, + }, + { + id: cr5.id, + type: "max", + value: max2, + }, + { + id: cr6.id, + type: "delete", + }, + ) + .commit() - const undelivered = await db.numbers.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) + const docNew = await db.u64s.find(id) + const doc1 = await db.u64s.find(cr1.id) + const doc2 = await db.u64s.find(cr2.id) + const doc3 = await db.u64s.find(cr3.id) + const doc4 = await db.u64s.find(cr4.id) + const doc5 = await db.u64s.find(cr5.id) + const doc6 = await db.u64s.find(cr6.id) + + assert(docNew?.value.value === value.value) + assert(doc1?.value.value === initial.value + addition.value) + assert(doc2?.value.value === min1.value) + assert(doc3?.value.value === initial.value) + assert(doc4?.value.value === max1.value) + assert(doc5?.value.value === initial.value) + assert(doc6 === null) + }) }) - }) - - await t.step("Should successfully parse and add documents", async () => { - await useDb(async (db) => { - const cr1 = await db - .atomic((schema) => schema.z_users) - .add(mockUser1) - .commit() - - const cr2 = await db - .atomic((schema) => schema.users) - .set("id2", mockUser1) - .commit() - - const cr3 = await db - .atomic((schema) => schema.users) - .mutate({ - type: "set", - id: "id3", - value: mockUser1, + + await t.step("Should enqueue message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" + + const db = kvdex(kv, { + numbers: collection(model()), }) - .commit() - assert(cr1.ok) - assert(cr2.ok) - assert(cr3.ok) - }) - }) + const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) - await t.step("Should fail to parse and adding documents", async () => { - await useDb((db) => { - let assertion1 = false - let assertion2 = false - let assertion3 = false + let assertion = false - try { - db - .atomic((schema) => schema.z_users) - .add(mockUserInvalid) - } catch (_) { - assertion1 = true - } + kv.listenQueue((msg) => { + const qMsg = msg as QueueMessage + assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data + }) - try { - db - .atomic((schema) => schema.z_users) - .set("id2", mockUserInvalid) - } catch (_) { - assertion2 = true - } + await db + .atomic((schema) => schema.numbers) + .enqueue("data", { + idsIfUndelivered: [undeliveredId], + }) + .commit() + + await sleep(100) + + const undelivered = await db.numbers.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + }) + }) - try { - db + await t.step("Should successfully parse and add documents", async () => { + await useDb(async (db) => { + const cr1 = await db .atomic((schema) => schema.z_users) + .add(mockUser1) + .commit() + + const cr2 = await db + .atomic((schema) => schema.users) + .set("id2", mockUser1) + .commit() + + const cr3 = await db + .atomic((schema) => schema.users) .mutate({ type: "set", id: "id3", - value: mockUserInvalid, + value: mockUser1, }) - } catch (_) { - assertion3 = true - } + .commit() + + assert(cr1.ok) + assert(cr2.ok) + assert(cr3.ok) + }) + }) - assert(assertion1) - assert(assertion2) - assert(assertion3) + await t.step("Should fail to parse and adding documents", async () => { + await useDb((db) => { + let assertion1 = false + let assertion2 = false + let assertion3 = false + + try { + db + .atomic((schema) => schema.z_users) + .add(mockUserInvalid) + } catch (_) { + assertion1 = true + } + + try { + db + .atomic((schema) => schema.z_users) + .set("id2", mockUserInvalid) + } catch (_) { + assertion2 = true + } + + try { + db + .atomic((schema) => schema.z_users) + .mutate({ + type: "set", + id: "id3", + value: mockUserInvalid, + }) + } catch (_) { + assertion3 = true + } + + assert(assertion1) + assert(assertion2) + assert(assertion3) + }) }) - }) + }, }) diff --git a/tests/db/cron.test.ts b/tests/db/cron.test.ts index 1eff4ff..473a56e 100644 --- a/tests/db/cron.test.ts +++ b/tests/db/cron.test.ts @@ -2,36 +2,40 @@ import { kvdex } from "../../mod.ts" import { assert } from "../deps.ts" import { sleep } from "../utils.ts" -Deno.test("db - cron", async (t) => { - await t.step("Should perform cron jobs given amount of times", async () => { - const kv = await Deno.openKv(":memory:") - const db = kvdex(kv, {}) - - let count1 = 0 - let count2 = 0 - let count3 = 0 - - db.cron(() => count1++, { - interval: 10, - exitOn: ({ count }) => count === 2, +Deno.test({ + name: "db - cron", + sanitizeOps: false, + fn: async (t) => { + await t.step("Should perform cron jobs given amount of times", async () => { + const kv = await Deno.openKv(":memory:") + const db = kvdex(kv, {}) + + let count1 = 0 + let count2 = 0 + let count3 = 0 + + db.cron(() => count1++, { + interval: 10, + exitOn: ({ count }) => count === 2, + }) + + db.cron(() => count2++, { + interval: 10, + exitOn: ({ isFirstJob }) => isFirstJob, + }) + + db.cron(() => count3++, { + interval: 10, + exitOn: ({ previousInterval }) => previousInterval > 0, + }) + + await sleep(1_000) + + assert(count1 === 2) + assert(count2 === 0) + assert(count3 === 1) + + kv.close() }) - - db.cron(() => count2++, { - interval: 10, - exitOn: ({ isFirstJob }) => isFirstJob, - }) - - db.cron(() => count3++, { - interval: 10, - exitOn: ({ previousInterval }) => previousInterval > 0, - }) - - await sleep(1_000) - - assert(count1 === 2) - assert(count2 === 0) - assert(count3 === 1) - - kv.close() - }) + }, }) diff --git a/tests/db/deleteAll.test.ts b/tests/db/deleteAll.test.ts index 0b3e447..f295b37 100644 --- a/tests/db/deleteAll.test.ts +++ b/tests/db/deleteAll.test.ts @@ -8,12 +8,13 @@ Deno.test("db - deleteAll", async (t) => { await useDb(async (db) => { const users = generateUsers(100) const u64s = [ - new Deno.KvU64(0n), - new Deno.KvU64(0n), + new Deno.KvU64(10n), + new Deno.KvU64(20n), ] const crs1 = await db.i_users.addMany(users) const crs2 = await db.l_users.addMany(users) + const crs3 = await db.u64s.addMany(u64s) assert(crs1.every((cr) => cr.ok)) diff --git a/tests/db/enqueue.test.ts b/tests/db/enqueue.test.ts index 7ca1c4a..4d6a51c 100644 --- a/tests/db/enqueue.test.ts +++ b/tests/db/enqueue.test.ts @@ -10,63 +10,67 @@ import { createHandlerId } from "../../src/utils.ts" import { assert } from "../deps.ts" import { sleep, useKv } from "../utils.ts" -Deno.test("db - enqueue", async (t) => { - await t.step("Should enqueue message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" - - const db = kvdex(kv, { - numbers: collection(model()), - }) +Deno.test({ + name: "db - enqueue", + sanitizeOps: false, + fn: async (t) => { + await t.step("Should enqueue message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" - const handlerId = createHandlerId([KVDEX_KEY_PREFIX], undefined) + const db = kvdex(kv, { + numbers: collection(model()), + }) - let assertion = false + const handlerId = createHandlerId([KVDEX_KEY_PREFIX], undefined) - kv.listenQueue((msg) => { - const qMsg = msg as QueueMessage - assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data - }) + let assertion = false - await db.enqueue("data", { - idsIfUndelivered: [undeliveredId], - }) + kv.listenQueue((msg) => { + const qMsg = msg as QueueMessage + assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data + }) + + await db.enqueue("data", { + idsIfUndelivered: [undeliveredId], + }) - await sleep(100) + await sleep(100) - const undelivered = await db.numbers.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) + const undelivered = await db.numbers.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + }) }) - }) - await t.step("Should enqueue message in correct topic", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" - const topic = "topic" + await t.step("Should enqueue message in correct topic", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" + const topic = "topic" - const db = kvdex(kv, { - numbers: collection(model()), - }) + const db = kvdex(kv, { + numbers: collection(model()), + }) - let assertion1 = false - let assertion2 = true + let assertion1 = false + let assertion2 = true - db.listenQueue(() => assertion1 = true, { topic }) + db.listenQueue(() => assertion1 = true, { topic }) - db.listenQueue(() => assertion2 = false) + db.listenQueue(() => assertion2 = false) - await db.enqueue("data", { - idsIfUndelivered: [undeliveredId], - topic, - }) + await db.enqueue("data", { + idsIfUndelivered: [undeliveredId], + topic, + }) - await sleep(100) + await sleep(100) - const undelivered = await db.numbers.findUndelivered(undeliveredId) - assert(assertion1 || typeof undelivered?.value === typeof data) - assert(assertion2) + const undelivered = await db.numbers.findUndelivered(undeliveredId) + assert(assertion1 || typeof undelivered?.value === typeof data) + assert(assertion2) + }) }) - }) + }, }) diff --git a/tests/db/listenQueue.test.ts b/tests/db/listenQueue.test.ts index a99c722..6cc552d 100644 --- a/tests/db/listenQueue.test.ts +++ b/tests/db/listenQueue.test.ts @@ -10,49 +10,53 @@ import { createHandlerId } from "../../src/utils.ts" import { assert } from "../deps.ts" import { sleep, useKv } from "../utils.ts" -Deno.test("db - listenQueue", async (t) => { - await t.step("Should receive message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const db = kvdex(kv, {}) - - const handlerId = createHandlerId([KVDEX_KEY_PREFIX], undefined) - - let assertion = false - - db.listenQueue((msgData) => { - assertion = msgData === data +Deno.test({ + name: "db - listenQueue", + sanitizeOps: false, + fn: async (t) => { + await t.step("Should receive message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const db = kvdex(kv, {}) + + const handlerId = createHandlerId([KVDEX_KEY_PREFIX], undefined) + + let assertion = false + + db.listenQueue((msgData) => { + assertion = msgData === data + }) + + await kv.enqueue({ + __handlerId__: handlerId, + __data__: data, + } as QueueMessage) + + await sleep(100) + assert(assertion) }) - - await kv.enqueue({ - __handlerId__: handlerId, - __data__: data, - } as QueueMessage) - - await sleep(100) - assert(assertion) }) - }) - await t.step("Should not receive collection queue message", async () => { - await useKv(async (kv) => { - const data = "data" + await t.step("Should not receive collection queue message", async () => { + await useKv(async (kv) => { + const data = "data" - const db = kvdex(kv, { - numbers: collection(model()), - }) + const db = kvdex(kv, { + numbers: collection(model()), + }) - let assertion = true + let assertion = true - db.listenQueue(() => { - assertion = false - }) + db.listenQueue(() => { + assertion = false + }) - await db.numbers.enqueue(data) + await db.numbers.enqueue(data) - await sleep(100) + await sleep(100) - assert(assertion) + assert(assertion) + }) }) - }) + }, }) diff --git a/tests/db/setInterval.test.ts b/tests/db/setInterval.test.ts new file mode 100644 index 0000000..3bfbebc --- /dev/null +++ b/tests/db/setInterval.test.ts @@ -0,0 +1,43 @@ +import { kvdex } from "../../mod.ts" +import { assert } from "../deps.ts" +import { sleep, useKv } from "../utils.ts" + +Deno.test({ + name: "db - setInterval", + sanitizeOps: false, + fn: async (t) => { + await t.step( + "Should run callback function given amount of times", + async () => { + await useKv(async (kv) => { + const db = kvdex(kv, {}) + + let count1 = 0 + let count2 = 0 + let count3 = 0 + + db.setInterval(() => count1++, { + interval: 10, + exitOn: ({ count }) => count === 2, + }) + + db.setInterval(() => count2++, { + interval: () => Math.random() * 20, + exitOn: ({ previousTimestamp }) => previousTimestamp === null, + }) + + db.setInterval(() => count3++, { + interval: 10, + exitOn: ({ previousInterval }) => previousInterval > 0, + }) + + await sleep(1_000) + + assert(count1 === 2) + assert(count2 === 0) + assert(count3 === 1) + }) + }, + ) + }, +}) diff --git a/tests/indexable_collection/enqueue.test.ts b/tests/indexable_collection/enqueue.test.ts index 7c96c40..2032f11 100644 --- a/tests/indexable_collection/enqueue.test.ts +++ b/tests/indexable_collection/enqueue.test.ts @@ -10,59 +10,63 @@ import { assert } from "../deps.ts" import { User } from "../models.ts" import { sleep, useDb, useKv } from "../utils.ts" -Deno.test("indexable_collection - enqueue", async (t) => { - await t.step("Should enqueue message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" +Deno.test({ + name: "indexable_collection - enqueue", + sanitizeOps: false, + fn: async (t) => { + await t.step("Should enqueue message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" - const db = kvdex(kv, { - i_users: indexableCollection(model(), { indices: {} }), - }) + const db = kvdex(kv, { + i_users: indexableCollection(model(), { indices: {} }), + }) - const handlerId = createHandlerId(db.i_users._keys.baseKey, undefined) + const handlerId = createHandlerId(db.i_users._keys.baseKey, undefined) - let assertion = false + let assertion = false - kv.listenQueue((msg) => { - const qMsg = msg as QueueMessage - assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data - }) + kv.listenQueue((msg) => { + const qMsg = msg as QueueMessage + assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data + }) - await db.i_users.enqueue(data, { - idsIfUndelivered: [undeliveredId], - }) + await db.i_users.enqueue(data, { + idsIfUndelivered: [undeliveredId], + }) - await sleep(100) + await sleep(100) - const undelivered = await db.i_users.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) + const undelivered = await db.i_users.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + }) }) - }) - await t.step("Should enqueue message in correct topic", async () => { - await useDb(async (db) => { - const data = "data" - const undeliveredId = "undelivered" - const topic = "topic" + await t.step("Should enqueue message in correct topic", async () => { + await useDb(async (db) => { + const data = "data" + const undeliveredId = "undelivered" + const topic = "topic" - let assertion1 = false - let assertion2 = true + let assertion1 = false + let assertion2 = true - db.i_users.listenQueue(() => assertion1 = true, { topic }) + db.i_users.listenQueue(() => assertion1 = true, { topic }) - db.i_users.listenQueue(() => assertion2 = false) + db.i_users.listenQueue(() => assertion2 = false) - await db.i_users.enqueue("data", { - idsIfUndelivered: [undeliveredId], - topic, - }) + await db.i_users.enqueue("data", { + idsIfUndelivered: [undeliveredId], + topic, + }) - await sleep(100) + await sleep(100) - const undelivered = await db.i_users.findUndelivered(undeliveredId) - assert(assertion1 || typeof undelivered?.value === typeof data) - assert(assertion2) + const undelivered = await db.i_users.findUndelivered(undeliveredId) + assert(assertion1 || typeof undelivered?.value === typeof data) + assert(assertion2) + }) }) - }) + }, }) diff --git a/tests/indexable_collection/listenQueue.test.ts b/tests/indexable_collection/listenQueue.test.ts index e05ebca..0b9110c 100644 --- a/tests/indexable_collection/listenQueue.test.ts +++ b/tests/indexable_collection/listenQueue.test.ts @@ -14,59 +14,67 @@ import { assert } from "../deps.ts" import { User } from "../models.ts" import { sleep, useKv } from "../utils.ts" -Deno.test("indexable_collection - listenQueue", async (t) => { - await t.step("Should receive message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "id" +Deno.test({ + name: "indexable_collection - listenQueue", + sanitizeOps: false, + fn: async (t) => { + await t.step("Should receive message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "id" - const db = kvdex(kv, { - i_users: indexableCollection(model(), { indices: {} }), - }) + const db = kvdex(kv, { + i_users: indexableCollection(model(), { indices: {} }), + }) - const handlerId = createHandlerId(db.i_users._keys.baseKey, undefined) + const handlerId = createHandlerId(db.i_users._keys.baseKey, undefined) - let assertion = false + let assertion = false - db.i_users.listenQueue((msgData) => { - assertion = msgData === data - }) + db.i_users.listenQueue((msgData) => { + assertion = msgData === data + }) - const msg: QueueMessage = { - __handlerId__: handlerId, - __data__: data, - } + const msg: QueueMessage = { + __handlerId__: handlerId, + __data__: data, + } - await kv.enqueue(msg, { - keysIfUndelivered: [ - extendKey([KVDEX_KEY_PREFIX], UNDELIVERED_KEY_PREFIX, undeliveredId), - ], - }) + await kv.enqueue(msg, { + keysIfUndelivered: [ + extendKey( + [KVDEX_KEY_PREFIX], + UNDELIVERED_KEY_PREFIX, + undeliveredId, + ), + ], + }) - await sleep(100) + await sleep(100) - const undelivered = await db.i_users.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) + const undelivered = await db.i_users.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + }) }) - }) - await t.step("Should not receive db queue message", async () => { - await useKv(async (kv) => { - const db = kvdex(kv, { - i_users: indexableCollection(model(), { indices: {} }), - }) + await t.step("Should not receive db queue message", async () => { + await useKv(async (kv) => { + const db = kvdex(kv, { + i_users: indexableCollection(model(), { indices: {} }), + }) - let assertion = true + let assertion = true - db.i_users.listenQueue(() => { - assertion = false - }) + db.i_users.listenQueue(() => { + assertion = false + }) - await db.enqueue("data") + await db.enqueue("data") - await sleep(100) + await sleep(100) - assert(assertion) + assert(assertion) + }) }) - }) + }, }) diff --git a/tests/large_collection/enqueue.test.ts b/tests/large_collection/enqueue.test.ts index 2f872b6..c337db4 100644 --- a/tests/large_collection/enqueue.test.ts +++ b/tests/large_collection/enqueue.test.ts @@ -10,59 +10,63 @@ import { assert } from "../deps.ts" import { User } from "../models.ts" import { sleep, useDb, useKv } from "../utils.ts" -Deno.test("large_collection - enqueue", async (t) => { - await t.step("Should enqueue message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" +Deno.test({ + name: "large_collection - enqueue", + sanitizeOps: false, + fn: async (t) => { + await t.step("Should enqueue message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" - const db = kvdex(kv, { - l_users: largeCollection(model()), - }) + const db = kvdex(kv, { + l_users: largeCollection(model()), + }) - const handlerId = createHandlerId(db.l_users._keys.baseKey, undefined) + const handlerId = createHandlerId(db.l_users._keys.baseKey, undefined) - let assertion = false + let assertion = false - kv.listenQueue((msg) => { - const qMsg = msg as QueueMessage - assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data - }) + kv.listenQueue((msg) => { + const qMsg = msg as QueueMessage + assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data + }) - await db.l_users.enqueue(data, { - idsIfUndelivered: [undeliveredId], - }) + await db.l_users.enqueue(data, { + idsIfUndelivered: [undeliveredId], + }) - await sleep(100) + await sleep(100) - const undelivered = await db.l_users.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) + const undelivered = await db.l_users.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + }) }) - }) - await t.step("Should enqueue message in correct topic", async () => { - await useDb(async (db) => { - const data = "data" - const undeliveredId = "undelivered" - const topic = "topic" + await t.step("Should enqueue message in correct topic", async () => { + await useDb(async (db) => { + const data = "data" + const undeliveredId = "undelivered" + const topic = "topic" - let assertion1 = false - let assertion2 = true + let assertion1 = false + let assertion2 = true - db.l_users.listenQueue(() => assertion1 = true, { topic }) + db.l_users.listenQueue(() => assertion1 = true, { topic }) - db.l_users.listenQueue(() => assertion2 = false) + db.l_users.listenQueue(() => assertion2 = false) - await db.l_users.enqueue("data", { - idsIfUndelivered: [undeliveredId], - topic, - }) + await db.l_users.enqueue("data", { + idsIfUndelivered: [undeliveredId], + topic, + }) - await sleep(100) + await sleep(100) - const undelivered = await db.l_users.findUndelivered(undeliveredId) - assert(assertion1 || typeof undelivered?.value === typeof data) - assert(assertion2) + const undelivered = await db.l_users.findUndelivered(undeliveredId) + assert(assertion1 || typeof undelivered?.value === typeof data) + assert(assertion2) + }) }) - }) + }, }) diff --git a/tests/large_collection/listenQueue.test.ts b/tests/large_collection/listenQueue.test.ts index 33a4d98..bdd4272 100644 --- a/tests/large_collection/listenQueue.test.ts +++ b/tests/large_collection/listenQueue.test.ts @@ -14,59 +14,67 @@ import { assert } from "../deps.ts" import { User } from "../models.ts" import { sleep, useKv } from "../utils.ts" -Deno.test("large_collection - listenQueue", async (t) => { - await t.step("Should receive message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "id" +Deno.test({ + name: "large_collection - listenQueue", + sanitizeOps: false, + fn: async (t) => { + await t.step("Should receive message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "id" - const db = kvdex(kv, { - l_users: largeCollection(model()), - }) + const db = kvdex(kv, { + l_users: largeCollection(model()), + }) - let assertion = false + let assertion = false - db.l_users.listenQueue((msgData) => { - assertion = msgData === data - }) + db.l_users.listenQueue((msgData) => { + assertion = msgData === data + }) - const handlerId = createHandlerId(db.l_users._keys.baseKey, undefined) + const handlerId = createHandlerId(db.l_users._keys.baseKey, undefined) - const msg: QueueMessage = { - __handlerId__: handlerId, - __data__: data, - } + const msg: QueueMessage = { + __handlerId__: handlerId, + __data__: data, + } - await kv.enqueue(msg, { - keysIfUndelivered: [ - extendKey([KVDEX_KEY_PREFIX], UNDELIVERED_KEY_PREFIX, undeliveredId), - ], - }) + await kv.enqueue(msg, { + keysIfUndelivered: [ + extendKey( + [KVDEX_KEY_PREFIX], + UNDELIVERED_KEY_PREFIX, + undeliveredId, + ), + ], + }) - await sleep(100) + await sleep(100) - const undelivered = await db.l_users.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) + const undelivered = await db.l_users.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + }) }) - }) - await t.step("Should not receive db queue message", async () => { - await useKv(async (kv) => { - const db = kvdex(kv, { - l_users: largeCollection(model()), - }) + await t.step("Should not receive db queue message", async () => { + await useKv(async (kv) => { + const db = kvdex(kv, { + l_users: largeCollection(model()), + }) - let assertion = true + let assertion = true - db.l_users.listenQueue(() => { - assertion = false - }) + db.l_users.listenQueue(() => { + assertion = false + }) - await db.enqueue("data") + await db.enqueue("data") - await sleep(100) + await sleep(100) - assert(assertion) + assert(assertion) + }) }) - }) + }, }) From f97990719ce85bfdb8e692500a6040daa0c1cac7 Mon Sep 17 00:00:00 2001 From: oliver-oloughlin Date: Fri, 3 Nov 2023 16:24:40 +0100 Subject: [PATCH 2/3] fix: fixed broken tests for deno 1.38.0 --- .github/workflows/test.yml | 5 +- README.md | 4 +- deno.jsonc | 2 +- src/atomic_wrapper.ts | 5 +- src/collection.ts | 20 +- src/indexable_collection.ts | 4 +- src/kvdex.ts | 18 +- src/large_collection.ts | 43 +- src/types.ts | 9 +- src/utils.ts | 50 +- tests/collection/addMany.test.ts | 25 +- tests/collection/count.test.ts | 4 +- tests/collection/delete.test.ts | 6 +- tests/collection/deleteMany.test.ts | 4 +- tests/collection/enqueue.test.ts | 86 +-- tests/collection/findMany.test.ts | 11 +- tests/collection/forEach.test.ts | 4 +- tests/collection/getMany.test.ts | 4 +- tests/collection/listenQueue.test.ts | 108 ++-- tests/collection/map.test.ts | 4 +- tests/collection/properties.test.ts | 24 +- tests/collection/updateMany.test.ts | 50 +- tests/db/atomic.test.ts | 546 +++++++++--------- tests/db/countAll.test.ts | 12 +- tests/db/cron.test.ts | 24 +- tests/db/deleteAll.test.ts | 13 +- tests/db/enqueue.test.ts | 92 +-- tests/db/listenQueue.test.ts | 78 +-- tests/db/setInterval.test.ts | 63 +- tests/indexable_collection/addMany.test.ts | 32 +- tests/indexable_collection/count.test.ts | 4 +- tests/indexable_collection/delete.test.ts | 8 +- tests/indexable_collection/deleteMany.test.ts | 4 +- tests/indexable_collection/enqueue.test.ts | 86 +-- tests/indexable_collection/findMany.test.ts | 13 +- tests/indexable_collection/forEach.test.ts | 4 +- tests/indexable_collection/getMany.test.ts | 4 +- .../indexable_collection/listenQueue.test.ts | 108 ++-- tests/indexable_collection/map.test.ts | 4 +- tests/indexable_collection/properties.test.ts | 24 +- .../updateBySecondaryIndex.test.ts | 8 +- tests/indexable_collection/updateMany.test.ts | 20 +- tests/large_collection/addMany.test.ts | 18 +- tests/large_collection/count.test.ts | 4 +- tests/large_collection/delete.test.ts | 8 +- tests/large_collection/deleteMany.test.ts | 4 +- tests/large_collection/enqueue.test.ts | 86 +-- tests/large_collection/findMany.test.ts | 13 +- tests/large_collection/forEach.test.ts | 4 +- tests/large_collection/getMany.test.ts | 4 +- tests/large_collection/listenQueue.test.ts | 108 ++-- tests/large_collection/map.test.ts | 4 +- tests/large_collection/properties.test.ts | 24 +- tests/large_collection/updateMany.test.ts | 50 +- tests/utils.ts | 25 +- 55 files changed, 979 insertions(+), 1010 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 234b8e7..5f6b4f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,10 @@ jobs: with: deno-version: v1.x - - name: Verify formatting + - name: Check Types + run: deno check --unstable mod.ts + + - name: Check formatting run: deno fmt --check - name: Run linter diff --git a/README.md b/README.md index 1e86ec9..a2c6a76 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ `kvdex` is a high-level abstraction layer for Deno KV with zero third-party dependencies. It's purpose is to enhance the experience of using Deno's KV store through additional features such as indexing, strongly typed collections, and -intervalled queue callbacks, while maintaining as much of the native -functionality as possible, like atomic operations and queue listeners. +queue-based intervals, while maintaining as much of the native functionality as +possible, like atomic operations and queue listeners. ## Highlights diff --git a/deno.jsonc b/deno.jsonc index 0621a65..8d9cfd3 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -1,7 +1,7 @@ { "lock": false, "tasks": { - "test": "deno lint && deno fmt && deno test -A --unstable" + "test": "deno check --unstable mod.ts && deno fmt --check && deno lint && deno test -A --unstable" }, "fmt": { "semiColons": false diff --git a/src/atomic_wrapper.ts b/src/atomic_wrapper.ts index 4f4dc39..18fef60 100644 --- a/src/atomic_wrapper.ts +++ b/src/atomic_wrapper.ts @@ -10,7 +10,7 @@ export class AtomicWrapper implements Deno.AtomicOperation { constructor( kv: Deno.Kv, - atomicBatchSize = ATOMIC_OPERATION_MUTATION_LIMIT / 2, + atomicBatchSize = ATOMIC_OPERATION_MUTATION_LIMIT / 4, ) { this.kv = kv this.current = kv.atomic() @@ -106,10 +106,9 @@ export class AtomicWrapper implements Deno.AtomicOperation { this.count++ // Add current operation to atomics list if batch size is reached, reset current and count - if (this.count >= this.atomicBatchSize - 1) { + if (this.count % this.atomicBatchSize === this.atomicBatchSize - 1) { this.atomics.push(this.current) this.current = this.kv.atomic() - this.count = 0 } } } diff --git a/src/collection.ts b/src/collection.ts index 24b4c74..b3758f2 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -1,4 +1,5 @@ import { + //ATOMIC_OPERATION_CONSERVATIVE_MUTATION_LIMIT, ID_KEY_PREFIX, KVDEX_KEY_PREFIX, UNDELIVERED_KEY_PREFIX, @@ -287,10 +288,9 @@ export class Collection< */ async delete(...ids: KvId[]) { // Perform delete operation for each id - await allFulfilled(ids.map(async (id) => { - const key = extendKey(this._keys.idKey, id) - await this.kv.delete(key) - })) + const atomic = new AtomicWrapper(this.kv) + ids.forEach((id) => atomic.delete(extendKey(this._keys.idKey, id))) + await atomic.commit() } /** @@ -418,8 +418,16 @@ export class Collection< throw errors } - // Return commit results - return results + // If a commit has failed, return commit error + if (!results.every((cr) => cr.ok)) { + return { ok: false } + } + + // Return commit result + return { + ok: true, + versionstamp: "0", + } } /** diff --git a/src/indexable_collection.ts b/src/indexable_collection.ts index 6a7d889..a5b41c1 100644 --- a/src/indexable_collection.ts +++ b/src/indexable_collection.ts @@ -34,6 +34,7 @@ import { setIndices, } from "./utils.ts" import { Document } from "./document.ts" +import { AtomicWrapper } from "./atomic_wrapper.ts" /** * Create an indexable collection builder function. @@ -240,7 +241,8 @@ export class IndexableCollection< } // Perform delete using atomic operation - const atomic = this.kv.atomic().delete(idKey) + const atomic = new AtomicWrapper(this.kv) + atomic.delete(idKey) deleteIndices(id, value, atomic, this) await atomic.commit() })) diff --git a/src/kvdex.ts b/src/kvdex.ts index 2c618a2..5dd32ab 100644 --- a/src/kvdex.ts +++ b/src/kvdex.ts @@ -197,7 +197,6 @@ export class KvDex> { // Collect all kvdex keys const keys: Deno.KvKey[] = [] for await (const { key } of iter) { - console.log(key) keys.push(key) } @@ -276,7 +275,7 @@ export class KvDex> { this.queueHandlers.set(handlerId, handlers) // Activate idempotent listener - this.idempotentListener() + return this.idempotentListener() } /** @@ -361,7 +360,7 @@ export class KvDex> { * * @param job - Work that will be run for each job interval. * @param options - Cron options. - * @returns Cron job ID. + * @returns Listener promise. * * @deprecated This method will be removed in a future release. Use kvdex.setInterval() instead, or the Deno.cron() API for cron jobs. */ @@ -398,7 +397,7 @@ export class KvDex> { } // Add cron job listener - this.listenQueue(async (msg) => { + const listener = this.listenQueue(async (msg) => { // Check if exit criteria is met, end repeating cron job if true let exit = false try { @@ -450,8 +449,8 @@ export class KvDex> { enqueueTimestamp: new Date(), }, options?.startDelay) - // Return the cron job id - return id + // Return listener + return listener } /** @@ -484,7 +483,7 @@ export class KvDex> { * * @param fn - Callback function. * @param options - Set interval options. - * @returns The interval id. + * @returns Listener promise. */ async setInterval( fn: (msg: IntervalMessage) => unknown, @@ -519,7 +518,7 @@ export class KvDex> { } // Add interval listener - this.listenQueue(async (msg) => { + const listener = this.listenQueue(async (msg) => { // Check if exit criteria is met, terminate interval if true let exit = false try { @@ -571,7 +570,8 @@ export class KvDex> { previousTimestamp: null, }, options?.startDelay) - return id + // Return listener + return listener } } diff --git a/src/large_collection.ts b/src/large_collection.ts index 155b945..c18d26d 100644 --- a/src/large_collection.ts +++ b/src/large_collection.ts @@ -1,5 +1,6 @@ import { Collection } from "./collection.ts" import { + ATOMIC_OPERATION_CONSERVATIVE_MUTATION_LIMIT, ID_KEY_PREFIX, KVDEX_KEY_PREFIX, LARGE_COLLECTION_STRING_LIMIT, @@ -27,10 +28,10 @@ import { extendKey, getDocumentId, kvGetMany, - useAtomics, } from "./utils.ts" import { Document } from "./document.ts" import { CorruptedDocumentDataError } from "./errors.ts" +import { AtomicWrapper } from "./atomic_wrapper.ts" /** * Create a large collection builder function. @@ -173,14 +174,18 @@ export class LargeCollection< return } - // Delete document entry - await this.kv.delete(idKey) + // Create atomic operation and delete all documetn entries + const atomic = new AtomicWrapper(this.kv) + atomic.delete(idKey) - // Delete document parts - await useAtomics(this.kv, value.ids, (segId, atomic) => { - const key = extendKey(this._keys.segmentKey, id, segId) - return atomic.delete(key) - }) + const keys = value.ids.map((segId) => + extendKey(this._keys.segmentKey, id, segId) + ) + + keys.forEach((key) => atomic.delete(key)) + + // Commit the operation + await atomic.commit() })) } @@ -288,26 +293,30 @@ export class LargeCollection< } // Set start index, initiate commit result and keys lists - let index = 0 const keys: KvKey[] = [] // Execute set operations for json parts, capture keys and commit results - const crs = await useAtomics(this.kv, jsonParts, (str, atomic) => { + const atomic = new AtomicWrapper( + this.kv, + ATOMIC_OPERATION_CONSERVATIVE_MUTATION_LIMIT, + ) + + jsonParts.forEach((str, index) => { const key = extendKey(this._keys.segmentKey, docId, index) keys.push(key) - index++ - - return atomic - .set(key, str, options) + atomic.set(key, str, options) }) + const segmentsCr = await atomic.commit() + // Determine whether setting json parts was successful - const success = crs.length > 0 && crs.every((cr) => cr.ok) const retry = options?.retry ?? 0 // If not successful, delete all json part entries - if (!success) { - await allFulfilled(keys.map((key) => this.kv.delete(key))) + if (!segmentsCr.ok) { + const op = new AtomicWrapper(this.kv) + keys.forEach((key) => op.delete(key)) + await op.commit() // Retry operation if there are remaining attempts if (retry > 0) { diff --git a/src/types.ts b/src/types.ts index 1c8952e..9fa9409 100644 --- a/src/types.ts +++ b/src/types.ts @@ -269,7 +269,9 @@ export type ListOptions = Deno.KvListOptions & { /** Id of document to end at. */ endId?: KvId +} +export type AtomicListOptions = ListOptions & { /** Batch size of atomic operations where applicable */ atomicBatchSize?: number } @@ -284,9 +286,12 @@ export type FindManyOptions = NonNullable[1]> export type UpdateManyOptions = ListOptions & SetOptions -export type CountAllOptions = Pick +export type CountAllOptions = Pick, "consistency"> -export type DeleteAllOptions = Pick, "atomicBatchSize"> +export type DeleteAllOptions = Pick< + AtomicListOptions, + "atomicBatchSize" +> export type EnqueueOptions = & Omit< diff --git a/src/utils.ts b/src/utils.ts index 3ae5fa6..c469fe3 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,8 +1,4 @@ -import { - ATOMIC_OPERATION_CONSERVATIVE_MUTATION_LIMIT, - GET_MANY_KEY_LIMIT, - UNDELIVERED_KEY_PREFIX, -} from "./constants.ts" +import { GET_MANY_KEY_LIMIT, UNDELIVERED_KEY_PREFIX } from "./constants.ts" import type { IndexableCollection } from "./indexable_collection.ts" import type { AtomicSetOptions, @@ -304,50 +300,6 @@ export async function kvGetMany( return slicedEntries.flat() } -/** - * Use optimized atomic operations without hitting mutation limit. - * - * @param kv - Deno KV instance. - * @param elements - Pool of elements. - * @param fn - Callback function to be called for each element. - * @returns Promise that resolves to list of atomic commit results. - */ -export async function useAtomics( - kv: Deno.Kv, - elements: T[], - fn: (value: T, op: Deno.AtomicOperation) => Deno.AtomicOperation, -) { - // Initiialize sliced elements list - const slicedElements: T[][] = [] - - // Slice elements based on atomic mutations limit - for ( - let i = 0; - i < elements.length; - i += ATOMIC_OPERATION_CONSERVATIVE_MUTATION_LIMIT - ) { - slicedElements.push( - elements.slice(i, i + ATOMIC_OPERATION_CONSERVATIVE_MUTATION_LIMIT), - ) - } - - // Invoke callback function for each element and execute atomic operation - return await allFulfilled(slicedElements.map(async (elements) => { - try { - let atomic = kv.atomic() - - elements.forEach((value) => { - atomic = fn(value, atomic) - }) - - return await atomic.commit() - } catch (e) { - console.error(e) - return { ok: false } as Deno.KvCommitError - } - })) -} - /** * Get a list of result values from a list of promises. * Only returns the values of fulfilled promises. diff --git a/tests/collection/addMany.test.ts b/tests/collection/addMany.test.ts index 4c5ce1b..88d0604 100644 --- a/tests/collection/addMany.test.ts +++ b/tests/collection/addMany.test.ts @@ -7,16 +7,15 @@ Deno.test("collection - addMany", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.users.addMany(users) + assert(cr.ok) - const docs = await db.users.findMany( - crs.map((cr) => cr.ok ? cr.id : ""), - ) - assert(docs.length === users.length) + const { result } = await db.users.getMany() + + assert(result.length === users.length) assert( users.every((user) => - docs.some((doc) => doc.value.username === user.username) + result.some((doc) => doc.value.username === user.username) ), ) }) @@ -28,16 +27,14 @@ Deno.test("collection - addMany", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.z_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.z_users.addMany(users) + assert(cr.ok) - const docs = await db.z_users.findMany( - crs.map((cr) => cr.ok ? cr.id : ""), - ) - assert(docs.length === users.length) + const { result } = await db.z_users.getMany() + assert(result.length === users.length) assert( users.every((user) => - docs.some((doc) => doc.value.username === user.username) + result.some((doc) => doc.value.username === user.username) ), ) }) diff --git a/tests/collection/count.test.ts b/tests/collection/count.test.ts index 7f5225d..55d7bd1 100644 --- a/tests/collection/count.test.ts +++ b/tests/collection/count.test.ts @@ -11,8 +11,8 @@ Deno.test("collection - count", async (t) => { assert(count1 === 0) const users = generateUsers(1_000) - const crs = await db.users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.users.addMany(users) + assert(cr.ok) const count2 = await db.users.count() assert(count2 === users.length) diff --git a/tests/collection/delete.test.ts b/tests/collection/delete.test.ts index 89e86b9..cb3884b 100644 --- a/tests/collection/delete.test.ts +++ b/tests/collection/delete.test.ts @@ -31,11 +31,13 @@ Deno.test("collection - delete", async (t) => { const users = generateUsers(1_000) const crs = await db.users.addMany(users) const count1 = await db.users.count() + const { result: ids } = await db.users.map((doc) => doc.id) - assert(crs.every((cr) => cr.ok)) + assert(crs.ok) assert(count1 === users.length) + assert(ids.length === users.length) - await db.users.delete(...crs.map((cr) => cr.ok ? cr.id : "")) + await db.users.delete(...ids) const count2 = await db.users.count() assert(count2 === 0) diff --git a/tests/collection/deleteMany.test.ts b/tests/collection/deleteMany.test.ts index 0a673b2..3310f5d 100644 --- a/tests/collection/deleteMany.test.ts +++ b/tests/collection/deleteMany.test.ts @@ -5,8 +5,8 @@ Deno.test("collection - deleteMany", async (t) => { await t.step("Should delete all documents from the collection", async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.users.addMany(users) + assert(cr.ok) const count1 = await db.users.count() assert(count1 === users.length) diff --git a/tests/collection/enqueue.test.ts b/tests/collection/enqueue.test.ts index 99065f2..9767a35 100644 --- a/tests/collection/enqueue.test.ts +++ b/tests/collection/enqueue.test.ts @@ -9,63 +9,63 @@ import { createHandlerId } from "../../src/utils.ts" import { assert } from "../deps.ts" import { sleep, useDb, useKv } from "../utils.ts" -Deno.test({ - name: "collection - enqueue", - fn: async (t) => { - await t.step("Should enqueue message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" +Deno.test("collection - enqueue", async (t) => { + await t.step("Should enqueue message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" - const db = kvdex(kv, { - numbers: collection(model()), - }) + const db = kvdex(kv, { + numbers: collection(model()), + }) - const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) + const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) - let assertion = false + let assertion = false - kv.listenQueue((msg) => { - const qMsg = msg as QueueMessage - assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data - }) + const listener = kv.listenQueue((msg) => { + const qMsg = msg as QueueMessage + assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data + }) - await db.numbers.enqueue(data, { - idsIfUndelivered: [undeliveredId], - }) + await db.numbers.enqueue(data, { + idsIfUndelivered: [undeliveredId], + }) - await sleep(100) + await sleep(100) - const undelivered = await db.numbers.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) - }) + const undelivered = await db.numbers.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + + return async () => await listener }) + }) + + await t.step("Should enqueue message in correct topic", async () => { + await useDb(async (db) => { + const data = "data" + const undeliveredId = "undelivered" + const topic = "topic" - await t.step("Should enqueue message in correct topic", async () => { - await useDb(async (db) => { - const data = "data" - const undeliveredId = "undelivered" - const topic = "topic" + let assertion1 = false + let assertion2 = true - let assertion1 = false - let assertion2 = true + const l1 = db.users.listenQueue(() => assertion1 = true, { topic }) - db.users.listenQueue(() => assertion1 = true, { topic }) + const l2 = db.users.listenQueue(() => assertion2 = false) - db.users.listenQueue(() => assertion2 = false) + await db.users.enqueue("data", { + idsIfUndelivered: [undeliveredId], + topic, + }) - await db.users.enqueue("data", { - idsIfUndelivered: [undeliveredId], - topic, - }) + await sleep(100) - await sleep(100) + const undelivered = await db.users.findUndelivered(undeliveredId) + assert(assertion1 || typeof undelivered?.value === typeof data) + assert(assertion2) - const undelivered = await db.users.findUndelivered(undeliveredId) - assert(assertion1 || typeof undelivered?.value === typeof data) - assert(assertion2) - }) + return async () => await Promise.all([l1, l2]) }) - }, - sanitizeOps: false, + }) }) diff --git a/tests/collection/findMany.test.ts b/tests/collection/findMany.test.ts index e64bc9e..bb91ecb 100644 --- a/tests/collection/findMany.test.ts +++ b/tests/collection/findMany.test.ts @@ -5,10 +5,11 @@ Deno.test("collection - findMany", async (t) => { await t.step("Should find all documents", async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.users.addMany(users) + assert(cr.ok) - const docs = await db.users.findMany(crs.map((cr) => cr.ok ? cr.id : "")) + const { result: ids } = await db.users.map((doc) => doc.id) + const docs = await db.users.findMany(ids) assert(docs.length === users.length) assert( users.every((user) => @@ -21,8 +22,8 @@ Deno.test("collection - findMany", async (t) => { await t.step("Should not find any documents", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.users.addMany(users) + assert(cr.ok) const docs = await db.users.findMany(["", "", ""]) assert(docs.length === 0) diff --git a/tests/collection/forEach.test.ts b/tests/collection/forEach.test.ts index 9873c86..b68e208 100644 --- a/tests/collection/forEach.test.ts +++ b/tests/collection/forEach.test.ts @@ -10,8 +10,8 @@ Deno.test("collection - forEach", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.users.addMany(users) + assert(cr.ok) const docs: Document[] = [] await db.users.forEach((doc) => docs.push(doc)) diff --git a/tests/collection/getMany.test.ts b/tests/collection/getMany.test.ts index 922bdd8..90eb1de 100644 --- a/tests/collection/getMany.test.ts +++ b/tests/collection/getMany.test.ts @@ -6,8 +6,8 @@ Deno.test("collection - getMany", async (t) => { await t.step("Should get all documents", async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.users.addMany(users) + assert(cr.ok) const { result } = await db.users.getMany() assert(result.length === users.length) diff --git a/tests/collection/listenQueue.test.ts b/tests/collection/listenQueue.test.ts index e8b1b7b..c9a1fbb 100644 --- a/tests/collection/listenQueue.test.ts +++ b/tests/collection/listenQueue.test.ts @@ -13,67 +13,67 @@ import { createHandlerId, extendKey } from "../../src/utils.ts" import { assert } from "../deps.ts" import { sleep, useKv } from "../utils.ts" -Deno.test({ - name: "collection - listenQueue", - fn: async (t) => { - await t.step("Should receive message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "id" - - const db = kvdex(kv, { - numbers: collection(model()), - }) - - const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) - - let assertion = false - - db.numbers.listenQueue((msgData) => { - assertion = msgData === data - }) - - const msg: QueueMessage = { - __handlerId__: handlerId, - __data__: data, - } - - await kv.enqueue(msg, { - keysIfUndelivered: [ - extendKey( - [KVDEX_KEY_PREFIX], - UNDELIVERED_KEY_PREFIX, - undeliveredId, - ), - ], - }) - - await sleep(100) - - const undelivered = await db.numbers.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) +Deno.test("collection - listenQueue", async (t) => { + await t.step("Should receive message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "id" + + const db = kvdex(kv, { + numbers: collection(model()), }) - }) - await t.step("Should not receive db queue message", async () => { - await useKv(async (kv) => { - const db = kvdex(kv, { - numbers: collection(model()), - }) + const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) + + let assertion = false + + const listener = db.numbers.listenQueue((msgData) => { + assertion = msgData === data + }) + + const msg: QueueMessage = { + __handlerId__: handlerId, + __data__: data, + } - let assertion = true + await kv.enqueue(msg, { + keysIfUndelivered: [ + extendKey( + [KVDEX_KEY_PREFIX], + UNDELIVERED_KEY_PREFIX, + undeliveredId, + ), + ], + }) - db.numbers.listenQueue(() => { - assertion = false - }) + await sleep(100) - await db.enqueue("data") + const undelivered = await db.numbers.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) - await sleep(100) + return async () => await listener + }) + }) - assert(assertion) + await t.step("Should not receive db queue message", async () => { + await useKv(async (kv) => { + const db = kvdex(kv, { + numbers: collection(model()), }) + + let assertion = true + + const listener = db.numbers.listenQueue(() => { + assertion = false + }) + + await db.enqueue("data") + + await sleep(100) + + assert(assertion) + + return async () => await listener }) - }, - sanitizeOps: false, + }) }) diff --git a/tests/collection/map.test.ts b/tests/collection/map.test.ts index 09d7edb..676c86f 100644 --- a/tests/collection/map.test.ts +++ b/tests/collection/map.test.ts @@ -8,8 +8,8 @@ Deno.test("collection - map", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.users.addMany(users) + assert(cr.ok) const { result } = await db.users.map((doc) => doc.value.username) diff --git a/tests/collection/properties.test.ts b/tests/collection/properties.test.ts index 81fddc5..a979670 100644 --- a/tests/collection/properties.test.ts +++ b/tests/collection/properties.test.ts @@ -40,8 +40,8 @@ Deno.test("collection - properties", async (t) => { await t.step("Should select using pagination", async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.users.addMany(users) + assert(cr.ok) const selected: Document[] = [] let cursor: string | undefined = undefined @@ -66,9 +66,9 @@ Deno.test("collection - properties", async (t) => { await t.step("Should select filtered", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.users.addMany(users) + const cr = await db.users.addMany(users) const count1 = await db.users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const sliced = users.slice(5, 7) @@ -92,9 +92,9 @@ Deno.test("collection - properties", async (t) => { await t.step("Should select in reverse", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.users.addMany(users) + const cr = await db.users.addMany(users) const count1 = await db.users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const query1 = await db.users.getMany() @@ -110,9 +110,9 @@ Deno.test("collection - properties", async (t) => { await t.step("Should select from start id", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.users.addMany(users) + const cr = await db.users.addMany(users) const count1 = await db.users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const index = 5 @@ -134,9 +134,9 @@ Deno.test("collection - properties", async (t) => { await t.step("Should select until end id", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.users.addMany(users) + const cr = await db.users.addMany(users) const count1 = await db.users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const index = 5 @@ -158,9 +158,9 @@ Deno.test("collection - properties", async (t) => { await t.step("Should select from start id to end id", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.users.addMany(users) + const cr = await db.users.addMany(users) const count1 = await db.users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const index1 = 5 diff --git a/tests/collection/updateMany.test.ts b/tests/collection/updateMany.test.ts index 46b544e..f71ca9b 100644 --- a/tests/collection/updateMany.test.ts +++ b/tests/collection/updateMany.test.ts @@ -1,7 +1,6 @@ -import { collection, CommitResult, kvdex, model } from "../../mod.ts" +import { collection, kvdex, model } from "../../mod.ts" import { assert } from "../deps.ts" import { mockUser1, mockUserInvalid } from "../mocks.ts" -import { User } from "../models.ts" import { generateNumbers, generateUsers, useDb, useKv } from "../utils.ts" Deno.test("collection - updateMany", async (t) => { @@ -10,12 +9,12 @@ Deno.test("collection - updateMany", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.users.addMany(users) + assert(cr.ok) - const okCrs = crs.filter((cr) => cr.ok) as CommitResult[] - const ids = okCrs.map((cr) => cr.id) - const versionstamps = okCrs.map((cr) => cr.versionstamp) + const docs = await db.users.getMany() + const ids = docs.result.map((doc) => doc.id) + const versionstamps = docs.result.map((doc) => doc.versionstamp) const updateData = { address: { @@ -65,20 +64,25 @@ Deno.test("collection - updateMany", async (t) => { dates.push(new Date("2000-01-01")) } - const crs1 = await db.numbers.addMany(numbers) as CommitResult[] - const crs2 = await db.arrays.addMany(arrays) as CommitResult[] - const crs3 = await db.dates.addMany(dates) as CommitResult[] + const cr1 = await db.numbers.addMany(numbers) + const cr2 = await db.arrays.addMany(arrays) + const cr3 = await db.dates.addMany(dates) - assert(crs1.every((cr) => cr.ok)) - assert(crs2.every((cr) => cr.ok)) - assert(crs3.every((cr) => cr.ok)) + assert(cr1.ok) + assert(cr2.ok) + assert(cr3.ok) - const ids1 = crs1.map((cr) => cr.id) - const versionstamps1 = crs1.map((cr) => cr.versionstamp) - const ids2 = crs2.map((cr) => cr.id) - const versionstamps2 = crs2.map((cr) => cr.versionstamp) - const ids3 = crs3.map((cr) => cr.id) - const versionstamps3 = crs3.map((cr) => cr.versionstamp) + const docs1 = await db.numbers.getMany() + const docs2 = await db.arrays.getMany() + const docs3 = await db.dates.getMany() + + const ids1 = docs1.result.map((doc) => doc.id) + const ids2 = docs2.result.map((doc) => doc.id) + const ids3 = docs3.result.map((doc) => doc.id) + + const versionstamps1 = docs1.result.map((doc) => doc.versionstamp) + const versionstamps2 = docs2.result.map((doc) => doc.versionstamp) + const versionstamps3 = docs3.result.map((doc) => doc.versionstamp) const val1 = 20 const val2 = ["4"] @@ -118,8 +122,8 @@ Deno.test("collection - updateMany", async (t) => { const users = generateUsers(10) let assertion = true - const crs = await db.z_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.z_users.addMany(users) + assert(cr.ok) await db.z_users.updateMany(mockUser1).catch(() => assertion = false) @@ -132,8 +136,8 @@ Deno.test("collection - updateMany", async (t) => { const users = generateUsers(10) let assertion = false - const crs = await db.z_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.z_users.addMany(users) + assert(cr.ok) await db.z_users.updateMany(mockUserInvalid).catch(() => assertion = true) diff --git a/tests/db/atomic.test.ts b/tests/db/atomic.test.ts index c689b6c..0e43f68 100644 --- a/tests/db/atomic.test.ts +++ b/tests/db/atomic.test.ts @@ -10,351 +10,349 @@ import { assert } from "../deps.ts" import { mockUser1, mockUser2, mockUserInvalid } from "../mocks.ts" import { sleep, useDb, useKv } from "../utils.ts" -Deno.test({ - name: "db - atomic", - sanitizeOps: false, - fn: async (t) => { - await t.step("Should add documents to collection", async () => { +Deno.test("db - atomic", async (t) => { + await t.step("Should add documents to collection", async () => { + await useDb(async (db) => { + const cr = await db + .atomic((schema) => schema.users) + .add(mockUser1) + .add(mockUser2) + .commit() + + assert(cr.ok) + + const count = await db.users.count() + assert(count === 2) + }) + }) + + await t.step( + "Should only set first document with colliding ids", + async () => { await useDb(async (db) => { + const id = "id" + const cr = await db .atomic((schema) => schema.users) - .add(mockUser1) - .add(mockUser2) + .set(id, mockUser1) + .set(id, mockUser2) .commit() assert(cr.ok) const count = await db.users.count() - assert(count === 2) + assert(count === 1) }) + }, + ) + + await t.step("Should delete document", async () => { + await useDb(async (db) => { + const cr1 = await db.users.add(mockUser1) + assert(cr1.ok) + + const cr2 = await db + .atomic((schema) => schema.users) + .delete(cr1.id) + .commit() + + assert(cr2.ok) + + const count = await db.users.count() + const doc = await db.users.find(cr1.id) + assert(count === 0) + assert(doc === null) }) + }) - await t.step( - "Should only set first document with colliding ids", - async () => { - await useDb(async (db) => { - const id = "id" + await t.step("Should perform sum operation", async () => { + await useDb(async (db) => { + const initial = 100n + const additional = 10n - const cr = await db - .atomic((schema) => schema.users) - .set(id, mockUser1) - .set(id, mockUser2) - .commit() + const cr1 = await db.u64s.add(new Deno.KvU64(initial)) + assert(cr1.ok) - assert(cr.ok) + const cr2 = await db + .atomic((schema) => schema.u64s) + .sum(cr1.id, additional) + .commit() - const count = await db.users.count() - assert(count === 1) - }) - }, - ) + assert(cr2.ok) + + const doc = await db.u64s.find(cr1.id) + assert(doc?.value.value === initial + additional) + }) + }) - await t.step("Should delete document", async () => { + await t.step( + "Should perform min operation and set document value to the given value", + async () => { await useDb(async (db) => { - const cr1 = await db.users.add(mockUser1) + const initial = 100n + const min = 10n + + const cr1 = await db.u64s.add(new Deno.KvU64(initial)) assert(cr1.ok) const cr2 = await db - .atomic((schema) => schema.users) - .delete(cr1.id) + .atomic((schema) => schema.u64s) + .min(cr1.id, min) .commit() assert(cr2.ok) - const count = await db.users.count() - const doc = await db.users.find(cr1.id) - assert(count === 0) - assert(doc === null) + const doc = await db.u64s.find(cr1.id) + assert(doc?.value.value === min) }) - }) + }, + ) - await t.step("Should perform sum operation", async () => { + await t.step( + "Should perform min operation and set document value to the existing value", + async () => { await useDb(async (db) => { const initial = 100n - const additional = 10n + const min = 200n const cr1 = await db.u64s.add(new Deno.KvU64(initial)) assert(cr1.ok) const cr2 = await db .atomic((schema) => schema.u64s) - .sum(cr1.id, additional) + .min(cr1.id, min) .commit() assert(cr2.ok) const doc = await db.u64s.find(cr1.id) - assert(doc?.value.value === initial + additional) + assert(doc?.value.value === initial) }) - }) - - await t.step( - "Should perform min operation and set document value to the given value", - async () => { - await useDb(async (db) => { - const initial = 100n - const min = 10n + }, + ) - const cr1 = await db.u64s.add(new Deno.KvU64(initial)) - assert(cr1.ok) - - const cr2 = await db - .atomic((schema) => schema.u64s) - .min(cr1.id, min) - .commit() - - assert(cr2.ok) - - const doc = await db.u64s.find(cr1.id) - assert(doc?.value.value === min) - }) - }, - ) - - await t.step( - "Should perform min operation and set document value to the existing value", - async () => { - await useDb(async (db) => { - const initial = 100n - const min = 200n - - const cr1 = await db.u64s.add(new Deno.KvU64(initial)) - assert(cr1.ok) - - const cr2 = await db - .atomic((schema) => schema.u64s) - .min(cr1.id, min) - .commit() + await t.step( + "Should perform max operation and set document value to the given value", + async () => { + await useDb(async (db) => { + const initial = 100n + const max = 200n - assert(cr2.ok) + const cr1 = await db.u64s.add(new Deno.KvU64(initial)) + assert(cr1.ok) - const doc = await db.u64s.find(cr1.id) - assert(doc?.value.value === initial) - }) - }, - ) + const cr2 = await db + .atomic((schema) => schema.u64s) + .max(cr1.id, max) + .commit() - await t.step( - "Should perform max operation and set document value to the given value", - async () => { - await useDb(async (db) => { - const initial = 100n - const max = 200n + assert(cr2.ok) - const cr1 = await db.u64s.add(new Deno.KvU64(initial)) - assert(cr1.ok) + const doc = await db.u64s.find(cr1.id) + assert(doc?.value.value === max) + }) + }, + ) - const cr2 = await db - .atomic((schema) => schema.u64s) - .max(cr1.id, max) - .commit() + await t.step( + "Should perform max operation and set document value to the existing value", + async () => { + await useDb(async (db) => { + const initial = 100n + const max = 10n - assert(cr2.ok) + const cr1 = await db.u64s.add(new Deno.KvU64(initial)) + assert(cr1.ok) - const doc = await db.u64s.find(cr1.id) - assert(doc?.value.value === max) - }) - }, - ) + const cr2 = await db + .atomic((schema) => schema.u64s) + .max(cr1.id, max) + .commit() - await t.step( - "Should perform max operation and set document value to the existing value", - async () => { - await useDb(async (db) => { - const initial = 100n - const max = 10n + assert(cr2.ok) - const cr1 = await db.u64s.add(new Deno.KvU64(initial)) - assert(cr1.ok) + const doc = await db.u64s.find(cr1.id) + assert(doc?.value.value === initial) + }) + }, + ) + + await t.step("Should perform mutation operations", async () => { + await useDb(async (db) => { + const initial = new Deno.KvU64(100n) + const value = new Deno.KvU64(200n) + const id = "id" + const addition = new Deno.KvU64(100n) + const min1 = new Deno.KvU64(10n) + const min2 = new Deno.KvU64(200n) + const max1 = new Deno.KvU64(200n) + const max2 = new Deno.KvU64(10n) + + const cr1 = await db.u64s.add(initial) + const cr2 = await db.u64s.add(initial) + const cr3 = await db.u64s.add(initial) + const cr4 = await db.u64s.add(initial) + const cr5 = await db.u64s.add(initial) + const cr6 = await db.u64s.add(initial) + + assert(cr1.ok && cr2.ok && cr3.ok && cr4.ok && cr5.ok && cr6.ok) + + await db + .atomic((schema) => schema.u64s) + .mutate( + { + id, + type: "set", + value, + }, + { + id: cr1.id, + type: "sum", + value: addition, + }, + { + id: cr2.id, + type: "min", + value: min1, + }, + { + id: cr3.id, + type: "min", + value: min2, + }, + { + id: cr4.id, + type: "max", + value: max1, + }, + { + id: cr5.id, + type: "max", + value: max2, + }, + { + id: cr6.id, + type: "delete", + }, + ) + .commit() + + const docNew = await db.u64s.find(id) + const doc1 = await db.u64s.find(cr1.id) + const doc2 = await db.u64s.find(cr2.id) + const doc3 = await db.u64s.find(cr3.id) + const doc4 = await db.u64s.find(cr4.id) + const doc5 = await db.u64s.find(cr5.id) + const doc6 = await db.u64s.find(cr6.id) + + assert(docNew?.value.value === value.value) + assert(doc1?.value.value === initial.value + addition.value) + assert(doc2?.value.value === min1.value) + assert(doc3?.value.value === initial.value) + assert(doc4?.value.value === max1.value) + assert(doc5?.value.value === initial.value) + assert(doc6 === null) + }) + }) - const cr2 = await db - .atomic((schema) => schema.u64s) - .max(cr1.id, max) - .commit() + await t.step("Should enqueue message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" - assert(cr2.ok) + const db = kvdex(kv, { + numbers: collection(model()), + }) - const doc = await db.u64s.find(cr1.id) - assert(doc?.value.value === initial) - }) - }, - ) + const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) - await t.step("Should perform mutation operations", async () => { - await useDb(async (db) => { - const initial = new Deno.KvU64(100n) - const value = new Deno.KvU64(200n) - const id = "id" - const addition = new Deno.KvU64(100n) - const min1 = new Deno.KvU64(10n) - const min2 = new Deno.KvU64(200n) - const max1 = new Deno.KvU64(200n) - const max2 = new Deno.KvU64(10n) - - const cr1 = await db.u64s.add(initial) - const cr2 = await db.u64s.add(initial) - const cr3 = await db.u64s.add(initial) - const cr4 = await db.u64s.add(initial) - const cr5 = await db.u64s.add(initial) - const cr6 = await db.u64s.add(initial) - - assert(cr1.ok && cr2.ok && cr3.ok && cr4.ok && cr5.ok && cr6.ok) - - await db - .atomic((schema) => schema.u64s) - .mutate( - { - id, - type: "set", - value, - }, - { - id: cr1.id, - type: "sum", - value: addition, - }, - { - id: cr2.id, - type: "min", - value: min1, - }, - { - id: cr3.id, - type: "min", - value: min2, - }, - { - id: cr4.id, - type: "max", - value: max1, - }, - { - id: cr5.id, - type: "max", - value: max2, - }, - { - id: cr6.id, - type: "delete", - }, - ) - .commit() + let assertion = false - const docNew = await db.u64s.find(id) - const doc1 = await db.u64s.find(cr1.id) - const doc2 = await db.u64s.find(cr2.id) - const doc3 = await db.u64s.find(cr3.id) - const doc4 = await db.u64s.find(cr4.id) - const doc5 = await db.u64s.find(cr5.id) - const doc6 = await db.u64s.find(cr6.id) - - assert(docNew?.value.value === value.value) - assert(doc1?.value.value === initial.value + addition.value) - assert(doc2?.value.value === min1.value) - assert(doc3?.value.value === initial.value) - assert(doc4?.value.value === max1.value) - assert(doc5?.value.value === initial.value) - assert(doc6 === null) + const listener = kv.listenQueue((msg) => { + const qMsg = msg as QueueMessage + assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data }) - }) - - await t.step("Should enqueue message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" - const db = kvdex(kv, { - numbers: collection(model()), + await db + .atomic((schema) => schema.numbers) + .enqueue("data", { + idsIfUndelivered: [undeliveredId], }) + .commit() - const handlerId = createHandlerId(db.numbers._keys.baseKey, undefined) + await sleep(100) - let assertion = false + const undelivered = await db.numbers.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) - kv.listenQueue((msg) => { - const qMsg = msg as QueueMessage - assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data + return async () => await listener + }) + }) + + await t.step("Should successfully parse and add documents", async () => { + await useDb(async (db) => { + const cr1 = await db + .atomic((schema) => schema.z_users) + .add(mockUser1) + .commit() + + const cr2 = await db + .atomic((schema) => schema.users) + .set("id2", mockUser1) + .commit() + + const cr3 = await db + .atomic((schema) => schema.users) + .mutate({ + type: "set", + id: "id3", + value: mockUser1, }) + .commit() - await db - .atomic((schema) => schema.numbers) - .enqueue("data", { - idsIfUndelivered: [undeliveredId], - }) - .commit() - - await sleep(100) - - const undelivered = await db.numbers.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) - }) + assert(cr1.ok) + assert(cr2.ok) + assert(cr3.ok) }) + }) - await t.step("Should successfully parse and add documents", async () => { - await useDb(async (db) => { - const cr1 = await db + await t.step("Should fail to parse and adding documents", async () => { + await useDb((db) => { + let assertion1 = false + let assertion2 = false + let assertion3 = false + + try { + db .atomic((schema) => schema.z_users) - .add(mockUser1) - .commit() + .add(mockUserInvalid) + } catch (_) { + assertion1 = true + } - const cr2 = await db - .atomic((schema) => schema.users) - .set("id2", mockUser1) - .commit() + try { + db + .atomic((schema) => schema.z_users) + .set("id2", mockUserInvalid) + } catch (_) { + assertion2 = true + } - const cr3 = await db - .atomic((schema) => schema.users) + try { + db + .atomic((schema) => schema.z_users) .mutate({ type: "set", id: "id3", - value: mockUser1, + value: mockUserInvalid, }) - .commit() - - assert(cr1.ok) - assert(cr2.ok) - assert(cr3.ok) - }) - }) + } catch (_) { + assertion3 = true + } - await t.step("Should fail to parse and adding documents", async () => { - await useDb((db) => { - let assertion1 = false - let assertion2 = false - let assertion3 = false - - try { - db - .atomic((schema) => schema.z_users) - .add(mockUserInvalid) - } catch (_) { - assertion1 = true - } - - try { - db - .atomic((schema) => schema.z_users) - .set("id2", mockUserInvalid) - } catch (_) { - assertion2 = true - } - - try { - db - .atomic((schema) => schema.z_users) - .mutate({ - type: "set", - id: "id3", - value: mockUserInvalid, - }) - } catch (_) { - assertion3 = true - } - - assert(assertion1) - assert(assertion2) - assert(assertion3) - }) + assert(assertion1) + assert(assertion2) + assert(assertion3) }) - }, + }) }) diff --git a/tests/db/countAll.test.ts b/tests/db/countAll.test.ts index 92a812c..62be794 100644 --- a/tests/db/countAll.test.ts +++ b/tests/db/countAll.test.ts @@ -8,16 +8,16 @@ Deno.test("db - countAll", async (t) => { await useDb(async (db) => { const users = generateUsers(10) - const crs1 = await db.i_users.addMany(users) - const crs2 = await db.l_users.addMany(users) - const crs3 = await db.u64s.addMany([ + const cr1 = await db.i_users.addMany(users) + const cr2 = await db.l_users.addMany(users) + const cr3 = await db.u64s.addMany([ new Deno.KvU64(0n), new Deno.KvU64(0n), ]) - assert(crs1.every((cr) => cr.ok)) - assert(crs2.every((cr) => cr.ok)) - assert(crs3.every((cr) => cr.ok)) + assert(cr1.ok) + assert(cr2.ok) + assert(cr3.ok) const count = await db.countAll() assert(count === 22) diff --git a/tests/db/cron.test.ts b/tests/db/cron.test.ts index 473a56e..10e877c 100644 --- a/tests/db/cron.test.ts +++ b/tests/db/cron.test.ts @@ -1,30 +1,24 @@ -import { kvdex } from "../../mod.ts" import { assert } from "../deps.ts" -import { sleep } from "../utils.ts" - -Deno.test({ - name: "db - cron", - sanitizeOps: false, - fn: async (t) => { - await t.step("Should perform cron jobs given amount of times", async () => { - const kv = await Deno.openKv(":memory:") - const db = kvdex(kv, {}) +import { sleep, useDb } from "../utils.ts" +Deno.test("db - cron", async (t) => { + await t.step("Should perform cron jobs given amount of times", async () => { + await useDb(async (db) => { let count1 = 0 let count2 = 0 let count3 = 0 - db.cron(() => count1++, { + const l1 = db.cron(() => count1++, { interval: 10, exitOn: ({ count }) => count === 2, }) - db.cron(() => count2++, { + const l2 = db.cron(() => count2++, { interval: 10, exitOn: ({ isFirstJob }) => isFirstJob, }) - db.cron(() => count3++, { + const l3 = db.cron(() => count3++, { interval: 10, exitOn: ({ previousInterval }) => previousInterval > 0, }) @@ -35,7 +29,7 @@ Deno.test({ assert(count2 === 0) assert(count3 === 1) - kv.close() + return async () => await Promise.all([l1, l2, l3]) }) - }, + }) }) diff --git a/tests/db/deleteAll.test.ts b/tests/db/deleteAll.test.ts index f295b37..c046920 100644 --- a/tests/db/deleteAll.test.ts +++ b/tests/db/deleteAll.test.ts @@ -12,14 +12,13 @@ Deno.test("db - deleteAll", async (t) => { new Deno.KvU64(20n), ] - const crs1 = await db.i_users.addMany(users) - const crs2 = await db.l_users.addMany(users) + const cr1 = await db.i_users.addMany(users) + const cr2 = await db.l_users.addMany(users) + const cr3 = await db.u64s.addMany(u64s) - const crs3 = await db.u64s.addMany(u64s) - - assert(crs1.every((cr) => cr.ok)) - assert(crs2.every((cr) => cr.ok)) - assert(crs3.every((cr) => cr.ok)) + assert(cr1.ok) + assert(cr2.ok) + assert(cr3.ok) const count1 = await db.countAll() assert(count1 === users.length * 2 + u64s.length) diff --git a/tests/db/enqueue.test.ts b/tests/db/enqueue.test.ts index 4d6a51c..6d0a04d 100644 --- a/tests/db/enqueue.test.ts +++ b/tests/db/enqueue.test.ts @@ -10,67 +10,67 @@ import { createHandlerId } from "../../src/utils.ts" import { assert } from "../deps.ts" import { sleep, useKv } from "../utils.ts" -Deno.test({ - name: "db - enqueue", - sanitizeOps: false, - fn: async (t) => { - await t.step("Should enqueue message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" +Deno.test("db - enqueue", async (t) => { + await t.step("Should enqueue message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" + + const db = kvdex(kv, { + numbers: collection(model()), + }) - const db = kvdex(kv, { - numbers: collection(model()), - }) + const handlerId = createHandlerId([KVDEX_KEY_PREFIX], undefined) - const handlerId = createHandlerId([KVDEX_KEY_PREFIX], undefined) + let assertion = false - let assertion = false + const listener = kv.listenQueue((msg) => { + const qMsg = msg as QueueMessage + assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data + }) - kv.listenQueue((msg) => { - const qMsg = msg as QueueMessage - assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data - }) + await db.enqueue("data", { + idsIfUndelivered: [undeliveredId], + }) - await db.enqueue("data", { - idsIfUndelivered: [undeliveredId], - }) + await sleep(100) - await sleep(100) + const undelivered = await db.numbers.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) - const undelivered = await db.numbers.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) - }) + return async () => await listener }) + }) - await t.step("Should enqueue message in correct topic", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" - const topic = "topic" + await t.step("Should enqueue message in correct topic", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" + const topic = "topic" - const db = kvdex(kv, { - numbers: collection(model()), - }) + const db = kvdex(kv, { + numbers: collection(model()), + }) + + let assertion1 = false + let assertion2 = true - let assertion1 = false - let assertion2 = true + const l1 = db.listenQueue(() => assertion1 = true, { topic }) - db.listenQueue(() => assertion1 = true, { topic }) + const l2 = db.listenQueue(() => assertion2 = false) - db.listenQueue(() => assertion2 = false) + await db.enqueue("data", { + idsIfUndelivered: [undeliveredId], + topic, + }) - await db.enqueue("data", { - idsIfUndelivered: [undeliveredId], - topic, - }) + await sleep(100) - await sleep(100) + const undelivered = await db.numbers.findUndelivered(undeliveredId) + assert(assertion1 || typeof undelivered?.value === typeof data) + assert(assertion2) - const undelivered = await db.numbers.findUndelivered(undeliveredId) - assert(assertion1 || typeof undelivered?.value === typeof data) - assert(assertion2) - }) + return async () => await Promise.all([l1, l2]) }) - }, + }) }) diff --git a/tests/db/listenQueue.test.ts b/tests/db/listenQueue.test.ts index 6cc552d..8cdc597 100644 --- a/tests/db/listenQueue.test.ts +++ b/tests/db/listenQueue.test.ts @@ -10,53 +10,53 @@ import { createHandlerId } from "../../src/utils.ts" import { assert } from "../deps.ts" import { sleep, useKv } from "../utils.ts" -Deno.test({ - name: "db - listenQueue", - sanitizeOps: false, - fn: async (t) => { - await t.step("Should receive message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const db = kvdex(kv, {}) - - const handlerId = createHandlerId([KVDEX_KEY_PREFIX], undefined) - - let assertion = false - - db.listenQueue((msgData) => { - assertion = msgData === data - }) - - await kv.enqueue({ - __handlerId__: handlerId, - __data__: data, - } as QueueMessage) - - await sleep(100) - assert(assertion) +Deno.test("db - listenQueue", async (t) => { + await t.step("Should receive message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const db = kvdex(kv, {}) + + const handlerId = createHandlerId([KVDEX_KEY_PREFIX], undefined) + + let assertion = false + + const listener = db.listenQueue((msgData) => { + assertion = msgData === data }) - }) - await t.step("Should not receive collection queue message", async () => { - await useKv(async (kv) => { - const data = "data" + await kv.enqueue({ + __handlerId__: handlerId, + __data__: data, + } as QueueMessage) - const db = kvdex(kv, { - numbers: collection(model()), - }) + await sleep(100) + assert(assertion) - let assertion = true + return async () => await listener + }) + }) - db.listenQueue(() => { - assertion = false - }) + await t.step("Should not receive collection queue message", async () => { + await useKv(async (kv) => { + const data = "data" - await db.numbers.enqueue(data) + const db = kvdex(kv, { + numbers: collection(model()), + }) - await sleep(100) + let assertion = true - assert(assertion) + const listener = db.listenQueue(() => { + assertion = false }) + + await db.numbers.enqueue(data) + + await sleep(100) + + assert(assertion) + + return async () => await listener }) - }, + }) }) diff --git a/tests/db/setInterval.test.ts b/tests/db/setInterval.test.ts index 3bfbebc..a443e18 100644 --- a/tests/db/setInterval.test.ts +++ b/tests/db/setInterval.test.ts @@ -1,43 +1,38 @@ -import { kvdex } from "../../mod.ts" import { assert } from "../deps.ts" -import { sleep, useKv } from "../utils.ts" +import { sleep, useDb } from "../utils.ts" -Deno.test({ - name: "db - setInterval", - sanitizeOps: false, - fn: async (t) => { - await t.step( - "Should run callback function given amount of times", - async () => { - await useKv(async (kv) => { - const db = kvdex(kv, {}) +Deno.test("db - setInterval", async (t) => { + await t.step( + "Should run callback function given amount of times", + async () => { + await useDb(async (db) => { + let count1 = 0 + let count2 = 0 + let count3 = 0 - let count1 = 0 - let count2 = 0 - let count3 = 0 + const l1 = db.setInterval(() => count1++, { + interval: 10, + exitOn: ({ count }) => count === 2, + }) - db.setInterval(() => count1++, { - interval: 10, - exitOn: ({ count }) => count === 2, - }) + const l2 = db.setInterval(() => count2++, { + interval: () => Math.random() * 20, + exitOn: ({ previousTimestamp }) => previousTimestamp === null, + }) - db.setInterval(() => count2++, { - interval: () => Math.random() * 20, - exitOn: ({ previousTimestamp }) => previousTimestamp === null, - }) + const l3 = db.setInterval(() => count3++, { + interval: 10, + exitOn: ({ previousInterval }) => previousInterval > 0, + }) - db.setInterval(() => count3++, { - interval: 10, - exitOn: ({ previousInterval }) => previousInterval > 0, - }) + await sleep(1_000) - await sleep(1_000) + assert(count1 === 2) + assert(count2 === 0) + assert(count3 === 1) - assert(count1 === 2) - assert(count2 === 0) - assert(count3 === 1) - }) - }, - ) - }, + return async () => await Promise.all([l1, l2, l3]) + }) + }, + ) }) diff --git a/tests/indexable_collection/addMany.test.ts b/tests/indexable_collection/addMany.test.ts index 6c2081c..4ee56dc 100644 --- a/tests/indexable_collection/addMany.test.ts +++ b/tests/indexable_collection/addMany.test.ts @@ -8,16 +8,15 @@ Deno.test("indexable_collection - addMany", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.i_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.i_users.addMany(users) + assert(cr.ok) - const docs = await db.i_users.findMany( - crs.map((cr) => cr.ok ? cr.id : ""), - ) - assert(docs.length === users.length) + const { result } = await db.i_users.getMany() + + assert(result.length === users.length) assert( users.every((user) => - docs.some((doc) => doc.value.username === user.username) + result.some((doc) => doc.value.username === user.username) ), ) }) @@ -28,12 +27,10 @@ Deno.test("indexable_collection - addMany", async (t) => { "Should not add documents with colliding primary indices", async () => { await useDb(async (db) => { - const crs = await db.i_users.addMany([mockUser1, mockUser1]) + const cr = await db.i_users.addMany([mockUser1, mockUser1]) const count = await db.i_users.count() - assert(crs.length === 2) - assert(crs.some((cr) => cr.ok)) - assert(crs.some((cr) => !cr.ok)) + assert(!cr.ok) assert(count === 1) }) }, @@ -44,16 +41,15 @@ Deno.test("indexable_collection - addMany", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.zi_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.zi_users.addMany(users) + assert(cr.ok) - const docs = await db.zi_users.findMany( - crs.map((cr) => cr.ok ? cr.id : ""), - ) - assert(docs.length === users.length) + const { result } = await db.zi_users.getMany() + + assert(result.length === users.length) assert( users.every((user) => - docs.some((doc) => doc.value.username === user.username) + result.some((doc) => doc.value.username === user.username) ), ) }) diff --git a/tests/indexable_collection/count.test.ts b/tests/indexable_collection/count.test.ts index d505433..073c461 100644 --- a/tests/indexable_collection/count.test.ts +++ b/tests/indexable_collection/count.test.ts @@ -11,8 +11,8 @@ Deno.test("indexable_collection - count", async (t) => { assert(count1 === 0) const users = generateUsers(1_000) - const crs = await db.i_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.i_users.addMany(users) + assert(cr.ok) const count2 = await db.i_users.count() assert(count2 === users.length) diff --git a/tests/indexable_collection/delete.test.ts b/tests/indexable_collection/delete.test.ts index a203cde..30d8789 100644 --- a/tests/indexable_collection/delete.test.ts +++ b/tests/indexable_collection/delete.test.ts @@ -49,13 +49,15 @@ Deno.test("indexable_collection - delete", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.i_users.addMany(users) + const cr = await db.i_users.addMany(users) const count1 = await db.i_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) - await db.i_users.delete(...crs.map((cr) => cr.ok ? cr.id : "")) + const { result: ids } = await db.i_users.map((doc) => doc.id) + + await db.i_users.delete(...ids) const count2 = await db.i_users.count() assert(count2 === 0) diff --git a/tests/indexable_collection/deleteMany.test.ts b/tests/indexable_collection/deleteMany.test.ts index bb3bbf1..d86ad38 100644 --- a/tests/indexable_collection/deleteMany.test.ts +++ b/tests/indexable_collection/deleteMany.test.ts @@ -9,7 +9,7 @@ Deno.test("indexable_collection - deleteMany", async (t) => { const users = generateUsers(1_000) const user1 = users[0] - const crs = await db.i_users.addMany(users) + const cr = await db.i_users.addMany(users) const count1 = await db.i_users.count() const byPrimary1 = await db.i_users.findByPrimaryIndex( "username", @@ -20,7 +20,7 @@ Deno.test("indexable_collection - deleteMany", async (t) => { user1.age, ) - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) assert(byPrimary1?.value.username === user1.username) assert(bySecondary1.result.length > 0) diff --git a/tests/indexable_collection/enqueue.test.ts b/tests/indexable_collection/enqueue.test.ts index 2032f11..e957471 100644 --- a/tests/indexable_collection/enqueue.test.ts +++ b/tests/indexable_collection/enqueue.test.ts @@ -10,63 +10,63 @@ import { assert } from "../deps.ts" import { User } from "../models.ts" import { sleep, useDb, useKv } from "../utils.ts" -Deno.test({ - name: "indexable_collection - enqueue", - sanitizeOps: false, - fn: async (t) => { - await t.step("Should enqueue message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" +Deno.test("indexable_collection - enqueue", async (t) => { + await t.step("Should enqueue message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" - const db = kvdex(kv, { - i_users: indexableCollection(model(), { indices: {} }), - }) + const db = kvdex(kv, { + i_users: indexableCollection(model(), { indices: {} }), + }) - const handlerId = createHandlerId(db.i_users._keys.baseKey, undefined) + const handlerId = createHandlerId(db.i_users._keys.baseKey, undefined) - let assertion = false + let assertion = false - kv.listenQueue((msg) => { - const qMsg = msg as QueueMessage - assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data - }) + const listener = kv.listenQueue((msg) => { + const qMsg = msg as QueueMessage + assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data + }) - await db.i_users.enqueue(data, { - idsIfUndelivered: [undeliveredId], - }) + await db.i_users.enqueue(data, { + idsIfUndelivered: [undeliveredId], + }) - await sleep(100) + await sleep(100) - const undelivered = await db.i_users.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) - }) + const undelivered = await db.i_users.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + + return async () => await listener }) + }) + + await t.step("Should enqueue message in correct topic", async () => { + await useDb(async (db) => { + const data = "data" + const undeliveredId = "undelivered" + const topic = "topic" - await t.step("Should enqueue message in correct topic", async () => { - await useDb(async (db) => { - const data = "data" - const undeliveredId = "undelivered" - const topic = "topic" + let assertion1 = false + let assertion2 = true - let assertion1 = false - let assertion2 = true + const l1 = db.i_users.listenQueue(() => assertion1 = true, { topic }) - db.i_users.listenQueue(() => assertion1 = true, { topic }) + const l2 = db.i_users.listenQueue(() => assertion2 = false) - db.i_users.listenQueue(() => assertion2 = false) + await db.i_users.enqueue("data", { + idsIfUndelivered: [undeliveredId], + topic, + }) - await db.i_users.enqueue("data", { - idsIfUndelivered: [undeliveredId], - topic, - }) + await sleep(100) - await sleep(100) + const undelivered = await db.i_users.findUndelivered(undeliveredId) + assert(assertion1 || typeof undelivered?.value === typeof data) + assert(assertion2) - const undelivered = await db.i_users.findUndelivered(undeliveredId) - assert(assertion1 || typeof undelivered?.value === typeof data) - assert(assertion2) - }) + return async () => await Promise.all([l1, l2]) }) - }, + }) }) diff --git a/tests/indexable_collection/findMany.test.ts b/tests/indexable_collection/findMany.test.ts index fb2e552..7140b6a 100644 --- a/tests/indexable_collection/findMany.test.ts +++ b/tests/indexable_collection/findMany.test.ts @@ -5,12 +5,11 @@ Deno.test("indexable_collection - findMany", async (t) => { await t.step("Should find all documents", async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.i_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.i_users.addMany(users) + assert(cr.ok) + + const { result: docs } = await db.i_users.getMany() - const docs = await db.i_users.findMany( - crs.map((cr) => cr.ok ? cr.id : ""), - ) assert(docs.length === users.length) assert( users.every((user) => @@ -23,8 +22,8 @@ Deno.test("indexable_collection - findMany", async (t) => { await t.step("Should not find any documents", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.i_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.i_users.addMany(users) + assert(cr.ok) const docs = await db.i_users.findMany(["", "", ""]) assert(docs.length === 0) diff --git a/tests/indexable_collection/forEach.test.ts b/tests/indexable_collection/forEach.test.ts index 9ca9a8b..5f9fb5a 100644 --- a/tests/indexable_collection/forEach.test.ts +++ b/tests/indexable_collection/forEach.test.ts @@ -10,8 +10,8 @@ Deno.test("indexable_collection - forEach", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.i_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.i_users.addMany(users) + assert(cr.ok) const docs: Document[] = [] await db.i_users.forEach((doc) => docs.push(doc)) diff --git a/tests/indexable_collection/getMany.test.ts b/tests/indexable_collection/getMany.test.ts index c3c2e07..2c0ef9e 100644 --- a/tests/indexable_collection/getMany.test.ts +++ b/tests/indexable_collection/getMany.test.ts @@ -6,8 +6,8 @@ Deno.test("indexable_collection - getMany", async (t) => { await t.step("Should get all documents", async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.i_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.i_users.addMany(users) + assert(cr.ok) const { result } = await db.i_users.getMany() assert(result.length === users.length) diff --git a/tests/indexable_collection/listenQueue.test.ts b/tests/indexable_collection/listenQueue.test.ts index 0b9110c..ee9a8f8 100644 --- a/tests/indexable_collection/listenQueue.test.ts +++ b/tests/indexable_collection/listenQueue.test.ts @@ -14,67 +14,67 @@ import { assert } from "../deps.ts" import { User } from "../models.ts" import { sleep, useKv } from "../utils.ts" -Deno.test({ - name: "indexable_collection - listenQueue", - sanitizeOps: false, - fn: async (t) => { - await t.step("Should receive message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "id" - - const db = kvdex(kv, { - i_users: indexableCollection(model(), { indices: {} }), - }) - - const handlerId = createHandlerId(db.i_users._keys.baseKey, undefined) - - let assertion = false - - db.i_users.listenQueue((msgData) => { - assertion = msgData === data - }) - - const msg: QueueMessage = { - __handlerId__: handlerId, - __data__: data, - } - - await kv.enqueue(msg, { - keysIfUndelivered: [ - extendKey( - [KVDEX_KEY_PREFIX], - UNDELIVERED_KEY_PREFIX, - undeliveredId, - ), - ], - }) - - await sleep(100) - - const undelivered = await db.i_users.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) +Deno.test("indexable_collection - listenQueue", async (t) => { + await t.step("Should receive message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "id" + + const db = kvdex(kv, { + i_users: indexableCollection(model(), { indices: {} }), }) - }) - await t.step("Should not receive db queue message", async () => { - await useKv(async (kv) => { - const db = kvdex(kv, { - i_users: indexableCollection(model(), { indices: {} }), - }) + const handlerId = createHandlerId(db.i_users._keys.baseKey, undefined) + + let assertion = false + + const listener = db.i_users.listenQueue((msgData) => { + assertion = msgData === data + }) + + const msg: QueueMessage = { + __handlerId__: handlerId, + __data__: data, + } - let assertion = true + await kv.enqueue(msg, { + keysIfUndelivered: [ + extendKey( + [KVDEX_KEY_PREFIX], + UNDELIVERED_KEY_PREFIX, + undeliveredId, + ), + ], + }) - db.i_users.listenQueue(() => { - assertion = false - }) + await sleep(100) - await db.enqueue("data") + const undelivered = await db.i_users.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) - await sleep(100) + return async () => await listener + }) + }) - assert(assertion) + await t.step("Should not receive db queue message", async () => { + await useKv(async (kv) => { + const db = kvdex(kv, { + i_users: indexableCollection(model(), { indices: {} }), }) + + let assertion = true + + const listener = db.i_users.listenQueue(() => { + assertion = false + }) + + await db.enqueue("data") + + await sleep(100) + + assert(assertion) + + return async () => await listener }) - }, + }) }) diff --git a/tests/indexable_collection/map.test.ts b/tests/indexable_collection/map.test.ts index 0278215..6609c20 100644 --- a/tests/indexable_collection/map.test.ts +++ b/tests/indexable_collection/map.test.ts @@ -8,8 +8,8 @@ Deno.test("indexable_collection - map", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.i_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.i_users.addMany(users) + assert(cr.ok) const { result } = await db.i_users.map((doc) => doc.value.username) diff --git a/tests/indexable_collection/properties.test.ts b/tests/indexable_collection/properties.test.ts index b408e2a..53da877 100644 --- a/tests/indexable_collection/properties.test.ts +++ b/tests/indexable_collection/properties.test.ts @@ -55,8 +55,8 @@ Deno.test("indexable_collection - properties", async (t) => { await t.step("Should select using pagination", async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.i_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.i_users.addMany(users) + assert(cr.ok) const selected: Document[] = [] let cursor: string | undefined = undefined @@ -81,9 +81,9 @@ Deno.test("indexable_collection - properties", async (t) => { await t.step("Should select filtered", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.i_users.addMany(users) + const cr = await db.i_users.addMany(users) const count1 = await db.i_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const sliced = users.slice(5, 7) @@ -107,9 +107,9 @@ Deno.test("indexable_collection - properties", async (t) => { await t.step("Should select in reverse", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.i_users.addMany(users) + const cr = await db.i_users.addMany(users) const count1 = await db.i_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const query1 = await db.i_users.getMany() @@ -125,9 +125,9 @@ Deno.test("indexable_collection - properties", async (t) => { await t.step("Should select from start id", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.i_users.addMany(users) + const cr = await db.i_users.addMany(users) const count1 = await db.i_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const index = 5 @@ -149,9 +149,9 @@ Deno.test("indexable_collection - properties", async (t) => { await t.step("Should select until end id", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.i_users.addMany(users) + const cr = await db.i_users.addMany(users) const count1 = await db.i_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const index = 5 @@ -173,9 +173,9 @@ Deno.test("indexable_collection - properties", async (t) => { await t.step("Should select from start id to end id", async () => { await useDb(async (db) => { const users = generateUsers(10) - const crs = await db.i_users.addMany(users) + const cr = await db.i_users.addMany(users) const count1 = await db.i_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const index1 = 5 diff --git a/tests/indexable_collection/updateBySecondaryIndex.test.ts b/tests/indexable_collection/updateBySecondaryIndex.test.ts index d8eddfd..b21ef1a 100644 --- a/tests/indexable_collection/updateBySecondaryIndex.test.ts +++ b/tests/indexable_collection/updateBySecondaryIndex.test.ts @@ -49,8 +49,8 @@ Deno.test("indexable_collection - updateBySecondaryIndex", async (t) => { await useDb(async (db) => { let assertion = true - const crs = await db.zi_users.addMany([mockUser1, mockUser2]) - assert(crs.every((cr) => cr.ok)) + const cr = await db.zi_users.addMany([mockUser1, mockUser2]) + assert(cr.ok) await db.zi_users.updateBySecondaryIndex("age", mockUser1.age, mockUser1) .catch(() => assertion = false) @@ -63,8 +63,8 @@ Deno.test("indexable_collection - updateBySecondaryIndex", async (t) => { await useDb(async (db) => { let assertion = false - const crs = await db.zi_users.addMany([mockUser1, mockUser2]) - assert(crs.every((cr) => cr.ok)) + const cr = await db.zi_users.addMany([mockUser1, mockUser2]) + assert(cr.ok) await db.zi_users.updateBySecondaryIndex( "age", diff --git a/tests/indexable_collection/updateMany.test.ts b/tests/indexable_collection/updateMany.test.ts index 960475d..e3e1cc9 100644 --- a/tests/indexable_collection/updateMany.test.ts +++ b/tests/indexable_collection/updateMany.test.ts @@ -1,7 +1,5 @@ -import { CommitResult } from "../../mod.ts" import { assert } from "../deps.ts" import { mockUser1, mockUserInvalid } from "../mocks.ts" -import { User } from "../models.ts" import { generateUsers, useDb } from "../utils.ts" Deno.test("indexable_collection - updateMany", async (t) => { @@ -10,12 +8,12 @@ Deno.test("indexable_collection - updateMany", async (t) => { async () => { await useDb(async (db) => { const users = generateUsers(1_000) - const crs = await db.i_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.i_users.addMany(users) + assert(cr.ok) - const okCrs = crs.filter((cr) => cr.ok) as CommitResult[] - const ids = okCrs.map((cr) => cr.id) - const versionstamps = okCrs.map((cr) => cr.versionstamp) + const { result: docs } = await db.i_users.getMany() + const ids = docs.map((doc) => doc.id) + const versionstamps = docs.map((doc) => doc.versionstamp) const updateData = { address: { @@ -48,8 +46,8 @@ Deno.test("indexable_collection - updateMany", async (t) => { const users = generateUsers(10) let assertion = true - const crs = await db.zi_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.zi_users.addMany(users) + assert(cr.ok) await db.zi_users.updateMany(mockUser1).catch(() => assertion = false) @@ -62,8 +60,8 @@ Deno.test("indexable_collection - updateMany", async (t) => { const users = generateUsers(10) let assertion = false - const crs = await db.zi_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.zi_users.addMany(users) + assert(cr.ok) await db.zi_users.updateMany(mockUserInvalid).catch(() => assertion = true diff --git a/tests/large_collection/addMany.test.ts b/tests/large_collection/addMany.test.ts index bc9fa06..ff808d4 100644 --- a/tests/large_collection/addMany.test.ts +++ b/tests/large_collection/addMany.test.ts @@ -7,12 +7,11 @@ Deno.test("large_collection - addMany", async (t) => { async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) - const crs = await db.l_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.l_users.addMany(users) + assert(cr.ok) + + const { result: docs } = await db.l_users.getMany() - const docs = await db.l_users.findMany( - crs.map((cr) => cr.ok ? cr.id : ""), - ) assert(docs.length === users.length) assert( users.every((user) => @@ -28,12 +27,11 @@ Deno.test("large_collection - addMany", async (t) => { async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) - const crs = await db.zl_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.zl_users.addMany(users) + assert(cr.ok) + + const { result: docs } = await db.zl_users.getMany() - const docs = await db.zl_users.findMany( - crs.map((cr) => cr.ok ? cr.id : ""), - ) assert(docs.length === users.length) assert( users.every((user) => diff --git a/tests/large_collection/count.test.ts b/tests/large_collection/count.test.ts index 904cf4e..e135556 100644 --- a/tests/large_collection/count.test.ts +++ b/tests/large_collection/count.test.ts @@ -11,8 +11,8 @@ Deno.test("large_collection - count", async (t) => { assert(count1 === 0) const users = generateLargeUsers(1_000) - const crs = await db.l_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.l_users.addMany(users) + assert(cr.ok) const count2 = await db.l_users.count() assert(count2 === users.length) diff --git a/tests/large_collection/delete.test.ts b/tests/large_collection/delete.test.ts index 9abb0b1..c2a6c58 100644 --- a/tests/large_collection/delete.test.ts +++ b/tests/large_collection/delete.test.ts @@ -29,13 +29,15 @@ Deno.test("large_collection - delete", async (t) => { async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) - const crs = await db.l_users.addMany(users) + const cr = await db.l_users.addMany(users) const count1 = await db.l_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) - await db.l_users.delete(...crs.map((cr) => cr.ok ? cr.id : "")) + const { result: ids } = await db.l_users.map((doc) => doc.id) + + await db.l_users.delete(...ids) const count2 = await db.l_users.count() assert(count2 === 0) diff --git a/tests/large_collection/deleteMany.test.ts b/tests/large_collection/deleteMany.test.ts index c46b3ba..5b39e6b 100644 --- a/tests/large_collection/deleteMany.test.ts +++ b/tests/large_collection/deleteMany.test.ts @@ -5,8 +5,8 @@ Deno.test("large_collection - deleteMany", async (t) => { await t.step("Should delete all documents from the collection", async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) - const crs = await db.l_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.l_users.addMany(users) + assert(cr.ok) const count1 = await db.l_users.count() assert(count1 === users.length) diff --git a/tests/large_collection/enqueue.test.ts b/tests/large_collection/enqueue.test.ts index c337db4..d26fbaa 100644 --- a/tests/large_collection/enqueue.test.ts +++ b/tests/large_collection/enqueue.test.ts @@ -10,63 +10,63 @@ import { assert } from "../deps.ts" import { User } from "../models.ts" import { sleep, useDb, useKv } from "../utils.ts" -Deno.test({ - name: "large_collection - enqueue", - sanitizeOps: false, - fn: async (t) => { - await t.step("Should enqueue message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "undelivered" +Deno.test("large_collection - enqueue", async (t) => { + await t.step("Should enqueue message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "undelivered" - const db = kvdex(kv, { - l_users: largeCollection(model()), - }) + const db = kvdex(kv, { + l_users: largeCollection(model()), + }) - const handlerId = createHandlerId(db.l_users._keys.baseKey, undefined) + const handlerId = createHandlerId(db.l_users._keys.baseKey, undefined) - let assertion = false + let assertion = false - kv.listenQueue((msg) => { - const qMsg = msg as QueueMessage - assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data - }) + const listener = kv.listenQueue((msg) => { + const qMsg = msg as QueueMessage + assertion = qMsg.__handlerId__ === handlerId && qMsg.__data__ === data + }) - await db.l_users.enqueue(data, { - idsIfUndelivered: [undeliveredId], - }) + await db.l_users.enqueue(data, { + idsIfUndelivered: [undeliveredId], + }) - await sleep(100) + await sleep(100) - const undelivered = await db.l_users.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) - }) + const undelivered = await db.l_users.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) + + return async () => await listener }) + }) + + await t.step("Should enqueue message in correct topic", async () => { + await useDb(async (db) => { + const data = "data" + const undeliveredId = "undelivered" + const topic = "topic" - await t.step("Should enqueue message in correct topic", async () => { - await useDb(async (db) => { - const data = "data" - const undeliveredId = "undelivered" - const topic = "topic" + let assertion1 = false + let assertion2 = true - let assertion1 = false - let assertion2 = true + const l1 = db.l_users.listenQueue(() => assertion1 = true, { topic }) - db.l_users.listenQueue(() => assertion1 = true, { topic }) + const l2 = db.l_users.listenQueue(() => assertion2 = false) - db.l_users.listenQueue(() => assertion2 = false) + await db.l_users.enqueue("data", { + idsIfUndelivered: [undeliveredId], + topic, + }) - await db.l_users.enqueue("data", { - idsIfUndelivered: [undeliveredId], - topic, - }) + await sleep(100) - await sleep(100) + const undelivered = await db.l_users.findUndelivered(undeliveredId) + assert(assertion1 || typeof undelivered?.value === typeof data) + assert(assertion2) - const undelivered = await db.l_users.findUndelivered(undeliveredId) - assert(assertion1 || typeof undelivered?.value === typeof data) - assert(assertion2) - }) + return async () => await Promise.all([l1, l2]) }) - }, + }) }) diff --git a/tests/large_collection/findMany.test.ts b/tests/large_collection/findMany.test.ts index 003dc62..5743832 100644 --- a/tests/large_collection/findMany.test.ts +++ b/tests/large_collection/findMany.test.ts @@ -5,12 +5,11 @@ Deno.test("large_collection - findMany", async (t) => { await t.step("Should find all documents", async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) - const crs = await db.l_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.l_users.addMany(users) + assert(cr.ok) + + const { result: docs } = await db.l_users.getMany() - const docs = await db.l_users.findMany( - crs.map((cr) => cr.ok ? cr.id : ""), - ) assert(docs.length === users.length) assert( users.every((user) => @@ -23,8 +22,8 @@ Deno.test("large_collection - findMany", async (t) => { await t.step("Should not find any documents", async () => { await useDb(async (db) => { const users = generateLargeUsers(10) - const crs = await db.l_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.l_users.addMany(users) + assert(cr.ok) const docs = await db.l_users.findMany(["", "", ""]) assert(docs.length === 0) diff --git a/tests/large_collection/forEach.test.ts b/tests/large_collection/forEach.test.ts index ee70616..c6cd1a1 100644 --- a/tests/large_collection/forEach.test.ts +++ b/tests/large_collection/forEach.test.ts @@ -10,8 +10,8 @@ Deno.test("large_collection - forEach", async (t) => { async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) - const crs = await db.l_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.l_users.addMany(users) + assert(cr.ok) const docs: Document[] = [] await db.l_users.forEach((doc) => docs.push(doc)) diff --git a/tests/large_collection/getMany.test.ts b/tests/large_collection/getMany.test.ts index bba7328..54686d0 100644 --- a/tests/large_collection/getMany.test.ts +++ b/tests/large_collection/getMany.test.ts @@ -6,8 +6,8 @@ Deno.test("large_collection - getMany", async (t) => { await t.step("Should get all documents", async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) - const crs = await db.l_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.l_users.addMany(users) + assert(cr.ok) const { result } = await db.l_users.getMany() assert(result.length === users.length) diff --git a/tests/large_collection/listenQueue.test.ts b/tests/large_collection/listenQueue.test.ts index bdd4272..3c8d7ce 100644 --- a/tests/large_collection/listenQueue.test.ts +++ b/tests/large_collection/listenQueue.test.ts @@ -14,67 +14,67 @@ import { assert } from "../deps.ts" import { User } from "../models.ts" import { sleep, useKv } from "../utils.ts" -Deno.test({ - name: "large_collection - listenQueue", - sanitizeOps: false, - fn: async (t) => { - await t.step("Should receive message with string data", async () => { - await useKv(async (kv) => { - const data = "data" - const undeliveredId = "id" - - const db = kvdex(kv, { - l_users: largeCollection(model()), - }) - - let assertion = false - - db.l_users.listenQueue((msgData) => { - assertion = msgData === data - }) - - const handlerId = createHandlerId(db.l_users._keys.baseKey, undefined) - - const msg: QueueMessage = { - __handlerId__: handlerId, - __data__: data, - } - - await kv.enqueue(msg, { - keysIfUndelivered: [ - extendKey( - [KVDEX_KEY_PREFIX], - UNDELIVERED_KEY_PREFIX, - undeliveredId, - ), - ], - }) - - await sleep(100) - - const undelivered = await db.l_users.findUndelivered(undeliveredId) - assert(assertion || typeof undelivered?.value === typeof data) +Deno.test("large_collection - listenQueue", async (t) => { + await t.step("Should receive message with string data", async () => { + await useKv(async (kv) => { + const data = "data" + const undeliveredId = "id" + + const db = kvdex(kv, { + l_users: largeCollection(model()), }) - }) - await t.step("Should not receive db queue message", async () => { - await useKv(async (kv) => { - const db = kvdex(kv, { - l_users: largeCollection(model()), - }) + let assertion = false + + const listener = db.l_users.listenQueue((msgData) => { + assertion = msgData === data + }) + + const handlerId = createHandlerId(db.l_users._keys.baseKey, undefined) + + const msg: QueueMessage = { + __handlerId__: handlerId, + __data__: data, + } - let assertion = true + await kv.enqueue(msg, { + keysIfUndelivered: [ + extendKey( + [KVDEX_KEY_PREFIX], + UNDELIVERED_KEY_PREFIX, + undeliveredId, + ), + ], + }) - db.l_users.listenQueue(() => { - assertion = false - }) + await sleep(100) - await db.enqueue("data") + const undelivered = await db.l_users.findUndelivered(undeliveredId) + assert(assertion || typeof undelivered?.value === typeof data) - await sleep(100) + return async () => await listener + }) + }) - assert(assertion) + await t.step("Should not receive db queue message", async () => { + await useKv(async (kv) => { + const db = kvdex(kv, { + l_users: largeCollection(model()), }) + + let assertion = true + + const listener = db.l_users.listenQueue(() => { + assertion = false + }) + + await db.enqueue("data") + + await sleep(100) + + assert(assertion) + + return async () => await listener }) - }, + }) }) diff --git a/tests/large_collection/map.test.ts b/tests/large_collection/map.test.ts index 1a56fb3..9d10d7a 100644 --- a/tests/large_collection/map.test.ts +++ b/tests/large_collection/map.test.ts @@ -8,8 +8,8 @@ Deno.test("large_collection - map", async (t) => { async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) - const crs = await db.l_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.l_users.addMany(users) + assert(cr.ok) const { result } = await db.l_users.map((doc) => doc.value.username) diff --git a/tests/large_collection/properties.test.ts b/tests/large_collection/properties.test.ts index 575c274..d1f8617 100644 --- a/tests/large_collection/properties.test.ts +++ b/tests/large_collection/properties.test.ts @@ -46,8 +46,8 @@ Deno.test("large_collection - properties", async (t) => { await t.step("Should select using pagination", async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) - const crs = await db.l_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.l_users.addMany(users) + assert(cr.ok) const selected: Document[] = [] let cursor: string | undefined = undefined @@ -72,9 +72,9 @@ Deno.test("large_collection - properties", async (t) => { await t.step("Should select filtered", async () => { await useDb(async (db) => { const users = generateLargeUsers(10) - const crs = await db.l_users.addMany(users) + const cr = await db.l_users.addMany(users) const count1 = await db.l_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const sliced = users.slice(5, 7) @@ -98,9 +98,9 @@ Deno.test("large_collection - properties", async (t) => { await t.step("Should select in reverse", async () => { await useDb(async (db) => { const users = generateLargeUsers(10) - const crs = await db.l_users.addMany(users) + const cr = await db.l_users.addMany(users) const count1 = await db.l_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const query1 = await db.l_users.getMany() @@ -116,9 +116,9 @@ Deno.test("large_collection - properties", async (t) => { await t.step("Should select from start id", async () => { await useDb(async (db) => { const users = generateLargeUsers(10) - const crs = await db.l_users.addMany(users) + const cr = await db.l_users.addMany(users) const count1 = await db.l_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const index = 5 @@ -140,9 +140,9 @@ Deno.test("large_collection - properties", async (t) => { await t.step("Should select until end id", async () => { await useDb(async (db) => { const users = generateLargeUsers(10) - const crs = await db.l_users.addMany(users) + const cr = await db.l_users.addMany(users) const count1 = await db.l_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const index = 5 @@ -164,9 +164,9 @@ Deno.test("large_collection - properties", async (t) => { await t.step("Should select from start id to end id", async () => { await useDb(async (db) => { const users = generateLargeUsers(10) - const crs = await db.l_users.addMany(users) + const cr = await db.l_users.addMany(users) const count1 = await db.l_users.count() - assert(crs.every((cr) => cr.ok)) + assert(cr.ok) assert(count1 === users.length) const index1 = 5 diff --git a/tests/large_collection/updateMany.test.ts b/tests/large_collection/updateMany.test.ts index 6d09460..11b6b5a 100644 --- a/tests/large_collection/updateMany.test.ts +++ b/tests/large_collection/updateMany.test.ts @@ -1,7 +1,6 @@ -import { collection, CommitResult, kvdex, model } from "../../mod.ts" +import { collection, kvdex, model } from "../../mod.ts" import { assert } from "../deps.ts" import { mockUser1, mockUserInvalid } from "../mocks.ts" -import { User } from "../models.ts" import { generateLargeUsers, generateNumbers, useDb, useKv } from "../utils.ts" Deno.test("large_collection - updateMany", async (t) => { @@ -10,12 +9,12 @@ Deno.test("large_collection - updateMany", async (t) => { async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) - const crs = await db.l_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.l_users.addMany(users) + assert(cr.ok) - const okCrs = crs.filter((cr) => cr.ok) as CommitResult[] - const ids = okCrs.map((cr) => cr.id) - const versionstamps = okCrs.map((cr) => cr.versionstamp) + const { result: docs } = await db.l_users.getMany() + const ids = docs.map((doc) => doc.id) + const versionstamps = docs.map((doc) => doc.versionstamp) const updateData = { address: { @@ -65,20 +64,25 @@ Deno.test("large_collection - updateMany", async (t) => { dates.push(new Date("2000-01-01")) } - const crs1 = await db.numbers.addMany(numbers) as CommitResult[] - const crs2 = await db.arrays.addMany(arrays) as CommitResult[] - const crs3 = await db.dates.addMany(dates) as CommitResult[] + const cr1 = await db.numbers.addMany(numbers) + const cr2 = await db.arrays.addMany(arrays) + const cr3 = await db.dates.addMany(dates) - assert(crs1.every((cr) => cr.ok)) - assert(crs2.every((cr) => cr.ok)) - assert(crs3.every((cr) => cr.ok)) + assert(cr1.ok) + assert(cr2.ok) + assert(cr3.ok) - const ids1 = crs1.map((cr) => cr.id) - const versionstamps1 = crs1.map((cr) => cr.versionstamp) - const ids2 = crs2.map((cr) => cr.id) - const versionstamps2 = crs2.map((cr) => cr.versionstamp) - const ids3 = crs3.map((cr) => cr.id) - const versionstamps3 = crs3.map((cr) => cr.versionstamp) + const { result: docs1 } = await db.numbers.getMany() + const { result: docs2 } = await db.arrays.getMany() + const { result: docs3 } = await db.dates.getMany() + + const ids1 = docs1.map((doc) => doc.id) + const ids2 = docs2.map((doc) => doc.id) + const ids3 = docs3.map((doc) => doc.id) + + const versionstamps1 = docs1.map((doc) => doc.versionstamp) + const versionstamps2 = docs2.map((doc) => doc.versionstamp) + const versionstamps3 = docs3.map((doc) => doc.versionstamp) const val1 = 20 const val2 = ["4"] @@ -118,8 +122,8 @@ Deno.test("large_collection - updateMany", async (t) => { const users = generateLargeUsers(10) let assertion = true - const crs = await db.zl_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.zl_users.addMany(users) + assert(cr.ok) await db.zl_users.updateMany(mockUser1).catch(() => assertion = false) @@ -132,8 +136,8 @@ Deno.test("large_collection - updateMany", async (t) => { const users = generateLargeUsers(10) let assertion = false - const crs = await db.zl_users.addMany(users) - assert(crs.every((cr) => cr.ok)) + const cr = await db.zl_users.addMany(users) + assert(cr.ok) await db.zl_users.updateMany(mockUserInvalid).catch(() => assertion = true diff --git a/tests/utils.ts b/tests/utils.ts index 59569e6..7bf923a 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -44,21 +44,29 @@ export function createDb(kv: Deno.Kv) { } // Temporary use functions -export async function useKv(fn: (kv: Deno.Kv) => unknown) { +export async function useKv( + fn: (kv: Deno.Kv) => unknown, +) { const kv = await Deno.openKv(":memory:") - await fn(kv) + const result = await fn(kv) kv.close() + + if (typeof result === "function") { + await result() + } } -export async function useDb(fn: (db: ReturnType) => unknown) { +export async function useDb( + fn: (db: ReturnType) => unknown, +) { await useKv(async (kv) => { const db = createDb(kv) - await fn(db) + return await fn(db) }) } // Generator functions -export function generateUsers(n: number) { +export function generateLargeUsers(n: number) { const users: User[] = [] let country = "" @@ -83,7 +91,7 @@ export function generateUsers(n: number) { return users } -export function generateLargeUsers(n: number) { +export function generateUsers(n: number) { const users: User[] = [] for (let i = 0; i < n; i++) { @@ -106,11 +114,6 @@ export function generateLargeUsers(n: number) { export function generateInvalidUsers(n: number) { const users: User[] = [] - let country = "" - for (let i = 0; i < 50_000; i++) { - country += "A" - } - for (let i = 0; i < n; i++) { users.push({ username: 100, From b28f518bc42c7cb2937d9c4a4805e324a189791f Mon Sep 17 00:00:00 2001 From: oliver-oloughlin Date: Fri, 3 Nov 2023 16:42:04 +0100 Subject: [PATCH 3/3] feat: improved performance of collection.count() --- src/collection.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/collection.ts b/src/collection.ts index b3758f2..5bd833a 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -588,8 +588,19 @@ export class Collection< * @returns A promise that resolves to a number representing the performed count. */ async count(options?: CountOptions) { - // Initiate count variable, increment for each document entry, return result + // Initiate count result let result = 0 + + // Perform efficient count if counting all document entries + if (selectsAll(options)) { + const iter = this.kv.list({ prefix: this._keys.idKey }, options) + for await (const _ of iter) { + result++ + } + return result + } + + // Perform count using many documents handler await this.handleMany(this._keys.idKey, () => result++, options) return result }