Skip to content

Commit

Permalink
keyv - feat: adding in support for many with setMany and set (#1314)
Browse files Browse the repository at this point in the history
* keyv - feat: adding in support for many with setMany and set

* updating setMany with redis and keyv

* Update index.ts

* adding in support for when setMany is not on the storage adapter

* fixing types

* lint fix on array

* updating eslint

* fixing Generic Storage Adapter

* adding in error checks

* adding in set with Array
  • Loading branch information
jaredwray authored Feb 25, 2025
1 parent 9ad8992 commit 90ccb19
Show file tree
Hide file tree
Showing 6 changed files with 181 additions and 17 deletions.
3 changes: 2 additions & 1 deletion packages/keyv/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,10 @@
"@keyv/serialize": "workspace:^"
},
"devDependencies": {
"@faker-js/faker": "^9.5.0",
"@keyv/compress-brotli": "workspace:^",
"@keyv/compress-lz4": "workspace:^",
"@keyv/compress-gzip": "workspace:^",
"@keyv/compress-lz4": "workspace:^",
"@keyv/memcache": "workspace:^",
"@keyv/mongo": "workspace:^",
"@keyv/sqlite": "workspace:^",
Expand Down
36 changes: 31 additions & 5 deletions packages/keyv/src/generic-store.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import EventManager from './event-manager.js';
import {type KeyvStoreAdapter, type StoredData} from './index.js';
import {
Keyv, type KeyvStoreAdapter, type KeyvEntry, type StoredData,
} from './index.js';

export type KeyvGenericStoreOptions = {
namespace?: string | (() => string);
Expand Down Expand Up @@ -121,13 +123,25 @@ export class KeyvGenericStore extends EventManager implements KeyvStoreAdapter {
return undefined;
}

return data.value as T;
return data as T;
}

async set(key: string, value: any, ttl?: number) {
async set(key: string, value: any, ttl?: number): Promise<boolean> {
const keyPrefix = this.getKeyPrefix(key, this.getNamespace());
const data = {value, expires: ttl ? Date.now() + ttl : undefined};
this._store.set(keyPrefix, data, ttl);
return true;
}

async setMany(entries: KeyvEntry[]): Promise<boolean[]> {
const results: boolean[] = [];
for (const entry of entries) {
// eslint-disable-next-line no-await-in-loop
const result = await this.set(entry.key, entry.value, entry.ttl);
results.push(result);
}

return results;
}

async delete(key: string): Promise<boolean> {
Expand All @@ -149,10 +163,10 @@ export class KeyvGenericStore extends EventManager implements KeyvStoreAdapter {
for (const key of keys) {
// eslint-disable-next-line no-await-in-loop
const value = await this.get(key);
values.push(value as T);
values.push(value);
}

return values;
return values as Array<StoredData<T | undefined>>;
}

async deleteMany(keys: string[]): Promise<boolean> {
Expand All @@ -174,3 +188,15 @@ export class KeyvGenericStore extends EventManager implements KeyvStoreAdapter {
throw new Error('Method not implemented.');
}
}

/**
* Create a Keyv instance with a generic store that is optimized for in-memory storage.
* This removes Keyv serialization and deserialization overhead, keyPrefix from Keyv.
*/
export function createKeyv(store: Map<any, any> | KeyvMapType, options?: KeyvGenericStoreOptions) {
const genericStore = new KeyvGenericStore(store, options);
const keyv = new Keyv({store: genericStore, useKeyPrefix: false});
keyv.serialize = undefined;
keyv.deserialize = undefined;
return keyv;
}
51 changes: 49 additions & 2 deletions packages/keyv/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,21 @@ export enum KeyvHooks {
POST_DELETE = 'postDelete',
}

export type KeyvEntry = {
/**
* Key to set.
*/
key: string;
/**
* Value to set.
*/
value: any;
/**
* Time to live in milliseconds.
*/
ttl?: number;
};

export type StoredDataNoRaw<Value> = Value | undefined;

export type StoredDataRaw<Value> = DeserializedData<Value> | undefined;
Expand All @@ -55,6 +70,7 @@ export type KeyvStoreAdapter = {
): Promise<Array<StoredData<Value | undefined>>>;
disconnect?(): Promise<void>;
deleteMany?(key: string[]): Promise<boolean>;
setMany?(data: KeyvEntry[]): Promise<boolean[]>;
iterator?<Value>(namespace?: string): AsyncGenerator<Array<string | Awaited<Value> | undefined>, void>;
} & IEventEmitter;

Expand Down Expand Up @@ -523,12 +539,18 @@ export class Keyv<GenericValue = any> extends EventManager {

/**
* Set an item to the store
* @param {string} key the key to use
* @param {string | Array<KeyvEntry>} key the key to use. If you pass in an array of KeyvEntry it will set many items
* @param {Value} value the value of the key
* @param {number} [ttl] time to live in milliseconds
* @returns {boolean} if it sets then it will return a true. On failure will return false.
*/
async set<Value = GenericValue>(key: string, value: Value, ttl?: number): Promise<boolean> {
async set<Value = GenericValue>(key: KeyvEntry[]): Promise<boolean[]>;
async set<Value = GenericValue>(key: string, value: Value, ttl?: number): Promise<boolean>;
async set<Value = GenericValue>(key: string | KeyvEntry[], value?: Value, ttl?: number): Promise<boolean | boolean[]> {
if (Array.isArray(key)) {
return this.setMany(key);
}

this.hooks.trigger(KeyvHooks.PRE_SET, {key, value, ttl});
const keyPrefixed = this._getKeyPrefix(key);
if (ttl === undefined) {
Expand Down Expand Up @@ -569,6 +591,31 @@ export class Keyv<GenericValue = any> extends EventManager {
return result;
}

async setMany<Value = GenericValue>(entries: KeyvEntry[]): Promise<boolean[]> {
let results: boolean[] = [];

try {
// If the store has a setMany method then use it
if (this._store.setMany !== undefined) {
results = await this._store.setMany(entries);
return results;
}

const promises: Array<Promise<boolean>> = [];
for (const entry of entries) {
promises.push(this.set(entry.key, entry.value, entry.ttl));
}

const promiseResults = await Promise.allSettled(promises);
results = promiseResults.map(result => (result as PromiseFulfilledResult<any>).value);
} catch (error) {
this.emit('error', error);
results = entries.map(() => false);
}

return results;
}

/**
* Delete an Entry
* @param {string | string[]} key the key to be deleted. if an array it will delete many items
Expand Down
19 changes: 14 additions & 5 deletions packages/keyv/test/generic-store.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,16 @@ describe('Keyv Generic set / get / has Operations', () => {
const store = new Map();
const keyv = new KeyvGenericStore(store);
await keyv.set('key1', 'value1');
expect(await keyv.get('key1')).toBe('value1');
expect(await keyv.get('key1')).toStrictEqual({value: 'value1', expires: undefined});
});

test('should set many keys', async () => {
const store = new Map();
const keyv = new KeyvGenericStore(store);
const result = await keyv.setMany([{key: 'key1', value: 'value1'}, {key: 'key2', value: 'value2'}]);
expect(await keyv.get('key1')).toStrictEqual({expires: undefined, value: 'value1'});
expect(await keyv.get('key2')).toStrictEqual({expires: undefined, value: 'value2'});
expect(result.length).toBe(2);
});

test('should get undefined for a non-existent key', async () => {
Expand Down Expand Up @@ -116,9 +125,9 @@ describe('Keyv Generic set / get / has Operations', () => {
await keyv.set('key2', 'value2');
await keyv.set('key3', 'value3');
const values = await keyv.getMany(['key1', 'key2', 'key3', 'key4']);
expect(values[0] as string).toBe('value1');
expect(values[1] as string).toBe('value2');
expect(values[2] as string).toBe('value3');
expect(values[0]).toStrictEqual({value: 'value1', expires: undefined});
expect(values[1]).toStrictEqual({value: 'value2', expires: undefined});
expect(values[2]).toStrictEqual({value: 'value3', expires: undefined});
expect(values[3]).toBe(undefined);
});
});
Expand Down Expand Up @@ -149,7 +158,7 @@ describe('Keyv Generic Delete / Clear Operations', () => {
await keyv.deleteMany(['key1', 'key2']);
expect(await keyv.get('key1')).toBe(undefined);
expect(await keyv.get('key2')).toBe(undefined);
expect(await keyv.get('key3')).toBe('value3');
expect(await keyv.get('key3')).toStrictEqual({value: 'value3', expires: undefined});
});

test('should emit error on delete many keys', async () => {
Expand Down
78 changes: 78 additions & 0 deletions packages/keyv/test/keyv-test.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import {
describe, test, expect, beforeEach,
} from 'vitest';
import {faker} from '@faker-js/faker';
import {Keyv} from '../src/index.js';
import {createKeyv} from '../src/generic-store.js';

describe('Keyv', async () => {
type TestData = {
key: string;
value: string;
};

let testData: TestData[] = [];

beforeEach(() => {
testData = [];
for (let i = 0; i < 5; i++) {
testData.push({
key: faker.string.alphanumeric(10),
value: faker.string.alphanumeric(10),
});
}
});

describe('setMany', async () => {
test('the function exists', async () => {
const keyv = new Keyv();
expect(keyv.setMany).toBeDefined();
});

test('returns a promise that is empty if nothing is sent in', async () => {
const keyv = new Keyv();
const result = await keyv.setMany([]);
expect(result.length).toEqual(0);
});

test('returns multiple responses on in memory storage', async () => {
const keyv = new Keyv();
const result = await keyv.setMany(testData);
expect(result.length).toEqual(testData.length);
const resultValue = await keyv.get(testData[0].key);
expect(resultValue).toEqual(testData[0].value);
});

test('should use the store to set multiple keys', async () => {
const map = new Map();
const keyv = createKeyv(map);
const result = await keyv.setMany(testData);
expect(result.length).toEqual(testData.length);
const resultValue = await keyv.get(testData[0].key);
expect(resultValue).toEqual(testData[0].value);
});

test('should emit and return false on error', async () => {
const map = new Map();
map.set = () => {
throw new Error('Test Error');
};

const keyv = createKeyv(map);
let errorEmitted = false;
keyv.on('error', () => {
errorEmitted = true;
});

const result = await keyv.setMany(testData);
expect(result).toEqual([false, false, false, false, false]);
expect(errorEmitted).toBe(true);
});

test('should set many items on keyv set function with array', async () => {
const keyv = createKeyv(new Map());
const result = await keyv.set(testData);
expect(result.length).toBe(5);
});
});
});
11 changes: 7 additions & 4 deletions packages/redis/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
type RedisFunctions,
type RedisScripts,
} from 'redis';
import {Keyv, type KeyvStoreAdapter} from 'keyv';
import {Keyv, type KeyvStoreAdapter, type KeyvEntry} from 'keyv';
import calculateSlot from 'cluster-key-slot';

export type KeyvRedisOptions = {
Expand Down Expand Up @@ -245,22 +245,25 @@ export default class KeyvRedis<T> extends EventEmitter implements KeyvStoreAdapt

/**
* Will set many key value pairs in the store. TTL is in milliseconds. This will be done as a single transaction.
* @param {Array<KeyvRedisEntry<string>>} entries - the key value pairs to set with optional ttl
* @param {KeyvEntry[]} entries - the key value pairs to set with optional ttl
*/
public async setMany(entries: Array<KeyvRedisEntry<string>>): Promise<void> {
public async setMany(entries: KeyvEntry[]): Promise<boolean[]> {
const client = await this.getClient();
const multi = client.multi();
for (const {key, value, ttl} of entries) {
const prefixedKey = this.createKeyPrefix(key, this._namespace);
if (ttl) {
// eslint-disable-next-line @typescript-eslint/naming-convention
// eslint-disable-next-line @typescript-eslint/naming-convention, @typescript-eslint/no-unsafe-argument
multi.set(prefixedKey, value, {PX: ttl});
} else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
multi.set(prefixedKey, value);
}
}

await multi.exec();

return entries.map(() => true);
}

/**
Expand Down

0 comments on commit 90ccb19

Please sign in to comment.