From 0aa7860d1172a6b9e10a40a9ba00b6365127b85a Mon Sep 17 00:00:00 2001 From: elmarti Date: Mon, 1 May 2023 19:26:12 +0100 Subject: [PATCH 1/4] :boom: indexeddb indexes IndexedDB requires a bit more configuration than the current implementation of an array of strings - this has been updated to use the fields unique, name, column and type. Type is optional as there's no specification for this at present, but as we move onto the other adapters, more configuration is needed --- src/interfaces/collection-config.interface.ts | 3 +- src/interfaces/index-config.interface.ts | 6 ++++ .../__tests__/indexeddb-persistence.test.ts | 33 ++++++++++++++++--- .../indexeddb/indexeddb-persistence.ts | 13 ++++++-- 4 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 src/interfaces/index-config.interface.ts diff --git a/src/interfaces/collection-config.interface.ts b/src/interfaces/collection-config.interface.ts index 2259799..e1ca74c 100644 --- a/src/interfaces/collection-config.interface.ts +++ b/src/interfaces/collection-config.interface.ts @@ -1,6 +1,7 @@ import { IColumnConfig } from './column-config.interface'; +import { IIndexConfig } from './index-config.interface'; export interface ICollectionConfig { columns: Array; - indexes: Array; + indexes: Array; } \ No newline at end of file diff --git a/src/interfaces/index-config.interface.ts b/src/interfaces/index-config.interface.ts new file mode 100644 index 0000000..2148300 --- /dev/null +++ b/src/interfaces/index-config.interface.ts @@ -0,0 +1,6 @@ +export interface IIndexConfig { + column: string; + type?: string;//tbc + unique: boolean; + name: string; +} \ No newline at end of file diff --git a/src/modules/persistence/indexeddb/__tests__/indexeddb-persistence.test.ts b/src/modules/persistence/indexeddb/__tests__/indexeddb-persistence.test.ts index 34ea9e1..17b7e5a 100644 --- a/src/modules/persistence/indexeddb/__tests__/indexeddb-persistence.test.ts +++ b/src/modules/persistence/indexeddb/__tests__/indexeddb-persistence.test.ts @@ -1,4 +1,5 @@ import { ICamaConfig } from "../../../../interfaces/cama-config.interface"; +import { ICollectionConfig } from "../../../../interfaces/collection-config.interface"; import { ILogger } from "../../../../interfaces/logger.interface"; import { PersistenceAdapterEnum } from "../../../../interfaces/perisistence-adapter.enum"; import IndexedDbPersistence from "../indexeddb-persistence"; @@ -10,14 +11,38 @@ describe("IndexedDbPersistence", () => { let indexedDbPersistence: IndexedDbPersistence; + const mockCollectionConfig: ICollectionConfig = { + indexes: [ + { name: 'index1', column: 'column1', unique: false }, + { name: 'index2', column: 'column2', unique: true }, + ], + columns: [] + }; + beforeEach(() => { - indexedDbPersistence = new IndexedDbPersistence(mockConfig, mockLogger, mockCollectionName); + indexedDbPersistence = new IndexedDbPersistence(mockConfig, mockLogger, mockCollectionName, mockCollectionConfig); }); - afterEach(() => { - indexedDbPersistence.destroy(); - }); + describe("indexes", () => { + it("should create indexes in the object store", async () => { + await indexedDbPersistence.update([]); // Ensure the database is initialized + + //@ts-ignore + const tx = indexedDbPersistence.db.transaction(mockCollectionName, 'readonly'); + const store = tx.objectStore(mockCollectionName); + + const indexNames = Array.from(store.indexNames); + + expect(indexNames.length).toBe(mockCollectionConfig.indexes.length); + + for (const indexConfig of mockCollectionConfig.indexes) { + expect(indexNames).toContain(indexConfig.name); + const index = store.index(indexConfig.name); + expect(index.unique).toBe(indexConfig.unique); + } + }); + }); describe("getData", () => { it("should get data from IndexedDB", async () => { const data = ["test-data-1", "test-data-2", "test-data-3"]; diff --git a/src/modules/persistence/indexeddb/indexeddb-persistence.ts b/src/modules/persistence/indexeddb/indexeddb-persistence.ts index 631df01..3ff73db 100644 --- a/src/modules/persistence/indexeddb/indexeddb-persistence.ts +++ b/src/modules/persistence/indexeddb/indexeddb-persistence.ts @@ -5,10 +5,11 @@ import { ICamaConfig } from '../../../interfaces/cama-config.interface'; import { ILogger } from '../../../interfaces/logger.interface'; import { IDBPDatabase, openDB } from 'idb'; +import { ICollectionConfig } from '../../../interfaces/collection-config.interface'; @injectable() export default class IndexedDbPersistence implements IPersistenceAdapter{ - private db?: IDBPDatabase; + public db?: IDBPDatabase; private dbName? = ""; private destroyed = false; private storeName = ""; @@ -17,7 +18,8 @@ export default class IndexedDbPersistence implements IPersistenceAdapter{ constructor( @inject(TYPES.CamaConfig) private config: ICamaConfig, @inject(TYPES.Logger) private logger:ILogger, - @inject(TYPES.CollectionName) private collectionName: string + @inject(TYPES.CollectionName) private collectionName: string, + @inject(TYPES.CollectionConfig) private collectionConfig: ICollectionConfig, ) { this.dbName = this.config.path || 'cama'; this.storeName = collectionName; @@ -25,9 +27,14 @@ export default class IndexedDbPersistence implements IPersistenceAdapter{ upgrade: async (db: IDBPDatabase) => { if (!db.objectStoreNames.contains(collectionName)) { const store = db.createObjectStore(collectionName); + store.put([], 'data'); + for(const index of this.collectionConfig.indexes){ + store.createIndex(index.name,index.column, {unique: !!index.unique}) + } } - await this.getData() + await this.getData(); + } } ).then(db => { From ca82551e479cc614170018157ff7a4e6b296cad6 Mon Sep 17 00:00:00 2001 From: elmarti Date: Mon, 1 May 2023 19:50:18 +0100 Subject: [PATCH 2/4] :test_tube: add initial implementation for localStorage testing --- jest.config.js | 3 ++- yarn.lock | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/jest.config.js b/jest.config.js index acff245..780a36d 100644 --- a/jest.config.js +++ b/jest.config.js @@ -6,6 +6,7 @@ module.exports = { testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(t|j)s$', moduleFileExtensions: ['ts', 'js', 'json', 'node'], "setupFiles": [ - "fake-indexeddb/auto" + "fake-indexeddb/auto", + "jest-localstorage-mock" ] }; diff --git a/yarn.lock b/yarn.lock index 5404d2b..9cac1e2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3641,6 +3641,11 @@ jest-leak-detector@^26.6.2: jest-get-type "^26.3.0" pretty-format "^26.6.2" +jest-localstorage-mock@^2.4.26: + version "2.4.26" + resolved "https://registry.yarnpkg.com/jest-localstorage-mock/-/jest-localstorage-mock-2.4.26.tgz#7d57fb3555f2ed5b7ed16fd8423fd81f95e9e8db" + integrity sha512-owAJrYnjulVlMIXOYQIPRCCn3MmqI3GzgfZCXdD3/pmwrIvFMXcKVWZ+aMc44IzaASapg0Z4SEFxR+v5qxDA2w== + jest-matcher-utils@^26.6.2: version "26.6.2" resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-26.6.2.tgz#8e6fd6e863c8b2d31ac6472eeb237bc595e53e7a" From 4734bc0525dde99237bb3bf8a628ba92977a5ad3 Mon Sep 17 00:00:00 2001 From: elmarti Date: Mon, 1 May 2023 19:50:48 +0100 Subject: [PATCH 3/4] :test_tube: initial tests for localstorage --- .../localstorage-persistence.test.ts | 79 ++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/src/modules/persistence/localstorage/__tests__/localstorage-persistence.test.ts b/src/modules/persistence/localstorage/__tests__/localstorage-persistence.test.ts index 26a994e..aed75ca 100644 --- a/src/modules/persistence/localstorage/__tests__/localstorage-persistence.test.ts +++ b/src/modules/persistence/localstorage/__tests__/localstorage-persistence.test.ts @@ -1,3 +1,78 @@ -it('works', () => { - expect(1).toBe(1); +import { ICamaConfig } from "../../../../interfaces/cama-config.interface"; +import { ILogger } from "../../../../interfaces/logger.interface"; +import { PersistenceAdapterEnum } from "../../../../interfaces/perisistence-adapter.enum"; +import { ICollectionConfig } from "../../../../interfaces/collection-config.interface"; +import LocalstoragePersistence from "../localstorage-persistence"; +//@ts-ignore +import * as jls from 'jest-localstorage-mock'; +describe("LocalstoragePersistence", () => { + const mockConfig: ICamaConfig = { path: "test-path", persistenceAdapter: PersistenceAdapterEnum.InMemory }; + const mockLogger: ILogger = { log: jest.fn(), startTimer: jest.fn(), endTimer: jest.fn() }; + const mockCollectionName = "test-collection"; + const mockCollectionConfig: ICollectionConfig = { + indexes: [ + { name: 'index1', column: 'column1' }, + { name: 'index2', column: 'column2' } + ], + columns: [] + }; + + let localstoragePersistence: LocalstoragePersistence; + + beforeEach(() => { + + localstoragePersistence = new LocalstoragePersistence(mockConfig, mockLogger, mockCollectionName, mockCollectionConfig); + }); + + afterEach(() => { + localstoragePersistence.destroy(); + }); + + describe("getData", () => { + it("should get data from localStorage", async () => { + const data = [{ column1: "a", column2: "b" }, { column1: "c", column2: "d" }, { column1: "e", column2: "f" }]; + + await localstoragePersistence.update(data); + + const result = await localstoragePersistence.getData(); + + expect(result).toEqual(data); + }); + }); + + describe("update", () => { + it("should update data in localStorage", async () => { + const data = [{ column1: "a", column2: "b" }, { column1: "c", column2: "d" }, { column1: "e", column2: "f" }]; + const updatedData = [{ column1: "g", column2: "h" }, { column1: "i", column2: "j" }]; + + await localstoragePersistence.update(data); + await localstoragePersistence.update(updatedData); + + const result = await localstoragePersistence.getData(); + + expect(result).toEqual(updatedData); + }); + }); + + describe("insert", () => { + it("should insert data into localStorage and update indexes", async () => { + const data = [{ column1: "a", column2: "b" }, { column1: "c", column2: "d" }, { column1: "e", column2: "f" }]; + const newData = [{ column1: "g", column2: "h" }, { column1: "i", column2: "j" }]; + + await localstoragePersistence.update(data); + await localstoragePersistence.insert(newData); + + const result = await localstoragePersistence.getData(); + + expect(result).toEqual([...data, ...newData]); + + // Check if indexes have been updated + for (const indexConfig of mockCollectionConfig.indexes) { + expect(jls.localStorage.setItem).toHaveBeenCalledWith( + expect.stringContaining(`-index-${indexConfig.column}`), + expect.any(String) + ); + } + }); + }); }); From 8c744a9331ba82f6ea56c77c35fdd2c31a67b6e3 Mon Sep 17 00:00:00 2001 From: elmarti Date: Tue, 2 May 2023 15:31:09 +0100 Subject: [PATCH 4/4] :sparkles: localStorage indexing --- package.json | 1 + src/interfaces/index-config.interface.ts | 2 +- .../localstorage/localstorage-persistence.ts | 55 ++++++++++++-- src/modules/query/query.service.ts | 72 +++++++++++++------ 4 files changed, 104 insertions(+), 26 deletions(-) diff --git a/package.json b/package.json index ed6270e..59ed369 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "eslint-plugin-jest": "^24.1.0", "fake-indexeddb": "^4.0.1", "jest": "^26.6.0", + "jest-localstorage-mock": "^2.4.26", "prettier": "^2.3.0", "semantic-release-gitmoji": "^1.4.2", "ts-jest": "^26.4.1", diff --git a/src/interfaces/index-config.interface.ts b/src/interfaces/index-config.interface.ts index 2148300..1e33260 100644 --- a/src/interfaces/index-config.interface.ts +++ b/src/interfaces/index-config.interface.ts @@ -1,6 +1,6 @@ export interface IIndexConfig { column: string; type?: string;//tbc - unique: boolean; + unique?: boolean; name: string; } \ No newline at end of file diff --git a/src/modules/persistence/localstorage/localstorage-persistence.ts b/src/modules/persistence/localstorage/localstorage-persistence.ts index 5b1c903..21df89d 100644 --- a/src/modules/persistence/localstorage/localstorage-persistence.ts +++ b/src/modules/persistence/localstorage/localstorage-persistence.ts @@ -4,20 +4,59 @@ import { TYPES } from '../../../types'; import { ICamaConfig } from '../../../interfaces/cama-config.interface'; import { ILogger } from '../../../interfaces/logger.interface'; +import { ICollectionConfig } from '../../../interfaces/collection-config.interface'; @injectable() export default class LocalstoragePersistence implements IPersistenceAdapter{ private readonly dbName; private destroyed = false; private cache: any = null; + private indexes = new Map(); constructor( @inject(TYPES.CamaConfig) private config: ICamaConfig, @inject(TYPES.Logger) private logger:ILogger, @inject(TYPES.CollectionName) private collectionName: string, + @inject(TYPES.CollectionConfig) private collectionConfig: ICollectionConfig, ) { this.dbName = this.config.path || 'cama'; + this.initIndexes(); this.getData(); + + } + private async initIndexes(): Promise { + for (const index of this.collectionConfig.indexes) { + const indexData = await this.loadIndex(index.column); + this.indexes.set(index.column, indexData); + } + } + public getIndex(column: string): Map { + return this.indexes.get(column); + } + private async loadIndex(column: string): Promise> { + //@ts-ignore + const rawData = await window.localStorage.getItem(`${this.dbName}-${this.collectionName}-index-${column}`); + const data = rawData ? JSON.parse(rawData) : []; + return new Map(data); + } + + private async saveIndex(column: string): Promise { + const indexData = this.indexes.get(column); + if (indexData) { + const data = Array.from(indexData.entries()); + //@ts-ignore + await window.localStorage.setItem(`${this.dbName}-${this.collectionName}-index-${column}`, JSON.stringify(data)); + } + } + + private updateIndexes(row: any, index: number): void { + for (const indexConfig of this.collectionConfig.indexes) { + const columnIndex = this.indexes.get(indexConfig.column); + if (columnIndex) { + columnIndex.set(row[indexConfig.column], index); + } + } } + async destroy(): Promise { //@ts-ignore window.localStorage.removeItem(`${this.dbName}-${this.collectionName}-data`); @@ -40,10 +79,18 @@ export default class LocalstoragePersistence implements IPersistenceAdapter{ } async insert(rows: Array): Promise { this.checkDestroyed(); - const data = await this.getData(); - data.push(...rows); - await this.setData(data); - + const data = await this.getData(); + for (const row of rows) { + const index = data.length; + data.push(row); + this.updateIndexes(row, index); + } + await this.setData(data); + + // Save the updated indexes + for (const indexConfig of this.collectionConfig.indexes) { + await this.saveIndex(indexConfig.column); + } } private async loadData(){ //@ts-ignore diff --git a/src/modules/query/query.service.ts b/src/modules/query/query.service.ts index 08827d2..71fb374 100644 --- a/src/modules/query/query.service.ts +++ b/src/modules/query/query.service.ts @@ -10,6 +10,8 @@ import { ILogger } from '../../interfaces/logger.interface'; import { IFilterResult } from '../../interfaces/filter-result.interface'; import { ICollectionMeta } from '../../interfaces/collection-meta.interface'; import { LogLevel } from '../../interfaces/logger-level.enum'; +import LocalstoragePersistence from '../persistence/localstorage/localstorage-persistence'; +import { ICollectionConfig } from '../../interfaces/collection-config.interface'; const obop = require('obop')(); @injectable() @@ -19,39 +21,67 @@ export class QueryService implements IQueryService{ constructor( @inject(TYPES.CollectionMeta) private collectionMeta: ICollectionMeta, @inject(TYPES.PersistenceAdapter) private persistenceAdapter: IPersistenceAdapter, - @inject(TYPES.Logger) private logger:ILogger) { + @inject(TYPES.Logger) private logger:ILogger, + @inject(TYPES.CollectionConfig) private collectionConfig: ICollectionConfig) { } - /** + /** * Handle filtering of queries * @param query - The query to be applied to the dataset * @param options - Options for further data manipulation */ - async filter(query: any = {}, options: IQueryOptions = {}): Promise> { + async filter(query: any = {}, options: IQueryOptions = {}): Promise> { - const filterResult:any = { + const filterResult:any = {}; - }; - let data = await this.persistenceAdapter.getData(); - if(Object.keys(query).length > 0){ - data = data.filter(sift(query)); - } - filterResult['totalCount'] = data.length; - if(options.sort){ - data = sort(data).by(options.sort); - } - if(options.offset){ - data = data.slice(options.offset, data.length); - } - if(options.limit){ - data = data.slice(0, options.limit); + let data = await this.persistenceAdapter.getData(); + + // Check if the persistenceAdapter is an instance of LocalstoragePersistence + if (this.persistenceAdapter instanceof LocalstoragePersistence) { + const indexedColumns = new Set(this.collectionConfig.indexes.map(index => index.column)); + + // Check if the query keys have any indexed columns + const queryKeys = Object.keys(query); + const indexedQueryKeys = queryKeys.filter(key => indexedColumns.has(key)); + + if (indexedQueryKeys.length > 0) { + // Use the index to get the relevant rows + const rowIndexes = new Set(); + indexedQueryKeys.forEach(key => { + const value = query[key]; + //@ts-ignore + const indexMap = this.persistenceAdapter.getIndex(key); + const rowIndex = indexMap.get(value); + + if (rowIndex !== undefined) { + rowIndexes.add(rowIndex); + } + }); + + // Filter the data using the rowIndexes + data = Array.from(rowIndexes).map(index => data[index]); } - filterResult['count'] = data.length; - filterResult['rows'] = data; - return filterResult; } + if (Object.keys(query).length > 0) { + data = data.filter(sift(query)); + } + filterResult['totalCount'] = data.length; + if (options.sort) { + data = sort(data).by(options.sort); + } + if (options.offset) { + data = data.slice(options.offset, data.length); + } + if (options.limit) { + data = data.slice(0, options.limit); + } + filterResult['count'] = data.length; + filterResult['rows'] = data; + return filterResult; +} + async update(query: any, delta: any): Promise { const data = await this.persistenceAdapter.getData(); this.logger.log(LogLevel.Debug, "Iterating pages");