WIP 💥 indexing

jest.config.js

@@ -6,6 +6,7 @@ module.exports = {
   testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(t|j)s$',
   moduleFileExtensions: ['ts', 'js', 'json', 'node'],
   "setupFiles": [
-    "fake-indexeddb/auto"
+    "fake-indexeddb/auto",
+    "jest-localstorage-mock"
 ]
 };

package.json

@@ -45,6 +45,7 @@
     "eslint-plugin-jest": "^24.1.0",
     "fake-indexeddb": "^4.0.1",
     "jest": "^26.6.0",
+    "jest-localstorage-mock": "^2.4.26",
     "prettier": "^2.3.0",
     "semantic-release-gitmoji": "^1.4.2",
     "ts-jest": "^26.4.1",

src/interfaces/collection-config.interface.ts

@@ -1,6 +1,7 @@
 import { IColumnConfig } from './column-config.interface';
+import { IIndexConfig } from './index-config.interface';
 
 export interface ICollectionConfig {
   columns: Array<IColumnConfig>;
-  indexes: Array<string>;
+  indexes: Array<IIndexConfig>;
 }

src/interfaces/index-config.interface.ts

@@ -0,0 +1,6 @@
+export interface IIndexConfig {
+    column: string;
+    type?: string;//tbc 
+    unique?: boolean;
+    name: string;
+}

src/modules/persistence/indexeddb/__tests__/indexeddb-persistence.test.ts

@@ -1,4 +1,5 @@
 import { ICamaConfig } from "../../../../interfaces/cama-config.interface";
+import { ICollectionConfig } from "../../../../interfaces/collection-config.interface";
 import { ILogger } from "../../../../interfaces/logger.interface";
 import { PersistenceAdapterEnum } from "../../../../interfaces/perisistence-adapter.enum";
 import IndexedDbPersistence from "../indexeddb-persistence";
@@ -10,14 +11,38 @@ describe("IndexedDbPersistence", () => {
 
   let indexedDbPersistence: IndexedDbPersistence;
 
+  const mockCollectionConfig: ICollectionConfig = {
+    indexes: [
+      { name: 'index1', column: 'column1', unique: false },
+      { name: 'index2', column: 'column2', unique: true },
+    ],
+    columns: []
+  };
+
   beforeEach(() => {
-    indexedDbPersistence = new IndexedDbPersistence(mockConfig, mockLogger, mockCollectionName);
+    indexedDbPersistence = new IndexedDbPersistence(mockConfig, mockLogger, mockCollectionName, mockCollectionConfig);
   });
 
-  afterEach(() => {
-    indexedDbPersistence.destroy();
-  });
 
+  describe("indexes", () => {
+    it("should create indexes in the object store", async () => {
+      await indexedDbPersistence.update([]); // Ensure the database is initialized
+
+      //@ts-ignore
+      const tx = indexedDbPersistence.db.transaction(mockCollectionName, 'readonly');
+      const store = tx.objectStore(mockCollectionName);
+
+      const indexNames = Array.from(store.indexNames);
+
+      expect(indexNames.length).toBe(mockCollectionConfig.indexes.length);
+
+      for (const indexConfig of mockCollectionConfig.indexes) {
+        expect(indexNames).toContain(indexConfig.name);
+        const index = store.index(indexConfig.name);
+        expect(index.unique).toBe(indexConfig.unique);
+      }
+    });
+  });
   describe("getData", () => {
     it("should get data from IndexedDB", async () => {
       const data = ["test-data-1", "test-data-2", "test-data-3"];

src/modules/persistence/indexeddb/indexeddb-persistence.ts

@@ -5,10 +5,11 @@ import { ICamaConfig } from '../../../interfaces/cama-config.interface';
 
 import { ILogger } from '../../../interfaces/logger.interface';
 import { IDBPDatabase, openDB } from 'idb';
+import { ICollectionConfig } from '../../../interfaces/collection-config.interface';
 
 @injectable()
 export default class IndexedDbPersistence implements IPersistenceAdapter{
-  private db?: IDBPDatabase;
+  public db?: IDBPDatabase;
   private dbName? = "";
   private destroyed = false;
   private storeName = "";
@@ -17,17 +18,23 @@ export default class IndexedDbPersistence implements IPersistenceAdapter{
   constructor(
     @inject(TYPES.CamaConfig) private config: ICamaConfig,
     @inject(TYPES.Logger) private logger:ILogger,
-    @inject(TYPES.CollectionName) private collectionName: string
+    @inject(TYPES.CollectionName) private collectionName: string,
+    @inject(TYPES.CollectionConfig) private collectionConfig: ICollectionConfig,
   ) {
       this.dbName = this.config.path || 'cama';
       this.storeName = collectionName;
       this.initPromise = openDB(this.dbName, 1, {
         upgrade: async (db: IDBPDatabase) => {
           if (!db.objectStoreNames.contains(collectionName)) {
             const store = db.createObjectStore(collectionName);
+
             store.put([], 'data');
+            for(const index of this.collectionConfig.indexes){
+              store.createIndex(index.name,index.column, {unique: !!index.unique})
+            }
           }
-          await this.getData()
+          await this.getData();
+
         }
       }
     ).then(db => {

src/modules/persistence/localstorage/__tests__/localstorage-persistence.test.ts

@@ -1,3 +1,78 @@
-it('works', () => {
-  expect(1).toBe(1);
+import { ICamaConfig } from "../../../../interfaces/cama-config.interface";
+import { ILogger } from "../../../../interfaces/logger.interface";
+import { PersistenceAdapterEnum } from "../../../../interfaces/perisistence-adapter.enum";
+import { ICollectionConfig } from "../../../../interfaces/collection-config.interface";
+import LocalstoragePersistence from "../localstorage-persistence";
+//@ts-ignore
+import * as jls from 'jest-localstorage-mock';
+describe("LocalstoragePersistence", () => {
+  const mockConfig: ICamaConfig = { path: "test-path", persistenceAdapter: PersistenceAdapterEnum.InMemory };
+  const mockLogger: ILogger = { log: jest.fn(), startTimer: jest.fn(), endTimer: jest.fn() };
+  const mockCollectionName = "test-collection";
+  const mockCollectionConfig: ICollectionConfig = {
+    indexes: [
+      { name: 'index1', column: 'column1' },
+      { name: 'index2', column: 'column2' }
+    ],
+    columns: []
+  };
+
+  let localstoragePersistence: LocalstoragePersistence;
+
+  beforeEach(() => {
+
+    localstoragePersistence = new LocalstoragePersistence(mockConfig, mockLogger, mockCollectionName, mockCollectionConfig);
+  });
+
+  afterEach(() => {
+    localstoragePersistence.destroy();
+  });
+
+  describe("getData", () => {
+    it("should get data from localStorage", async () => {
+      const data = [{ column1: "a", column2: "b" }, { column1: "c", column2: "d" }, { column1: "e", column2: "f" }];
+
+      await localstoragePersistence.update(data);
+
+      const result = await localstoragePersistence.getData();
+
+      expect(result).toEqual(data);
+    });
+  });
+
+  describe("update", () => {
+    it("should update data in localStorage", async () => {
+      const data = [{ column1: "a", column2: "b" }, { column1: "c", column2: "d" }, { column1: "e", column2: "f" }];
+      const updatedData = [{ column1: "g", column2: "h" }, { column1: "i", column2: "j" }];
+
+      await localstoragePersistence.update(data);
+      await localstoragePersistence.update(updatedData);
+
+      const result = await localstoragePersistence.getData();
+
+      expect(result).toEqual(updatedData);
+    });
+  });
+
+  describe("insert", () => {
+    it("should insert data into localStorage and update indexes", async () => {
+      const data = [{ column1: "a", column2: "b" }, { column1: "c", column2: "d" }, { column1: "e", column2: "f" }];
+      const newData = [{ column1: "g", column2: "h" }, { column1: "i", column2: "j" }];
+
+      await localstoragePersistence.update(data);
+      await localstoragePersistence.insert(newData);
+
+      const result = await localstoragePersistence.getData();
+
+      expect(result).toEqual([...data, ...newData]);
+
+      // Check if indexes have been updated
+      for (const indexConfig of mockCollectionConfig.indexes) {
+        expect(jls.localStorage.setItem).toHaveBeenCalledWith(
+          expect.stringContaining(`-index-${indexConfig.column}`),
+          expect.any(String)
+        );
+      }
+    });
+  });
 });

src/modules/persistence/localstorage/localstorage-persistence.ts

@@ -4,20 +4,59 @@ import { TYPES } from '../../../types';
 import { ICamaConfig } from '../../../interfaces/cama-config.interface';
 
 import { ILogger } from '../../../interfaces/logger.interface';
+import { ICollectionConfig } from '../../../interfaces/collection-config.interface';
 
 @injectable()
 export default class LocalstoragePersistence implements IPersistenceAdapter{
   private readonly dbName;
   private destroyed = false;
   private cache: any = null;
+  private indexes = new Map();
   constructor(
     @inject(TYPES.CamaConfig) private config: ICamaConfig,
     @inject(TYPES.Logger) private logger:ILogger,
     @inject(TYPES.CollectionName) private collectionName: string,
+    @inject(TYPES.CollectionConfig) private collectionConfig: ICollectionConfig,
   ) {
     this.dbName = this.config.path || 'cama';
+    this.initIndexes();
     this.getData();
+
+  }
+  private async initIndexes(): Promise<void> {
+    for (const index of this.collectionConfig.indexes) {
+      const indexData = await this.loadIndex(index.column);
+      this.indexes.set(index.column, indexData);
+    }
+  }
+  public getIndex(column: string): Map<any, number> {
+    return this.indexes.get(column);
+  }
+  private async loadIndex(column: string): Promise<Map<any, number>> {
+    //@ts-ignore
+    const rawData = await window.localStorage.getItem(`${this.dbName}-${this.collectionName}-index-${column}`);
+    const data = rawData ? JSON.parse(rawData) : [];
+    return new Map(data);
+  }
+
+  private async saveIndex(column: string): Promise<void> {
+    const indexData = this.indexes.get(column);
+    if (indexData) {
+      const data = Array.from(indexData.entries());
+      //@ts-ignore
+      await window.localStorage.setItem(`${this.dbName}-${this.collectionName}-index-${column}`, JSON.stringify(data));
+    }
+  }
+
+  private updateIndexes(row: any, index: number): void {
+    for (const indexConfig of this.collectionConfig.indexes) {
+      const columnIndex = this.indexes.get(indexConfig.column);
+      if (columnIndex) {
+        columnIndex.set(row[indexConfig.column], index);
+      }
+    }
   }
+
   async destroy(): Promise<void> {
     //@ts-ignore
     window.localStorage.removeItem(`${this.dbName}-${this.collectionName}-data`);
@@ -40,10 +79,18 @@ export default class LocalstoragePersistence implements IPersistenceAdapter{
   }
   async insert(rows: Array<any>): Promise<any> {
     this.checkDestroyed();
-      const data = await this.getData();
-      data.push(...rows);
-      await this.setData(data);
-
+    const data = await this.getData();
+    for (const row of rows) {
+      const index = data.length;
+      data.push(row);
+      this.updateIndexes(row, index);
+    }
+    await this.setData(data);
+
+    // Save the updated indexes
+    for (const indexConfig of this.collectionConfig.indexes) {
+      await this.saveIndex(indexConfig.column);
+    }
   }
   private async loadData(){
     //@ts-ignore

src/modules/query/query.service.ts

@@ -10,6 +10,8 @@ import { ILogger } from '../../interfaces/logger.interface';
 import { IFilterResult } from '../../interfaces/filter-result.interface';
 import { ICollectionMeta } from '../../interfaces/collection-meta.interface';
 import { LogLevel } from '../../interfaces/logger-level.enum';
+import LocalstoragePersistence from '../persistence/localstorage/localstorage-persistence';
+import { ICollectionConfig } from '../../interfaces/collection-config.interface';
 const obop = require('obop')();
 
 @injectable()
@@ -19,39 +21,67 @@ export class QueryService<T> implements IQueryService<T>{
   constructor(
     @inject(TYPES.CollectionMeta) private collectionMeta: ICollectionMeta,
     @inject(TYPES.PersistenceAdapter) private persistenceAdapter: IPersistenceAdapter,
-              @inject(TYPES.Logger) private logger:ILogger) {
+              @inject(TYPES.Logger) private logger:ILogger,
+              @inject(TYPES.CollectionConfig) private collectionConfig: ICollectionConfig) {
 
   }
 
-  /**
+ /**
    * Handle filtering of queries
    * @param query - The query to be applied to the dataset
    * @param options - Options for further data manipulation
    */
-  async filter(query: any = {}, options: IQueryOptions = {}): Promise<IFilterResult<T>> {
+ async filter(query: any = {}, options: IQueryOptions = {}): Promise<IFilterResult<T>> {
 
-    const filterResult:any = {
+  const filterResult:any = {};
 
-    };
-    let data = await this.persistenceAdapter.getData();
-    if(Object.keys(query).length > 0){
-      data = data.filter(sift(query));
-    }
-    filterResult['totalCount'] = data.length;
-    if(options.sort){
-      data = sort(data).by(options.sort);
-    }
-    if(options.offset){
-      data = data.slice(options.offset, data.length);
-    }
-    if(options.limit){
-      data = data.slice(0, options.limit);
+  let data = await this.persistenceAdapter.getData();
+
+  // Check if the persistenceAdapter is an instance of LocalstoragePersistence
+  if (this.persistenceAdapter instanceof LocalstoragePersistence) {
+    const indexedColumns = new Set(this.collectionConfig.indexes.map(index => index.column));
+
+    // Check if the query keys have any indexed columns
+    const queryKeys = Object.keys(query);
+    const indexedQueryKeys = queryKeys.filter(key => indexedColumns.has(key));
+
+    if (indexedQueryKeys.length > 0) {
+      // Use the index to get the relevant rows
+      const rowIndexes = new Set<number>();
+      indexedQueryKeys.forEach(key => {
+        const value = query[key];
+        //@ts-ignore
+        const indexMap = this.persistenceAdapter.getIndex(key);
+        const rowIndex = indexMap.get(value);
+
+        if (rowIndex !== undefined) {
+          rowIndexes.add(rowIndex);
+        }
+      });
+
+      // Filter the data using the rowIndexes
+      data = Array.from(rowIndexes).map(index => data[index]);
     }
-    filterResult['count'] = data.length;
-    filterResult['rows'] = data;
-    return  filterResult;
   }
 
+  if (Object.keys(query).length > 0) {
+    data = data.filter(sift(query));
+  }
+  filterResult['totalCount'] = data.length;
+  if (options.sort) {
+    data = sort(data).by(options.sort);
+  }
+  if (options.offset) {
+    data = data.slice(options.offset, data.length);
+  }
+  if (options.limit) {
+    data = data.slice(0, options.limit);
+  }
+  filterResult['count'] = data.length;
+  filterResult['rows'] = data;
+  return filterResult;
+}
+
   async update(query: any, delta: any): Promise<void> {
     const data = await this.persistenceAdapter.getData();
     this.logger.log(LogLevel.Debug, "Iterating pages");

yarn.lock

@@ -3641,6 +3641,11 @@ jest-leak-detector@^26.6.2:
     jest-get-type "^26.3.0"
     pretty-format "^26.6.2"
 
+jest-localstorage-mock@^2.4.26:
+  version "2.4.26"
+  resolved "https://registry.yarnpkg.com/jest-localstorage-mock/-/jest-localstorage-mock-2.4.26.tgz#7d57fb3555f2ed5b7ed16fd8423fd81f95e9e8db"
+  integrity sha512-owAJrYnjulVlMIXOYQIPRCCn3MmqI3GzgfZCXdD3/pmwrIvFMXcKVWZ+aMc44IzaASapg0Z4SEFxR+v5qxDA2w==
+
 jest-matcher-utils@^26.6.2:
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-26.6.2.tgz#8e6fd6e863c8b2d31ac6472eeb237bc595e53e7a"