article: vector database

CHANGELOG.md

@@ -2,7 +2,8 @@
 # RxDB Changelog
 
 <!-- CHANGELOG NEWEST -->
-
+- FIX RxPipeline tries to store metadata that does not match the json schema.
+- ADD utilities function for vector search.
 <!-- ADD new changes here! -->
 
 <!-- /CHANGELOG NEWEST -->

docs-src/sidebars.js

@@ -139,8 +139,9 @@ const sidebars = {
         'downsides-of-offline-first',
         'slow-indexeddb',
         'why-nosql',
-        'react-native-database',
         'alternatives',
+        'react-native-database',
+        'articles/javascript-vector-database',
         'articles/angular-database',
         'articles/browser-database',
         'articles/browser-storage',

docs-src/src/css/custom.css

@@ -67,9 +67,6 @@ html[data-theme="dark"] {
     font-size: 1.75rem;
 }
 
-.markdown>p {
-    font-size: 1rem;
-}
 
 .footer {
     background-color: var(--ifm-navbar-background-color);

orga/before-next-major.md

@@ -19,6 +19,11 @@ Set the default to 14 and also remove all occurences of `jsonPositionSize`.
 In the migration-storage plugin we run a catch on `oldStorageInstance.cleanup(0)` to fix v14->v15 migration.
 We should remove that catch in the next major release.
 
+
+## Change the `RX_PIPELINE_CHECKPOINT_CONTEXT` to `rx-pipeline-checkpoint` in the rx-pipeline.ts file
+
+This was not possible before because it requires adding the new value to the schema enum.
+
 ---------------------------------
 # Maybe later (not sure if should be done)
 

package.json

@@ -309,6 +309,12 @@
       "import": "./dist/esm/plugins/pipeline/index.js",
       "default": "./dist/esm/plugins/pipeline/index.js"
     },
+    "./plugins/vector": {
+      "types": "./dist/types/plugins/vector/index.d.ts",
+      "require": "./dist/cjs/plugins/vector/index.js",
+      "import": "./dist/esm/plugins/vector/index.js",
+      "default": "./dist/esm/plugins/vector/index.js"
+    },
     "./plugins/validate-ajv": {
       "types": "./dist/types/plugins/validate-ajv/index.d.ts",
       "require": "./dist/cjs/plugins/validate-ajv/index.js",
@@ -588,4 +594,4 @@
     "webpack-cli": "5.1.4",
     "webpack-dev-server": "5.0.4"
   }
-}
+}

src/plugins/pipeline/rx-pipeline.ts

@@ -32,8 +32,10 @@ import { getChangedDocumentsSince } from '../../rx-storage-helper.ts';
 import { mapDocumentsDataToCacheDocs } from '../../doc-cache.ts';
 import { getPrimaryKeyOfInternalDocument } from '../../rx-database-internal-store.ts';
 import { FLAGGED_FUNCTIONS, blockFlaggedFunctionKey, releaseFlaggedFunctionKey } from './flagged-functions.ts';
-export const RX_PIPELINE_CHECKPOINT_CONTEXT = 'rx-pipeline-checkpoint';
 
+export const RX_PIPELINE_CHECKPOINT_CONTEXT = 'OTHER';
+// TODO change the context in the next major version.
+// export const RX_PIPELINE_CHECKPOINT_CONTEXT = 'rx-pipeline-checkpoint';
 
 export class RxPipeline<RxDocType> {
     processQueue = PROMISE_RESOLVE_VOID;
@@ -191,7 +193,7 @@ export class RxPipeline<RxDocType> {
             const writeResult = await insternalStore.bulkWrite([{
                 previous: checkpointDoc,
                 document: newDoc,
-            }], RX_PIPELINE_CHECKPOINT_CONTEXT);
+            }], 'rx-pipeline');
             if (writeResult.error.length > 0) {
                 throw writeResult.error;
             }
@@ -243,7 +245,7 @@ export async function setCheckpointDoc<RxDocType>(
     const writeResult = await insternalStore.bulkWrite([{
         previous,
         document: newDoc,
-    }], RX_PIPELINE_CHECKPOINT_CONTEXT);
+    }], 'rx-pipeline');
     if (writeResult.error.length > 0) {
         throw writeResult.error;
     }

src/plugins/utils/utils-array.ts

@@ -173,3 +173,9 @@ export function uniqueArray(arrArg: string[]): string[] {
     });
 }
 
+
+export function sortByObjectNumberProperty<T>(property: keyof T) {
+    return (a: T, b: T) => {
+        return (b as any)[property] - (a as any)[property];
+    }
+}

src/plugins/vector/index.ts

@@ -0,0 +1,2 @@
+export type * from './types.ts';
+export * from './vector-distance.ts';

src/plugins/vector/types.ts

@@ -0,0 +1 @@
+export type Vector = number[];

src/plugins/vector/vector-distance.ts

@@ -0,0 +1,34 @@
+import type { Vector } from './types';
+
+/**
+ * Vector comparison methods
+ * @link https://www.restack.io/p/vector-database-knowledge-answer-javascript-cat-ai
+ * @returns 
+ */
+export function euclideanDistance(A: Vector, B: Vector): number {
+    return Math.sqrt(A.reduce((sum, a, i) => sum + Math.pow(a - B[i], 2), 0));
+}
+export function manhattanDistance(A: Vector, B: Vector) {
+    return A.reduce((sum, a, i) => sum + Math.abs(a - B[i]), 0);
+}
+
+
+export function cosineSimilarity(A: Vector, B: Vector): number {
+    const dotProduct = A.reduce((sum, a, i) => sum + a * B[i], 0);
+    const magnitudeA = Math.sqrt(A.reduce((sum, a) => sum + a * a, 0));
+    const magnitudeB = Math.sqrt(B.reduce((sum, b) => sum + b * b, 0));
+    return dotProduct / (magnitudeA * magnitudeB);
+}
+
+
+/**
+ * @link https://github.com/vector5ai/vector5db/blob/main/src/metrics/JaccardSimilarity.ts
+ */
+export function jaccardSimilarity(a: number[], b: number[]): number {
+    const setA = new Set(a);
+    const setB = new Set(b);
+    const intersection = new Set([...setA].filter((x) => setB.has(x))).size;
+    const union = new Set([...setA, ...setB]).size;
+
+    return 1 - (intersection / union);
+}

test/unit/rx-pipeline.test.ts

@@ -8,12 +8,14 @@ import {
     randomCouchString
 } from '../../plugins/core/index.mjs';
 import {
-    HumanWithTimestampDocumentType
+    HumanWithTimestampDocumentType,
+    getConfig
 } from '../../plugins/test-utils/index.mjs';
 import {
     schemaObjects,
     humansCollection
 } from '../../plugins/test-utils/index.mjs';
+import { wrappedValidateAjvStorage } from '../../plugins/validate-ajv/index.mjs';
 import { RxDBPipelinePlugin } from '../../plugins/pipeline/index.mjs';
 addRxPlugin(RxDBPipelinePlugin);
 import { RxDBLeaderElectionPlugin } from '../../plugins/leader-election/index.mjs';
@@ -71,6 +73,35 @@ describeParallel('rx-pipeline.test.js', () => {
             await c1.database.destroy();
             await c2.database.destroy();
         });
+        it('write some document depending on another with schema validator', async () => {
+            const storage = wrappedValidateAjvStorage({
+                storage: getConfig().storage.getStorage()
+            });
+            const c1 = await humansCollection.create(0, undefined, undefined, undefined, storage);
+            const c2 = await humansCollection.create(0, undefined, undefined, undefined, storage);
+            await c1.addPipeline({
+                destination: c2,
+                handler: async (docs) => {
+                    for (const doc of docs) {
+                        await c2.insert(schemaObjects.humanData(doc.passportId));
+                    }
+                },
+                identifier: randomCouchString(10)
+            });
+
+            await c1.insert(schemaObjects.humanData('foobar'));
+
+            /**
+             * Here we run the query on the destination directly after
+             * a write to the source. The pipeline should automatically halt
+             * the reads to the destination until the pipeline is idle.
+             */
+            const doc2 = await c2.findOne().exec(true);
+            assert.strictEqual(doc2.passportId, 'foobar');
+
+            await c1.database.destroy();
+            await c2.database.destroy();
+        });
         // it('write some document depending on another', async () => {
         //     const dbs = await multipleOnSameDB(0);
         //     const c1 = dbs.collection;