Storage adapter for MongoDB

packages/automerge-repo-storage-mongodb/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2019-2023, Ink & Switch LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/automerge-repo-storage-mongodb/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@automerge/automerge-repo-storage-mongodb",
+  "version": "2.0.0-alpha.11",
+  "description": "MongoDB storage adapter for Automerge Repo",
+  "repository": "https://github.com/automerge/automerge-repo/tree/master/packages/automerge-repo-storage-mongodb",
+  "author": "Kræn Hansen <[email protected]>",
+  "license": "MIT",
+  "type": "module",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsc",
+    "watch": "npm-watch build",
+    "test": "vitest"
+  },
+  "dependencies": {
+    "@automerge/automerge-repo": "workspace:*",
+    "mongodb": "^6.8.0"
+  },
+  "watch": {
+    "build": {
+      "patterns": "./src/**/*",
+      "extensions": [
+        ".ts"
+      ]
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/automerge-repo-storage-mongodb/src/index.ts

@@ -0,0 +1,159 @@
+/**
+ * @packageDocumentation
+ * A `StorageAdapter` which stores data in MongoDB
+ */
+
+import {
+  Chunk,
+  StorageAdapterInterface,
+  type StorageKey,
+} from "@automerge/automerge-repo/slim"
+import { MongoClient, MongoClientOptions, Collection, BSON } from "mongodb"
+import assert from "node:assert"
+
+// function storageKeyToString(key: StorageKey) {
+//   return key.join("/")
+// }
+
+/**
+ * Assert that the argument passed is an array of strings
+ * NOTE: This is necessary to enable passing this directly to the database
+ */
+function assertStorageKey(value: unknown): asserts value is StorageKey {
+  assert(Array.isArray(value), "Expected an array")
+  for (const element of value) {
+    assert(typeof element === "string", "Expected an array of strings")
+  }
+}
+
+/**
+ * Builds a query filter from a key prefix using "array index position"
+ * See https://www.mongodb.com/docs/manual/tutorial/query-arrays/#query-for-an-element-by-the-array-index-position
+ */
+function buildKeyPrefixFilter(keyPrefix: StorageKey) {
+  return Object.fromEntries(keyPrefix.map((part, i) => {
+    return [`key.${i}`, part]
+  }))
+}
+
+type ChunkDocument = { key: StorageKey; data: BSON.Binary }
+
+type MongoDBStorageAdapterOptions = {
+  dbName?: string
+  collectionName?: string
+  /** The strategy for storing and querying keys */
+  keyStorageStrategy?: "array" | "string"
+}
+
+export class MongoDBStorageAdapter implements StorageAdapterInterface {
+  #client: MongoClient
+  #dbName: string
+  #collectionName: string
+  #collection: Promise<Collection<ChunkDocument>> | undefined = undefined
+
+  /**
+   * @param url - The url of the MongoDB server.
+   * @param options - Additional options to pass when instantiating the MongoDB client.
+   */
+  constructor(
+    url: string,
+    options?: MongoDBStorageAdapterOptions & MongoClientOptions
+  )
+
+  /**
+   * @param client - The MongoDB client.
+   * @param options - Additional options.
+   */
+  constructor(client: MongoClient, options?: MongoDBStorageAdapterOptions)
+
+  /**
+   * @param url - The url of the MongoDB server.
+   * @param options - Additional options to pass when instantiating the MongoDB client.
+   */
+  constructor(
+    urlOrClient: MongoClient | string,
+    {
+      dbName = "automerge",
+      collectionName = "chunks",
+      keyStorageStrategy = "array",
+      ...clientOptions
+    }: MongoDBStorageAdapterOptions & MongoClientOptions = {}
+  ) {
+    this.#dbName = dbName
+    this.#collectionName = collectionName
+    if (typeof urlOrClient === "string") {
+      this.#client = new MongoClient(urlOrClient, clientOptions)
+    } else if (urlOrClient instanceof MongoClient) {
+      this.#client = urlOrClient
+    } else {
+      throw new TypeError(
+        "Expected first argument to be either a client or a url string"
+      )
+    }
+    if (keyStorageStrategy === "string") {
+      throw new Error("Using strings for key storage isn't implemented yet")
+    }
+  }
+
+  async load(key: StorageKey): Promise<Uint8Array | undefined> {
+    assertStorageKey(key)
+    const collection = await this.collection
+    const result = await collection.findOne({ key })
+    if (result) {
+      return new Uint8Array(result.data.value())
+    } else {
+      return undefined
+    }
+  }
+
+  async save(key: StorageKey, data: Uint8Array): Promise<void> {
+    assertStorageKey(key)
+    const collection = await this.collection
+    await collection.updateOne(
+      { key: key },
+      { $set: { data: new BSON.Binary(data) } },
+      { upsert: true }
+    )
+  }
+
+  async remove(key: StorageKey): Promise<void> {
+    assertStorageKey(key)
+    const collection = await this.collection
+    await collection.deleteOne({ key })
+  }
+
+  async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
+    assertStorageKey(keyPrefix)
+    const collection = await this.collection
+    const query = buildKeyPrefixFilter(keyPrefix)
+    const cursor = await collection.find(query)
+    if (cursor) {
+      const result: Chunk[] = []
+      for await (const { key, data } of cursor) {
+        result.push({ key, data: new Uint8Array(data.value()) })
+      }
+      return result
+    } else {
+      return []
+    }
+  }
+
+  async removeRange(keyPrefix: StorageKey): Promise<void> {
+    assertStorageKey(keyPrefix)
+    const collection = await this.collection
+    const query = buildKeyPrefixFilter(keyPrefix)
+    await collection.deleteMany(query)
+  }
+
+  private get collection(): Promise<Collection<ChunkDocument>> {
+    // Lazily connects the client and constructs the db and collection
+    if (!this.#collection) {
+      this.#collection = this.#client
+        .connect()
+        .then(client =>
+          client.db(this.#dbName).collection(this.#collectionName)
+        )
+    }
+    return this.#collection
+  }
+}

packages/automerge-repo-storage-mongodb/test/MongoDBStorageAdapter.test.ts

@@ -0,0 +1,70 @@
+import cp from "node:child_process"
+import { beforeAll, describe } from "vitest"
+import { MongoClient } from "mongodb"
+import { runStorageAdapterTests } from "../../automerge-repo/src/helpers/tests/storage-adapter-tests"
+import { MongoDBStorageAdapter } from "../src"
+
+const MONGODB_URL = "mongodb://localhost:27017"
+const MONGODB_DB_NAME = "automerge-repo-test-db"
+const MONGODB_COLLECTION_NAME = "automerge-repo-storage"
+const MONGODB_DOCKER_CONTAINER_NAME = "automerge-repo-test-mongo"
+
+/**
+ * @returns true if the docker cli is available
+ */
+function hasDocker() {
+  try {
+    cp.execFileSync("docker", ["--version"], { encoding: "utf8" })
+    return true
+  } catch {
+    return false
+  }
+}
+
+let client: MongoClient
+
+describe.skipIf(!hasDocker())("MongoDBStorageAdapter", () => {
+  beforeAll(async () => {
+    try {
+      cp.execFileSync("docker", ["kill", MONGODB_DOCKER_CONTAINER_NAME])
+    } catch {
+      // Will fail if no dockers are running
+    }
+    const mongoContainerId = cp
+      .execFileSync(
+        "docker",
+        [
+          "run",
+          "--rm",
+          "--detach",
+          "--publish",
+          "27017:27017",
+          "--name",
+          MONGODB_DOCKER_CONTAINER_NAME,
+          "mongo",
+        ],
+        { encoding: "utf8" }
+      )
+      .trim()
+
+    client = new MongoClient(MONGODB_URL)
+    await client.connect()
+
+    return async () => {
+      cp.execFileSync("docker", ["kill", mongoContainerId])
+    }
+  })
+  // TODO: Check if docker is available and skip tests if not
+  const setup = async () => {
+    const adapter = new MongoDBStorageAdapter(MONGODB_URL, {
+      dbName: MONGODB_DB_NAME,
+      collectionName: MONGODB_COLLECTION_NAME,
+    })
+    const teardown = async () => {
+      await client.db(MONGODB_DB_NAME).dropCollection(MONGODB_COLLECTION_NAME)
+    }
+    return { adapter, teardown }
+  }
+
+  runStorageAdapterTests(setup)
+})

packages/automerge-repo-storage-mongodb/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "NodeNext",
+    "moduleResolution": "Node16",
+    "declaration": true,
+    "declarationMap": true,
+    "outDir": "./dist",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true
+  },
+  "include": ["src/**/*.ts"]
+}

packages/automerge-repo-storage-mongodb/typedoc.json

@@ -0,0 +1,5 @@
+{
+  "extends": ["../../typedoc.base.json"],
+  "entryPoints": ["src/index.ts"],
+  "readme": "none"
+}

packages/automerge-repo-storage-mongodb/vitest.config.ts

@@ -0,0 +1,11 @@
+import { defineConfig, mergeConfig } from "vitest/config"
+import rootConfig from "../../vitest.config"
+
+export default mergeConfig(
+  rootConfig,
+  defineConfig({
+    test: {
+      environment: "node",
+    },
+  })
+)