Update State Storage to use Linked Merkle Tree

docs/sequencer/spec.md

@@ -0,0 +1,76 @@
+## Merkle Tree Stores
+
+Object we need to store:
+(Nodes, Leaves, MaximumIndex)
+
+Level 1:
+Async stores: (InMemory*, Redis*)
+
+Schema:
+Record<path, { leaf, index }> 
+
+write<object>
+get<object>Async
+getMaximumIndexAsync
+getLeafLessOrEqualAsync(path) (gives us either our current leaf or previous leaf in case of insert)
+
+openTransaction()
+commit()
+
+( getLeafByIndex )
+
+Level 2:
+CachedStore: implements Sync, parent: Async
+
+Sync:
+set<object>
+getNode
+getLeaf(path) => { leaf: LinkedLeaf, index: bigint }
+getMaximumIndex
+getLeafLessOrEqual(path) => { leaf: LinkedLeaf, index: bigint }
+
+Cached:
+preloadMerkleWitness(index)
+preloadKeys(paths: string[])
+mergeIntoParent()
+
+Level 3:
+SyncCachedStore: implements Sync, parent: Sync
+mergeIntoParent()
+
+preLoading:
+input: path
+```
+const leaf = parent.getLeaf(path)
+if(leaf !== undefined) {
+  super.cache(leaf);
+  // Update
+  preloadMerkleWitness(leaf.index);
+} else {
+  // Insert
+  const previousLeaf = parent.getLeafLessOrEqual(path);
+  super.cache(previousLeaf);
+  preloadMerkleWitness(previousLeaf.index);
+  const maximumIndex = this.preloadAndGetMaximumINndex(); // super.getMaximumINdex() ?? await parent.getMaximumIndexASync()
+  preloadMerkleWitness(maximumIndex);
+}
+
+```
+
+Sync interface:
+Union of LinkedMerkleTreeStore (rename to LinkedLeafStore) + MerkleTreeStore
+
+Async
+Level 1 methods
+
+InMemoryLeafStore - subset that does leafs + maximumindex
+InMemoryMerkleStore - subset that does only merkle nodes
+
+-> future Redis
+
+InMemoryAsyncLinkedMerkleTreeStore - implements Async 
+uses inmemory implementations
+
+
+
+

packages/api/src/graphql/VanillaGraphqlModules.ts

@@ -6,7 +6,7 @@ import { QueryGraphqlModule } from "./modules/QueryGraphqlModule";
 import { BatchStorageResolver } from "./modules/BatchStorageResolver";
 import { NodeStatusResolver } from "./modules/NodeStatusResolver";
 import { BlockResolver } from "./modules/BlockResolver";
-import { MerkleWitnessResolver } from "./modules/MerkleWitnessResolver";
+import { LinkedMerkleWitnessResolver as MerkleWitnessResolver } from "./modules/LinkedMerkleWitnessResolver";
 
 export type VanillaGraphqlModulesRecord = {
   MempoolResolver: typeof MempoolResolver;

packages/api/src/graphql/modules/LeafResolver.ts

@@ -0,0 +1,28 @@
+import { Field, ObjectType } from "type-graphql";
+import { LinkedLeafStruct } from "@proto-kit/common";
+
+@ObjectType()
+export class LeafDTO {
+  public static fromServiceLayerModel(leaf: LinkedLeafStruct) {
+    return new LeafDTO(
+      leaf.value.toString(),
+      leaf.path.toString(),
+      leaf.nextPath.toString()
+    );
+  }
+
+  @Field()
+  value: string;
+
+  @Field()
+  path: string;
+
+  @Field()
+  nextPath: string;
+
+  private constructor(value: string, path: string, nextPath: string) {
+    this.value = value;
+    this.path = path;
+    this.nextPath = nextPath;
+  }
+}

packages/api/src/graphql/modules/LinkedMerkleWitnessResolver.ts

@@ -0,0 +1,62 @@
+import { Arg, Field, ObjectType, Query } from "type-graphql";
+import { inject } from "tsyringe";
+import {
+  LinkedLeafAndMerkleWitness,
+  LinkedMerkleTree,
+} from "@proto-kit/common";
+import { CachedLinkedMerkleTreeStore } from "@proto-kit/sequencer/dist/state/merkle/CachedLinkedMerkleTreeStore";
+import { AsyncLinkedMerkleTreeStore } from "@proto-kit/sequencer/dist/state/async/AsyncLinkedMerkleTreeStore";
+
+import { GraphqlModule, graphqlModule } from "../GraphqlModule";
+
+import { MerkleWitnessDTO } from "./MerkleWitnessResolver";
+import { LeafDTO } from "./LeafResolver";
+
+@ObjectType()
+export class LinkedMerkleWitnessDTO {
+  public static fromServiceLayerObject(witness: LinkedLeafAndMerkleWitness) {
+    const { leaf, merkleWitness } = witness;
+    const leafDTO = LeafDTO.fromServiceLayerModel(leaf);
+    const witnessDTO = MerkleWitnessDTO.fromServiceLayerObject(merkleWitness);
+    return new LinkedMerkleWitnessDTO(leafDTO, witnessDTO);
+  }
+
+  public constructor(leaf: LeafDTO, witness: MerkleWitnessDTO) {
+    this.leaf = leaf;
+    this.merkleWitness = new MerkleWitnessDTO(
+      witness.siblings,
+      witness.isLefts
+    );
+  }
+
+  @Field(() => LeafDTO)
+  public leaf: LeafDTO;
+
+  @Field(() => MerkleWitnessDTO)
+  public merkleWitness: MerkleWitnessDTO;
+}
+
+@graphqlModule()
+export class LinkedMerkleWitnessResolver extends GraphqlModule<object> {
+  public constructor(
+    @inject("AsyncMerkleStore")
+    private readonly treeStore: AsyncLinkedMerkleTreeStore
+  ) {
+    super();
+  }
+
+  @Query(() => LinkedMerkleWitnessDTO, {
+    description:
+      "Allows retrieval of merkle witnesses corresponding to a specific path in the appchain's state tree. These proves are generally retrieved from the current 'proven' state",
+  })
+  public async witness(@Arg("path") path: string) {
+    const syncStore = await CachedLinkedMerkleTreeStore.new(this.treeStore);
+
+    const tree = new LinkedMerkleTree(syncStore);
+    await syncStore.preloadKey(BigInt(path));
+
+    const witness = tree.getWitness(BigInt(path));
+
+    return LinkedMerkleWitnessDTO.fromServiceLayerObject(witness);
+  }
+}

packages/api/src/index.ts

@@ -9,4 +9,5 @@ export * from "./graphql/modules/NodeStatusResolver";
 export * from "./graphql/modules/AdvancedNodeStatusResolver";
 export * from "./graphql/services/NodeStatusService";
 export * from "./graphql/modules/MerkleWitnessResolver";
+export * from "./graphql/modules/LinkedMerkleWitnessResolver";
 export * from "./graphql/VanillaGraphqlModules";

packages/common/package.json

@@ -20,7 +20,8 @@
     "lodash": "^4.17.21",
     "loglevel": "^1.8.1",
     "reflect-metadata": "^0.1.13",
-    "typescript-memoize": "^1.1.1"
+    "typescript-memoize": "^1.1.1",
+    "ts-mixer": "^6.0.3"
   },
   "peerDependencies": {
     "o1js": "^1.1.0",

packages/common/src/index.ts

@@ -13,7 +13,11 @@ export * from "./log";
 export * from "./events/EventEmittingComponent";
 export * from "./events/EventEmitter";
 export * from "./trees/MerkleTreeStore";
+export * from "./trees/LinkedMerkleTreeStore";
 export * from "./trees/InMemoryMerkleTreeStorage";
 export * from "./trees/RollupMerkleTree";
+export * from "./trees/LinkedMerkleTree";
+export * from "./trees/InMemoryLinkedLeafStore";
 export * from "./events/EventEmitterProxy";
 export * from "./trees/MockAsyncMerkleStore";
+export * from "./trees/InMemoryLinkedMerkleLeafStore";

packages/common/src/trees/InMemoryLinkedLeafStore.ts

@@ -0,0 +1,40 @@
+import { LinkedLeafStore, LinkedLeaf } from "./LinkedMerkleTreeStore";
+
+export class InMemoryLinkedLeafStore implements LinkedLeafStore {
+  public leaves: {
+    [key: string]: { leaf: LinkedLeaf; index: bigint };
+  } = {};
+
+  public maximumIndex?: bigint;
+
+  public getLeaf(
+    path: bigint
+  ): { leaf: LinkedLeaf; index: bigint } | undefined {
+    return this.leaves[path.toString()];
+  }
+
+  public setLeaf(index: bigint, value: LinkedLeaf): void {
+    const leaf = this.getLeaf(value.path);
+    if (leaf !== undefined && leaf?.index !== index) {
+      throw new Error("Cannot change index of existing leaf");
+    }
+    this.leaves[value.path.toString()] = { leaf: value, index: index };
+    if (this.maximumIndex === undefined || index > this.maximumIndex) {
+      this.maximumIndex = index;
+    }
+  }
+
+  public getMaximumIndex(): bigint | undefined {
+    return this.maximumIndex;
+  }
+
+  // This gets the leaf with the closest path.
+  public getLeafLessOrEqual(
+    path: bigint
+  ): { leaf: LinkedLeaf; index: bigint } | undefined {
+    return Object.values(this.leaves).find(
+      (storedLeaf) =>
+        storedLeaf.leaf.nextPath > path && storedLeaf.leaf.path <= path
+    );
+  }
+}

packages/common/src/trees/InMemoryLinkedMerkleLeafStore.ts

@@ -0,0 +1,9 @@
+import { Mixin } from "ts-mixer";
+
+import { InMemoryLinkedLeafStore } from "./InMemoryLinkedLeafStore";
+import { InMemoryMerkleTreeStorage } from "./InMemoryMerkleTreeStorage";
+
+export class InMemoryLinkedMerkleLeafStore extends Mixin(
+  InMemoryLinkedLeafStore,
+  InMemoryMerkleTreeStorage
+) {}

packages/common/src/trees/InMemoryMerkleTreeStorage.ts

@@ -1,7 +1,7 @@
 import { MerkleTreeStore } from "./MerkleTreeStore";
 
 export class InMemoryMerkleTreeStorage implements MerkleTreeStore {
-  protected nodes: {
+  public nodes: {
     [key: number]: {
       [key: string]: bigint;
     };