feat: Add Substrate indexing

.eslintignore

@@ -1,2 +1,3 @@
 src/model/**
 src/abi/**
+src/indexer/substrateIndexer/types/**

package.json

@@ -28,9 +28,13 @@
     "@subsquid/evm-processor": "^1.17.1",
     "@subsquid/evm-typegen": "^4.0.0",
     "@subsquid/graphql-server": "^4.5.1",
+    "@subsquid/ss58": "^2.0.2",
+    "@subsquid/substrate-processor": "^8.5.1",
+    "@subsquid/substrate-runtime": "^2.0.0",
+    "@subsquid/substrate-typegen": "^8.1.0",
     "@subsquid/typeorm-codegen": "^1.3.3",
     "@subsquid/typeorm-migration": "^1.3.0",
-    "@subsquid/typeorm-store": "^1.3.0",
+    "@subsquid/typeorm-store": "^1.5.1",
     "ethers": "^6.11.1",
     "license-check-and-add": "^4.0.5",
     "pg": "^8.11.5",
@@ -46,6 +50,7 @@
     "@subsquid/substrate-metadata-explorer": "^3.1.2",
     "@types/chai": "^4.3.16",
     "@types/mocha": "^10.0.7",
+    "@types/node": "^20.14.12",
     "@types/sinon": "^17.0.3",
     "chai": "^4.3.7",
     "eslint": "^8.21.0",

src/indexer/config/index.ts

@@ -16,6 +16,8 @@ import { EVMParser } from "../evmIndexer/evmParser";
 import { ContractType } from "../evmIndexer/evmTypes";
 import { getContract } from "../evmIndexer/utils";
 import type { IParser, IProcessor } from "../indexer";
+import { SubstrateParser } from "../substrateIndexer/substrateParser";
+import { createSubstrateProvider } from "../substrateIndexer/utils";
 
 import type { EnvVariables } from "./validator";
 
@@ -40,6 +42,7 @@ export type Domain = DomainSDK & {
   startBlock: number;
   resources: Array<Resource>;
   blockConfirmations: number;
+  gateway?: string;
 };
 
 export enum HandlerType {
@@ -76,7 +79,11 @@ export async function getConfig(envVars: EnvVariables): Promise<Config> {
   const rpcMap = createRpcMap(envVars.rpcUrls);
   const parserMap = await initializeParserMap(sharedConfig, rpcMap);
 
-  const domainConfig = getDomainConfig(sharedConfig, envVars.domainId);
+  const domainConfig = getDomainConfig(
+    sharedConfig,
+    envVars.domainId,
+    envVars.domainGateway,
+  );
   const parser = getDomainParser(domainConfig.id, parserMap);
 
   parser.setParsers(parserMap);
@@ -125,46 +132,58 @@ async function initializeParserMap(
       decimals: domain.nativeTokenDecimals,
     });
 
-    if (domain.type === Network.EVM) {
-      const rpcUrl = rpcMap.get(domain.id);
-      if (!rpcUrl) {
-        throw new Error(
-          `Unsupported or missing RPC URL for domain ID: ${domain.id}`,
-        );
-      }
-      const provider = new ethers.JsonRpcProvider(rpcUrl);
-
-      for (const resource of domain.resources as EvmResource[]) {
-        if (
-          resource.type == ResourceType.FUNGIBLE &&
-          resource.address !== NATIVE_TOKEN_ADDRESS
-        ) {
-          const token = getContract(
-            provider,
-            resource.address,
-            ContractType.ERC20,
-          );
-          const symbol = (await token.symbol()) as string;
-          const decimals = Number(await token.decimals());
-
-          tokenMap.set(resource.address.toLowerCase(), { symbol, decimals });
+    const rpcUrl = rpcMap.get(domain.id);
+    if (!rpcUrl) {
+      throw new Error(
+        `Unsupported or missing RPC URL for domain ID: ${domain.id}`,
+      );
+    }
+    switch (domain.type) {
+      case Network.EVM: {
+        const provider = new ethers.JsonRpcProvider(rpcUrl);
+
+        for (const resource of domain.resources as EvmResource[]) {
+          if (
+            resource.type == ResourceType.FUNGIBLE &&
+            resource.address != NATIVE_TOKEN_ADDRESS
+          ) {
+            const token = getContract(
+              provider,
+              resource.address,
+              ContractType.ERC20,
+            );
+            const symbol = (await token.symbol()) as string;
+            const decimals = Number(await token.decimals());
+
+            tokenMap.set(resource.address.toLowerCase(), { symbol, decimals });
+          }
         }
+        parserMap.set(domain.id, new EVMParser(provider, tokenMap));
+        break;
+      }
+      case Network.SUBSTRATE: {
+        const provider = await createSubstrateProvider(rpcUrl);
+        parserMap.set(domain.id, new SubstrateParser(provider));
+        break;
       }
-      parserMap.set(domain.id, new EVMParser(provider, tokenMap));
     }
   }
 
   return parserMap;
 }
 
-function getDomainConfig(sharedConfig: SharedConfig, domainId: number): Domain {
+function getDomainConfig(
+  sharedConfig: SharedConfig,
+  domainId: number,
+  domainGateway: string,
+): Domain {
   const domainConfig = sharedConfig.domains.find(
     (domain) => domain.id === domainId,
   );
   if (!domainConfig) {
     throw new Error(`No configuration found for domain ID: ${domainId}`);
   }
-
+  domainConfig.gateway = domainGateway;
   return domainConfig;
 }
 

src/indexer/evmIndexer/evmParser.ts

@@ -8,20 +8,20 @@ import { randomUUID } from "crypto";
 import { ResourceType } from "@buildwithsygma/core";
 import type { Log } from "@subsquid/evm-processor";
 import { assertNotNull, decodeHex } from "@subsquid/evm-processor";
-import type { BigNumberish, JsonRpcProvider, Provider } from "ethers";
-import { AbiCoder, ethers, formatUnits } from "ethers";
+import type { JsonRpcProvider, Provider } from "ethers";
+import { ethers } from "ethers";
 
 import * as bridge from "../../abi/bridge";
+import { decodeAmountOrTokenId, generateTransferID } from "../../indexer/utils";
 import { logger } from "../../utils/logger";
 import type { Domain, Token } from "../config";
 import type { IParser } from "../indexer";
 import type {
   DecodedDepositLog,
-  DecodedFailedHandlerExecution,
+  DecodedFailedHandlerExecutionLog,
   DecodedProposalExecutionLog,
   FeeData,
 } from "../types";
-import { generateTransferID } from "../utils";
 
 import { ContractType } from "./evmTypes";
 import { getContract } from "./utils";
@@ -64,8 +64,8 @@ export class EVMParser implements IParser {
         `Resource with ID ${event.resourceID} not found in shared configuration`,
       );
     }
-    const resourceType = resource.type || "";
-    const resourceDecimals = resource.decimals || 18;
+    const resourceType = resource.type ?? "";
+    const resourceDecimals = resource.decimals ?? 18;
 
     const transaction = assertNotNull(log.transaction, "Missing transaction");
 
@@ -87,11 +87,7 @@ export class EVMParser implements IParser {
       depositData: event.data,
       handlerResponse: event.handlerResponse,
       transferType: resourceType,
-      amount: this.decodeAmountsOrTokenId(
-        event.data,
-        resourceDecimals,
-        resourceType,
-      ),
+      amount: decodeAmountOrTokenId(event.data, resourceDecimals, resourceType),
       fee: await this.getFee(event, fromDomain, this.provider),
     };
   }
@@ -121,7 +117,7 @@ export class EVMParser implements IParser {
   public parseFailedHandlerExecution(
     log: Log,
     toDomain: Domain,
-  ): DecodedFailedHandlerExecution {
+  ): DecodedFailedHandlerExecutionLog {
     const event = bridge.events.FailedHandlerExecution.decode(log);
     const transaction = assertNotNull(log.transaction, "Missing transaction");
 
@@ -133,6 +129,7 @@ export class EVMParser implements IParser {
     } catch (err) {
       errMsg = "Unknown error type, raw data:" + event.lowLevelData.toString();
     }
+
     return {
       id: generateTransferID(
         event.depositNonce.toString(),
@@ -169,8 +166,6 @@ export class EVMParser implements IParser {
         logger.error(`Unsupported resource type: ${resourceType}`);
         return "";
     }
-
-    console.log(recipient);
     return recipient;
   }
 
@@ -199,9 +194,9 @@ export class EVMParser implements IParser {
         id: randomUUID(),
         tokenAddress: fee.tokenAddress,
         tokenSymbol:
-          this.tokens.get(fee.tokenAddress.toLowerCase())?.symbol || "",
+          this.tokens.get(fee.tokenAddress.toLowerCase())?.symbol ?? "",
         decimals:
-          this.tokens.get(fee.tokenAddress.toLowerCase())?.decimals || 18,
+          this.tokens.get(fee.tokenAddress.toLowerCase())?.decimals ?? 18,
         amount: fee.fee.toString(),
       };
     } catch (err) {
@@ -216,31 +211,6 @@ export class EVMParser implements IParser {
     }
   }
 
-  private decodeAmountsOrTokenId(
-    data: string,
-    decimals: number,
-    resourceType: ResourceType,
-  ): string {
-    switch (resourceType) {
-      case ResourceType.FUNGIBLE: {
-        const amount = AbiCoder.defaultAbiCoder().decode(
-          ["uint256"],
-          data,
-        )[0] as BigNumberish;
-        return formatUnits(amount, decimals);
-      }
-      case ResourceType.NON_FUNGIBLE: {
-        const tokenId = AbiCoder.defaultAbiCoder().decode(
-          ["uint256"],
-          data,
-        )[0] as bigint;
-        return tokenId.toString();
-      }
-      default:
-        return "";
-    }
-  }
-
   private decodeGenericCall(genericCallData: Buffer): string {
     // 32 + 2 + 1 + 1 + 20 + 20
     const lenExecuteFuncSignature = Number(

src/indexer/evmIndexer/evmProcessor.ts

@@ -2,14 +2,19 @@
 The Licensed Work is (c) 2024 Sygma
 SPDX-License-Identifier: LGPL-3.0-only
 */
+import type {
+  DataHandlerContext,
+  EvmBatchProcessorFields,
+} from "@subsquid/evm-processor";
 import { EvmBatchProcessor } from "@subsquid/evm-processor";
+import type { Store } from "@subsquid/typeorm-store";
 
 import * as bridge from "../../abi/bridge";
 import type { Domain } from "../config";
-import type { Context, DecodedEvents, IParser, IProcessor } from "../indexer";
+import type { DecodedEvents, IParser, IProcessor } from "../indexer";
 import type {
   DecodedDepositLog,
-  DecodedFailedHandlerExecution,
+  DecodedFailedHandlerExecutionLog,
   DecodedProposalExecutionLog,
 } from "../types";
 
@@ -44,8 +49,8 @@ export class EVMProcessor implements IProcessor {
         transaction: true,
       });
 
-    if (process.env.DOMAIN_GATEWAY) {
-      evmProcessor.setGateway(process.env.DOMAIN_GATEWAY);
+    if (domain.gateway) {
+      evmProcessor.setGateway(domain.gateway);
     }
     return evmProcessor;
   }
@@ -56,7 +61,7 @@ export class EVMProcessor implements IProcessor {
   ): Promise<DecodedEvents> {
     const deposits: DecodedDepositLog[] = [];
     const executions: DecodedProposalExecutionLog[] = [];
-    const failedHandlerExecutions: DecodedFailedHandlerExecution[] = [];
+    const failedHandlerExecutions: DecodedFailedHandlerExecutionLog[] = [];
     for (const block of ctx.blocks) {
       for (const log of block.logs) {
         if (log.topics[0] === bridge.events.Deposit.topic) {
@@ -72,6 +77,9 @@ export class EVMProcessor implements IProcessor {
         }
       }
     }
-    return { deposits, executions, failedHandlerExecutions };
+    return { deposits, executions, failedHandlerExecutions, fees: [] };
   }
 }
+
+export type Fields = EvmBatchProcessorFields<EvmBatchProcessor>;
+export type Context = DataHandlerContext<Store, Fields>;