feat: refactor startup

.env

@@ -1,7 +1,11 @@
-DB_NAME=squid
-DB_PASS=squid
+DB_NAME=postgres
+DB_PASS=postgres
 DB_PORT=5432
 DB_USERNAME=postgres
-DB_HOST=localhost
+DB_HOST=db
+SHARED_CONFIG_URL="https://ipfs.io/ipfs/bafkreiasnla2ya55of6nwm3swjstip4q2ixfa3t6tvixyibclfovxnerte"
+1_METADATA='{"id": 1, "rpcUrl": "http://evm1:8545"}'
+2_METADATA='{"id": 2, "rpcUrl": "http://evm2:8545"}'
+3_METADATA='{"id": 3, "rpcUrl": "ws://substrate-pallet:9944"}'
 
-SHARED_CONFIG_URL="https://ipfs.io/ipfs/bafkreiasnla2ya55of6nwm3swjstip4q2ixfa3t6tvixyibclfovxnerte"
+ENV_DOMAINS='[1,2,3]'

Dockerfile

@@ -33,12 +33,11 @@ COPY --from=builder /squid/package*.json ./
 COPY --from=builder /squid/lib ./lib
 COPY --from=builder /squid/envs ./envs
 COPY --from=builder /squid/db ./db
-COPY --from=builder /squid/start-prod.sh ./start-prod.sh
 COPY --from=builder /squid/commands.json ./
 
 RUN corepack yarn global add @subsquid/commands && mv $(which squid-commands) /usr/local/bin/sqd
 
 LABEL org.opencontainers.image.source https://github.com/sygmaprotocol/squid-indexer/
 EXPOSE 8000
 
-ENTRYPOINT sh /start-prod.sh
+ENTRYPOINT ["sh", "-c", "sqd migration:apply && node ./lib/main.js"]

docker-compose.yml

@@ -37,45 +37,21 @@ services:
       POSTGRES_PASSWORD: ${DB_PASS}
     ports:
       - "23798:${DB_PORT}"
-    env_file: ./envs/.env.init
-  indexer-init:
+    env_file: ./envs/.env.indexer.example
+  indexer:
     build: 
       context: .
       dockerfile: ./Dockerfile 
     restart: on-failure
     depends_on: 
       - db
-    env_file: ./envs/.env.init
-  indexer_evm1:
-    build: 
-      context: .
-      dockerfile: ./Dockerfile 
-    restart: on-failure
-    depends_on: 
-      - indexer-init
-    env_file: ./envs/.env.evm-1
-  indexer_evm2:
-    build: 
-      context: .
-      dockerfile: ./Dockerfile 
-    restart: on-failure
-    depends_on: 
-      - indexer-init
-    env_file: ./envs/.env.evm-2
-  indexer_substrate:
-    build: 
-      context: .
-      dockerfile: ./Dockerfile 
-    restart: on-failure
-    depends_on: 
-      - indexer-init
-    env_file: ./envs/.env.substrate
+    env_file: ./envs/.env.indexer.example
   api:
     build: 
       context: .
       dockerfile: ./Dockerfile.api
     depends_on: 
-      - indexer-init
-    env_file: ./envs/.env.api
+      - indexer
+    env_file: ./envs/.env.indexer.example
     ports:
-      - "8000:8000"
+      - "8000:8000"

envs/.env.indexer.example

@@ -0,0 +1,11 @@
+DB_NAME=postgres
+DB_PASS=postgres
+DB_PORT=5432
+DB_USERNAME=postgres
+DB_HOST=db
+SHARED_CONFIG_URL="https://ipfs.io/ipfs/bafkreiasnla2ya55of6nwm3swjstip4q2ixfa3t6tvixyibclfovxnerte"
+1_METADATA='{"id": 1, "rpcUrl": "http://evm1:8545"}'
+2_METADATA='{"id": 2, "rpcUrl": "http://evm2:8545"}'
+3_METADATA='{"id": 3, "rpcUrl": "ws://substrate-pallet:9944"}'
+
+ENV_DOMAINS='[1,2,3]'

src/api/App.ts

@@ -22,7 +22,6 @@ import { routesPlugin } from "./services/plugins/routes";
 
 export class App {
   public readonly instance: FastifyInstance;
-
   protected constructor(instance: FastifyInstance) {
     this.instance = instance;
   }

src/api/controllers/TransfersController.ts

@@ -16,7 +16,6 @@ import { TransfersService } from "../services/dataAccess/transfers.service";
 
 export class TransfersController {
   private transfersService: TransfersService;
-
   constructor(dataSource: DataSource) {
     this.transfersService = new TransfersService(dataSource);
   }

src/indexer/config/envLoader.ts

@@ -19,10 +19,10 @@ export type DomainMetadata = {
 
 export type EnvVariables = {
   sharedConfigURL: string;
-  domainMetadata: DomainMetadata;
   dbConfig: DbConfig;
   logLevel: string;
   version: string;
+  envDomains: number[];
 };
 
 export function getEnv(): EnvVariables {
@@ -31,17 +31,6 @@ export function getEnv(): EnvVariables {
     throw new Error(`SHARED_CONFIG_URL is not defined in the environment.`);
   }
 
-  const domainId = Number(process.env.DOMAIN_ID);
-  if (isNaN(domainId)) {
-    throw new Error(`DOMAIN_ID environment variable is invalid or not set.`);
-  }
-
-  const domainMetadata = process.env[`${domainId}_METADATA`];
-  if (!domainMetadata) {
-    throw new Error(`Environment variable ${domainId}_METADATA is not set`);
-  }
-  const parsedDomainMetadata = JSON.parse(domainMetadata) as DomainMetadata;
-
   const dbHost = process.env.DB_HOST;
   if (!dbHost) {
     throw new Error(`DB_HOST is not defined in the environment.`);
@@ -67,16 +56,17 @@ export function getEnv(): EnvVariables {
     throw new Error(`DB_PASS is not defined in the environment.`);
   }
 
+  const envDomains = process.env.ENV_DOMAINS;
+  if (!envDomains) {
+    throw new Error(`ENV_DOMAINS is not defined in the environment.`);
+  }
+  const parsedEnvDomains = JSON.parse(envDomains) as number[];
+
   const logLevel = process.env.LOG_LEVEL || "debug";
   const version = process.env.VERSION || "unknown";
 
   return {
     sharedConfigURL,
-    domainMetadata: {
-      domainId: domainId,
-      domainGateway: parsedDomainMetadata.domainGateway,
-      rpcUrl: parsedDomainMetadata.rpcUrl,
-    },
     dbConfig: {
       host: dbHost,
       name: dbName,
@@ -86,5 +76,14 @@ export function getEnv(): EnvVariables {
     },
     logLevel,
     version,
+    envDomains: parsedEnvDomains,
   };
 }
+
+export function getDomainMetadata(domainID: string): DomainMetadata {
+  const domainMetadata = process.env[`${domainID}_METADATA`];
+  if (!domainMetadata) {
+    throw new Error(`Domain metadata not configured for domain: ${domainID}`);
+  }
+  return JSON.parse(domainMetadata) as DomainMetadata;
+}

src/indexer/config/index.ts

@@ -10,8 +10,6 @@ import type {
 
 import { logger } from "../../utils/logger";
 
-import type { EnvVariables } from "./envLoader";
-
 export type SharedConfig = {
   domains: Array<Domain>;
 };
@@ -42,21 +40,6 @@ type Handler = {
   address: string;
 };
 
-export async function getDomainConfig(envVars: EnvVariables): Promise<Domain> {
-  const sharedConfig = await fetchSharedConfig(envVars.sharedConfigURL);
-
-  const domainConfig = sharedConfig.domains.find(
-    (domain) => domain.id === envVars.domainMetadata.domainId,
-  );
-  if (!domainConfig) {
-    throw new Error(
-      `No configuration found for domain ID: ${envVars.domainMetadata.domainId}`,
-    );
-  }
-  domainConfig.gateway = envVars.domainMetadata.domainGateway;
-  return domainConfig;
-}
-
 export async function fetchSharedConfig(url: string): Promise<SharedConfig> {
   try {
     const response = await fetch(url);

src/indexer/evmIndexer/evmParser.ts

@@ -10,6 +10,7 @@ import type { Log } from "@subsquid/evm-processor";
 import { assertNotNull } from "@subsquid/evm-processor";
 import type { JsonRpcProvider, Provider } from "ethers";
 import { ethers } from "ethers";
+import type winston from "winston";
 
 import * as bridge from "../../abi/bridge";
 import {
@@ -19,7 +20,6 @@ import {
 } from "../../indexer/utils";
 import { Domain, Resource, Route, Token } from "../../model";
 import { NotFoundError } from "../../utils/error";
-import { logger } from "../../utils/logger";
 import type { Domain as DomainType } from "../config";
 import type { IParser } from "../indexer";
 import type {
@@ -41,8 +41,10 @@ type FeeDataResponse = {
 export class EVMParser implements IParser {
   private STATIC_FEE_DATA = "0x00";
   private provider: JsonRpcProvider;
-  constructor(provider: JsonRpcProvider) {
+  private logger: winston.Logger;
+  constructor(provider: JsonRpcProvider, logger: winston.Logger) {
     this.provider = provider;
+    this.logger = logger;
   }
 
   public async parseDeposit(
@@ -241,7 +243,7 @@ export class EVMParser implements IParser {
         amount: fee.fee.toString(),
       };
     } catch (err) {
-      logger.error("Calculating fee failed", err);
+      this.logger.error("Calculating fee failed", err);
       return {
         tokenAddress: "",
         amount: "0",

src/indexer/evmIndexer/evmProcessor.ts

@@ -8,10 +8,10 @@ import type {
 } from "@subsquid/evm-processor";
 import { EvmBatchProcessor } from "@subsquid/evm-processor";
 import type { Store } from "@subsquid/typeorm-store";
+import type winston from "winston";
 
 import * as bridge from "../../abi/bridge";
 import { NotFoundError } from "../../utils/error";
-import { logger } from "../../utils/logger";
 import type { Domain } from "../config";
 import type { IParser, IProcessor } from "../indexer";
 import type {
@@ -25,9 +25,11 @@ import type {
 export class EVMProcessor implements IProcessor {
   private parser: IParser;
   private rpcUrl: string;
-  constructor(parser: IParser, rpcUrl: string) {
+  private logger: winston.Logger;
+  constructor(parser: IParser, rpcUrl: string, logger: winston.Logger) {
     this.parser = parser;
     this.rpcUrl = rpcUrl;
+    this.logger = logger;
   }
   public getProcessor(domain: Domain): EvmBatchProcessor {
     const evmProcessor = new EvmBatchProcessor()
@@ -69,6 +71,9 @@ export class EVMProcessor implements IProcessor {
     const fees: FeeCollectedData[] = [];
 
     for (const block of ctx.blocks) {
+      this.logger.info(
+        `Processing block ${block.header.height} on networ ${domain.name}(${domain.id})`,
+      );
       for (const log of block.logs) {
         try {
           switch (log.topics[0]) {
@@ -97,12 +102,12 @@ export class EVMProcessor implements IProcessor {
             }
 
             default:
-              logger.error(`Unsupported log topic: ${log.topics[0]}`);
+              this.logger.error(`Unsupported log topic: ${log.topics[0]}`);
               break;
           }
         } catch (error) {
           if (error instanceof NotFoundError) {
-            logger.error(error.message);
+            this.logger.error(error.message);
           } else {
             throw error;
           }

src/indexer/indexer.ts

@@ -11,6 +11,7 @@ import type {
 import type { SubstrateBatchProcessor } from "@subsquid/substrate-processor";
 import { TypeormDatabase } from "@subsquid/typeorm-store";
 import { In } from "typeorm";
+import type winston from "winston";
 
 import {
   Transfer,
@@ -20,7 +21,6 @@ import {
   TransferStatus,
   Fee,
 } from "../model";
-import { logger } from "../utils/logger";
 
 import type { Domain } from "./config";
 import type { Context as EvmContext } from "./evmIndexer/evmProcessor";
@@ -69,10 +69,12 @@ export interface IProcessor {
 export class Indexer {
   private domain: Domain;
   private processor: IProcessor;
+  private logger: winston.Logger;
 
-  constructor(processor: IProcessor, domain: Domain) {
+  constructor(processor: IProcessor, domain: Domain, logger: winston.Logger) {
     this.processor = processor;
     this.domain = domain;
+    this.logger = logger;
   }
 
   public startProcessing(): void {
@@ -83,6 +85,7 @@ export class Indexer {
         isolationLevel: "READ COMMITTED",
       }),
       async (ctx) => {
+        ctx.log = ctx.log.child({ domain: this.domain.id.toString() });
         const decodedEvents = await this.processor.processEvents(
           ctx,
           this.domain,
@@ -240,7 +243,7 @@ export class Indexer {
         relations: { deposit: true },
       });
       if (!transfer?.deposit) {
-        logger.warn(
+        this.logger.warn(
           `Deposit for the fee with txHash: ${f.txIdentifier} not found, skipping...`,
         );
         continue;