Indexers v2

docker-compose.yml

@@ -25,6 +25,42 @@ services:
         limits:
           cpus: "4.0"
           memory: 8192M
+
+  arbitrum-mainnet-indexer:
+    build:
+      context: ./indexers-v2
+      dockerfile: Dockerfile
+    networks:
+      - data
+    depends_on:
+      - db
+    restart: always
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_NAME: arbitrum_mainnet
+      DB_PASS: $PG_PASSWORD
+      GQL_PORT: 4350
+      RPC_ENDPOINT: wss://arbitrum-one-rpc.publicnode.com
+      NETWORK_NAME: arbitrum_mainnet
+
+  base-mainnet-indexer:
+    build:
+      context: ./indexers-v2
+      dockerfile: Dockerfile
+    networks:
+      - data
+    depends_on:
+      - db
+    restart: always
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_NAME: base_mainnet
+      DB_PASS: $PG_PASSWORD
+      GQL_PORT: 4350
+      RPC_ENDPOINT: https://mainnet.base.org
+      NETWORK_NAME: base_mainnet
 
   eth-mainnet-processor:
     image: ghcr.io/synthetixio/data/indexer:${VERSION}-mainnet

indexers-v2/Dockerfile

@@ -0,0 +1,16 @@
+FROM node:16-alpine
+
+WORKDIR /app
+
+COPY package*.json ./
+COPY requirements.txt .
+
+RUN apk add --no-cache python3 py3-pip
+RUN apk add --no-cache build-base && npm ci && apk del build-base
+
+COPY . .
+
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir -r requirements.txt
+
+ENTRYPOINT ["sh", "-c", "python3 main.py --network_name \"$NETWORK_NAME\" --rpc_endpoint \"$RPC_ENDPOINT\" && npm run generate:processor && npm run build && (npm run generate:migration || true) && npm run start"]

indexers-v2/commands.json

@@ -0,0 +1,93 @@
+{
+    "$schema": "https://cdn.subsquid.io/schemas/commands.json",
+    "commands": {
+      "clean": {
+        "description": "delete all build artifacts",
+        "cmd": ["npx", "--yes", "rimraf", "lib"]
+      },
+      "generate": {
+        "description": "Generate a squid from an ABI file",
+        "cmd": ["squid-gen-abi"]
+      },
+      "squid-gen-abi": {
+        "description": "Generate a squid from an ABI file",
+        "cmd": ["squid-gen-abi"],
+        "hidden": true
+      },
+      "build": {
+        "description": "Build the squid project",
+        "deps": ["clean"],
+        "cmd": ["tsc"]
+      },
+      "up": {
+        "description": "Start a PG database",
+        "cmd": ["docker-compose", "up", "-d"]
+      },
+      "down": {
+        "description": "Drop a PG database",
+        "cmd": ["docker-compose", "down"]
+      },
+      "migration:apply": {
+        "description": "Apply the DB migrations",
+        "cmd": ["squid-typeorm-migration", "apply"]
+      },
+      "migration:generate": {
+        "description": "Generate a DB migration matching the TypeORM entities",
+        "deps": ["build", "migration:clean"],
+        "cmd": ["squid-typeorm-migration", "generate"],
+      },
+      "migration:clean": {
+        "description": "Clean the migrations folder",
+        "cmd": ["npx", "--yes", "rimraf", "./db/migrations"],
+      },
+      "migration": {
+        "deps": ["build"],
+        "cmd": ["squid-typeorm-migration", "generate"],
+        "hidden": true
+      },
+      "codegen": {
+        "description": "Generate TypeORM entities from the schema file",
+        "cmd": ["squid-typeorm-codegen"]
+      },
+      "typegen": {
+        "description": "Generate data access classes for an ABI file(s) in the ./abi folder",
+        "cmd": ["squid-evm-typegen", "./src/abi", {"glob": "./abi/*.json"}, "--multicall"]
+      },
+      "process": {
+        "description": "Load .env and start the squid processor",
+        "deps": ["build", "migration:apply"],
+        "cmd": ["node", "--require=dotenv/config", "lib/main.js"]
+      },
+      "process:prod": {
+        "description": "Start the squid processor",
+        "deps": ["migration:apply"],
+        "cmd": ["node", "lib/main.js"],
+        "hidden": true
+      },
+      "serve": {
+        "description": "Start the GraphQL API server",
+        "cmd": ["squid-graphql-server"]
+      },
+      "serve:prod": {
+        "description": "Start the GraphQL API server with caching and limits",
+        "cmd": ["squid-graphql-server",
+          "--dumb-cache", "in-memory",
+          "--dumb-cache-ttl", "1000",
+          "--dumb-cache-size", "100",
+          "--dumb-cache-max-age", "1000" ]
+      },
+      "check-updates": {
+        "cmd": ["npx", "--yes", "npm-check-updates", "--filter=/subsquid/", "--upgrade"],
+        "hidden": true
+      },
+      "bump": {
+        "description": "Bump @subsquid packages to the latest versions",
+        "deps": ["check-updates"],
+        "cmd": ["npm", "i", "-f"]
+      },
+      "open": {
+        "description": "Open a local browser window",
+        "cmd": ["npx", "--yes", "opener"]
+      }
+    }
+  }

indexers-v2/main.py

@@ -0,0 +1,143 @@
+import json
+import sys
+import os
+import argparse
+import yaml
+from synthetix import Synthetix
+
+
+def save_abi(abi, contract_name):
+    os.makedirs("abi", exist_ok=True)
+    with open(f"abi/{contract_name}.json", "w") as file:
+        json.dump(abi, file, indent=2)
+
+
+def create_squidgen_config(rpc_url, archive_url, contracts_info):
+    config = {
+        "archive": archive_url,
+        "finalityConfirmation": 1,
+        "chain": {"url": rpc_url, "rateLimit": 10},
+        "target": {"type": "postgres"},
+        "contracts": [],
+    }
+
+    for contract in contracts_info:
+        name = contract["name"]
+        address = contract["address"]
+        contract_config = {
+            "name": name,
+            "address": address,
+            "abi": f"./abi/{name}.json",
+            "events": True,
+            "functions": False,
+        }
+        config["contracts"].append(contract_config)
+
+    return config
+
+
+def create_squid_config(network_name):
+    return {
+        "manifestVersion": "subsquid.io/v0.1",
+        "name": network_name,
+        "version": 1,
+        "description": "A squid indexer generated from an ABI template",
+        "build": None,
+        "deploy": {
+            "addons": {"postgres": None},
+            "processor": {"cmd": ["node", "lib/main"]},
+            "api": {
+                "cmd": [
+                    "npx",
+                    "squid-graphql-server",
+                    "--dumb-cache",
+                    "in-memory",
+                    "--dumb-cache-ttl",
+                    "1000",
+                    "--dumb-cache-size",
+                    "100",
+                    "--dumb-cache-max-age",
+                    "1000",
+                ]
+            },
+        },
+    }
+
+
+def write_yaml(config, filename):
+    with open(filename, "w") as file:
+        yaml.dump(config, file, default_flow_style=False)
+
+
+def load_network_config(path):
+    with open(f"{path}/network_config.yaml", "r") as file:
+        return yaml.safe_load(file)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate Squid configuration files for a given network"
+    )
+    parser.add_argument("--network_name", type=str, help="Network name", required=True)
+    parser.add_argument("--rpc_endpoint", type=str, help="RPC URL", required=True)
+    args = parser.parse_args()
+
+    network_name = args.network_name
+    path = f"networks/{network_name}"
+    network_config = load_network_config(path)[network_name]
+
+    if not network_config:
+        print(f"Network '{network_name}' not found in {path}/network_config.yaml")
+        sys.exit(1)
+
+    rpc_endpoint = args.rpc_endpoint
+    if not rpc_endpoint:
+        print("RPC_ENDPOINT environment variable is not set")
+        sys.exit(1)
+    archive_url = network_config["archive_url"]
+
+    contracts = []
+
+    if "contracts_from_sdk" in network_config:
+        contracts_from_sdk = network_config["contracts_from_sdk"]
+
+        if "cannon_config" in network_config:
+            snx = Synthetix(
+                provider_rpc=rpc_endpoint,
+                network_id=network_config["network_id"],
+                cannon_config=network_config["cannon_config"],
+            )
+        else:
+            snx = Synthetix(
+                provider_rpc=rpc_endpoint,
+                network_id=network_config["network_id"],
+            )
+
+        for contract in contracts_from_sdk:
+            name = contract["name"]
+            package = contract["package"]
+            contract_data = snx.contracts[package][name]
+            if name == "buyback_snx":
+                name = "BuybackSnx"
+            save_abi(contract_data["abi"], name)
+            contracts.append({"name": name, "address": contract_data["address"]})
+
+    if "contracts_from_abi" in network_config:
+        contracts_from_abi = network_config["contracts_from_abi"]
+
+        for contract in contracts_from_abi:
+            name = contract["name"]
+            with open(f"{path}/abi/{name}.json", "r") as file:
+                contract_abi = json.load(file)
+            save_abi(contract_abi, name)
+            contracts.append({"name": name, "address": contract["address"]})
+
+    squidgen_config = create_squidgen_config(rpc_endpoint, archive_url, contracts)
+    write_yaml(squidgen_config, "squidgen.yaml")
+
+    squid_config = create_squid_config(args.network_name)
+    write_yaml(squid_config, "squid.yaml")
+
+    print(
+        f"squidgen.yaml, squid.yaml, and ABI files have been generated for {args.network_name}"
+    )

indexers-v2/networks/arbitrum_mainnet/network_config.yaml

@@ -0,0 +1,9 @@
+arbitrum_mainnet:
+  archive_url: "https://v2.archive.subsquid.io/network/arbitrum-one"
+  contracts:
+    - ["CoreProxy", "system"]
+    - ["AccountProxy", "system"]
+    - ["SpotMarketProxy", "spotFactory"]
+    - ["PerpsMarketProxy", "perpsFactory"]
+    - ["PerpsAccountProxy", "perpsFactory"]
+  network_id: 42161