diff --git a/README.md b/README.md index 341dd5d..8d339be 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,16 @@ -# Frak Indexer +# Infra blockchain -Frak Indexer is an open-source project designed to index events from Frak smart contracts on the Arbitrum network. It combines the power of Ponder for building robust crypto apps with eRPC for efficient RPC caching and load balancing, all deployed using SST (Serverless Stack) on AWS infrastructure. +This repository contain all the blockchain related infra code for the [Frak](https://frak.id) project. ## Architecture Overview The Frak Indexer consists of two main components: -1. **Ponder Service**: An open-source backend framework for building robust, performant, and maintainable crypto apps. In this project, it's used for indexing blockchain events from Frak smart contracts. +1.**[eRPC](https://github.com/erpc/erpc)**: RPC request caching and load balancing across multiple node providers -2. **eRPC Service**: A fault-tolerant EVM RPC load balancer with reorg-aware permanent caching and auto-discovery of node providers. It provides a layer of caching on top of other RPCs, enhancing performance and reliability. +2.**[Ponder](https://github.com/ponder-sh/ponder)**: Indexing blockchain events from Frak smart contracts. -Both services are deployed as containerized applications on AWS ECS (Elastic Container Service) using Fargate, with an Application Load Balancer (ALB) routing traffic between them. - -## Deployment Architecture - -- **VPC**: A dedicated VPC is created to house all components. -- **ECS Cluster**: Both Ponder and eRPC services run in the same ECS cluster. -- **Application Load Balancer**: - - Listens on port 80 - - Routes traffic based on path patterns: - - `/rpc-main/*` -> eRPC service (for cached RPC requests) - - `/*` (all other paths) -> Ponder service (for indexed data access) -- **CloudFront Distribution**: Sits in front of the ALB to provide additional caching and global content delivery. - -## Key Features - -- **Efficient Indexing**: Utilizes Ponder to create a robust and maintainable indexing solution for Frak smart contract events. -- **Optimized RPC Access**: eRPC provides caching and load balancing across multiple RPC providers, improving performance and reliability. -- **Scalable**: Utilizes AWS Fargate for serverless container management. -- **High Availability**: Deployed across multiple availability zones. -- **Secure**: Uses AWS security groups and VPC for network isolation. -- **Observable**: Includes CloudWatch logs and metrics for monitoring. -- **Maintainable**: Infrastructure as Code (IaC) using SST for easy updates and version control. - -## Contributing - -We welcome contributions to the Frak Indexer project! +Both services are deployed as containerized applications on AWS ECS (Elastic Container Service) using Fargate, with a master Application Load Balancer (ALB) routing traffic between them. ## License diff --git a/bun.lockb b/bun.lockb index 8430dfa..6f455b1 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/infra/common.ts b/infra/common.ts index cb09669..bdd1b67 100644 --- a/infra/common.ts +++ b/infra/common.ts @@ -1,4 +1,5 @@ import * as aws from "@pulumi/aws"; +import { Output } from "@pulumi/pulumi"; // Get the VPC const { id: vpcId } = await aws.ec2.getVpc({ @@ -10,3 +11,17 @@ export const vpc = sst.aws.Vpc.get("MasterVpc", vpcId); export const cluster = await aws.ecs.getCluster({ clusterName: `master-cluster-${$app.stage}`, }); + +/** + * Build the postgres DB for the current env + */ +export const database = + $app.stage !== "production" + ? sst.aws.Postgres.get("blockchain", { + id: "frak-indexer-production-blockchaininstance", + }) + : new sst.aws.Postgres("blockchain", { + vpc: Output.create(vpc).apply((v) => ({ + subnets: v.privateSubnets, + })), + }); diff --git a/infra/erpc.ts b/infra/erpc.ts index 822f85a..19872e2 100644 --- a/infra/erpc.ts +++ b/infra/erpc.ts @@ -1,6 +1,6 @@ import * as aws from "@pulumi/aws"; import { all } from "@pulumi/pulumi"; -import { cluster, vpc } from "./common.ts"; +import { cluster, database, vpc } from "./common.ts"; import { ServiceTargets } from "./components/ServiceTargets.ts"; import { SstService } from "./utils.ts"; @@ -32,6 +32,19 @@ const erpcServiceTargets = new ServiceTargets("ErpcServiceDomain", { }, }); +/** + * Build the erpc database URL + */ +const dbUrl = all([ + database.host, + database.port, + database.username, + database.password, + database.database, +]).apply(([host, port, username, password, database]) => { + return `postgres://${username}:${password}@${host}:${port}/${database}`; +}); + // Create the erpc service (only on prod stage) export const erpcService = new SstService("Erpc", { vpc, @@ -60,6 +73,7 @@ export const erpcService = new SstService("Erpc", { // Env environment: { ERPC_LOG_LEVEL: "warn", + ERPC_DATABASE_URL: dbUrl, }, // SSM secrets ssm: { @@ -79,9 +93,6 @@ export const erpcService = new SstService("Erpc", { "arn:aws:ssm:eu-west-1:262732185023:parameter/sst/frak-indexer/.fallback/Secret/PONDER_RPC_SECRET/value", NEXUS_RPC_SECRET: "arn:aws:ssm:eu-west-1:262732185023:parameter/sst/frak-indexer/.fallback/Secret/NEXUS_RPC_SECRET/value", - // Postgres db - ERPC_DATABASE_URL: - "arn:aws:ssm:eu-west-1:262732185023:parameter/indexer/sst/Secret/ERPC_DATABASE_URL/value", }, // Tell the service registry to forward requests to the 8080 port serviceRegistry: { diff --git a/package.json b/package.json index 56e3538..e451099 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,5 @@ "engines": { "node": ">=18.14" }, - "packageManager": "bun@1.1.38", "workspaces": ["packages/*"] } diff --git a/packages/erpc/Dockerfile b/packages/erpc/Dockerfile index 8691943..a5ac687 100644 --- a/packages/erpc/Dockerfile +++ b/packages/erpc/Dockerfile @@ -1,11 +1,17 @@ -# Base image is erpc -FROM ghcr.io/erpc/erpc:0.0.24 +# Config bundler step +FROM oven/bun:latest AS bundler +RUN mkdir -p /temp/dev -# Copy the config -COPY erpc.yaml /root/erpc.yaml +# Bundle everything in a single erpc.js file +COPY . /temp/dev +RUN cd /temp/dev && bun install --production +RUN cd /temp/dev && bun build --outfile ./erpc.js --minify --target node --external "@erpc-cloud/*" src/index.ts -EXPOSE 8080/tcp -EXPOSE 4001/tcp +# Final image +FROM ghcr.io/erpc/erpc:main AS FINAL + +# Copy the bundled config +COPY --from=bundler ./temp/dev/erpc.js /root/erpc.js # Run the server CMD ["./erpc-server"] \ No newline at end of file diff --git a/packages/erpc/erpc-config.ts b/packages/erpc/erpc-config.ts deleted file mode 100644 index 379d86d..0000000 --- a/packages/erpc/erpc-config.ts +++ /dev/null @@ -1,315 +0,0 @@ -import { - type ProjectConfig, - type UpstreamConfig, - buildAlchemyUpstream, - buildEnvioUpstream, - buildEvmNetworks, - buildEvmUpstream, - buildPimlicoUpstream, - buildProject, - buildRateLimit, - buildSecretAuthStrategy, - bundlersMethods, - envVariable, -} from "@konfeature/erpc-config-generator"; -import { - type RpcMethodWithRegex, - buildErpcConfig, -} from "@konfeature/erpc-config-generator"; -import type { EIP1474Methods } from "viem"; -import { arbitrum, arbitrumSepolia } from "viem/chains"; - -/* -------------------------------------------------------------------------- */ -/* Config generator for the Frak eRPC config */ -/* -------------------------------------------------------------------------- */ - -// Build every rate limits -const envioRateLimits = buildRateLimit({ - id: "envion-rate-limit", - rules: [ - { - method: "*", - maxCount: 400, - period: "1s", - }, - ], -}); -const alchemyRateLimits = buildRateLimit({ - id: "alchemy-rate-limit", - rules: [ - { - method: "*", - maxCount: 200, - period: "1s", - }, - ], -}); -const blockPiRateLimits = buildRateLimit({ - id: "block-pi-rate-limit", - rules: [ - { - method: "*", - maxCount: 250, - period: "1s", - }, - ], -}); -const pimlicoRateLimits = buildRateLimit({ - id: "pimlico-rate-limit", - rules: [ - { - method: "*", - maxCount: 400, - period: "1s", - }, - ], -}); -const drpcRpcRateLimits = buildRateLimit({ - id: "drpc-rate-limit", - rules: [ - { - method: "*", - maxCount: 200, - period: "1s", - }, - ], -}); -const llamaFreeRpcRateLimits = buildRateLimit({ - id: "llama-free-rpc-rate-limit", - rules: [ - { - method: "*", - maxCount: 50, - period: "1s", - }, - ], -}); -const tenderlyFreeRpcRateLimits = buildRateLimit({ - id: "tenderly-free-rpc-rate-limit", - rules: [ - { - method: "*", - maxCount: 50, - period: "1s", - }, - ], -}); - -// Each networks we will use -const mainnetNetworks = buildEvmNetworks({ - chains: [arbitrum], - generic: { - // Some failsafe config - failsafe: { - timeout: { - duration: "30s", - }, - retry: { - maxAttempts: 5, - delay: "500ms", - backoffMaxDelay: "10s", - backoffFactor: 0.5, - jitter: "200ms", - }, - hedge: { - delay: "5s", - maxCount: 2, - }, - }, - }, -}); -const testnetNetworks = buildEvmNetworks({ - chains: [arbitrumSepolia], - generic: { - // Some failsafe config - failsafe: { - hedge: { - delay: "5s", - maxCount: 2, - }, - }, - // Overide finality depth since arb sepolia could have huge reorgs - evm: { - finalityDepth: 2048, - }, - }, -}); -const networks = [...mainnetNetworks, ...testnetNetworks]; - -const pimlicoSpecificMethods: RpcMethodWithRegex[] = [ - ...bundlersMethods, - "pm_*", - "pimlico_*", -]; - -// Build each upstream we will use -// Envio only op for arbitrum sepolia, it's fcked up on arbitrum -// Disabled for now since it's returning incosistant data -const envioUpstream = buildEnvioUpstream({ - rateLimitBudget: envioRateLimits.id, - ignoreMethods: ["*"], - // todo: simple port of the vendors/evio.go stuff hereh - // since ts sdk doesn't support null value if ts definition doesn't give optional stuff - allowMethods: [ - "eth_chainId", - "eth_blockNumber", - "eth_getBlockByNumber", - "eth_getBlockByHash", - "eth_getTransactionByHash", - "eth_getTransactionByBlockHashAndIndex", - "eth_getTransactionByBlockNumberAndIndex", - "eth_getTransactionReceipt", - "eth_getBlockReceipts", - "eth_getLogs", - ], -}); -const alchemyUpstream = buildAlchemyUpstream({ - apiKey: envVariable("ALCHEMY_API_KEY"), - rateLimitBudget: alchemyRateLimits.id, - ignoreMethods: pimlicoSpecificMethods, -}); -const _blockpiArbSepoliaUpstream = buildEvmUpstream({ - id: "blockpi-arbSepolia", - endpoint: `https://arbitrum-sepolia.blockpi.network/v1/rpc/${envVariable("BLOCKPI_API_KEY_ARB_SEPOLIA")}`, - rateLimitBudget: blockPiRateLimits.id, - ignoreMethods: pimlicoSpecificMethods, -}); -const _blockpiArbUpstream = buildEvmUpstream({ - id: "blockpi-arb", - endpoint: `https://arbitrum.blockpi.network/v1/rpc/${envVariable("BLOCKPI_API_KEY_ARB")}`, - rateLimitBudget: blockPiRateLimits.id, - ignoreMethods: pimlicoSpecificMethods, -}); -const pimlicoUpstream = buildPimlicoUpstream({ - apiKey: envVariable("PIMLICO_API_KEY"), - rateLimitBudget: pimlicoRateLimits.id, - ignoreMethods: ["*"], - allowMethods: pimlicoSpecificMethods, -}); -const llamaFreeRpcUpstreamArb = buildEvmUpstream({ - id: "llama-arbitrum-free-rpc", - endpoint: "https://arbitrum.llamarpc.com", - rateLimitBudget: llamaFreeRpcRateLimits.id, - ignoreMethods: ["*"], - allowMethods: ["eth_chainId", "eth_getBlockByNumber", "eth_call"], -}); -const tenderlyFreeRpcUpstreamArbSepolia = buildEvmUpstream({ - id: "tenderly-arbitrum-sepolia-free-rpc", - endpoint: "https://arbitrum-sepolia.gateway.tenderly.co", - rateLimitBudget: tenderlyFreeRpcRateLimits.id, - ignoreMethods: ["*"], - allowMethods: ["eth_chainId", "eth_getBlockByNumber", "eth_call"], -}); -const drpcUpstream: UpstreamConfig = { - id: "drpc-rpc", - type: "evm+drpc", - vendorName: "drpc", - endpoint: `drpc://${envVariable("DRPC_API_KEY")}`, - rateLimitBudget: drpcRpcRateLimits.id, - ignoreMethods: ["*"], - allowMethods: [ - "eth_chainId", - "eth_getBlockByNumber", - "eth_getLogs", - "eth_call", - ], -}; - -// Build the ponder indexing project -const ponderProject: ProjectConfig = buildProject({ - id: "ponder-rpc", - networks, - upstreams: [ - alchemyUpstream, - llamaFreeRpcUpstreamArb, - drpcUpstream, - envioUpstream, - ], - auth: { - strategies: [ - buildSecretAuthStrategy({ - secret: { - value: envVariable("PONDER_RPC_SECRET"), - }, - }), - ], - }, -}); - -// Build the ponder indexing project -const ponderDevProject: ProjectConfig = buildProject({ - id: "ponder-dev-rpc", - networks, - upstreams: [tenderlyFreeRpcUpstreamArbSepolia, drpcUpstream, envioUpstream], - auth: { - strategies: [ - buildSecretAuthStrategy({ - secret: { - value: envVariable("PONDER_RPC_SECRET"), - }, - }), - ], - }, -}); - -// Build the nexus rpc project -// todo: add authentication + more restrictie cors origin -const nexusProject: ProjectConfig = buildProject({ - id: "nexus-rpc", - networks, - upstreams: [alchemyUpstream, pimlicoUpstream], - cors: { - allowedOrigins: ["*"], - allowedMethods: ["GET", "POST", "OPTIONS"], - allowedHeaders: ["Content-Type", "Authorization"], - exposedHeaders: ["X-Request-ID"], - allowCredentials: true, - maxAge: 3600, - }, - auth: { - strategies: [ - buildSecretAuthStrategy({ - secret: { - value: envVariable("NEXUS_RPC_SECRET"), - }, - }), - ], - }, -}); - -// Build the global config -export default buildErpcConfig({ - config: { - logLevel: envVariable("ERPC_LOG_LEVEL"), - database: { - evmJsonRpcCache: { - driver: "postgresql", - postgresql: { - connectionUri: envVariable("ERPC_DATABASE_URL"), - table: "rpc_cache", - }, - }, - }, - server: { - httpPort: 8080, - maxTimeout: "60s", - listenV6: false, - }, - metrics: { - enabled: true, - listenV6: false, - }, - projects: [ponderProject, ponderDevProject, nexusProject], - rateLimiters: { - budgets: [ - envioRateLimits, - alchemyRateLimits, - pimlicoRateLimits, - blockPiRateLimits, - drpcRpcRateLimits, - llamaFreeRpcRateLimits, - tenderlyFreeRpcRateLimits, - ], - }, - }, -}); diff --git a/packages/erpc/erpc.yaml b/packages/erpc/erpc.yaml deleted file mode 100644 index 88eafc1..0000000 --- a/packages/erpc/erpc.yaml +++ /dev/null @@ -1,233 +0,0 @@ -# Config generated using: https://github.com/KONFeature/erpc-config-generator -logLevel: ${ERPC_LOG_LEVEL} -server: - httpPort: 8080 - httpHostV4: 0.0.0.0 - httpHostV6: "[::]" - listenV4: true - listenV6: false - maxTimeout: 60s -metrics: - port: 4001 - hostV4: 0.0.0.0 - hostV6: "[::]" - listenV4: true - listenV6: false - enabled: true -database: - evmJsonRpcCache: - driver: postgresql - postgresql: - connectionUri: ${ERPC_DATABASE_URL} - table: rpc_cache -projects: - - rateLimitBudget: "" - id: ponder-rpc - networks: &var1 - - failsafe: - timeout: - duration: 30s - retry: - maxAttempts: 5 - delay: 500ms - backoffMaxDelay: 10s - backoffFactor: 0.5 - jitter: 200ms - hedge: - delay: 5s - maxCount: 2 - architecture: evm - rateLimitBudget: "" - evm: - chainId: 42161 - finalityDepth: 1024 - blockTrackerInterval: "" - - failsafe: - hedge: - delay: 5s - maxCount: 2 - evm: - chainId: 421614 - finalityDepth: 2048 - blockTrackerInterval: "" - architecture: evm - rateLimitBudget: "" - upstreams: - - &var4 - id: alchemy - endpoint: evm+alchemy://${ALCHEMY_API_KEY} - type: evm+alchemy - rateLimitBudget: alchemy-rate-limit - vendorName: Alchemy - ignoreMethods: &var5 - - eth_estimateUserOperationGas - - eth_getUserOperationByHash - - eth_getUserOperationReceipt - - eth_sendUserOperation - - eth_supportedEntryPoints - - pm_* - - pimlico_* - allowMethods: [] - autoIgnoreUnsupportedMethods: true - - id: llama-arbitrum-free-rpc - endpoint: https://arbitrum.llamarpc.com - rateLimitBudget: llama-free-rpc-rate-limit - type: evm - vendorName: Generic Evm - ignoreMethods: - - "*" - allowMethods: - - eth_chainId - - eth_getBlockByNumber - - eth_call - autoIgnoreUnsupportedMethods: true - - &var2 - id: drpc-rpc - type: evm+drpc - vendorName: drpc - endpoint: drpc://${DRPC_API_KEY} - rateLimitBudget: drpc-rate-limit - ignoreMethods: - - "*" - allowMethods: - - eth_chainId - - eth_getBlockByNumber - - eth_getLogs - - eth_call - - &var3 - id: envio - endpoint: evm+envio://rpc.hypersync.xyz - rateLimitBudget: envion-rate-limit - type: evm+envio - vendorName: Envio - ignoreMethods: - - "*" - allowMethods: - - eth_chainId - - eth_blockNumber - - eth_getBlockByNumber - - eth_getBlockByHash - - eth_getTransactionByHash - - eth_getTransactionByBlockHashAndIndex - - eth_getTransactionByBlockNumberAndIndex - - eth_getTransactionReceipt - - eth_getBlockReceipts - - eth_getLogs - autoIgnoreUnsupportedMethods: true - auth: - strategies: - - allowMethods: - - "*" - ignoreMethods: [] - rateLimitBudget: "" - type: secret - secret: - value: ${PONDER_RPC_SECRET} - - rateLimitBudget: "" - id: ponder-dev-rpc - networks: *var1 - upstreams: - - id: tenderly-arbitrum-sepolia-free-rpc - endpoint: https://arbitrum-sepolia.gateway.tenderly.co - rateLimitBudget: tenderly-free-rpc-rate-limit - type: evm - vendorName: Generic Evm - ignoreMethods: - - "*" - allowMethods: - - eth_chainId - - eth_getBlockByNumber - - eth_call - autoIgnoreUnsupportedMethods: true - - *var2 - - *var3 - auth: - strategies: - - allowMethods: - - "*" - ignoreMethods: [] - rateLimitBudget: "" - type: secret - secret: - value: ${PONDER_RPC_SECRET} - - rateLimitBudget: "" - id: nexus-rpc - networks: *var1 - upstreams: - - *var4 - - id: pimlico - endpoint: evm+pimlico://${PIMLICO_API_KEY} - rateLimitBudget: pimlico-rate-limit - type: evm+pimlico - vendorName: Pimlico - ignoreMethods: - - "*" - allowMethods: *var5 - autoIgnoreUnsupportedMethods: true - cors: - allowedOrigins: - - "*" - allowedMethods: - - GET - - POST - - OPTIONS - allowedHeaders: - - Content-Type - - Authorization - exposedHeaders: - - X-Request-ID - allowCredentials: true - maxAge: 3600 - auth: - strategies: - - allowMethods: - - "*" - ignoreMethods: [] - rateLimitBudget: "" - type: secret - secret: - value: ${NEXUS_RPC_SECRET} -rateLimiters: - budgets: - - id: envion-rate-limit - rules: - - method: "*" - maxCount: 400 - period: 1s - waitTime: "" - - id: alchemy-rate-limit - rules: - - method: "*" - maxCount: 200 - period: 1s - waitTime: "" - - id: pimlico-rate-limit - rules: - - method: "*" - maxCount: 400 - period: 1s - waitTime: "" - - id: block-pi-rate-limit - rules: - - method: "*" - maxCount: 250 - period: 1s - waitTime: "" - - id: drpc-rate-limit - rules: - - method: "*" - maxCount: 200 - period: 1s - waitTime: "" - - id: llama-free-rpc-rate-limit - rules: - - method: "*" - maxCount: 50 - period: 1s - waitTime: "" - - id: tenderly-free-rpc-rate-limit - rules: - - method: "*" - maxCount: 50 - period: 1s - waitTime: "" diff --git a/packages/erpc/package.json b/packages/erpc/package.json index 358c760..36349a0 100644 --- a/packages/erpc/package.json +++ b/packages/erpc/package.json @@ -8,19 +8,20 @@ "build": "erpc-config", "build:check": "erpc-config validate", "lint": "biome lint .", - "typecheck": "tsc" + "typecheck": "tsc", + "docker:dev": "docker build --tag frak-erpc . && docker run --env-file ./.env.local -P frak-erpc" }, "devDependencies": { "@biomejs/biome": "1.9.4", - "@konfeature/erpc-config-generator": "0.0.13", - "@types/aws-lambda": "8.10.146", - "@types/node": "^22.10.1", - "sst": "3.3.44", - "typescript": "^5.7.2", - "viem": "^2.21.54" + "viem": "^2.21.54", + "sst": "3.3.29", + "typescript": "^5.7.2" + }, + "dependencies": { + "@konfeature/erpc-config-generator": "0.1.0", + "@erpc-cloud/config": "^0.0.11" }, "engines": { "node": ">=18.14" - }, - "packageManager": "bun@1.1.38" + } } \ No newline at end of file diff --git a/packages/erpc/src/index.ts b/packages/erpc/src/index.ts new file mode 100644 index 0000000..4f116ba --- /dev/null +++ b/packages/erpc/src/index.ts @@ -0,0 +1,137 @@ +import type { LogLevel } from "@erpc-cloud/config"; +import { initErpcConfig } from "@konfeature/erpc-config-generator"; +import { arbNetwork, arbSepoliaNetwork } from "./networks"; +import { + alchemyRateRules, + blockPiRateRules, + drpcRateRules, + envioRateRules, + llamaFreeRateRules, + pimlicoRateRules, + tenderlyFreeRateRules, +} from "./rateLimits"; +import { cacheConfig } from "./storage"; +import { + alchemyUpstream, + drpcUpstream, + envioUpstream, + llamaFreeUpstreamArb, + pimlicoUpstream, + tenderlyFreeUpstreamArbSepolia, +} from "./upstreams"; + +/** + * Build our top level erpc config + */ +export default initErpcConfig({ + logLevel: (process.env.ERPC_LOG_LEVEL ?? "info") as LogLevel, + database: { + evmJsonRpcCache: cacheConfig, + }, + server: { + httpPort: 8080, + maxTimeout: "60s", + listenV6: false, + }, + metrics: { + enabled: true, + listenV6: false, + }, + blablou: "test", +}) + .addRateLimiters({ + alchemy: alchemyRateRules, + envio: envioRateRules, + pimlico: pimlicoRateRules, + blockPi: blockPiRateRules, + drpc: drpcRateRules, + llamaFree: llamaFreeRateRules, + tenderlyFree: tenderlyFreeRateRules, + }) + // Add networks to the config + .decorate("networks", { + arbitrum: arbNetwork, + arbitrumSepolia: arbSepoliaNetwork, + }) + // Add upstreams to the config + .decorate("upstreams", { + envio: envioUpstream, + alchemy: alchemyUpstream, + pimlico: pimlicoUpstream, + drpc: drpcUpstream, + llamaFree: llamaFreeUpstreamArb, + tenderlyFree: tenderlyFreeUpstreamArbSepolia, + }) + // Add our ponder prod project + .addProject(({ store: { upstreams, networks } }) => ({ + id: "ponder-rpc", + networks: [networks.arbitrum], + upstreams: [ + upstreams.alchemy, + upstreams.envio, + upstreams.drpc, + upstreams.llamaFree, + ], + auth: { + strategies: [ + { + type: "secret", + secret: { + value: process.env.PONDER_RPC_SECRET ?? "a", + }, + }, + ], + }, + })) + // Add our ponder dev project + .addProject(({ store: { upstreams, networks } }) => ({ + id: "ponder-dev-rpc", + networks: [networks.arbitrumSepolia], + upstreams: [ + upstreams.alchemy, + upstreams.envio, + upstreams.drpc, + upstreams.tenderlyFree, + ], + auth: { + strategies: [ + { + type: "secret", + secret: { + value: process.env.PONDER_RPC_SECRET ?? "a", + }, + }, + ], + }, + })) + // Add our wallet project + .addProject(({ store: { upstreams, networks } }) => ({ + id: "nexus-rpc", + networks: [networks.arbitrum, networks.arbitrumSepolia], + upstreams: [ + upstreams.alchemy, + upstreams.drpc, + upstreams.llamaFree, + upstreams.tenderlyFree, + ], + auth: { + strategies: [ + { + type: "secret", + secret: { + value: process.env.NEXUS_RPC_SECRET ?? "a", + }, + }, + ], + }, + cors: { + allowedOrigins: ["*"], + allowedMethods: ["GET", "POST", "OPTIONS"], + allowedHeaders: ["Content-Type", "Authorization"], + exposedHeaders: ["X-Request-ID"], + allowCredentials: true, + maxAge: 3600, + }, + })) + // And bundle it altogether + .build(); diff --git a/packages/erpc/src/networks.ts b/packages/erpc/src/networks.ts new file mode 100644 index 0000000..2a1bd16 --- /dev/null +++ b/packages/erpc/src/networks.ts @@ -0,0 +1,47 @@ +import type { NetworkConfig } from "@erpc-cloud/config"; + +export const arbNetwork = { + architecture: "evm", + failsafe: { + timeout: { + duration: "30s", + }, + retry: { + maxAttempts: 5, + delay: "500ms", + backoffMaxDelay: "10s", + backoffFactor: 0.5, + jitter: "200ms", + }, + hedge: { + delay: "3s", + maxCount: 2, + }, + }, + evm: { + chainId: 42161, + }, +} as const satisfies NetworkConfig; + +export const arbSepoliaNetwork = { + architecture: "evm", + failsafe: { + timeout: { + duration: "120s", + }, + retry: { + maxAttempts: 3, + delay: "1s", + backoffMaxDelay: "30s", + backoffFactor: 0.5, + jitter: "200ms", + }, + hedge: { + delay: "5s", + maxCount: 2, + }, + }, + evm: { + chainId: 421614, + }, +} as const satisfies NetworkConfig; diff --git a/packages/erpc/src/rateLimits.ts b/packages/erpc/src/rateLimits.ts new file mode 100644 index 0000000..d00d47f --- /dev/null +++ b/packages/erpc/src/rateLimits.ts @@ -0,0 +1,45 @@ +import type { RateLimitRuleConfig } from "@erpc-cloud/config"; + +/** + * Build a generic rate limits rules, counting on the number of request per minutes + * @param count + */ +function genericRateLimitsRules(count: number): RateLimitRuleConfig { + return { + method: "*", + maxCount: count, + period: "1s", + waitTime: "30s", + }; +} + +type RuleExport = [RateLimitRuleConfig, ...RateLimitRuleConfig[]]; + +export const envioRateRules: RuleExport = [ + genericRateLimitsRules(400), + { + method: "eth_getLogs", + maxCount: 150, + period: "1s", + waitTime: "10s", + }, +]; + +export const alchemyRateRules: RuleExport = [ + genericRateLimitsRules(200), + { + method: "eth_getLogs", + maxCount: 30, + period: "1s", + waitTime: "10s", + }, +]; + +export const pimlicoRateRules: RuleExport = [genericRateLimitsRules(400)]; + +export const blockPiRateRules: RuleExport = [genericRateLimitsRules(100)]; + +export const drpcRateRules: RuleExport = [genericRateLimitsRules(100)]; + +export const llamaFreeRateRules: RuleExport = [genericRateLimitsRules(30)]; +export const tenderlyFreeRateRules: RuleExport = [genericRateLimitsRules(10)]; diff --git a/packages/erpc/src/storage.ts b/packages/erpc/src/storage.ts new file mode 100644 index 0000000..729b917 --- /dev/null +++ b/packages/erpc/src/storage.ts @@ -0,0 +1,89 @@ +import { + type CacheConfig, + CacheEmptyBehaviorAllow, + CacheEmptyBehaviorIgnore, + type CachePolicyConfig, + type ConnectorConfig, + DataFinalityStateFinalized, + DataFinalityStateRealtime, + DataFinalityStateUnfinalized, +} from "@erpc-cloud/config"; + +if (!process.env.ERPC_DATABASE_URL) { + throw new Error("Missing ERPC_DATABASE_URL environment variable"); +} + +/** + * The connectors we will use + */ +const connectors = [ + { + id: "pg-main", + driver: "postgresql", + postgresql: { + connectionUri: process.env.ERPC_DATABASE_URL as string, + table: "rpc_cache", + }, + }, + { + id: "memory-main", + driver: "memory", + memory: { + maxItems: 65_536, + }, + }, +] as const satisfies ConnectorConfig[]; + +/** + * Define the cache policies we will use + * todo: Should check with 4337 userOpGas price if it play nicely + * todo: Also find a way to cache 4337 related et_getCode method for longer period in memory? Only if non empty maybe? + */ +const cachePolicies = [ + // Cache all finalized data in the pg database + { + connector: "pg-main", + network: "*", + method: "*", + finality: DataFinalityStateFinalized, + empty: CacheEmptyBehaviorAllow, + }, + // Cache not finalized data for 2sec in the memory + { + connector: "memory-main", + network: "*", + method: "*", + finality: DataFinalityStateUnfinalized, + empty: CacheEmptyBehaviorIgnore, + // 2sec in nanoseconds + ttl: 2_000_000_000, + }, + // Cache realtime data for 2sec on the memory on arbitrum + { + connector: "memory-main", + network: "evm:42161", + method: "*", + finality: DataFinalityStateRealtime, + empty: CacheEmptyBehaviorIgnore, + // 2sec in nanoseconds + ttl: 2_000_000_000, + }, + // Cache realtime data for 30sec on arbitrum sepolia + { + connector: "memory-main", + network: "evm:421614", + method: "*", + finality: DataFinalityStateRealtime, + empty: CacheEmptyBehaviorIgnore, + // 30sec in nanoseconds + ttl: 30_000_000_000, + }, +] as const satisfies CachePolicyConfig[]; + +/** + * Export our final cache config + */ +export const cacheConfig = { + connectors, + policies: cachePolicies, +} as const satisfies CacheConfig; diff --git a/packages/erpc/src/upstreams.ts b/packages/erpc/src/upstreams.ts new file mode 100644 index 0000000..c2d2ba3 --- /dev/null +++ b/packages/erpc/src/upstreams.ts @@ -0,0 +1,112 @@ +import type { UpstreamConfig } from "@erpc-cloud/config"; + +if (!process.env.ALCHEMY_API_KEY) { + throw new Error("Missing ALCHEMY_API_KEY environment variable"); +} +if (!process.env.PIMLICO_API_KEY) { + throw new Error("Missing PIMLICO_API_KEY environment variable"); +} +if (!process.env.DRPC_API_KEY) { + throw new Error("Missing DRPC_API_KEY environment variable"); +} + +/** + * Method specifics for for the smart wallets + */ +const erc4337Methods = [ + "eth_estimateUserOperationGas", + "eth_getUserOperationByHash", + "eth_getUserOperationReceipt", + "eth_sendUserOperation", + "eth_supportedEntryPoints", + "pm_*", + "pimlico_*", +]; + +const indexingMethods = [ + "eth_chainId", + "eth_blockNumber", + "eth_getLogs", + "eth_getBlock*", + "eth_getTransaction*", +]; + +const freeRpcMethods = [ + "eth_chainId", + "eth_blockNumber", + "eth_call", + "eth_getCode", + "eth_getStorageAt", + "eth_getBlock*", + "eth_getTransaction*", +]; + +// Drpc methods are indexing + free rpc methods deduplicated +const drpcMethods = Array.from( + new Set([...indexingMethods, ...freeRpcMethods]) +); + +export const envioUpstream = { + endpoint: "evm+envio://rpc.hypersync.xyz", + type: "evm+envio", + vendorName: "Envio", + ignoreMethods: ["*"], + // Budget for rate limiting + rateLimitBudget: "envio", + // Only allow getLogs, getBlockBy and getTransactions* + allowMethods: indexingMethods, +} as const satisfies UpstreamConfig; + +export const alchemyUpstream = { + endpoint: `evm+alchemy://${process.env.ALCHEMY_API_KEY}`, + type: "evm+alchemy", + vendorName: "Alchemy", + // Budget for rate limiting + rateLimitBudget: "alchemy", + // Ignore all the pimlico + ignoreMethods: erc4337Methods, +} as const satisfies UpstreamConfig; + +export const pimlicoUpstream = { + endpoint: `evm+pimlico://${process.env.PIMLICO_API_KEY}`, + type: "evm+pimlico", + vendorName: "Pimlico", + // Budget for rate limiting + rateLimitBudget: "pimlico", + // Only allow the 4337 methods + ignoreMethods: ["*"], + allowMethods: erc4337Methods, +} as const satisfies UpstreamConfig; + +export const drpcUpstream = { + endpoint: `drpc://${process.env.DRPC_API_KEY}`, + type: "evm+drpc", + vendorName: "drpc", + // Budget for rate limiting + rateLimitBudget: "drpc", + // Only allow chainId, getBlockBy and getLogs + ignoreMethods: ["*"], + allowMethods: drpcMethods, +} as const satisfies UpstreamConfig; + +export const llamaFreeUpstreamArb = { + endpoint: "https://arbitrum.llamarpc.com", + type: "evm", + vendorName: "LlamaFree", + // Budget for rate limiting + rateLimitBudget: "llamaFree", + // Only allow chainId and getBlockBy + ignoreMethods: ["*"], + allowMethods: freeRpcMethods, +} as const satisfies UpstreamConfig; + +export const tenderlyFreeUpstreamArbSepolia = { + endpoint: "https://arbitrum-sepolia.gateway.tenderly.co", + type: "evm", + vendorName: "TenderlyFree", + // Budget for rate limiting + rateLimitBudget: "tenderlyFree", + // Only allow chainId and getBlockBy + ignoreMethods: ["*"], + allowMethods: freeRpcMethods, +} as const satisfies UpstreamConfig; diff --git a/packages/ponder/package.json b/packages/ponder/package.json index 4af5b27..d4113b2 100644 --- a/packages/ponder/package.json +++ b/packages/ponder/package.json @@ -33,6 +33,5 @@ }, "engines": { "node": ">=18.14" - }, - "packageManager": "bun@1.1.38" + } } \ No newline at end of file