Simulate in batches

src/lib/config.ts

@@ -168,7 +168,7 @@ const defaultBatch: RpcConfig['batch'] = {
     },
     multicall: {
         batchSize: 4_096,
-        wait: 100,
+        wait: 300,
     },
 };
 const defaultRetry: RpcConfig['retry'] = {
@@ -213,6 +213,7 @@ const defaultUnwrapConfig: RpcConfig['unwrap'] = {
 const defaultHarvestConfig: RpcConfig['harvest'] = {
     enabled: true,
     minTvlThresholdUsd: 100,
+    parallelSimulations: 5,
     profitabilityCheck: {
         enabled: false,
         minExpectedRewardsWei: bigintMultiplyFloat(ONE_ETHER, 0.002),

src/lib/harvest-chain.ts

@@ -1,5 +1,5 @@
 import { BeefyHarvestLensABI } from '../abi/BeefyHarvestLensABI';
-import { getReadOnlyRpcClient, getWalletAccount, getWalletClient } from '../lib/rpc-client';
+import { getReadOnlyRpcClient, getWalletClient } from '../lib/rpc-client';
 import { bigintMultiplyFloat } from '../util/bigint';
 import { rootLogger } from '../util/logger';
 import { getChainWNativeTokenAddress } from './addressbook';
@@ -49,7 +49,6 @@ export async function harvestChain({
     const wnative = getChainWNativeTokenAddress(chain);
     const publicClient = getReadOnlyRpcClient({ chain });
     const walletClient = getWalletClient({ chain });
-    const walletAccount = getWalletAccount({ chain });
     const rpcConfig = RPC_CONFIG[chain];
 
     const items = vaults.map(vault => ({
@@ -85,7 +84,10 @@ export async function harvestChain({
     const successfulSimulations = await reportOnMultipleHarvestAsyncCall(
         items,
         'simulation',
-        'parallel',
+        {
+            type: 'parallel-batched',
+            batchSize: rpcConfig.harvest.parallelSimulations,
+        },
         async item => {
             if (VAULT_IDS_WE_SHOULD_BLIND_HARVEST.includes(item.vault.id)) {
                 return {
@@ -109,11 +111,11 @@ export async function harvestChain({
 
             const {
                 result: { callReward, gasUsed, lastHarvest, paused, success, blockNumber, harvestResult },
-            } = await publicClient.simulateContract({
+            } = await publicClient.simulateContractInBatch({
                 ...harvestLensContract,
                 functionName: 'harvest',
                 args: [item.vault.strategyAddress, wnative] as const,
-                account: walletAccount,
+                //account: walletAccount, // setting the account disables multicall batching
             });
             const lastHarvestDate = new Date(Number(lastHarvest) * 1000);
             const timeSinceLastHarvestMs = now.getTime() - lastHarvestDate.getTime();
@@ -149,7 +151,7 @@ export async function harvestChain({
     const shouldHarvestDecisions = await reportOnMultipleHarvestAsyncCall(
         successfulSimulations,
         'decision',
-        'parallel',
+        { type: 'parallel' },
         async item => {
             if (item.vault.eol) {
                 return {
@@ -393,7 +395,7 @@ export async function harvestChain({
         msg: 'Harvesting strats',
         data: { chain, count: stratsToBeHarvested.length },
     });
-    await reportOnMultipleHarvestAsyncCall(stratsToBeHarvested, 'transaction', 'sequential', async item => {
+    await reportOnMultipleHarvestAsyncCall(stratsToBeHarvested, 'transaction', { type: 'sequential' }, async item => {
         let harvestParams: HarvestParameters = {
             strategyAddress: item.vault.strategyAddress,
             // mode fails to estimate gas because their eth_estimateGas method doesn't accept fee params

src/lib/reports.ts

@@ -2,7 +2,7 @@ import { get, set } from 'lodash';
 import { BaseError, TimeoutError } from 'viem';
 import { type Async, type AsyncSuccessType, promiseTimings } from '../util/async';
 import { rootLogger } from '../util/logger';
-import { runSequentially, splitPromiseResultsByStatus } from '../util/promise';
+import { type RunMode, runWithMode, splitPromiseResultsByStatus } from '../util/promise';
 import type { Prettify } from '../util/types';
 import { CENSOR_SECRETS_FROM_REPORTS } from './config';
 
@@ -77,7 +77,7 @@ export async function reportOnMultipleAsyncCall<
 >(
     items: TItem[],
     reportKey: TKey,
-    mode: 'parallel' | 'sequential',
+    mode: RunMode,
     make: (item: TItem) => Promise<TVal>
 ): Promise<Prettify<TItem & { [k in TKey]: TVal }>[]> {
     logger.info({
@@ -106,9 +106,7 @@ export async function reportOnMultipleAsyncCall<
         };
     };
 
-    const results = await (mode === 'parallel'
-        ? Promise.allSettled(items.map(processItem))
-        : runSequentially(items, processItem));
+    const results = await runWithMode(mode, items, processItem);
     const { fulfilled, rejected } = splitPromiseResultsByStatus(results);
 
     logger.info({

src/lib/rpc-actions/index.ts

@@ -8,6 +8,8 @@ import type {
     DeriveChain,
     ExtractAbiFunctionForArgs,
     ParseAccount,
+    SimulateContractParameters,
+    SimulateContractReturnType,
     Transport,
     Chain as ViemChain,
 } from 'viem';
@@ -22,11 +24,25 @@ import {
     type AggressivelyWriteContractReturnType,
     aggressivelyWriteContract,
 } from './aggressivelyWriteContract';
+import { simulateContractInBatch } from './simulateContractInBatch';
 
-type CustomRpcPublicActions<TChain extends ViemChain | undefined = ViemChain | undefined> = {
+type CustomRpcPublicActions<
+    TChain extends ViemChain | undefined = ViemChain | undefined,
+    TAccount extends Account | undefined = Account | undefined,
+> = {
     aggressivelyWaitForTransactionReceipt: (
         args: AggressivelyWaitForTransactionReceiptParameters
     ) => Promise<AggressivelyWaitForTransactionReceiptReturnType<TChain>>;
+
+    simulateContractInBatch: <
+        const abi extends Abi | readonly unknown[],
+        functionName extends ContractFunctionName<abi, 'nonpayable' | 'payable'>,
+        args extends ContractFunctionArgs<abi, 'nonpayable' | 'payable', functionName>,
+        chainOverride extends ViemChain | undefined,
+        accountOverride extends Account | Address | undefined = undefined,
+    >(
+        args: SimulateContractParameters<abi, functionName, args, TChain, chainOverride, accountOverride>
+    ) => Promise<SimulateContractReturnType<abi, functionName, args, TChain, TAccount, chainOverride, accountOverride>>;
 };
 export function createCustomRpcPublicActions({ chain }: { chain: Chain }) {
     return function customRpcPublicActions<
@@ -36,6 +52,7 @@ export function createCustomRpcPublicActions({ chain }: { chain: Chain }) {
     >(client: Client<TTransport, TChain, TAccount>): CustomRpcPublicActions</*TTransport,*/ TChain /*, TAccount*/> {
         return {
             aggressivelyWaitForTransactionReceipt: args => aggressivelyWaitForTransactionReceipt({ chain }, args),
+            simulateContractInBatch: args => simulateContractInBatch(client, args),
         };
     };
 }

src/lib/rpc-actions/simulateContractInBatch.ts

@@ -0,0 +1,86 @@
+import type { Abi, Address } from 'abitype';
+import {
+    type Account,
+    type BaseError,
+    type Chain,
+    type Client,
+    type ContractFunctionArgs,
+    type ContractFunctionName,
+    type SimulateContractParameters,
+    type SimulateContractReturnType,
+    type Transport,
+    decodeFunctionResult,
+    encodeFunctionData,
+    getContractError,
+} from 'viem';
+import { parseAccount } from 'viem/accounts';
+
+/**
+ * Fork of viem's `simulateContract` function with a few modifications to accept multicall batching.
+ *
+ * This would usually hide errors but we are mostly batching calls to a lens that is designed to never
+ * throw errors, so we can safely ignore them.
+ */
+export async function simulateContractInBatch<
+    chain extends Chain | undefined,
+    account extends Account | undefined,
+    const abi extends Abi | readonly unknown[],
+    functionName extends ContractFunctionName<abi, 'nonpayable' | 'payable'>,
+    const args extends ContractFunctionArgs<abi, 'nonpayable' | 'payable', functionName>,
+    chainOverride extends Chain | undefined = undefined,
+    accountOverride extends Account | Address | undefined = undefined,
+>(
+    client: Client<Transport, chain, account>,
+    parameters: SimulateContractParameters<abi, functionName, args, chain, chainOverride, accountOverride>
+): Promise<SimulateContractReturnType<abi, functionName, args, chain, account, chainOverride, accountOverride>> {
+    const { abi, address, args, dataSuffix, functionName, ...callRequest } = parameters as SimulateContractParameters;
+
+    const account = callRequest.account ? parseAccount(callRequest.account) : client.account;
+    const calldata = encodeFunctionData({ abi, args, functionName });
+    try {
+        // @ts-expect-error
+        const { data } = await client.call({
+            batch: client.batch,
+            data: `${calldata}${dataSuffix ? dataSuffix.replace('0x', '') : ''}`,
+            to: address,
+            ...callRequest,
+            account,
+        });
+        const result = decodeFunctionResult({
+            abi,
+            args,
+            functionName,
+            data: data || '0x',
+        });
+        const minimizedAbi = abi.filter(abiItem => 'name' in abiItem && abiItem.name === parameters.functionName);
+        return {
+            result,
+            request: {
+                abi: minimizedAbi,
+                address,
+                args,
+                dataSuffix,
+                functionName,
+                ...callRequest,
+                account,
+            },
+        } as unknown as SimulateContractReturnType<
+            abi,
+            functionName,
+            args,
+            chain,
+            account,
+            chainOverride,
+            accountOverride
+        >;
+    } catch (error) {
+        throw getContractError(error as BaseError, {
+            abi,
+            address,
+            args,
+            docsPath: '/docs/contract/simulateContractInBatch',
+            functionName,
+            sender: account?.address,
+        });
+    }
+}

src/lib/rpc-config.ts

@@ -75,6 +75,9 @@ export type RpcConfig = {
         // wether we should set the transaction gas limit
         setTransactionGasLimit: boolean;
 
+        // multicall config
+        parallelSimulations: number;
+
         // these parameters are used to know if we have enough gas to send a transaction
         balanceCheck: {
             // by how much we should multiply our given gas price

src/lib/rpc-transport.ts

@@ -6,16 +6,12 @@ const logger = rootLogger.child({ module: 'rpc-transport' });
 export function loggingHttpTransport(url?: string, config: HttpTransportConfig = {}): HttpTransport {
     return http(url, {
         onFetchRequest: async request => {
-            const content = await request.json();
+            const content = await request.clone().text();
             logger.trace({ msg: 'rpc.http: request', data: content });
-            // @ts-ignore: avoid `Body is unusable` error
-            request.json = async () => content;
         },
         onFetchResponse: async response => {
-            const content = await response.json();
+            const content = await response.clone().text();
             logger.debug({ msg: 'rpc.http: response', data: content });
-            // @ts-ignore: avoid `Body is unusable` error
-            response.json = async () => content;
         },
         ...config,
     });

src/script/eol-with-rewards.ts

@@ -6,7 +6,7 @@ import { getChainWNativeTokenAddress, getChainWNativeTokenDecimals } from '../li
 import { type Chain, allChainIds } from '../lib/chain';
 import { RPC_CONFIG } from '../lib/config';
 import { type AItem, type AKey, type AVal, reportOnMultipleAsyncCall, serializeReport } from '../lib/reports';
-import { getReadOnlyRpcClient, getWalletAccount } from '../lib/rpc-client';
+import { getReadOnlyRpcClient } from '../lib/rpc-client';
 import type { BeefyVault } from '../lib/vault';
 import { getVaultsToMonitorByChain } from '../lib/vault-list';
 import type { Async } from '../util/async';
@@ -246,7 +246,9 @@ async function main() {
                 const rewards = BigInt(cur.rewards || '0');
                 const wnativeDecimals = getChainWNativeTokenDecimals(cur.chain);
                 const divisor = BigInt(`1${'0'.repeat(wnativeDecimals)}`);
-                const rewardsEth = `${(rewards / divisor).toString()}.${rewards.toString().padStart(wnativeDecimals, '0')}`;
+                const rewardsEth = `${(rewards / divisor).toString()}.${rewards
+                    .toString()
+                    .padStart(wnativeDecimals, '0')}`;
                 acc[cur.chain].push({ vaultId: cur.vaultId, rewards, rewardsEth });
                 return acc;
             },
@@ -293,7 +295,6 @@ function reportOnMultipleEolRewardsAsyncCall<
 async function fetchLensResult(chain: Chain, vaults: BeefyVault[]) {
     const wnative = getChainWNativeTokenAddress(chain);
     const publicClient = getReadOnlyRpcClient({ chain });
-    const walletAccount = getWalletAccount({ chain });
 
     // we need the harvest lense
     const rpcConfig = RPC_CONFIG[chain];
@@ -310,14 +311,14 @@ async function fetchLensResult(chain: Chain, vaults: BeefyVault[]) {
         report: { vault, simulation: null } as EolWithRewardsReportItem,
     }));
 
-    await reportOnMultipleEolRewardsAsyncCall(items, 'simulation', 'parallel', async item => {
+    await reportOnMultipleEolRewardsAsyncCall(items, 'simulation', { type: 'parallel' }, async item => {
         const {
             result: { callReward, gasUsed, lastHarvest, paused, success, blockNumber, harvestResult },
-        } = await publicClient.simulateContract({
+        } = await publicClient.simulateContractInBatch({
             ...harvestLensContract,
             functionName: 'harvest',
             args: [item.vault.strategyAddress, wnative] as const,
-            account: walletAccount,
+            //account: walletAccount, // setting the account disables multicall batching
         });
 
         const now = new Date();

src/util/promise.ts

@@ -1,4 +1,4 @@
-import { get, isString } from 'lodash';
+import { chunk, get, isString } from 'lodash';
 import type { Prettify } from './types';
 
 export function sleep(ms: number) {
@@ -20,6 +20,16 @@ export function splitPromiseResultsByStatus<T>(results: PromiseSettledResult<T>[
     return { fulfilled, rejected };
 }
 
+export type RunMode = { type: 'parallel' } | { type: 'sequential' } | { type: 'parallel-batched'; batchSize: number };
+
+export function runWithMode<T, R>(mode: RunMode, items: T[], process: (item: T) => Promise<R>) {
+    return mode.type === 'parallel'
+        ? runParallel(items, process)
+        : mode.type === 'sequential'
+          ? runSequentially(items, process)
+          : runParallelBatches(items, mode.batchSize, process);
+}
+
 export async function runSequentially<T, R>(
     items: T[],
     process: (item: T) => Promise<R>
@@ -36,6 +46,31 @@ export async function runSequentially<T, R>(
     return results;
 }
 
+export async function runParallel<T, R>(
+    items: T[],
+    process: (item: T) => Promise<R>
+): Promise<PromiseSettledResult<R>[]> {
+    return Promise.allSettled(items.map(process));
+}
+
+export async function runParallelBatches<T, R>(
+    items: T[],
+    batchSize: number,
+    process: (item: T) => Promise<R>
+): Promise<PromiseSettledResult<R>[]> {
+    if (batchSize <= 0) {
+        throw new Error('Batch size must be greater than 0');
+    }
+    const batches = chunk(items, batchSize);
+    let results: PromiseSettledResult<R>[] = [];
+
+    for (const batch of batches) {
+        const batchResults = await runParallel(batch, process);
+        results = results.concat(batchResults);
+    }
+    return results;
+}
+
 // copy-pasted from node_modules/viem/src/utils/promise/withRetry.ts and node_modules/viem/src/utils/wait.ts
 export async function wait(time: number) {
     return new Promise(res => setTimeout(res, time));
@@ -53,7 +88,13 @@ export function withRetry<TData>(
         // The max number of times to retry.
         retryCount?: number;
         // Whether or not to retry when an error is thrown.
-        shouldRetry?: ({ count, error }: { count: number; error: Error }) => Promise<boolean> | boolean;
+        shouldRetry?: ({
+            count,
+            error,
+        }: {
+            count: number;
+            error: Error;
+        }) => Promise<boolean> | boolean;
     } = {}
 ) {
     return new Promise<TData>((resolve, reject) => {