From 76be46443aa86f2f5c21fe23e24b9d2f8d0255b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pr=C3=A9vost?= Date: Fri, 7 Jul 2023 11:01:13 +0200 Subject: [PATCH] Switch to viem --- data/interfaces/standard/Multicall3.json | 236 ----------------- data/interfaces/standard/Multicall3.ts | 236 +++++++++++++++++ package-lock.json | 141 +++++----- package.json | 4 +- src/protocol/beefy/script/beefy.ts | 4 +- src/protocol/common/utils/batch-rpc-calls.ts | 129 +++++++++ src/protocol/common/utils/rpc-chain-runner.ts | 7 +- src/protocol/common/utils/rpc-config.ts | 32 ++- src/protocol/common/utils/snapshot.ts | 246 ++++++++++++++++++ src/protocol/common/utils/viem/client.ts | 93 +++++++ src/protocol/common/utils/viem/http.ts | 111 ++++++++ src/types/rpc-config.ts | 5 +- src/utils/abi.ts | 5 +- src/utils/shared-resources/shared-rpc.ts | 17 +- tsconfig.json | 2 +- 15 files changed, 928 insertions(+), 340 deletions(-) delete mode 100644 data/interfaces/standard/Multicall3.json create mode 100644 data/interfaces/standard/Multicall3.ts create mode 100644 src/protocol/common/utils/snapshot.ts create mode 100644 src/protocol/common/utils/viem/client.ts create mode 100644 src/protocol/common/utils/viem/http.ts diff --git a/data/interfaces/standard/Multicall3.json b/data/interfaces/standard/Multicall3.json deleted file mode 100644 index d4f9b8c4..00000000 --- a/data/interfaces/standard/Multicall3.json +++ /dev/null @@ -1,236 +0,0 @@ -[ - { - "inputs": [ - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "aggregate", - "outputs": [ - { "internalType": "uint256", "name": "blockNumber", "type": "uint256" }, - { "internalType": "bytes[]", "name": "returnData", "type": "bytes[]" } - ], - "stateMutability": "payable", - "type": "function" - }, - { - "inputs": [ - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bool", "name": "allowFailure", "type": "bool" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call3[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "aggregate3", - "outputs": [ - { - "components": [ - { "internalType": "bool", "name": "success", "type": "bool" }, - { "internalType": "bytes", "name": "returnData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Result[]", - "name": "returnData", - "type": "tuple[]" - } - ], - "stateMutability": "payable", - "type": "function" - }, - { - "inputs": [ - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bool", "name": "allowFailure", "type": "bool" }, - { "internalType": "uint256", "name": "value", "type": "uint256" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call3Value[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "aggregate3Value", - "outputs": [ - { - "components": [ - { "internalType": "bool", "name": "success", "type": "bool" }, - { "internalType": "bytes", "name": "returnData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Result[]", - "name": "returnData", - "type": "tuple[]" - } - ], - "stateMutability": "payable", - "type": "function" - }, - { - "inputs": [ - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "blockAndAggregate", - "outputs": [ - { "internalType": "uint256", "name": "blockNumber", "type": "uint256" }, - { "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }, - { - "components": [ - { "internalType": "bool", "name": "success", "type": "bool" }, - { "internalType": "bytes", "name": "returnData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Result[]", - "name": "returnData", - "type": "tuple[]" - } - ], - "stateMutability": "payable", - "type": "function" - }, - { - "inputs": [], - "name": "getBasefee", - "outputs": [{ "internalType": "uint256", "name": "basefee", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" }], - "name": "getBlockHash", - "outputs": [{ "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getBlockNumber", - "outputs": [{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getChainId", - "outputs": [{ "internalType": "uint256", "name": "chainid", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getCurrentBlockCoinbase", - "outputs": [{ "internalType": "address", "name": "coinbase", "type": "address" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getCurrentBlockDifficulty", - "outputs": [{ "internalType": "uint256", "name": "difficulty", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getCurrentBlockGasLimit", - "outputs": [{ "internalType": "uint256", "name": "gaslimit", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getCurrentBlockTimestamp", - "outputs": [{ "internalType": "uint256", "name": "timestamp", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [{ "internalType": "address", "name": "addr", "type": "address" }], - "name": "getEthBalance", - "outputs": [{ "internalType": "uint256", "name": "balance", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getLastBlockHash", - "outputs": [{ "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [ - { "internalType": "bool", "name": "requireSuccess", "type": "bool" }, - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "tryAggregate", - "outputs": [ - { - "components": [ - { "internalType": "bool", "name": "success", "type": "bool" }, - { "internalType": "bytes", "name": "returnData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Result[]", - "name": "returnData", - "type": "tuple[]" - } - ], - "stateMutability": "payable", - "type": "function" - }, - { - "inputs": [ - { "internalType": "bool", "name": "requireSuccess", "type": "bool" }, - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "tryBlockAndAggregate", - "outputs": [ - { "internalType": "uint256", "name": "blockNumber", "type": "uint256" }, - { "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }, - { - "components": [ - { "internalType": "bool", "name": "success", "type": "bool" }, - { "internalType": "bytes", "name": "returnData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Result[]", - "name": "returnData", - "type": "tuple[]" - } - ], - "stateMutability": "payable", - "type": "function" - } -] diff --git a/data/interfaces/standard/Multicall3.ts b/data/interfaces/standard/Multicall3.ts new file mode 100644 index 00000000..51a6c6e6 --- /dev/null +++ b/data/interfaces/standard/Multicall3.ts @@ -0,0 +1,236 @@ +export default [ + { + inputs: [ + { + components: [ + { internalType: "address", name: "target", type: "address" }, + { internalType: "bytes", name: "callData", type: "bytes" }, + ], + internalType: "struct Multicall3.Call[]", + name: "calls", + type: "tuple[]", + }, + ], + name: "aggregate", + outputs: [ + { internalType: "uint256", name: "blockNumber", type: "uint256" }, + { internalType: "bytes[]", name: "returnData", type: "bytes[]" }, + ], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { + components: [ + { internalType: "address", name: "target", type: "address" }, + { internalType: "bool", name: "allowFailure", type: "bool" }, + { internalType: "bytes", name: "callData", type: "bytes" }, + ], + internalType: "struct Multicall3.Call3[]", + name: "calls", + type: "tuple[]", + }, + ], + name: "aggregate3", + outputs: [ + { + components: [ + { internalType: "bool", name: "success", type: "bool" }, + { internalType: "bytes", name: "returnData", type: "bytes" }, + ], + internalType: "struct Multicall3.Result[]", + name: "returnData", + type: "tuple[]", + }, + ], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { + components: [ + { internalType: "address", name: "target", type: "address" }, + { internalType: "bool", name: "allowFailure", type: "bool" }, + { internalType: "uint256", name: "value", type: "uint256" }, + { internalType: "bytes", name: "callData", type: "bytes" }, + ], + internalType: "struct Multicall3.Call3Value[]", + name: "calls", + type: "tuple[]", + }, + ], + name: "aggregate3Value", + outputs: [ + { + components: [ + { internalType: "bool", name: "success", type: "bool" }, + { internalType: "bytes", name: "returnData", type: "bytes" }, + ], + internalType: "struct Multicall3.Result[]", + name: "returnData", + type: "tuple[]", + }, + ], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { + components: [ + { internalType: "address", name: "target", type: "address" }, + { internalType: "bytes", name: "callData", type: "bytes" }, + ], + internalType: "struct Multicall3.Call[]", + name: "calls", + type: "tuple[]", + }, + ], + name: "blockAndAggregate", + outputs: [ + { internalType: "uint256", name: "blockNumber", type: "uint256" }, + { internalType: "bytes32", name: "blockHash", type: "bytes32" }, + { + components: [ + { internalType: "bool", name: "success", type: "bool" }, + { internalType: "bytes", name: "returnData", type: "bytes" }, + ], + internalType: "struct Multicall3.Result[]", + name: "returnData", + type: "tuple[]", + }, + ], + stateMutability: "payable", + type: "function", + }, + { + inputs: [], + name: "getBasefee", + outputs: [{ internalType: "uint256", name: "basefee", type: "uint256" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [{ internalType: "uint256", name: "blockNumber", type: "uint256" }], + name: "getBlockHash", + outputs: [{ internalType: "bytes32", name: "blockHash", type: "bytes32" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [], + name: "getBlockNumber", + outputs: [{ internalType: "uint256", name: "blockNumber", type: "uint256" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [], + name: "getChainId", + outputs: [{ internalType: "uint256", name: "chainid", type: "uint256" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [], + name: "getCurrentBlockCoinbase", + outputs: [{ internalType: "address", name: "coinbase", type: "address" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [], + name: "getCurrentBlockDifficulty", + outputs: [{ internalType: "uint256", name: "difficulty", type: "uint256" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [], + name: "getCurrentBlockGasLimit", + outputs: [{ internalType: "uint256", name: "gaslimit", type: "uint256" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [], + name: "getCurrentBlockTimestamp", + outputs: [{ internalType: "uint256", name: "timestamp", type: "uint256" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [{ internalType: "address", name: "addr", type: "address" }], + name: "getEthBalance", + outputs: [{ internalType: "uint256", name: "balance", type: "uint256" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [], + name: "getLastBlockHash", + outputs: [{ internalType: "bytes32", name: "blockHash", type: "bytes32" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { internalType: "bool", name: "requireSuccess", type: "bool" }, + { + components: [ + { internalType: "address", name: "target", type: "address" }, + { internalType: "bytes", name: "callData", type: "bytes" }, + ], + internalType: "struct Multicall3.Call[]", + name: "calls", + type: "tuple[]", + }, + ], + name: "tryAggregate", + outputs: [ + { + components: [ + { internalType: "bool", name: "success", type: "bool" }, + { internalType: "bytes", name: "returnData", type: "bytes" }, + ], + internalType: "struct Multicall3.Result[]", + name: "returnData", + type: "tuple[]", + }, + ], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "bool", name: "requireSuccess", type: "bool" }, + { + components: [ + { internalType: "address", name: "target", type: "address" }, + { internalType: "bytes", name: "callData", type: "bytes" }, + ], + internalType: "struct Multicall3.Call[]", + name: "calls", + type: "tuple[]", + }, + ], + name: "tryBlockAndAggregate", + outputs: [ + { internalType: "uint256", name: "blockNumber", type: "uint256" }, + { internalType: "bytes32", name: "blockHash", type: "bytes32" }, + { + components: [ + { internalType: "bool", name: "success", type: "bool" }, + { internalType: "bytes", name: "returnData", type: "bytes" }, + ], + internalType: "struct Multicall3.Result[]", + name: "returnData", + type: "tuple[]", + }, + ], + stateMutability: "payable", + type: "function", + }, +] as const; diff --git a/package-lock.json b/package-lock.json index fcff3f71..d45966f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "abstract-cache-redis": "^2.0.0", "async-lock": "^1.4.0", "axios": "^1.3.4", - "blockchain-addressbook": "^0.41.65", + "blockchain-addressbook": "^0.43.12", "decimal.js": "^10.4.3", "dotenv": "^16.0.3", "ethereum-multicall": "^2.16.1", @@ -45,7 +45,7 @@ "rxjs": "^7.8.0", "simple-git": "^3.17.0", "uuid": "^9.0.0", - "viem": "^0.3.17", + "viem": "^1.1.1", "yargs": "^17.7.1" }, "devDependencies": { @@ -1974,9 +1974,9 @@ "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==" }, "node_modules/@noble/curves": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-0.9.0.tgz", - "integrity": "sha512-OAdtHMXBp7Chl2lcTn/i7vnFX/q+hhTwDnek5NfYfZsY4LyaUuHCcoq2JlLY3BTFTLT+ZhYZalhF6ejlV7KnJQ==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.0.0.tgz", + "integrity": "sha512-2upgEu0iLiDVDZkNLeFV2+ht0BAVgQnEmCk6JsOch9Rp8xfkMCbvbAZlA2pBHQc73dbl+vFOXfqkf4uemdn0bw==", "funding": [ { "type": "individual", @@ -2332,9 +2332,9 @@ ] }, "node_modules/@scure/bip32": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.2.0.tgz", - "integrity": "sha512-O+vT/hBVk+ag2i6j2CDemwd1E1MtGt+7O1KzrPNsaNvSsiEK55MyPIxJIMI2PS8Ijj464B2VbQlpRoQXxw1uHg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.3.0.tgz", + "integrity": "sha512-bcKpo1oj54hGholplGLpqPHRbIsnbixFtc06nwuNM5/dwSXOq/AAYoIBRsBmnZJSdfeNW5rnff7NTAz3ZCqR9Q==", "funding": [ { "type": "individual", @@ -2342,25 +2342,11 @@ } ], "dependencies": { - "@noble/curves": "~0.8.3", + "@noble/curves": "~1.0.0", "@noble/hashes": "~1.3.0", "@scure/base": "~1.1.0" } }, - "node_modules/@scure/bip32/node_modules/@noble/curves": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-0.8.3.tgz", - "integrity": "sha512-OqaOf4RWDaCRuBKJLDURrgVxjLmneGsiCXGuzYB5y95YithZMA6w4uk34DHSm0rKMrrYiaeZj48/81EvaAScLQ==", - "funding": [ - { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - ], - "dependencies": { - "@noble/hashes": "1.3.0" - } - }, "node_modules/@scure/bip39": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.2.0.tgz", @@ -2696,9 +2682,9 @@ "dev": true }, "node_modules/@wagmi/chains": { - "version": "0.2.16", - "resolved": "https://registry.npmjs.org/@wagmi/chains/-/chains-0.2.16.tgz", - "integrity": "sha512-rkWaI2PxCnbD8G07ZZff5QXftnSkYL0h5f4DkHCG3fGYYr/ZDvmCL4bMae7j7A9sAif1csPPBmbCzHp3R5ogCQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@wagmi/chains/-/chains-1.1.0.tgz", + "integrity": "sha512-pWZlxBk0Ql8E7DV8DwqlbBpOyUdaG9UDlQPBxJNALuEK1I0tbQ3AVvSDnlsEIt06UPmPo5o27gzs3hwPQ/A+UA==", "funding": [ { "type": "gitcoin", @@ -2710,7 +2696,7 @@ } ], "peerDependencies": { - "typescript": ">=4.9.4" + "typescript": ">=5.0.4" }, "peerDependenciesMeta": { "typescript": { @@ -2725,11 +2711,11 @@ "dev": true }, "node_modules/abitype": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/abitype/-/abitype-0.7.1.tgz", - "integrity": "sha512-VBkRHTDZf9Myaek/dO3yMmOzB/y2s3Zo6nVU7yaw1G+TvCHAjwaJzNGN9yo4K5D8bU/VZXKP1EJpRhFr862PlQ==", + "version": "0.8.7", + "resolved": "https://registry.npmjs.org/abitype/-/abitype-0.8.7.tgz", + "integrity": "sha512-wQ7hV8Yg/yKmGyFpqrNZufCxbszDe5es4AZGYPBitocfSqXtjrTG9JMWFcc4N30ukl2ve48aBTwt7NJxVQdU3w==", "peerDependencies": { - "typescript": ">=4.9.4", + "typescript": ">=5.0.4", "zod": "^3 >=3.19.1" }, "peerDependenciesMeta": { @@ -3203,9 +3189,9 @@ "integrity": "sha512-s0IrSOzLlbvX7yp4WBfPITzpAU8sqQcpsmwXDiKwrG4r491vwCO/XpejasRNl0piBMe/DvP4Tz0mIS/X1DPJBQ==" }, "node_modules/blockchain-addressbook": { - "version": "0.41.65", - "resolved": "https://registry.npmjs.org/blockchain-addressbook/-/blockchain-addressbook-0.41.65.tgz", - "integrity": "sha512-6fOFqzRH97lH8N25gHTblain0ilpIO02/TmnbFMEIHvG81Ng4UJTUnrpjhiJuWBF0bhgYqPiAZFGRPyXqJrL0A==" + "version": "0.43.12", + "resolved": "https://registry.npmjs.org/blockchain-addressbook/-/blockchain-addressbook-0.43.12.tgz", + "integrity": "sha512-kNNzd3vYmX79XU4WNgE0E4YVvaRP/SNy78rFECeGyJ6FtlfW6le7fz2uO4pwSXReng0GFx8ccRCUyBIbnL+rzQ==" }, "node_modules/bn.js": { "version": "5.2.1", @@ -11407,15 +11393,15 @@ } }, "node_modules/typescript": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.2.tgz", - "integrity": "sha512-wVORMBGO/FAs/++blGNeAVdbNKtIh1rbBL2EyQ1+J9lClJ93KiiKe8PmFIVdXhHcyv44SL9oglmfeSsndo0jRw==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.3.tgz", + "integrity": "sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw==", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=12.20" + "node": ">=14.17" } }, "node_modules/typical": { @@ -11655,19 +11641,22 @@ } }, "node_modules/viem": { - "version": "0.3.17", - "resolved": "https://registry.npmjs.org/viem/-/viem-0.3.17.tgz", - "integrity": "sha512-WGJQ3rV2gMiHgwxLqeAZQ3HetSfsrkF//WezdJPnwUNXuic/c2jjB3MBk4XfYvuRUMccPfc37ORi03AmQxjscg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/viem/-/viem-1.1.1.tgz", + "integrity": "sha512-RVwDpDQYngaXRWnP3ZfmKHSis7ojN4q46mOApGRlfGrS/p56UD7/hA35zYxeq+X0dwVhCNhWuIqtV9XSsMtVfg==", "dependencies": { "@adraffy/ens-normalize": "1.9.0", - "@noble/curves": "0.9.0", + "@noble/curves": "1.0.0", "@noble/hashes": "1.3.0", - "@scure/bip32": "1.2.0", + "@scure/bip32": "1.3.0", "@scure/bip39": "1.2.0", - "@wagmi/chains": "0.2.16", - "abitype": "0.7.1", + "@wagmi/chains": "1.1.0", + "abitype": "0.8.7", "isomorphic-ws": "5.0.0", "ws": "8.12.0" + }, + "peerDependencies": { + "typescript": ">=5.0.4" } }, "node_modules/viem/node_modules/ws": { @@ -13378,9 +13367,9 @@ "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==" }, "@noble/curves": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-0.9.0.tgz", - "integrity": "sha512-OAdtHMXBp7Chl2lcTn/i7vnFX/q+hhTwDnek5NfYfZsY4LyaUuHCcoq2JlLY3BTFTLT+ZhYZalhF6ejlV7KnJQ==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.0.0.tgz", + "integrity": "sha512-2upgEu0iLiDVDZkNLeFV2+ht0BAVgQnEmCk6JsOch9Rp8xfkMCbvbAZlA2pBHQc73dbl+vFOXfqkf4uemdn0bw==", "requires": { "@noble/hashes": "1.3.0" } @@ -13635,23 +13624,13 @@ "integrity": "sha512-ZxOhsSyxYwLJj3pLZCefNitxsj093tb2vq90mp2txoYeBqbcjDjqFhyM8eUjq/uFm6zJ+mUuqxlS2FkuSY1MTA==" }, "@scure/bip32": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.2.0.tgz", - "integrity": "sha512-O+vT/hBVk+ag2i6j2CDemwd1E1MtGt+7O1KzrPNsaNvSsiEK55MyPIxJIMI2PS8Ijj464B2VbQlpRoQXxw1uHg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.3.0.tgz", + "integrity": "sha512-bcKpo1oj54hGholplGLpqPHRbIsnbixFtc06nwuNM5/dwSXOq/AAYoIBRsBmnZJSdfeNW5rnff7NTAz3ZCqR9Q==", "requires": { - "@noble/curves": "~0.8.3", + "@noble/curves": "~1.0.0", "@noble/hashes": "~1.3.0", "@scure/base": "~1.1.0" - }, - "dependencies": { - "@noble/curves": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-0.8.3.tgz", - "integrity": "sha512-OqaOf4RWDaCRuBKJLDURrgVxjLmneGsiCXGuzYB5y95YithZMA6w4uk34DHSm0rKMrrYiaeZj48/81EvaAScLQ==", - "requires": { - "@noble/hashes": "1.3.0" - } - } } }, "@scure/bip39": { @@ -13953,9 +13932,9 @@ "dev": true }, "@wagmi/chains": { - "version": "0.2.16", - "resolved": "https://registry.npmjs.org/@wagmi/chains/-/chains-0.2.16.tgz", - "integrity": "sha512-rkWaI2PxCnbD8G07ZZff5QXftnSkYL0h5f4DkHCG3fGYYr/ZDvmCL4bMae7j7A9sAif1csPPBmbCzHp3R5ogCQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@wagmi/chains/-/chains-1.1.0.tgz", + "integrity": "sha512-pWZlxBk0Ql8E7DV8DwqlbBpOyUdaG9UDlQPBxJNALuEK1I0tbQ3AVvSDnlsEIt06UPmPo5o27gzs3hwPQ/A+UA==", "requires": {} }, "abbrev": { @@ -13965,9 +13944,9 @@ "dev": true }, "abitype": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/abitype/-/abitype-0.7.1.tgz", - "integrity": "sha512-VBkRHTDZf9Myaek/dO3yMmOzB/y2s3Zo6nVU7yaw1G+TvCHAjwaJzNGN9yo4K5D8bU/VZXKP1EJpRhFr862PlQ==", + "version": "0.8.7", + "resolved": "https://registry.npmjs.org/abitype/-/abitype-0.8.7.tgz", + "integrity": "sha512-wQ7hV8Yg/yKmGyFpqrNZufCxbszDe5es4AZGYPBitocfSqXtjrTG9JMWFcc4N30ukl2ve48aBTwt7NJxVQdU3w==", "requires": {} }, "abort-controller": { @@ -14323,9 +14302,9 @@ "integrity": "sha512-s0IrSOzLlbvX7yp4WBfPITzpAU8sqQcpsmwXDiKwrG4r491vwCO/XpejasRNl0piBMe/DvP4Tz0mIS/X1DPJBQ==" }, "blockchain-addressbook": { - "version": "0.41.65", - "resolved": "https://registry.npmjs.org/blockchain-addressbook/-/blockchain-addressbook-0.41.65.tgz", - "integrity": "sha512-6fOFqzRH97lH8N25gHTblain0ilpIO02/TmnbFMEIHvG81Ng4UJTUnrpjhiJuWBF0bhgYqPiAZFGRPyXqJrL0A==" + "version": "0.43.12", + "resolved": "https://registry.npmjs.org/blockchain-addressbook/-/blockchain-addressbook-0.43.12.tgz", + "integrity": "sha512-kNNzd3vYmX79XU4WNgE0E4YVvaRP/SNy78rFECeGyJ6FtlfW6le7fz2uO4pwSXReng0GFx8ccRCUyBIbnL+rzQ==" }, "bn.js": { "version": "5.2.1", @@ -20534,9 +20513,9 @@ } }, "typescript": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.2.tgz", - "integrity": "sha512-wVORMBGO/FAs/++blGNeAVdbNKtIh1rbBL2EyQ1+J9lClJ93KiiKe8PmFIVdXhHcyv44SL9oglmfeSsndo0jRw==" + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.3.tgz", + "integrity": "sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw==" }, "typical": { "version": "4.0.0", @@ -20707,17 +20686,17 @@ } }, "viem": { - "version": "0.3.17", - "resolved": "https://registry.npmjs.org/viem/-/viem-0.3.17.tgz", - "integrity": "sha512-WGJQ3rV2gMiHgwxLqeAZQ3HetSfsrkF//WezdJPnwUNXuic/c2jjB3MBk4XfYvuRUMccPfc37ORi03AmQxjscg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/viem/-/viem-1.1.1.tgz", + "integrity": "sha512-RVwDpDQYngaXRWnP3ZfmKHSis7ojN4q46mOApGRlfGrS/p56UD7/hA35zYxeq+X0dwVhCNhWuIqtV9XSsMtVfg==", "requires": { "@adraffy/ens-normalize": "1.9.0", - "@noble/curves": "0.9.0", + "@noble/curves": "1.0.0", "@noble/hashes": "1.3.0", - "@scure/bip32": "1.2.0", + "@scure/bip32": "1.3.0", "@scure/bip39": "1.2.0", - "@wagmi/chains": "0.2.16", - "abitype": "0.7.1", + "@wagmi/chains": "1.1.0", + "abitype": "0.8.7", "isomorphic-ws": "5.0.0", "ws": "8.12.0" }, diff --git a/package.json b/package.json index a6bec4f0..de7ee5f4 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "abstract-cache-redis": "^2.0.0", "async-lock": "^1.4.0", "axios": "^1.3.4", - "blockchain-addressbook": "^0.41.65", + "blockchain-addressbook": "^0.43.12", "decimal.js": "^10.4.3", "dotenv": "^16.0.3", "ethereum-multicall": "^2.16.1", @@ -76,7 +76,7 @@ "rxjs": "^7.8.0", "simple-git": "^3.17.0", "uuid": "^9.0.0", - "viem": "^0.3.17", + "viem": "^1.1.1", "yargs": "^17.7.1" } } diff --git a/src/protocol/beefy/script/beefy.ts b/src/protocol/beefy/script/beefy.ts index b9e8675d..96777292 100644 --- a/src/protocol/beefy/script/beefy.ts +++ b/src/protocol/beefy/script/beefy.ts @@ -388,8 +388,8 @@ async function importIgnoreAddress(cmdParams: CmdParams) { function importBeefyDataPrices(cmdParams: CmdParams) { const behaviour = _createImportBehaviourFromCmdParams(cmdParams); async function getInputs() { - const rpcConfig = createRpcConfig("bsc", behaviour); // never used const streamConfig = createBatchStreamConfig("bsc", behaviour); + const rpcConfig = createRpcConfig("bsc", behaviour, streamConfig); // never used const ctx: ImportCtx = { chain: "bsc" as Chain, // not used here client: cmdParams.client, @@ -475,8 +475,8 @@ async function importInvestmentData(chain: Chain, cmdParams: CmdParams) { function importBeefyDataShareRate(chain: Chain, cmdParams: CmdParams) { const behaviour = _createImportBehaviourFromCmdParams(cmdParams); - const rpcConfig = createRpcConfig(chain, behaviour); const streamConfig = createBatchStreamConfig(chain, behaviour); + const rpcConfig = createRpcConfig(chain, behaviour, streamConfig); const ctx: ImportCtx = { chain, client: cmdParams.client, diff --git a/src/protocol/common/utils/batch-rpc-calls.ts b/src/protocol/common/utils/batch-rpc-calls.ts index 799fab5c..6b02a222 100644 --- a/src/protocol/common/utils/batch-rpc-calls.ts +++ b/src/protocol/common/utils/batch-rpc-calls.ts @@ -1,5 +1,6 @@ import { ethers } from "ethers"; import * as Rx from "rxjs"; +import { Prettify } from "viem/dist/types/types/utils"; import { RpcCallMethod } from "../../../types/rpc-config"; import { LogInfos, mergeLogsInfos, rootLogger } from "../../../utils/logger"; import { ProgrammerError } from "../../../utils/programmer-error"; @@ -7,6 +8,7 @@ import { RpcLimitations } from "../../../utils/rpc/rpc-limitations"; import { callLockProtectedRpc } from "../../../utils/shared-resources/shared-rpc"; import { ErrorEmitter, ErrorReport, ImportCtx } from "../types/import-context"; import { cloneBatchProvider } from "./rpc-config"; +import { CustomViemClient } from "./viem/client"; const logger = rootLogger.child({ module: "utils", component: "batch-rpc-calls" }); @@ -144,6 +146,109 @@ export function batchRpcCalls$, TRes, TQue ); } +export function batchRpcCallsViem$, TRes, TQueryObj, TQueryResp>(options: { + ctx: ImportCtx; + emitError: TErr; + getQuery: (obj: TObj) => TQueryObj; + processBatch: (provider: CustomViemClient, queryObjs: TQueryObj[]) => Promise>; + // we are doing this much rpc calls per input object + // this is used to calculate the input batch to send to the client + // and to know if we can inject the batch provider or if we should use the regular provider + rpcCallsPerInputObj: { + [method in RpcCallMethod]: number; + }; + logInfos: LogInfos; + formatOutput: (objs: TObj, results: TQueryResp) => TRes; +}) { + const { maxInputObjsPerBatch, canUseBatchProvider } = getBatchConfigFromLimitations({ + maxInputObjsPerBatch: options.ctx.streamConfig.maxInputTake, + rpcCallsPerInputObj: options.rpcCallsPerInputObj, + limitations: options.ctx.rpcConfig.rpcLimitations, + logInfos: options.logInfos, + }); + + const workConcurrency = options.ctx.rpcConfig.rpcLimitations.minDelayBetweenCalls === "no-limit" ? options.ctx.streamConfig.workConcurrency : 1; + return Rx.pipe( + // add object TS type + Rx.tap((_: TObj) => {}), + + // take a batch of items + Rx.bufferTime(options.ctx.streamConfig.maxInputWaitMs, undefined, maxInputObjsPerBatch), + Rx.filter((objs) => objs.length > 0), + + // for each batch, fetch the transfers + Rx.mergeMap(async (objs) => { + logger.trace( + mergeLogsInfos( + { + msg: "batchRpcCalls$ - batch", + data: { + objsCount: objs.length, + maxInputObjsPerBatch, + canUseBatchProvider, + workConcurrency, + maxInputWaitMs: options.ctx.streamConfig.maxInputWaitMs, + }, + }, + options.logInfos, + ), + ); + + const objAndCallParams = objs.map((obj) => ({ obj, query: options.getQuery(obj) })); + + const rpcClient = options.ctx.rpcConfig.getViemClient(canUseBatchProvider ? "batch" : "linear", options.logInfos, options.ctx.streamConfig); + + try { + logger.trace(mergeLogsInfos({ msg: "Ready to call RPC", data: { chain: options.ctx.chain } }, options.logInfos)); + + const contractCalls = objAndCallParams.map(({ query }) => query); + const resultMap = await options.processBatch(rpcClient, contractCalls); + + const outputs: TRes[] = []; + for (const { obj, query } of objAndCallParams) { + const errorReport = resultMap.errors.get(query); + if (errorReport) { + options.emitError(obj, errorReport); + continue; + } + + if (!resultMap.successes.has(query)) { + logger.error(mergeLogsInfos({ msg: "result not found", data: { chain: options.ctx.chain, obj, query, resultMap } }, options.logInfos)); + throw new ProgrammerError( + mergeLogsInfos({ msg: "result not found", data: { chain: options.ctx.chain, obj, query, resultMap } }, options.logInfos), + ); + } + const res = resultMap.successes.get(query) as TQueryResp; + outputs.push(options.formatOutput(obj, res)); + } + return outputs; + } catch (error: any) { + // here, none of the retrying worked, so we emit all the objects as in error + const report: ErrorReport = { + error, + infos: mergeLogsInfos({ msg: "Error doing batch rpc work", data: { chain: options.ctx.chain, err: error } }, options.logInfos), + }; + logger.debug(report.infos); + logger.debug(report.error); + for (const obj of objs) { + report.infos.data = { ...report.infos.data, obj }; + options.emitError(obj, report); + } + return Rx.EMPTY; + } + }, workConcurrency), + + Rx.tap( + (objs) => + Array.isArray(objs) && + logger.trace(mergeLogsInfos({ msg: "batchRpcCalls$ - done", data: { chain: options.ctx.chain, objsCount: objs.length } }, options.logInfos)), + ), + + // flatten + Rx.mergeAll(), + ); +} + export function getBatchConfigFromLimitations(options: { maxInputObjsPerBatch: number; rpcCallsPerInputObj: { @@ -191,3 +296,27 @@ export function getBatchConfigFromLimitations(options: { return { maxInputObjsPerBatch, canUseBatchProvider }; } + +export function mergeBatchCallResults( + resultA: RPCBatchCallResult, + resultB: RPCBatchCallResult, +): RPCBatchCallResult> { + const successes = new Map(); + const errors = new Map(); + const allKeys = Array.from(resultA.successes.keys()).concat(Array.from(resultB.successes.keys())); + for (const key of allKeys) { + const resA = resultA.successes.get(key); + const resB = resultB.successes.get(key); + const errA = resultA.errors.get(key); + const errB = resultB.errors.get(key); + + if (errA || errB) { + errors.set(key, errA || errB!); + } else if (!resA || !resB) { + throw new ProgrammerError({ msg: "mergeBatchCallResults - missing result", data: { key, resA, resB, errA, errB } }); + } else { + successes.set(key, { ...resA, ...resB }); + } + } + return { successes, errors }; +} diff --git a/src/protocol/common/utils/rpc-chain-runner.ts b/src/protocol/common/utils/rpc-chain-runner.ts index 7b392b7f..d00a8dd8 100644 --- a/src/protocol/common/utils/rpc-chain-runner.ts +++ b/src/protocol/common/utils/rpc-chain-runner.ts @@ -55,18 +55,19 @@ export function createChainRunner( chain: "bsc", }; + const streamConfig = createBatchStreamConfig(options.chain, options.behaviour); + // get our rpc configs and associated workers const rpcConfigs = options.behaviour.forceRpcUrl - ? [createRpcConfig(options.chain, options.behaviour)] + ? [createRpcConfig(options.chain, options.behaviour, streamConfig)] : getMultipleRpcConfigsForChain({ chain: options.chain, behaviour: options.behaviour, + streamConfig: streamConfig, }); logger.debug({ msg: "splitting inputs between rpcs", data: { rpcCount: rpcConfigs.length } }); - const streamConfig = createBatchStreamConfig(options.chain, options.behaviour); - const workers = rpcConfigs.map((rpcConfig) => { const ctx: ImportCtx = { chain: options.chain, diff --git a/src/protocol/common/utils/rpc-config.ts b/src/protocol/common/utils/rpc-config.ts index a6dc4ccc..7a300969 100644 --- a/src/protocol/common/utils/rpc-config.ts +++ b/src/protocol/common/utils/rpc-config.ts @@ -5,8 +5,9 @@ import { RpcConfig } from "../../../types/rpc-config"; import { getChainNetworkId } from "../../../utils/addressbook"; import { ETHERSCAN_API_KEY } from "../../../utils/config"; import { - addDebugLogsToProvider, JsonRpcProviderWithMultiAddressGetLogs, + MultiChainEtherscanProvider, + addDebugLogsToProvider, monkeyPatchAnkrBscLinearProvider, monkeyPatchArchiveNodeRpcProvider, monkeyPatchCeloProvider, @@ -15,17 +16,17 @@ import { monkeyPatchLayer2ReceiptFormat, monkeyPatchMissingEffectiveGasPriceReceiptFormat, monkeyPatchProviderToRetryUnderlyingNetworkChangedError, - MultiChainEtherscanProvider, } from "../../../utils/ethers"; -import { rootLogger } from "../../../utils/logger"; +import { LogInfos, rootLogger } from "../../../utils/logger"; import { ProgrammerError } from "../../../utils/programmer-error"; import { removeSecretsFromRpcUrl } from "../../../utils/rpc/remove-secrets-from-rpc-url"; -import { getBestRpcUrlsForChain, getRpcLimitations, RpcLimitations } from "../../../utils/rpc/rpc-limitations"; -import { ImportBehaviour } from "../types/import-context"; +import { RpcLimitations, getBestRpcUrlsForChain, getRpcLimitations } from "../../../utils/rpc/rpc-limitations"; +import { BatchStreamConfig, ImportBehaviour } from "../types/import-context"; +import { createViemPublicClient } from "./viem/client"; const logger = rootLogger.child({ module: "rpc-utils", component: "rpc-config" }); -export function getMultipleRpcConfigsForChain(options: { chain: Chain; behaviour: ImportBehaviour }): RpcConfig[] { +export function getMultipleRpcConfigsForChain(options: { chain: Chain; behaviour: ImportBehaviour; streamConfig: BatchStreamConfig }): RpcConfig[] { let rpcUrls = getBestRpcUrlsForChain(options.chain, options.behaviour); if (options.behaviour.rpcCount !== "all") { rpcUrls = rpcUrls.slice(0, options.behaviour.rpcCount); @@ -39,7 +40,7 @@ export function getMultipleRpcConfigsForChain(options: { chain: Chain; behaviour logger.debug({ msg: "Using RPC URLs", data: { chain: options.chain, rpcUrls: rpcUrls.map((url) => removeSecretsFromRpcUrl(options.chain, url)) } }); - return rpcUrls.map((rpcUrl) => createRpcConfig(options.chain, { ...options.behaviour, forceRpcUrl: rpcUrl })); + return rpcUrls.map((rpcUrl) => createRpcConfig(options.chain, { ...options.behaviour, forceRpcUrl: rpcUrl }, options.streamConfig)); } const defaultRpcOptions: Partial = { @@ -58,7 +59,7 @@ const defaultRpcOptions: Partial = { skipFetchSetup: true, }; -export function createRpcConfig(chain: Chain, behaviour: ImportBehaviour): RpcConfig { +export function createRpcConfig(chain: Chain, behaviour: ImportBehaviour, streamConfig: BatchStreamConfig): RpcConfig { const rpcUrls = getBestRpcUrlsForChain(chain, behaviour); const urlObj = new URL(behaviour.forceRpcUrl || rpcUrls[0]); @@ -73,11 +74,24 @@ export function createRpcConfig(chain: Chain, behaviour: ImportBehaviour): RpcCo const rpcOptions: ethers.utils.ConnectionInfo = { ...defaultRpcOptions, url: rpcUrl, user, password, timeout: behaviour.rpcTimeoutMs }; const networkish = { name: chain, chainId: getChainNetworkId(chain) }; + const limitations = getRpcLimitations(chain, rpcOptions.url, behaviour); + const rpcConfig: RpcConfig = { chain, linearProvider: new JsonRpcProviderWithMultiAddressGetLogs(rpcOptions, networkish), batchProvider: new ethers.providers.JsonRpcBatchProvider(rpcOptions, networkish), - rpcLimitations: getRpcLimitations(chain, rpcOptions.url, behaviour), + getViemClient: (type: "linear" | "batch", logInfos: LogInfos, streamConfig: BatchStreamConfig) => + createViemPublicClient({ + type, + chain, + rpcUrl, + logInfos, + limitations, + behaviour, + auth: user && password ? { user, password } : undefined, + streamConfig, + }), + rpcLimitations: limitations, }; // instantiate etherscan provider diff --git a/src/protocol/common/utils/snapshot.ts b/src/protocol/common/utils/snapshot.ts new file mode 100644 index 00000000..fb087913 --- /dev/null +++ b/src/protocol/common/utils/snapshot.ts @@ -0,0 +1,246 @@ +import Decimal from "decimal.js"; +import { ethers } from "ethers"; +import * as Rx from "rxjs"; +import { Hex, multicall3Abi, parseAbi } from "viem"; +import { RpcCallMethod } from "../../../types/rpc-config"; +import { Multicall3Abi } from "../../../utils/abi"; +import { MULTICALL3_ADDRESS_MAP } from "../../../utils/config"; +import { LogInfos, mergeLogsInfos, rootLogger } from "../../../utils/logger"; +import { ProgrammerError } from "../../../utils/programmer-error"; +import { Range, SupportedRangeTypes, isInRange } from "../../../utils/range"; +import { forkOnNullableField$ } from "../../../utils/rxjs/utils/exclude-null-field"; +import { fetchBlockDatetime$ } from "../connector/block-datetime"; +import { fetchBlock$ } from "../loader/blocks"; +import { ErrorEmitter, ErrorReport, ImportCtx, Throwable } from "../types/import-context"; +import { RPCBatchCallResult, batchRpcCallsViem$, mergeBatchCallResults } from "./batch-rpc-calls"; +import { isAddressBatchQueries, isJsonRpcBatchQueries } from "./query/optimizer-utils"; +import { AddressBatchOutput, JsonRpcBatchOutput, QueryOptimizerOutput } from "./query/query-types"; +import { CustomViemClient } from "./viem/client"; + +const logger = rootLogger.child({ module: "beefy", component: "share-rate-snapshots" }); + +export interface BeefySnapshotCallResult { + res: TRes; + blockNumber: number; + blockDatetime: Date; +} + +/** + * This is optimized for fetching queries returned by the optimizer + * Mostly used by the share rate snapshots so we can multicall multiple vaults ppfs at once + */ +export function singleBlockMulticallSnapshot$< + TObj, + TParams, + TResult, + TQueryContent, + TErr extends ErrorEmitter, + TRange extends SupportedRangeTypes, + TRes, +>(options: { + ctx: ImportCtx; + logInfos: LogInfos; + emitError: TErr; + getOptimizerQuery: (obj: TObj) => QueryOptimizerOutput; + getCallParams: (query: TQueryContent, blockNumber: number) => TParams; + rpcCallsPerInputObj: { + [method in RpcCallMethod]: number; + }; + processBatch: (client: CustomViemClient, batch: TParams[]) => Promise>; + formatOutput: (obj: TObj, results: BeefySnapshotCallResult[]) => TRes; +}): Rx.OperatorFunction { + const mcMap = MULTICALL3_ADDRESS_MAP[options.ctx.chain]; + type MultipleResult = { obj: TObj; result: TResult }; + type MaybeResult = { result: { blockNumber: number } | null }; + + const fetchMulticallBlockDatetimeIfPossible = async (client: CustomViemClient, blockNumber: bigint): Promise => { + if (!mcMap) { + return null; + } + const multicallAddress = mcMap.multicallAddress as Hex; + let blockDatetimePromise: Promise = Promise.resolve(null); + if (options.ctx.rpcConfig.rpcLimitations.canUseMulticallBlockTimestamp) { + blockDatetimePromise = client + .readContract({ + abi: Multicall3Abi, + address: multicallAddress, + functionName: "getCurrentBlockTimestamp", + }) + .then((res) => new Date(Number(res))); + } + return blockDatetimePromise; + }; + + const fetchJsonRpcBatch$: Rx.OperatorFunction< + { obj: TObj; query: JsonRpcBatchOutput }, + { + obj: TObj; + query: JsonRpcBatchOutput; + results: TResult[]; + } + > = Rx.pipe( + batchRpcCallsViem$({ + ctx: options.ctx, + emitError: (item, errReport) => options.emitError(item.obj, errReport), + logInfos: mergeLogsInfos({ msg: "maybe using multicall" }, options.logInfos), + rpcCallsPerInputObj: options.rpcCallsPerInputObj, + getQuery: ({ query }) => ({ + blockTag: getBlockTag(options.ctx, rangeToBlockNumber(query.range)), + param: options.getCallParams(query.obj, rangeToBlockNumber(query.range)), + }), + processBatch: async (client, params) => { + // also fetch datetime + const datetimePromises = params.map( + async (param) => [param, { blockDatetime: await fetchMulticallBlockDatetimeIfPossible(client, param.blockTag) }] as const, + ); + const results = await options.processBatch( + client, + params.map(({ param }) => param), + ); + const datetimes = await Promise.all(datetimePromises); + const mergedResult = mergeBatchCallResults(results, { successes: new Map(datetimes), errors: new Map() }); + + return mergedResult; + }, + formatOutput: (item, result) => ({ ...item, results: [result] }), + }), + ); + + const workConcurrency = options.ctx.rpcConfig.rpcLimitations.minDelayBetweenCalls === "no-limit" ? options.ctx.streamConfig.workConcurrency : 1; + + type ResultWithMabeBlockDatetime = { res: TRes; blockNumber: number; blockDatetime: Date | null }; + const fetchAddressBatch$: Rx.OperatorFunction< + { obj: TObj; query: AddressBatchOutput }, + { obj: TObj; query: AddressBatchOutput; results: MultipleResult[] } + > = Rx.pipe( + Rx.tap((item) => logger.trace({ msg: "item", data: item })), + + Rx.mergeMap(async (item) => { + if (!mcMap) { + throw new ProgrammerError({ + msg: "Cannot use this method, multicall contract is not defined for this chain", + data: { chain: options.ctx.chain }, + }); + } + + const multicallAddress = mcMap.multicallAddress as Hex; + const blockTag = getBlockTag(options.ctx, rangeToBlockNumber(item.query.range)); + const client = options.ctx.rpcConfig.getViemClient("batch", options.logInfos, options.ctx.streamConfig); + + try { + let blockDatetime: Date | null = null; + + let blockTimestampPromise: Promise = Promise.resolve(null); + if (options.ctx.rpcConfig.rpcLimitations.canUseMulticallBlockTimestamp) { + blockTimestampPromise = client + .readContract({ + abi: Multicall3Abi, + address: multicallAddress, + functionName: "getCurrentBlockTimestamp", + }) + .then((res) => new Date(Number(res))); + } + + const res = await options.processBatch(client, item.query); + + return [{ ...item, res, blockNumber: blockTag, blockDatetime }]; + } catch (e: unknown) { + const error = e as Throwable; + // here, none of the retrying worked, so we emit all the objects as in error + const report: ErrorReport = { + error, + infos: mergeLogsInfos({ msg: "Error during address batch query multicall", data: { chain: options.ctx.chain, err: error } }, logInfos), + }; + logger.debug(report.infos); + logger.debug(report.error); + options.emitError(item.obj, report); + return Rx.EMPTY; + } + }, workConcurrency), + Rx.concatAll(), + + Rx.tap((item) => item), + + // handle those chains where multicall is not able to fetch the datetime + forkOnNullableField$({ + key: "blockDatetime", + handleNulls$: Rx.pipe( + Rx.tap((item) => + logger.trace({ msg: "Could not use Multicall3.getCurrentBlockTimestamp, fetching the proper timestamp", data: { mcMap, item } }), + ), + fetchBlockDatetime$({ + ctx: options.ctx, + emitError: (item, errReport) => options.emitError(item.obj, errReport), + getBlockNumber: (item) => item.blockNumber, + formatOutput: (item, blockDatetime) => ({ ...item, blockDatetime }), + }), + ), + // sometimes the multicall result differ from other methods so if the block exists we + // want to use the same datetime we already used in the database to avoid duplicate rows + handleNonNulls$: Rx.pipe( + fetchBlock$({ + ctx: options.ctx, + chain: options.ctx.chain, + emitError: (item, errReport) => options.emitError(item.obj, errReport), + getBlockNumber: (item) => item.blockNumber, + formatOutput: (item, block) => ({ + ...item, + blockDatetime: block ? block.datetime : item.blockDatetime, + }), + }), + ), + }), + + Rx.map((item) => ({ + obj: item.obj, + query: item.query, + results: item.results.map( + (res): MultipleShareRateResult => ({ + product: res.product, + result: { + blockDatetime: item.blockDatetime, + blockNumber: item.blockNumber, + shareRate: res.shareRate, + }, + }), + ), + })), + ); + + return Rx.pipe( + // wrap obj + Rx.map((obj: TObj) => ({ obj, query: options.getQueryCallParams(obj) })), + + // handle different types of requests differently + Rx.connect((items$) => + Rx.merge( + items$.pipe( + Rx.filter((item): item is { obj: TObj; query: JsonRpcBatchOutput } => isJsonRpcBatchQueries(item.query)), + fetchJsonRpcBatch$, + ), + items$.pipe( + Rx.filter((item): item is { obj: TObj; query: AddressBatchOutput } => isAddressBatchQueries(item.query)), + fetchAddressBatch$, + ), + ), + ), + Rx.map(({ obj, results }) => options.formatOutput(obj, results)), + ); +} + +const getBlockTag = (ctx: ImportCtx, blockNumber: number): bigint => { + // read the next block for those chains who can't read their own writes + let blockTag = blockNumber; + if (!ctx.rpcConfig.rpcLimitations.stateChangeReadsOnSameBlock) { + blockTag = blockNumber + 1; + } + return BigInt(blockTag); +}; + +function rangeToBlockNumber(range: Range): number { + const midPoint = Math.round((range.from + range.to) / 2); + if (!isInRange(range, midPoint)) { + throw new ProgrammerError({ msg: "Midpoint is not in range, most likely an invalid range", data: { range, midPoint } }); + } + return midPoint; +} diff --git a/src/protocol/common/utils/viem/client.ts b/src/protocol/common/utils/viem/client.ts new file mode 100644 index 00000000..994b61b9 --- /dev/null +++ b/src/protocol/common/utils/viem/client.ts @@ -0,0 +1,93 @@ +import { PublicClient, createPublicClient } from "viem"; +import { Chain } from "../../../../types/chain"; +import { MULTICALL3_ADDRESS_MAP } from "../../../../utils/config"; +import { LogInfos } from "../../../../utils/logger"; +import { ProgrammerError } from "../../../../utils/programmer-error"; +import { removeSecretsFromRpcUrl } from "../../../../utils/rpc/remove-secrets-from-rpc-url"; +import { RpcLimitations } from "../../../../utils/rpc/rpc-limitations"; +import { BatchStreamConfig, ImportBehaviour } from "../../types/import-context"; +import { customHttp } from "./http"; + +export type CustomViemClient = PublicClient & { + onError: (handler: (error: unknown) => void) => void; + clone: () => CustomViemClient; +}; + +export const createViemPublicClient = ({ + type, + chain, + rpcUrl, + logInfos, + behaviour, + limitations, + streamConfig, + auth, +}: { + type: "linear" | "batch"; + chain: Chain; + rpcUrl: string; + logInfos: LogInfos; + behaviour: ImportBehaviour; + limitations: RpcLimitations; + streamConfig: BatchStreamConfig; + auth?: { user: string; password: string }; +}): CustomViemClient => { + const viemHttpTransport = customHttp({ chain, logInfos, limitations, streamConfig }, rpcUrl, { + batch: limitations.disableBatching || type === "linear" ? undefined : { batchSize: limitations.methods.eth_call || 1000, wait: 100 }, + fetchOptions: auth ? { headers: { Authorization: `Basic ${Buffer.from(`${auth.user}:${auth.password}`).toString("base64")}` } } : undefined, + key: removeSecretsFromRpcUrl(chain, rpcUrl), + name: removeSecretsFromRpcUrl(chain, rpcUrl), + retryCount: 0, // we do our own retry logic + retryDelay: 0, // we do our own retry logic + timeout: behaviour.rpcTimeoutMs, + }); + + const client: CustomViemClient = createPublicClient({ + transport: viemHttpTransport, + batch: + limitations.disableBatching || type === "linear" + ? undefined + : { + // multicall when available + multicall: MULTICALL3_ADDRESS_MAP[chain] !== null, + }, + }) as CustomViemClient; + + // monkey patch the client to add an onError callback + const defaultErrorHandler = (error: unknown) => {}; + let errorHandler = defaultErrorHandler; + + client.onError = (handler: (error: unknown) => void) => { + if (errorHandler !== defaultErrorHandler) { + throw new ProgrammerError("onError can only be set once"); + } + errorHandler = handler; + }; + const originalCall = client.call.bind(client); + const callWithErrorHandler: typeof client.call = async (params) => { + try { + return await originalCall(params); + } catch (error) { + errorHandler(error); + throw error; + } + }; + client.call = callWithErrorHandler; + + // create a clone method to create a new client with the same config + // this is useful to make it possible to have multiple parts of the codebase + // batching calls to the same RPC since batching works per client with viem + client.clone = () => + createViemPublicClient({ + type, + logInfos, + chain, + rpcUrl, + behaviour, + limitations, + streamConfig, + auth, + }); + + return client; +}; diff --git a/src/protocol/common/utils/viem/http.ts b/src/protocol/common/utils/viem/http.ts new file mode 100644 index 00000000..fc4e09a0 --- /dev/null +++ b/src/protocol/common/utils/viem/http.ts @@ -0,0 +1,111 @@ +import { RpcRequestError, UrlRequiredError, createTransport, type HttpTransport, type HttpTransportConfig } from "viem"; +import { createBatchScheduler } from "viem/src/utils/promise/createBatchScheduler"; +import { rpc, type RpcRequest } from "viem/src/utils/rpc.js"; +import { Chain } from "../../../../types/chain"; +import { LogInfos, mergeLogsInfos, rootLogger } from "../../../../utils/logger"; +import { removeSecretsFromRpcUrl } from "../../../../utils/rpc/remove-secrets-from-rpc-url"; +import { RpcLimitations } from "../../../../utils/rpc/rpc-limitations"; +import { callLockProtectedRpc } from "../../../../utils/shared-resources/shared-rpc"; +import { BatchStreamConfig } from "../../types/import-context"; + +const logger = rootLogger.child({ module: "utils", component: "viem-http" }); + +/** + * Updates viem http transport to add: + * - ethers like debug hooks + */ +export function customHttp( + { + chain: beefyChain, + logInfos, + limitations, + streamConfig, + }: { chain: Chain; logInfos: LogInfos; limitations: RpcLimitations; streamConfig: BatchStreamConfig }, + /** URL of the JSON-RPC API. Defaults to the chain's public RPC URL. */ + url?: string, + config: HttpTransportConfig = {}, +): HttpTransport { + const { batch, fetchOptions, key = "http", name = "HTTP JSON-RPC", retryDelay } = config; + return ({ chain, retryCount: retryCount_, timeout: timeout_ }) => { + const { batchSize = 1000, wait = 0 } = typeof batch === "object" ? batch : {}; + const retryCount = config.retryCount ?? retryCount_; + const timeout = timeout_ ?? config.timeout ?? 10_000; + const url_ = url || chain?.rpcUrls.default.http[0]; + if (!url_) throw new UrlRequiredError(); + + // ============================================= CHANGED HERE ============================================= + // changelist: + // - add logs + // - add callLockProtectedRpc + // - add onError callback and set it on the client + const rpcHttp: typeof rpc.http = async (url, options) => { + const safeToLogUrl = removeSecretsFromRpcUrl(beefyChain, url); + const response = await callLockProtectedRpc( + async () => { + try { + logger.trace({ msg: "RPC request", data: { request: options.body, rpcUrl: safeToLogUrl } }); + const res = await rpc.http(url, options); + logger.trace({ msg: "RPC response", data: { request: options.body, response, rpcUrl: safeToLogUrl } }); + return res; + } catch (error) { + logger.trace({ msg: "RPC error", data: { request: options.body, error, rpcUrl: safeToLogUrl } }); + throw error; + } + }, + { + chain: beefyChain, + rpcLimitations: limitations, + logInfos: mergeLogsInfos({ msg: "viem client", data: { chain: beefyChain, rpcUrl: safeToLogUrl } }, logInfos), + maxTotalRetryMs: streamConfig.maxTotalRetryMs, + noLockIfNoLimit: !!config.batch, // no lock when using batch provider because we are using a copy of the provider + provider: url, + }, + ); + + return response; + }; + // ======================================================================================================== + + return createTransport( + { + key, + name, + async request({ method, params }) { + const body = { method, params }; + + const { schedule } = createBatchScheduler({ + id: `${url}`, + wait, + shouldSplitBatch(requests) { + return requests.length > batchSize; + }, + fn: (body: RpcRequest[]) => + rpcHttp(url_, { + body, + fetchOptions, + timeout, + }), + }); + + const fn = async (body: RpcRequest) => (batch ? schedule(body) : [await rpcHttp(url_, { body, fetchOptions, timeout })]); + + const [{ error, result }] = await fn(body); + if (error) + throw new RpcRequestError({ + body, + error, + url: url_, + }); + return result; + }, + retryCount, + retryDelay, + timeout, + type: "http", + }, + { + url, + }, + ); + }; +} diff --git a/src/types/rpc-config.ts b/src/types/rpc-config.ts index d6d18687..c5be7cb2 100644 --- a/src/types/rpc-config.ts +++ b/src/types/rpc-config.ts @@ -1,5 +1,8 @@ import { ethers } from "ethers"; +import { BatchStreamConfig } from "../protocol/common/types/import-context"; +import { CustomViemClient } from "../protocol/common/utils/viem/client"; import { JsonRpcProviderWithMultiAddressGetLogs } from "../utils/ethers"; +import { LogInfos } from "../utils/logger"; import { RpcLimitations } from "../utils/rpc/rpc-limitations"; import { Chain } from "./chain"; @@ -9,8 +12,8 @@ export interface RpcConfig { // most should use the batch provider linearProvider: JsonRpcProviderWithMultiAddressGetLogs; batchProvider: ethers.providers.JsonRpcBatchProvider; + getViemClient: (type: "batch" | "linear", logInfos: LogInfos, streamConfig: BatchStreamConfig) => CustomViemClient; rpcLimitations: RpcLimitations; - etherscan?: { provider: ethers.providers.EtherscanProvider; limitations: RpcLimitations; diff --git a/src/utils/abi.ts b/src/utils/abi.ts index 52c141ba..5f24b61b 100644 --- a/src/utils/abi.ts +++ b/src/utils/abi.ts @@ -1,9 +1,10 @@ import { ethers } from "ethers"; import BeefyVaultV6Abi from "../../data/interfaces/beefy/BeefyVaultV6/BeefyVaultV6.json"; import ERC20Abi from "../../data/interfaces/standard/ERC20.json"; -import Multicall3Abi from "../../data/interfaces/standard/Multicall3.json"; +import Multicall3AbiRaw from "../../data/interfaces/standard/Multicall3"; // parse ABI only once by using a global interface variable export const BeefyVaultV6AbiInterface = new ethers.utils.Interface(BeefyVaultV6Abi); export const ERC20AbiInterface = new ethers.utils.Interface(ERC20Abi); -export const Multicall3AbiInterface = new ethers.utils.Interface(Multicall3Abi); +export const Multicall3AbiInterface = new ethers.utils.Interface(Multicall3AbiRaw); +export const Multicall3Abi = Multicall3AbiRaw; diff --git a/src/utils/shared-resources/shared-rpc.ts b/src/utils/shared-resources/shared-rpc.ts index 3631dd76..be8a78e4 100644 --- a/src/utils/shared-resources/shared-rpc.ts +++ b/src/utils/shared-resources/shared-rpc.ts @@ -1,6 +1,7 @@ import AsyncLock from "async-lock"; import { ethers } from "ethers"; import { backOff, IBackOffOptions } from "exponential-backoff"; +import { isString } from "lodash"; import { createHash } from "node:crypto"; import { Chain } from "../../types/chain"; import { sleep } from "../../utils/async"; @@ -29,7 +30,7 @@ type CallLockProtectedRpcOptions = { maxTotalRetryMs: number; logInfos: LogInfos; chain: Chain; - provider: ethers.providers.JsonRpcProvider | ethers.providers.JsonRpcBatchProvider | ethers.providers.EtherscanProvider; + provider: ethers.providers.JsonRpcProvider | ethers.providers.JsonRpcBatchProvider | ethers.providers.EtherscanProvider | string; rpcLimitations: RpcLimitations; // if true, we will not call the work function behind a AsyncLock // it's up to the caller to ensure the work function is properly batched @@ -58,7 +59,12 @@ export async function callLockProtectedRpc(work: () => Promise, opti const redlock = await getRedlock(); // create a string we can log as raw rpc url may contain an api key - const url = options.provider instanceof ethers.providers.EtherscanProvider ? options.provider.getBaseUrl() : options.provider.connection.url; + const url = + options.provider instanceof ethers.providers.EtherscanProvider + ? options.provider.getBaseUrl() + : isString(options.provider) + ? options.provider + : options.provider.connection.url; const publicRpcUrl = removeSecretsFromRpcUrl(options.chain, url); const rpcLockId = `${options.chain}:rpc:lock:${getRpcPublicUniqueId(options.chain, url)}`; const lastCallCacheKey = `${options.chain}:rpc:last-call-date:${getRpcPublicUniqueId(options.chain, url)}`; @@ -145,7 +151,12 @@ export async function callLockProtectedRpc(work: () => Promise, opti */ async function callNoLimitRpc(work: () => Promise, options: CallLockProtectedRpcOptions) { // create a string we can log as raw rpc url may contain an api key - const url = options.provider instanceof ethers.providers.EtherscanProvider ? options.provider.getBaseUrl() : options.provider.connection.url; + const url = + options.provider instanceof ethers.providers.EtherscanProvider + ? options.provider.getBaseUrl() + : isString(options.provider) + ? options.provider + : options.provider.connection.url; const publicRpcUrl = removeSecretsFromRpcUrl(options.chain, url); const rpcLockId = `${options.chain}:rpc:lock:${publicRpcUrl}`; diff --git a/tsconfig.json b/tsconfig.json index 765747ae..364904c1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,5 +16,5 @@ "noImplicitThis": true //"noImplicitReturns": true }, - "include": ["./src/main.ts", "./src/**/*.ts", "./data/interfaces/**/*.json", "./data/config/**/*.json"] + "include": ["./src/main.ts", "./src/**/*.ts", "./data/interfaces/**/*.json", "./data/config/**/*.json", "data/interfaces/standard/Multicall3.ts"] }