diff --git a/packages/util/package.json b/packages/util/package.json index 5d39706b..9701795f 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -5,14 +5,17 @@ "license": "AGPL-3.0", "dependencies": { "@vulcanize/solidity-mapper": "^0.1.0", + "csv-writer": "^1.6.0", "debug": "^4.3.1", "decimal.js": "^10.3.1", "ethers": "^5.2.0", "fs-extra": "^10.0.0", "lodash": "^4.17.21", + "node-fetch": "2", "pg-boss": "^6.1.0", "prom-client": "^14.0.1", - "toml": "^3.0.0" + "toml": "^3.0.0", + "yargs": "^17.0.1" }, "devDependencies": { "@types/fs-extra": "^9.0.11", @@ -36,6 +39,7 @@ "scripts": { "lint": "eslint .", "build": "tsc", - "build:contracts": "hardhat compile" + "build:contracts": "hardhat compile", + "estimate-event-counts": "ts-node src/estimate-event-counts.ts" } } diff --git a/packages/util/src/estimate-event-counts.ts b/packages/util/src/estimate-event-counts.ts new file mode 100644 index 00000000..56ebf089 --- /dev/null +++ b/packages/util/src/estimate-event-counts.ts @@ -0,0 +1,253 @@ +import fetch from 'node-fetch'; +import yargs from 'yargs'; +import assert from 'assert'; +import path from 'path'; +import fs from 'fs'; +import { createObjectCsvWriter } from 'csv-writer'; + +import { wait } from './misc'; + +const CACHE_DIR = 'out/cached-data'; +const RESULTS_DIR = 'out/estimation-results'; + +const FACTORY_ADDRESS = '0x1F98431c8aD98523631AE4a59f267346ea31F984'; +const NFPM_ADDRESS = '0xC36442b4a4522E871399CD717aBDD847Ab11FE88'; + +const FACTORY_EVENT_SIGNATURE_MAP = { + PoolCreated: '0x783cca1c0412dd0d695e784568c96da2e9c22ff989357a2e8b1d9b2b4e6b7118' +}; + +const NFPM_EVENT_SIGNATURE_MAP = { + IncreaseLiquidity: '0x3067048beee31b25b2f1681f88dac838c8bba36af25bfb2b7cf7473a5847e35f', + DecreaseLiquidity: '0x26f6a048ee9138f2c0ce266f322cb99228e8d619ae2bff30c67f8dcf9d2377b4', + Collect: '0x40d0efd1a53d60ecbf40971b9daf7dc90178c3aadc7aab1765632738fa8b8f01', + Transfer: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' +}; + +const POOL_EVENT_SIGNATURE_MAP = { + Initialize: '0x98636036cb66a9c19a37435efc1e90142190214e8abeb821bdba3f2990dd4c95', + Mint: '0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde', + Burn: '0x0c396cd989a39f4459b5fa1aed6a9a8dcdbc45908acfd67e028cd568da98982c', + Swap: '0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67' +}; + +const EVENTS = ['PoolCreated', 'IncreaseLiquidity', 'DecreaseLiquidity', 'Collect', 'Transfer', 'Initialize', 'Mint', 'Burn', 'Swap']; + +// Etherscan API limitation +const RESULT_SIZE_LIMIT = 1000; + +// Run: yarn estimate-event-counts --start-block [start-block] --end-block [end-block] --sample-size [sample-size] --interval [interval] --api-key +async function main () { + const argv = await yargs.parserConfiguration({ + 'parse-numbers': false + }).options({ + startBlock: { + type: 'number', + default: 12369621, + describe: 'Start block' + }, + endBlock: { + type: 'number', + default: 14171509, + describe: 'End block' + }, + sampleSize: { + type: 'number', + default: 50, + describe: 'Sample size' + }, + interval: { + type: 'number', + default: 100000, + describe: 'Interval' + }, + apiKey: { + type: 'string', + require: true, + demandOption: true, + describe: 'Etherscan API key' + } + }).argv; + + console.log('Start block: ', argv.startBlock); + console.log('End block: ', argv.endBlock); + console.log('Sample size: ', argv.sampleSize); + console.log('Interval: ', argv.interval); + + // Counts for the events in order: + // PoolCreated, IncreaseLiquidity, DecreaseLiquidity, Collect, Transfer, Initialize, Mint, Burn, Swap + const counts = [0, 0, 0, 0, 0, 0, 0, 0, 0]; + const prevCounts:number[] = new Array(7); + let totalEvents = 0; + + // Total number of blocks to be processed + const numberOfBlocks = argv.endBlock - argv.startBlock + 1; + + // Block range to query for + let fromBlock = argv.startBlock; + let toBlock = fromBlock + argv.sampleSize; + + // Continue till endBlock + while (toBlock <= argv.endBlock) { + // Event counts for the current sample + let sampleEventCounts = [0, 0, 0, 0, 0, 0, 0, 0, 0]; + + // Check cache for event counts + const cachedData = checkCachedData(fromBlock, toBlock); + + if (cachedData) { + assert(cachedData.counts, `Found invalid cached data (from: ${fromBlock}, to: ${toBlock})`); + + sampleEventCounts = cachedData.counts; + } else { + sampleEventCounts = await fetchSampleCounts(fromBlock, toBlock, argv.apiKey); + + cacheData(fromBlock, toBlock, sampleEventCounts); + } + + // Increment event counts + sampleEventCounts.forEach((count: number, index: number) => { + if (prevCounts[index] === undefined) { + counts[index] = count; + } else { + const meanCount = (prevCounts[index] + count) / 2; + counts[index] = counts[index] + Math.round((meanCount / argv.sampleSize) * argv.interval); + } + + prevCounts[index] = count; + }); + + // Calculate total number of events + totalEvents = counts.reduce((a, b) => { + return a + b; + }, 0); + + console.log(`Event counts till block ${fromBlock}:`, ...counts); + console.log('Total:', totalEvents); + + const blocksProcessed = fromBlock - argv.startBlock + 1; + const completePercentage = Math.round((blocksProcessed / numberOfBlocks) * 100); + console.log(`Processed ${blocksProcessed} of ${numberOfBlocks} blocks (${completePercentage}%)`); + + fromBlock = fromBlock + argv.interval; + toBlock = fromBlock + argv.sampleSize; + } + + // Export estimation results to a CSV + const csvPath = `${argv.startBlock}-${argv.endBlock}-${argv.sampleSize}-${argv.interval}.csv`; + await exportResult(csvPath, counts, totalEvents); +} + +async function fetchSampleCounts (fromBlock: number, toBlock: number, apiKey: string): Promise { + let data: any[] = []; + + // Fetch factory events + const factoryResponsePromise = fetch(`https://api.etherscan.io/api?module=logs&action=getLogs&fromBlock=${fromBlock}&toBlock=${toBlock}&address=${FACTORY_ADDRESS}&topic0=${FACTORY_EVENT_SIGNATURE_MAP.PoolCreated}&apikey=${apiKey}`); + + // Fetch NFPM events + const nfpmResponsePromises = Object.values(NFPM_EVENT_SIGNATURE_MAP).map(signature => { + return fetch(`https://api.etherscan.io/api?module=logs&action=getLogs&fromBlock=${fromBlock}&toBlock=${toBlock}&address=${NFPM_ADDRESS}&topic0=${signature}&apikey=${apiKey}`); + }); + + let responses = await Promise.all([...[factoryResponsePromise], ...nfpmResponsePromises]); + let dataPromises = responses.map(response => { + return response.json(); + }); + + let responseData = await Promise.all(dataPromises); + data = data.concat(responseData); + + // Wait for 1 sec according to API restrictions + await wait(1000); + + // Fetch pool events + const poolResponsePromises = Object.values(POOL_EVENT_SIGNATURE_MAP).map(signature => { + return fetch(`https://api.etherscan.io/api?module=logs&action=getLogs&fromBlock=${fromBlock}&toBlock=${toBlock}&topic0=${signature}&apikey=${apiKey}`); + }); + + responses = await Promise.all([...poolResponsePromises]); + dataPromises = responses.map(response => { + return response.json(); + }); + + responseData = await Promise.all(dataPromises); + data = data.concat(responseData); + + // Wait for 1 sec according to API restrictions + await wait(1000); + + // Calculate event counts + const sampleEventCounts: number[] = []; + + data.forEach((obj: any) => { + assert(Array.isArray(obj.result), `Result "${obj.result}" is not an array`); + + if (obj.result.length === RESULT_SIZE_LIMIT) { + console.log(`WARNING: Result size reached limit (${RESULT_SIZE_LIMIT}). Estimates might not be correct. Please reduce sample size.`); + } + + sampleEventCounts.push(obj.result.length); + }); + + return sampleEventCounts; +} + +function cacheData (fromBlock: number, toBlock: number, sampleEventCounts: number[]): void { + const jsonPath = `${fromBlock}-${toBlock}.json`; + const cacheFilePath = path.resolve(CACHE_DIR, jsonPath); + resolveDir(path.dirname(cacheFilePath)); + + const data = { + counts: sampleEventCounts + }; + + fs.writeFileSync(cacheFilePath, JSON.stringify(data)); +} + +function checkCachedData (fromBlock: number, toBlock: number): any { + const jsonPath = `${fromBlock}-${toBlock}.json`; + const cacheFilePath = path.resolve(CACHE_DIR, jsonPath); + + if (!fs.existsSync(cacheFilePath)) { + return; + } + + const cachedResponse = JSON.parse(fs.readFileSync(cacheFilePath, 'utf-8')); + + return cachedResponse; +} + +async function exportResult (csvPath: string, counts: number[], totalEvents: number): Promise { + const filePath = path.resolve(RESULTS_DIR, csvPath); + resolveDir(path.dirname(filePath)); + + const csvWriter = createObjectCsvWriter({ + path: filePath, + header: [ + { id: 'event', title: 'Event' }, + { id: 'count', title: 'Count' } + ] + }); + + const records = counts.map((count, index) => { + return { + event: EVENTS[index], + count + }; + }); + records.push({ event: 'Total', count: totalEvents }); + + await csvWriter.writeRecords(records); +} + +function resolveDir (dir: string): void { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } +} + +main().then(() => { + process.exit(); +}).catch(err => { + console.log(err); +}); diff --git a/yarn.lock b/yarn.lock index 745bb8bb..be34c282 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5305,6 +5305,11 @@ cssfilter@0.0.10: resolved "https://registry.yarnpkg.com/cssfilter/-/cssfilter-0.0.10.tgz#c6d2672632a2e5c83e013e6864a42ce8defd20ae" integrity sha1-xtJnJjKi5cg+AT5oZKQs6N79IK4= +csv-writer@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/csv-writer/-/csv-writer-1.6.0.tgz#d0cea44b6b4d7d3baa2ecc6f3f7209233514bcf9" + integrity sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g== + currently-unhandled@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea" @@ -10282,6 +10287,13 @@ node-environment-flags@1.0.6: object.getownpropertydescriptors "^2.0.3" semver "^5.7.0" +node-fetch@2: + version "2.6.7" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== + dependencies: + whatwg-url "^5.0.0" + node-fetch@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.1.2.tgz#ab884e8e7e57e38a944753cec706f788d1768bb5"