diff --git a/package-lock.json b/package-lock.json index 1bb6609..6993c61 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,7 @@ "web3": "^4.9.0", "web3-validator": "^2.0.6", "xrpl": "^3.1.0", - "zod": "^3.23.8" + "zod": "^3.24.1" }, "devDependencies": { "@types/lodash": "^4.17.13", @@ -4352,10 +4352,9 @@ } }, "node_modules/zod": { - "version": "3.23.8", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", - "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", - "license": "MIT", + "version": "3.24.1", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz", + "integrity": "sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==", "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index ecc0545..c5e1c0a 100644 --- a/package.json +++ b/package.json @@ -40,12 +40,12 @@ "web3": "^4.9.0", "web3-validator": "^2.0.6", "xrpl": "^3.1.0", - "zod": "^3.23.8" + "zod": "^3.24.1" }, "devDependencies": { + "@types/lodash": "^4.17.13", "@types/micro": "^10.0.0", "@types/mocha": "^10.0.6", - "@types/lodash": "^4.17.13", "@types/uuid": "^9.0.8", "@types/web3": "^1.2.2", "earl": "^1.3.0", @@ -56,4 +56,4 @@ "ts-node": "^10.9.2", "typescript": "^5.4.5" } -} \ No newline at end of file +} diff --git a/src/blockchains/eth/eth_types.ts b/src/blockchains/eth/eth_types.ts index e43c0f0..dd565f5 100644 --- a/src/blockchains/eth/eth_types.ts +++ b/src/blockchains/eth/eth_types.ts @@ -1,32 +1,41 @@ -export type TraceAction = { - author?: string, - address?: string, - rewardType?: string, - callType?: string, - from?: string, - to?: string, - value?: string, - balance?: string, - refundAddress?: string -} +import { z } from "zod" -export type TraceResult = { - gasUsed: string, - address?: string -} +const TraceActionSchema = z.object({ + author: z.string().optional(), + address: z.string().optional(), + rewardType: z.string().optional(), + callType: z.string().optional(), + from: z.string().optional(), + to: z.string().optional(), + value: z.string().optional(), + balance: z.string().optional(), + refundAddress: z.string().optional() +}); -export type Trace = { - action: TraceAction, - blockHash: string, - blockNumber: number, - result: TraceResult, - subtraces: number, - traceAddress: number[], - transactionHash: string, - transactionPosition: number, - type: string, - error?: string -} + +export type TraceAction = z.infer + +const TraceResultSchema = z.object({ + gasUsed: z.string(), + address: z.string().optional() +}) + +export type TraceResult = z.infer + +export const TraceSchema = z.object({ + action: TraceActionSchema, + blockHash: z.string(), + blockNumber: z.number(), + result: TraceResultSchema.optional().nullable(), + subtraces: z.number(), + traceAddress: z.array(z.number()), + transactionHash: z.string().optional(), + transactionPosition: z.number().optional(), + type: z.string(), + error: z.string().optional() +}) + +export type Trace = z.infer export type ETHBlock = { baseFeePerGas?: string, @@ -75,9 +84,9 @@ export type ETHTransfer = { valueExactBase36: string, blockNumber: number, timestamp: number, - transactionHash?: string, - transactionPosition?: number, - internalTxPosition?: number, + transactionHash: string, + transactionPosition: number, + internalTxPosition: number, type: string, primaryKey?: number, } diff --git a/src/blockchains/eth/eth_worker.ts b/src/blockchains/eth/eth_worker.ts index fd98c08..21a8b1a 100644 --- a/src/blockchains/eth/eth_worker.ts +++ b/src/blockchains/eth/eth_worker.ts @@ -2,7 +2,7 @@ import { logger } from '../../lib/logger'; import { constructRPCClient } from '../../lib/http_client'; import { injectDAOHackTransfers, DAO_HACK_FORK_BLOCK } from './lib/dao_hack'; import { getGenesisTransfers } from './lib/genesis_transfers'; -import { assignInternalTransactionPosition, transactionOrder } from './lib/util' +import { assignInternalTransactionPosition, doQAETHTransfers, transactionOrder, mergeSortedArrays } from './lib/util' import { BaseWorker } from '../../lib/worker_base'; import { Web3Interface, constructWeb3Wrapper, safeCastToNumber } from './lib/web3_wrapper'; import { decodeTransferTrace } from './lib/decode_transfers'; @@ -105,13 +105,15 @@ export class ETHWorker extends BaseWorker { logger.info(`Fetching transfer events for interval ${fromBlock}:${toBlock}`) const [traces, blocks, receipts] = await this.fetchData(fromBlock, toBlock) const events: (ETHTransfer | EOB)[] = this.transformPastEvents(fromBlock, toBlock, traces, blocks, receipts) - assignInternalTransactionPosition(events) - events.push(...collectEndOfBlocks(fromBlock, toBlock, blocks, this.web3Wrapper)) events.sort(transactionOrder) + doQAETHTransfers(events, fromBlock, toBlock) + assignInternalTransactionPosition(events) + const eobEvents = collectEndOfBlocks(fromBlock, toBlock, blocks, this.web3Wrapper) + const mergedEvents = mergeSortedArrays(events, eobEvents, transactionOrder) this.lastExportedBlock = toBlock - return events + return mergedEvents } async init(): Promise { diff --git a/src/blockchains/eth/lib/decode_transfers.ts b/src/blockchains/eth/lib/decode_transfers.ts index c9844ba..22b7376 100644 --- a/src/blockchains/eth/lib/decode_transfers.ts +++ b/src/blockchains/eth/lib/decode_transfers.ts @@ -16,8 +16,9 @@ export function decodeTransferTrace(trace: Trace, timestamp: number, web3Wrapper valueExactBase36: web3Wrapper.parseHexToBase36String(trace['action']['value']), blockNumber: trace['blockNumber'], timestamp: timestamp, - transactionHash: trace['transactionHash'], - transactionPosition: trace['transactionPosition'], + transactionHash: trace['transactionHash'] ? trace['transactionHash'] : `mining_${trace['action']['rewardType']}`, + transactionPosition: trace['transactionPosition'] ? trace['transactionPosition'] : 0, + internalTxPosition: 0, type: trace['type'] }; } @@ -26,7 +27,10 @@ export function decodeTransferTrace(trace: Trace, timestamp: number, web3Wrapper if (trace['type'] === 'create') { assertIsDefined(trace['action']['from'], "'from' field is expected in trace action on 'create' type"); assertIsDefined(trace['action']['value'], "'value' field is expected in trace action on 'create' type"); + assertIsDefined(trace['result'], "'result' field is expected in trace on 'create' type"); assertIsDefined(trace['result']['address'], "'address' field is expected in trace result on 'create' type"); + assertIsDefined(trace['transactionHash'], "'transactionHash' field is expected in trace on 'create' type"); + assertIsDefined(trace['transactionPosition'], "'transactionPosition' field is expected in trace on 'create' type"); return { from: trace['action']['from'], @@ -37,6 +41,7 @@ export function decodeTransferTrace(trace: Trace, timestamp: number, web3Wrapper timestamp: timestamp, transactionHash: trace['transactionHash'], transactionPosition: trace['transactionPosition'], + internalTxPosition: 0, type: trace['type'] }; } @@ -45,6 +50,8 @@ export function decodeTransferTrace(trace: Trace, timestamp: number, web3Wrapper assertIsDefined(trace['action']['refundAddress'], "'refundAddress' field is expected in trace action on 'suicide' type"); assertIsDefined(trace['action']['address'], "'address' field is expected in trace action on 'suicide' type"); assertIsDefined(trace['action']['balance'], "'balance' field is expected in trace action on 'suicide' type") + assertIsDefined(trace['transactionHash'], "'transactionHash' field is expected in trace on 'suicide' type"); + assertIsDefined(trace['transactionPosition'], "'transactionPosition' field is expected in trace on 'suicide' type"); return { from: trace['action']['address'], @@ -55,6 +62,7 @@ export function decodeTransferTrace(trace: Trace, timestamp: number, web3Wrapper timestamp: timestamp, transactionHash: trace['transactionHash'], transactionPosition: trace['transactionPosition'], + internalTxPosition: 0, type: trace['type'] }; } @@ -66,6 +74,8 @@ export function decodeTransferTrace(trace: Trace, timestamp: number, web3Wrapper assertIsDefined(trace['action']['from'], `'from' field is expected in trace action on ${trace['type']} type`); assertIsDefined(trace['action']['value'], `'value' field is expected in trace action on ${trace['type']} type`); assertIsDefined(trace['action']['to'], `'to' field is expected in trace action on ${trace['type']} type`); + assertIsDefined(trace['transactionHash'], `'transactionHash' field is expected in trace on ${trace['type']} type`); + assertIsDefined(trace['transactionPosition'], `'transactionPosition' field is expected in trace on ${trace['type']} type`); return { from: trace['action']['from'], @@ -76,6 +86,7 @@ export function decodeTransferTrace(trace: Trace, timestamp: number, web3Wrapper timestamp: timestamp, transactionHash: trace['transactionHash'], transactionPosition: trace['transactionPosition'], + internalTxPosition: 0, type: trace['type'] }; } diff --git a/src/blockchains/eth/lib/end_of_block.ts b/src/blockchains/eth/lib/end_of_block.ts index 6ccd3d3..fe0eac3 100644 --- a/src/blockchains/eth/lib/end_of_block.ts +++ b/src/blockchains/eth/lib/end_of_block.ts @@ -10,6 +10,7 @@ export type EOB = { timestamp: number, transactionHash: string, transactionPosition: number, + internalTxPosition: number, type: string, primaryKey?: number }; diff --git a/src/blockchains/eth/lib/fees_decoder.ts b/src/blockchains/eth/lib/fees_decoder.ts index 05dc188..3176a27 100644 --- a/src/blockchains/eth/lib/fees_decoder.ts +++ b/src/blockchains/eth/lib/fees_decoder.ts @@ -26,6 +26,8 @@ export class FeesDecoder { blockNumber: safeCastToNumber(this.web3Wrapper.parseHexToNumber(transaction.blockNumber)), timestamp: safeCastToNumber(this.web3Wrapper.parseHexToNumber(block.timestamp)), transactionHash: transaction.hash, + transactionPosition: safeCastToNumber(this.web3Wrapper.parseHexToNumber(transaction.transactionIndex)), + internalTxPosition: 0, type: 'fee' }]; } @@ -46,6 +48,8 @@ export class FeesDecoder { blockNumber: safeCastToNumber(this.web3Wrapper.parseHexToNumber(transaction.blockNumber)), timestamp: safeCastToNumber(this.web3Wrapper.parseHexToNumber(block.timestamp)), transactionHash: transaction.hash, + transactionPosition: safeCastToNumber(this.web3Wrapper.parseHexToNumber(transaction.transactionIndex)), + internalTxPosition: 0, type: 'fee_burnt' }; } @@ -75,6 +79,8 @@ export class FeesDecoder { blockNumber: safeCastToNumber(this.web3Wrapper.parseHexToNumber(transaction.blockNumber)), timestamp: safeCastToNumber(this.web3Wrapper.parseHexToNumber(block.timestamp)), transactionHash: transaction.hash, + transactionPosition: safeCastToNumber(this.web3Wrapper.parseHexToNumber(transaction.transactionIndex)), + internalTxPosition: 0, type: 'fee' }; } diff --git a/src/blockchains/eth/lib/fetch_data.ts b/src/blockchains/eth/lib/fetch_data.ts index 08792f6..36aced6 100644 --- a/src/blockchains/eth/lib/fetch_data.ts +++ b/src/blockchains/eth/lib/fetch_data.ts @@ -1,11 +1,26 @@ +import { z } from "zod" +import { logger } from '../../../lib/logger'; import { filterErrors } from './filter_errors'; import { Web3Interface } from './web3_wrapper'; -import { Trace, ETHBlock, ETHReceiptsMap, ETHReceipt } from '../eth_types'; +import { Trace, TraceSchema, ETHBlock, ETHReceiptsMap, ETHReceipt } from '../eth_types'; import { HTTPClientInterface } from '../../../types' -export function parseEthInternalTrx(result: Trace[], isETH: boolean, theMergeBlockNumber: number): Trace[] { - const traces = filterErrors(result); +export function parseEthInternalTrx(traceFilterResult: any[], isETH: boolean, theMergeBlockNumber: number): Trace[] { + const traces = filterErrors(traceFilterResult).map(t => { + try { + return TraceSchema.parse(t) + } + catch (error) { + if (error instanceof z.ZodError) { + logger.error('Validation failed:\n', error.toString()) + logger.error('Trying to parse object:\n', JSON.stringify(t)); + } else { + logger.error('An unexpected error occurred:', error); + } + throw error + } + }) return traces .filter((trace: Trace) => diff --git a/src/blockchains/eth/lib/genesis_transfers.ts b/src/blockchains/eth/lib/genesis_transfers.ts index 9e0edc2..0923744 100644 --- a/src/blockchains/eth/lib/genesis_transfers.ts +++ b/src/blockchains/eth/lib/genesis_transfers.ts @@ -12,10 +12,17 @@ const GENESIS_TIMESTAMP = 1438269973; export function getGenesisTransfers(web3Wrapper: Web3Interface): ETHTransfer[] { const result: ETHTransfer[] = []; + + const txHashMap: Map = new Map(); + GENESIS_TRANSFERS.forEach((transfer) => { const [id, from, to, amount] = transfer; const wei = web3Wrapper.etherToWei(amount); + // Used to construct incrementing internal transaction numbers + const currentCount = txHashMap.get(from) || 0; + + result.push({ from: 'GENESIS', to: to, @@ -24,6 +31,8 @@ export function getGenesisTransfers(web3Wrapper: Web3Interface): ETHTransfer[] { blockNumber: 0, timestamp: GENESIS_TIMESTAMP, transactionHash: from, + transactionPosition: currentCount, + internalTxPosition: 0, type: 'genesis' }); }); @@ -35,6 +44,9 @@ export function getGenesisTransfers(web3Wrapper: Web3Interface): ETHTransfer[] { valueExactBase36: BigInt('5000000000000000000').toString(36), blockNumber: 0, timestamp: GENESIS_TIMESTAMP, + transactionHash: 'GENESIS_mining_tx', + transactionPosition: 0, + internalTxPosition: 0, type: 'reward' }); diff --git a/src/blockchains/eth/lib/util.ts b/src/blockchains/eth/lib/util.ts index 41504c0..5cbb1f0 100644 --- a/src/blockchains/eth/lib/util.ts +++ b/src/blockchains/eth/lib/util.ts @@ -1,7 +1,8 @@ import { ETHTransfer } from '../eth_types'; +import { EOB } from './end_of_block' const { groupBy } = require('lodash'); -export function transactionOrder(a: ETHTransfer, b: ETHTransfer) { +export function transactionOrder(a: ETHTransfer | EOB, b: ETHTransfer | EOB) { if (a.blockNumber !== b.blockNumber) { return a.blockNumber - b.blockNumber } @@ -21,15 +22,85 @@ export function transactionOrder(a: ETHTransfer, b: ETHTransfer) { const ethTransferKey = (transfer: ETHTransfer) => `${transfer.blockNumber}-${transfer.transactionHash ?? ''}-${transfer.transactionPosition ?? ''}-${transfer.from}-${transfer.to}` -export function assignInternalTransactionPosition(transfers: ETHTransfer[], groupByKey: (transfer: ETHTransfer) => string = ethTransferKey): ETHTransfer[] { +export function assignInternalTransactionPosition(transfers: ETHTransfer[], groupByKey: (transfer: ETHTransfer) => string = ethTransferKey): void { const grouped = groupBy(transfers, groupByKey) const values: ETHTransfer[][] = Object.values(grouped) - return values.flatMap((transfersSameKey: ETHTransfer[]) => { + values.forEach((transfersSameKey: ETHTransfer[]) => { transfersSameKey.forEach((transfer: ETHTransfer, index: number) => { transfer.internalTxPosition = index }) - return transfersSameKey }) } +export function assertBlocksMatch(groupedTransfers: any, fromBlock: number, toBlock: number) { + const keys = Object.keys(groupedTransfers) + if (keys.length !== toBlock - fromBlock + 1) { + throw new Error(`Wrong number of blocks seen. Expected ${toBlock - fromBlock + 1} got ${keys.length}.`) + } + + for (let block = fromBlock; block <= toBlock; block++) { + if (!groupedTransfers.hasOwnProperty(block.toString())) { + throw new Error(`Missing transfers for block ${block}.`) + } + } +} + + +export function assertTransfersWithinBlock(transfersPerBlock: ETHTransfer[]) { + let expectedTxPosition = 0 + + for (const transfer of transfersPerBlock) { + if (transfer.transactionPosition !== expectedTxPosition) { + // We allow for multiple transfers withing a transaction. That is why we can see the same transaction position several times. + if (transfer.transactionPosition !== expectedTxPosition + 1) { + throw new Error(`Unexpected transaction position for transfer: ${JSON.stringify(transfer)}, expected tx position: ${expectedTxPosition} or ${expectedTxPosition + 1}`); + } + expectedTxPosition += 1 + } + } +} + + +/** + * Assert data quality guarantees on top of input Transfers + * + * Throw an error if: + * 1. A block number is missing + * 2. Within a block, a transactions number is missing + * + * @param transfers Ordered array of transfers + * @param fromBlock Block number indicating start of expected interval + * @param toBlock Block number indicating end of expected interval + */ +export function doQAETHTransfers(sortedTransfers: ETHTransfer[], fromBlock: number, toBlock: number) { + if (fromBlock > toBlock) { + throw new Error(`Invalid block range: fromBlock ${fromBlock} is greater than toBlock ${toBlock}`); + } + + const groupedTransfers = groupBy(sortedTransfers, (transfer: ETHTransfer) => transfer.blockNumber) + + assertBlocksMatch(groupedTransfers, fromBlock, toBlock); + + for (const key of Object.keys(groupedTransfers)) { + assertTransfersWithinBlock(groupedTransfers[key]) + } +} + +export function mergeSortedArrays(sortedArr1: T[], sortedArr2: T[], comparator: (a: T, b: T) => number = (a, b) => + a < b ? -1 : a > b ? 1 : 0): T[] { + const merged: T[] = []; + let i = 0, j = 0; + + while (i < sortedArr1.length && j < sortedArr2.length) { + if (comparator(sortedArr1[i], sortedArr2[j]) < 0) { + merged.push(sortedArr1[i++]); + } else { + merged.push(sortedArr2[j++]); + } + } + + return merged.concat(sortedArr1.slice(i), sortedArr2.slice(j)); +} + + diff --git a/src/blockchains/eth/lib/withdrawals_decoder.ts b/src/blockchains/eth/lib/withdrawals_decoder.ts index 46f6e58..6f46908 100644 --- a/src/blockchains/eth/lib/withdrawals_decoder.ts +++ b/src/blockchains/eth/lib/withdrawals_decoder.ts @@ -10,7 +10,7 @@ export class WithdrawalsDecoder { } getBeaconChainWithdrawals(withdrawals: BeaconChainWithdrawal[], blockNumber: number, blockTimestamp: number): ETHTransfer[] { - return withdrawals.map((withdrawal) => { + return withdrawals.map((withdrawal, index) => { const gweiAmount = this.web3Wrapper.gweiToWei(withdrawal.amount); return { from: constants.ETH_WITHDRAWAL, @@ -20,6 +20,8 @@ export class WithdrawalsDecoder { blockNumber: blockNumber, timestamp: blockTimestamp, transactionHash: `WITHDRAWAL_${blockNumber}`, + transactionPosition: 0, + internalTxPosition: index, type: 'beacon_withdrawal' }; }); diff --git a/src/blockchains/eth_contracts/lib/transform_create_traces.ts b/src/blockchains/eth_contracts/lib/transform_create_traces.ts index 8a8739d..5c9ab1d 100644 --- a/src/blockchains/eth_contracts/lib/transform_create_traces.ts +++ b/src/blockchains/eth_contracts/lib/transform_create_traces.ts @@ -27,7 +27,9 @@ function constructCreationOutput(parentAddress: string, createTraces: Trace[], b ContractCreationTrace[] { const result: ContractCreationTrace[] = createTraces.map(trace => { + assertIsDefined(trace.result, "'result' field is expected in trace on 'create' type") assertIsDefined(trace.result.address, "'address' field is expected in trace result on 'create' type") + assertIsDefined(trace.transactionHash, "'transactionHash' field is expected in trace result on 'create' type") const record: ContractCreationTrace = { address: trace.result.address, diff --git a/src/test/eth/decode_transfers.spec.ts b/src/test/eth/decode_transfers.spec.ts index 20fb5fa..ab4afec 100644 --- a/src/test/eth/decode_transfers.spec.ts +++ b/src/test/eth/decode_transfers.spec.ts @@ -3,6 +3,7 @@ process.env.IS_ETH = 'true'; import { decodeTransferTrace } from '../../blockchains/eth/lib/decode_transfers'; import { NODE_URL } from '../../blockchains/eth/lib/constants'; import { Web3Interface, constructWeb3WrapperNoCredentials } from '../../blockchains/eth/lib/web3_wrapper'; +import { ETHTransfer } from '../../blockchains/eth/eth_types'; describe('genesis transfers', function () { @@ -31,7 +32,7 @@ describe('genesis transfers', function () { const timestamp = 1000000; const result = decodeTransferTrace(suicide_trace, timestamp, web3Wrapper); - const result_expected = { + const result_expected: ETHTransfer = { 'from': '0xa6c3b7f6520a0ef594fc666d3874ec78c561cdbb', 'to': '0x245133ea0fb1b77fab5886d7ffb8046dfeff3858', 'value': 160000000000000000, @@ -39,6 +40,7 @@ describe('genesis transfers', function () { 'blockNumber': 711983, 'timestamp': 1000000, 'transactionHash': '0xd715da4f846e41be86ea87dc97b186cafea3b50c95d5d9d889ec522b248b207f', + 'internalTxPosition': 0, 'transactionPosition': 10, 'type': 'suicide' }; @@ -74,7 +76,7 @@ describe('genesis transfers', function () { const result = decodeTransferTrace(call_trace, timestamp, web3Wrapper); - const result_expected = { + const result_expected: ETHTransfer = { 'from': '0x48f2e6e5d0872da169c7f5823d5a2d5ea5f2b5e7', 'to': '0x7de5aba7de728950c92c57d08e20d4077161f12f', 'value': 1, @@ -83,6 +85,7 @@ describe('genesis transfers', function () { 'timestamp': 1450433505, 'transactionHash': '0x22f839c82ff455554ec8aa98ee2b9a03d0d5ed4707b46d4a0a217df7d58bda2c', 'transactionPosition': 10, + 'internalTxPosition': 0, 'type': 'call' }; @@ -112,17 +115,18 @@ describe('genesis transfers', function () { const timestamp = 1450433505; const result = decodeTransferTrace(reward_trace, timestamp, web3Wrapper); - const result_expected = { + const result_expected: ETHTransfer = { 'from': 'mining_block', 'to': '0x2a65aca4d5fc5b5c859090a6c34d164135398226', 'transactionHash': '0x22f839c82ff455554ec8aa98ee2b9a03d0d5ed4707b46d4a0a217df7d58bda2c', 'transactionPosition': 10, + 'internalTxPosition': 0, 'value': 5000000000000000000, 'valueExactBase36': '11zk02pzlmwow', 'blockNumber': 710093, 'timestamp': 1450433505, 'type': 'reward' - }; + } assert.deepStrictEqual(result, result_expected); }); @@ -150,7 +154,7 @@ describe('genesis transfers', function () { const timestamp = 1450435908; const result = decodeTransferTrace(create_trace, timestamp, web3Wrapper); - const result_expected = { + const result_expected: ETHTransfer = { 'from': '0x245133ea0fb1b77fab5886d7ffb8046dfeff3858', 'to': '0xa6c3b7f6520a0ef594fc666d3874ec78c561cdbb', 'value': 1500000000000000000, @@ -159,8 +163,9 @@ describe('genesis transfers', function () { 'timestamp': 1450435908, 'transactionHash': '0x6d39df3c46f19e8ef5e8bb3b81a063a29cb352675a00d66f0dc2117a1799add1', 'transactionPosition': 11, + 'internalTxPosition': 0, 'type': 'create' - }; + } assert.deepStrictEqual(result, result_expected); }); diff --git a/src/test/eth/fees_decoder.spec.ts b/src/test/eth/fees_decoder.spec.ts index 4e99601..c787c6f 100644 --- a/src/test/eth/fees_decoder.spec.ts +++ b/src/test/eth/fees_decoder.spec.ts @@ -1,7 +1,7 @@ import assert from 'assert'; import { Web3Interface, constructWeb3WrapperNoCredentials, safeCastToNumber } from '../../blockchains/eth/lib/web3_wrapper'; import { FeesDecoder } from '../../blockchains/eth/lib/fees_decoder'; -import { ETHBlock, ETHReceipt } from '../../blockchains/eth/eth_types'; +import { ETHBlock, ETHReceipt, ETHTransfer } from '../../blockchains/eth/eth_types'; import * as constants from '../../blockchains/eth/lib/constants'; /** @@ -174,7 +174,7 @@ describe('Fees decoder test', function () { safeCastToNumber(web3Wrapper.parseHexToNumber(block_json_post_london_zero_priority.number)), turnReceiptsToMap(receipts_json_post_london_no_priority), true); - const expected = [{ + const expected: ETHTransfer[] = [{ from: '0xea674fdde714fd979de3edf0f56aa9716b898ec8', to: constants.BURN_ADDRESS, value: 1049725694283000, @@ -182,6 +182,8 @@ describe('Fees decoder test', function () { blockNumber: 13447057, timestamp: 1634631172, transactionHash: '0xc8bebc11bbe703cdfb2a1a9599221baf4f19a1e20808866346791799d2dac7a9', + transactionPosition: 0, + internalTxPosition: 0, type: 'fee_burnt' } ]; @@ -194,12 +196,14 @@ describe('Fees decoder test', function () { safeCastToNumber(web3Wrapper.parseHexToNumber(block_json_post_london_with_priority.number)), turnReceiptsToMap(receipts_json_post_london_with_priority), true); - const expected = [{ + const expected: ETHTransfer[] = [{ blockNumber: 13447057, from: '0x8ae57a027c63fca8070d1bf38622321de8004c67', timestamp: 1634631172, to: constants.BURN_ADDRESS, transactionHash: '0x1e53bf3951f6cb70461df500ec75ed5d88d73bd44d88ca7faabaa4b1e65aec98', + transactionPosition: 164, + internalTxPosition: 0, type: 'fee_burnt', value: 3653345337731778, valueExactBase36: 'zz03ofi5du' @@ -212,6 +216,8 @@ describe('Fees decoder test', function () { blockNumber: 13447057, timestamp: 1634631172, transactionHash: '0x1e53bf3951f6cb70461df500ec75ed5d88d73bd44d88ca7faabaa4b1e65aec98', + transactionPosition: 164, + internalTxPosition: 0, type: 'fee' } ]; @@ -224,12 +230,14 @@ describe('Fees decoder test', function () { safeCastToNumber(web3Wrapper.parseHexToNumber(block_json_post_london_old_tx_type.number)), turnReceiptsToMap(receipts_json_post_london_old_tx_type), true); - const expected = [{ + const expected: ETHTransfer[] = [{ blockNumber: 13318440, from: '0xddfabcdc4d8ffc6d5beaf154f18b778f892a0740', timestamp: 1632888074, to: constants.BURN_ADDRESS, transactionHash: '0xec5b5841e0a425bf69553a0ccecfa58b053a63e30f5fbdd9ecbdee5e9fb0666c', + transactionPosition: 97, + internalTxPosition: 0, type: 'fee_burnt', value: 1391883443307000, valueExactBase36: 'dpdqfcs260' @@ -242,6 +250,8 @@ describe('Fees decoder test', function () { blockNumber: 13318440, timestamp: 1632888074, transactionHash: '0xec5b5841e0a425bf69553a0ccecfa58b053a63e30f5fbdd9ecbdee5e9fb0666c', + transactionPosition: 97, + internalTxPosition: 0, type: 'fee' } ]; @@ -254,7 +264,7 @@ describe('Fees decoder test', function () { safeCastToNumber(web3Wrapper.parseHexToNumber(block_json_pre_london.number)), turnReceiptsToMap(receipts_json_pre_london), true); - const expected = [{ + const expected: ETHTransfer[] = [{ from: '0x39fa8c5f2793459d6622857e7d9fbb4bd91766d3', to: '0x2a65aca4d5fc5b5c859090a6c34d164135398226', value: 2354887722000000, @@ -262,6 +272,8 @@ describe('Fees decoder test', function () { blockNumber: 1000000, timestamp: 1455404053, transactionHash: '0xea1093d492a1dcb1bef708f771a99a96ff05dcab81ca76c31940300177fcf49f', + transactionPosition: 0, + internalTxPosition: 0, type: 'fee' }]; diff --git a/src/test/eth/fetch_events.spec.ts b/src/test/eth/fetch_events.spec.ts index cf78082..8ad2fd0 100644 --- a/src/test/eth/fetch_events.spec.ts +++ b/src/test/eth/fetch_events.spec.ts @@ -113,6 +113,8 @@ describe('fetch past events', function () { blockNumber: 5711193, timestamp: 1527814787, transactionHash: '0x1a06a3a86d2897741f3ddd774df060a63d626b01197c62015f404e1f007fa04d', + transactionPosition: 0, + internalTxPosition: 0, type: 'fee' }; @@ -125,6 +127,7 @@ describe('fetch past events', function () { timestamp: 1527814787, transactionHash: '0x1a06a3a86d2897741f3ddd774df060a63d626b01197c62015f404e1f007fa04d', transactionPosition: 0, + internalTxPosition: 0, type: 'call' }; }); @@ -155,6 +158,8 @@ describe('fetch past events', function () { blockNumber: 0, timestamp: 1438269973, transactionHash: 'GENESIS_000d836201318ec6899a67540690382780743280', + transactionPosition: 0, + internalTxPosition: 0, type: 'genesis' }; diff --git a/src/test/eth/util.spec.ts b/src/test/eth/util.spec.ts index da214f5..b478a55 100644 --- a/src/test/eth/util.spec.ts +++ b/src/test/eth/util.spec.ts @@ -1,6 +1,6 @@ import { expect } from 'earl'; import { cloneDeep } from 'lodash'; -import { transactionOrder, assignInternalTransactionPosition } from "../../blockchains/eth/lib/util" +import { transactionOrder, assignInternalTransactionPosition, doQAETHTransfers } from "../../blockchains/eth/lib/util" import { ETHTransfer } from '../../blockchains/eth/eth_types'; describe('transactionOrder utils', () => { @@ -74,6 +74,7 @@ describe('transactionOrder utils', () => { blockNumber: 1, timestamp: 1000, transactionHash: "hash", + transactionPosition: 0, internalTxPosition: 5, type: "type" }, { @@ -84,6 +85,7 @@ describe('transactionOrder utils', () => { blockNumber: 1, timestamp: 2000, transactionHash: "hash", + transactionPosition: 0, internalTxPosition: 2, type: "type" } @@ -106,14 +108,15 @@ describe('assignInternalTransactionPosition utils', () => { timestamp: 1000, transactionHash: "hash", transactionPosition: 10, + internalTxPosition: 0, type: "type" }] const expected = cloneDeep(transfers) expected[0].internalTxPosition = 0 - const result = assignInternalTransactionPosition(transfers) + assignInternalTransactionPosition(transfers) - expect(result).toEqual(expected) + expect(transfers).toEqual(expected) }) it('zero position is not changed', () => { @@ -131,8 +134,8 @@ describe('assignInternalTransactionPosition utils', () => { }] const expected = cloneDeep(transfers) - const result = assignInternalTransactionPosition(transfers) - expect(result).toEqual(expected) + assignInternalTransactionPosition(transfers) + expect(transfers).toEqual(expected) }) it('two different records assigned correctly', () => { @@ -146,6 +149,7 @@ describe('assignInternalTransactionPosition utils', () => { timestamp: 1000, transactionHash: "hash1", transactionPosition: 1, + internalTxPosition: 0, type: "type" }, { @@ -157,14 +161,15 @@ describe('assignInternalTransactionPosition utils', () => { timestamp: 1000, transactionHash: "hash2", transactionPosition: 2, + internalTxPosition: 0, type: "type" }] const expected = cloneDeep(transfers) expected[0].internalTxPosition = 0 expected[1].internalTxPosition = 0 - const result = assignInternalTransactionPosition(transfers) - expect(result).toEqual(expected) + assignInternalTransactionPosition(transfers) + expect(transfers).toEqual(expected) }) it('two equal records assigned correctly', () => { @@ -178,6 +183,7 @@ describe('assignInternalTransactionPosition utils', () => { timestamp: 1000, transactionHash: "hash", transactionPosition: 10, + internalTxPosition: 0, type: "type" }, { @@ -189,13 +195,116 @@ describe('assignInternalTransactionPosition utils', () => { timestamp: 1000, transactionHash: "hash", transactionPosition: 10, + internalTxPosition: 0, type: "type" }] const expected = cloneDeep(transfers) expected[0].internalTxPosition = 0 expected[1].internalTxPosition = 1 - const result = assignInternalTransactionPosition(transfers) - expect(result).toEqual(expected) + assignInternalTransactionPosition(transfers) + expect(transfers).toEqual(expected) }) }) + + +describe('doQAETHTransfers', () => { + // Helper function to create ETHTransfer objects + const createTransfer = ( + from: string, + to: string, + value: number, + blockNumber: number, + transactionHash: string, + transactionPosition: number, + type: string = 'transfer' + ): ETHTransfer => ({ + from, to, value, valueExactBase36: value.toString(36), blockNumber, timestamp: 1000, transactionHash, + transactionPosition, internalTxPosition: 0, type, + }); + + it('Valid single block with single transaction', () => { + const transfers: ETHTransfer[] = [ + createTransfer('A', 'B', 1, 100, "hash", 0), + ] + + expect(() => doQAETHTransfers(transfers, 100, 100)).not.toThrow() + }) + + it('Valid transfers with multiple blocks and consecutive transactions', () => { + const transfers: ETHTransfer[] = [ + createTransfer('A', 'B', 10, 100, "hash1", 0), + createTransfer('C', 'D', 20, 100, "hash2", 1), + createTransfer('E', 'F', 10, 101, "hash1", 0), + createTransfer('G', 'H', 20, 101, "hash2", 1), + createTransfer('I', 'J', 10, 102, "hash1", 0), + ] + + expect(() => doQAETHTransfers(transfers, 100, 102)).not.toThrow() + }); + + it('Multiple transers with same transaction position', () => { + const transfers: ETHTransfer[] = [ + createTransfer('A', 'B', 10, 100, "hash", 0), + createTransfer('C', 'D', 20, 100, "hash", 0), + createTransfer('E', 'F', 10, 100, "hash", 0), + ] + + expect(() => doQAETHTransfers(transfers, 100, 100)).not.toThrow() + }); + + it('Throws error when a block in the range is missing', () => { + const transfers: ETHTransfer[] = [ + createTransfer('A', 'B', 10, 100, "hash", 0), + createTransfer('C', 'D', 20, 102, "hash", 0), // Missing block 101 + createTransfer('E', 'F', 10, 103, "hash", 0) + ] + + expect(() => doQAETHTransfers(transfers, 100, 103)).toThrow('Wrong number of blocks seen. Expected 4 got 3.') + }) + + it('Throws error when a transaction position is missing within a block', () => { + const transfers: ETHTransfer[] = [ + createTransfer('A', 'B', 10, 100, "hash", 0), + // Missing transactionPosition 1 in block 100 + createTransfer('E', 'F', 20, 100, "hash1", 2), + createTransfer('G', 'H', 10, 101, "hash2", 1), + ] + + expect(() => doQAETHTransfers(transfers, 100, 101)).toThrow() + }) + + it('Throws error when transfers array is empty', () => { + const transfers: ETHTransfer[] = [] + + expect(() => doQAETHTransfers(transfers, 100, 100)).toThrow() + }) + + it('Throws error when fromBlock is greater than toBlock', () => { + const transfers: ETHTransfer[] = [ + createTransfer('A', 'B', 1, 100, "hash", 0), + createTransfer('C', 'D', 2, 101, "hash", 0), + ] + + expect(() => doQAETHTransfers(transfers, 102, 100)).toThrow() + }) + + it('Throws error when the last block is missing', () => { + const transfers: ETHTransfer[] = [ + createTransfer('A', 'B', 1, 100, "hash", 0), + createTransfer('C', 'D', 2, 101, "hash", 0), + // Missing block 102 + ]; + + expect(() => doQAETHTransfers(transfers, 100, 102)).toThrow() + }) + + it('Throws error when the data for unexpected blocks is present', () => { + const transfers: ETHTransfer[] = [ + createTransfer('A', 'B', 1, 100, "hash", 0), + createTransfer('C', 'D', 2, 101, "hash", 0) + ]; + + expect(() => doQAETHTransfers(transfers, 102, 103)).toThrow() + }) +}); diff --git a/src/test/eth/withdrawals_decoder.spec.ts b/src/test/eth/withdrawals_decoder.spec.ts index d30dba9..1484eca 100644 --- a/src/test/eth/withdrawals_decoder.spec.ts +++ b/src/test/eth/withdrawals_decoder.spec.ts @@ -3,6 +3,7 @@ import assert from 'assert'; import { WithdrawalsDecoder } from '../../blockchains/eth/lib/withdrawals_decoder'; import { NODE_URL, ETH_WITHDRAWAL } from '../../blockchains/eth/lib/constants'; import { Web3Interface, constructWeb3WrapperNoCredentials } from '../../blockchains/eth/lib/web3_wrapper'; +import { ETHTransfer } from '../../blockchains/eth/eth_types'; const web3Wrapper: Web3Interface = constructWeb3WrapperNoCredentials(NODE_URL); @@ -19,7 +20,7 @@ describe('withdrawals decoder test', function () { const result = decoder.getBeaconChainWithdrawals([withdrawal], 18742200, 1702046471); - const expected = [{ + const expected: ETHTransfer[] = [{ from: ETH_WITHDRAWAL, to: withdrawal.address, value: 17482528000000000, @@ -27,6 +28,8 @@ describe('withdrawals decoder test', function () { blockNumber: 18742200, timestamp: 1702046471, transactionHash: 'WITHDRAWAL_18742200', + transactionPosition: 0, + internalTxPosition: 0, type: 'beacon_withdrawal' }]; diff --git a/src/test/eth/worker.spec.ts b/src/test/eth/worker.spec.ts index fd52405..ed47b1e 100644 --- a/src/test/eth/worker.spec.ts +++ b/src/test/eth/worker.spec.ts @@ -6,17 +6,42 @@ import { ETHBlock, ETHTransfer } from '../../blockchains/eth/eth_types'; import { expect } from 'earl' describe('Test worker', function () { - let feeResult: ETHTransfer; + let feeResultBlock5711191: ETHTransfer; + let feeResultBlock5711192: ETHTransfer; + let feeResultBlock5711193: ETHTransfer; let callResult: ETHTransfer; let endOfBlock: EOB; - let eobWithPrimaryKey: EOB & { primaryKey: number }; let worker = new ETHWorker(constants); let blockInfos = new Map() - let feeResultWithPrimaryKey: ETHTransfer; - let callResultWithPrimaryKey: ETHTransfer; beforeEach(function () { - feeResult = { + feeResultBlock5711191 = { + from: '0x03b16ab6e23bdbeeab719d8e4c49d63674876253', + to: '0x829bd824b016326a401d083b33d092293333a830', + value: 14086000000000000, + valueExactBase36: '3up2j2e99ts', + blockNumber: 5711191, + timestamp: 1527814787, + transactionHash: '0x1a06a3a86d2897741f3ddd774df060a63d626b01197c62015f404e1f007fa04d', + transactionPosition: 0, + internalTxPosition: 0, + type: 'fee' + } satisfies ETHTransfer + + feeResultBlock5711192 = { + from: '0x03b16ab6e23bdbeeab719d8e4c49d63674876253', + to: '0x829bd824b016326a401d083b33d092293333a830', + value: 14086000000000000, + valueExactBase36: '3up2j2e99ts', + blockNumber: 5711192, + timestamp: 1527814787, + transactionHash: '0x1a06a3a86d2897741f3ddd774df060a63d626b01197c62015f404e1f007fa04d', + transactionPosition: 0, + internalTxPosition: 0, + type: 'fee' + } satisfies ETHTransfer + + feeResultBlock5711193 = { from: '0x03b16ab6e23bdbeeab719d8e4c49d63674876253', to: '0x829bd824b016326a401d083b33d092293333a830', value: 14086000000000000, @@ -24,8 +49,10 @@ describe('Test worker', function () { blockNumber: 5711193, timestamp: 1527814787, transactionHash: '0x1a06a3a86d2897741f3ddd774df060a63d626b01197c62015f404e1f007fa04d', + transactionPosition: 0, + internalTxPosition: 0, type: 'fee' - } satisfies ETHTransfer; + } satisfies ETHTransfer callResult = { from: '0x03b16ab6e23bdbeeab719d8e4c49d63674876253', @@ -36,17 +63,11 @@ describe('Test worker', function () { timestamp: 1527814787, transactionHash: '0x1a06a3a86d2897741f3ddd774df060a63d626b01197c62015f404e1f007fa04d', transactionPosition: 0, + internalTxPosition: 0, type: 'call' - } satisfies ETHTransfer; - endOfBlock = endOfBlockEvent(5711193); - feeResultWithPrimaryKey = v8.deserialize(v8.serialize(feeResult)); - feeResultWithPrimaryKey.primaryKey = 1; + } satisfies ETHTransfer - callResultWithPrimaryKey = v8.deserialize(v8.serialize(callResult)); - callResultWithPrimaryKey.primaryKey = 2; - - eobWithPrimaryKey = v8.deserialize(v8.serialize(endOfBlock)); - eobWithPrimaryKey.primaryKey = 3; + endOfBlock = endOfBlockEvent(5711193); blockInfos.set(5711191, ethBlockEvent(5711191)) blockInfos.set(5711192, ethBlockEvent(5711192)) @@ -62,7 +83,7 @@ describe('Test worker', function () { return Promise.resolve([[], blockInfos, {}]); }; worker.transformPastEvents = function () { - return [feeResult, callResult]; + return [feeResultBlock5711191, feeResultBlock5711192, feeResultBlock5711193, callResult]; }; const result = await worker.work(); @@ -72,8 +93,8 @@ describe('Test worker', function () { // so there should be 3 EOB const blocks = result.map((value) => value.blockNumber); const types = result.map((value) => value.type); - expect(blocks).toEqual([5711191, 5711192, 5711193, 5711193, 5711193]); - expect(types).toEqual(["EOB", "EOB", "fee", "call", "EOB"]); + expect(blocks).toEqual([5711191, 5711191, 5711192, 5711192, 5711193, 5711193, 5711193]); + expect(types).toEqual(["fee", "EOB", "fee", "EOB", "fee", "call", "EOB"]); }) }); @@ -103,6 +124,7 @@ function endOfBlockEvent(blockNumber: number): EOB { timestamp: 1438271100, transactionHash: "0x0000000000000000000000000000000000000000", transactionPosition: 2147483647, + internalTxPosition: 0, type: "EOB" }; }