Internal tx pos

package.json

@@ -45,6 +45,7 @@
   "devDependencies": {
     "@types/micro": "^10.0.0",
     "@types/mocha": "^10.0.6",
+    "@types/lodash": "^4.17.13",
     "@types/uuid": "^9.0.8",
     "@types/web3": "^1.2.2",
     "earl": "^1.3.0",
@@ -55,4 +56,4 @@
     "ts-node": "^10.9.2",
     "typescript": "^5.4.5"
   }
-}
+}

src/blockchains/eth/eth_types.ts

@@ -77,6 +77,7 @@ export type ETHTransfer = {
   timestamp: number,
   transactionHash?: string,
   transactionPosition?: number,
+  internalTxPosition?: number,
   type: string,
   primaryKey?: number,
 }

src/blockchains/eth/eth_worker.ts

@@ -2,7 +2,7 @@ import { logger } from '../../lib/logger';
 import { constructRPCClient } from '../../lib/http_client';
 import { injectDAOHackTransfers, DAO_HACK_FORK_BLOCK } from './lib/dao_hack';
 import { getGenesisTransfers } from './lib/genesis_transfers';
-import { transactionOrder, stableSort } from './lib/util';
+import { assignInternalTransactionPosition, transactionOrder } from './lib/util'
 import { BaseWorker } from '../../lib/worker_base';
 import { Web3Interface, constructWeb3Wrapper, safeCastToNumber } from './lib/web3_wrapper';
 import { decodeTransferTrace } from './lib/decode_transfers';
@@ -101,32 +101,26 @@ export class ETHWorker extends BaseWorker {
     setWorkerSleepTime(this, workerContext);
     if (workerContext === NO_WORK_SLEEP) return [];
 
-    const { fromBlock, toBlock } = nextIntervalCalculator(this);
-    logger.info(`Fetching transfer events for interval ${fromBlock}:${toBlock}`);
-    const [traces, blocks, receipts] = await this.fetchData(fromBlock, toBlock);
-    let events: (ETHTransfer | EOB)[] = this.transformPastEvents(fromBlock, toBlock, traces, blocks, receipts);
-
+    const { fromBlock, toBlock } = nextIntervalCalculator(this)
+    logger.info(`Fetching transfer events for interval ${fromBlock}:${toBlock}`)
+    const [traces, blocks, receipts] = await this.fetchData(fromBlock, toBlock)
+    const events: (ETHTransfer | EOB)[] = this.transformPastEvents(fromBlock, toBlock, traces, blocks, receipts)
+    assignInternalTransactionPosition(events)
     events.push(...collectEndOfBlocks(fromBlock, toBlock, blocks, this.web3Wrapper))
-    if (events.length > 0) {
-      stableSort(events, transactionOrder);
-      extendEventsWithPrimaryKey(events, this.lastPrimaryKey);
-
-      this.lastPrimaryKey += events.length;
-    }
+    events.sort(transactionOrder)
 
-    this.lastExportedBlock = toBlock;
+    this.lastExportedBlock = toBlock
 
-    return events;
+    return events
   }
 
   async init(): Promise<void> {
-    this.lastConfirmedBlock = await this.web3Wrapper.getBlockNumber() - this.settings.CONFIRMATIONS;
+    this.lastConfirmedBlock = await this.web3Wrapper.getBlockNumber() - this.settings.CONFIRMATIONS
   }
 }
 
-export function extendEventsWithPrimaryKey<T extends { primaryKey?: number }>(events: T[], lastPrimaryKey: number) {
-  for (let i = 0; i < events.length; i++) {
-    events[i].primaryKey = lastPrimaryKey + i + 1;
-  }
-}
+
+
+
+
 

src/blockchains/eth/lib/decode_transfers.ts

@@ -16,6 +16,8 @@ export function decodeTransferTrace(trace: Trace, timestamp: number, web3Wrapper
       valueExactBase36: web3Wrapper.parseHexToBase36String(trace['action']['value']),
       blockNumber: trace['blockNumber'],
       timestamp: timestamp,
+      transactionHash: trace['transactionHash'],
+      transactionPosition: trace['transactionPosition'],
       type: trace['type']
     };
   }

src/blockchains/eth/lib/end_of_block.ts

@@ -7,7 +7,7 @@ export type EOB = {
     value: number,
     valueExactBase36: string,
     blockNumber: number,
-    timestamp: number, 
+    timestamp: number,
     transactionHash: string,
     transactionPosition: number,
     type: string,
@@ -27,12 +27,13 @@ export const collectEndOfBlocks = (from: number, to: number, blockInfos: Map<num
             timestamp: safeCastToNumber(web3Wrapper.parseHexToNumber(blockInfos.get(blockNumber)!!.timestamp)),
             transactionHash: "0x0000000000000000000000000000000000000000",
             transactionPosition: maxTxPosition,
+            internalTxPosition: 0,
             type: "EOB"
-        }})
-    }
+        }
+    })
+}
 
-const maxTxPosition = Math.pow(2, 31) - 1 // max int32    
+const maxTxPosition = Math.pow(2, 31) - 1 // max int32
 // from - inclusive, to - exclusive
 const range = (from: number, to: number, step: number) =>
     [...Array(Math.floor((to - from) / step) + 1)].map((_, i) => from + i * step);
-

src/blockchains/eth/lib/util.ts

@@ -0,0 +1,35 @@
+import { ETHTransfer } from '../eth_types';
+const { groupBy } = require('lodash');
+
+export function transactionOrder(a: ETHTransfer, b: ETHTransfer) {
+  if (a.blockNumber !== b.blockNumber) {
+    return a.blockNumber - b.blockNumber
+  }
+
+  const transactionPositionA = (a.transactionPosition !== undefined) ? a.transactionPosition : -1
+  const transactionPositionB = (b.transactionPosition !== undefined) ? b.transactionPosition : -1
+
+  if (transactionPositionA !== transactionPositionB) {
+    return transactionPositionA - transactionPositionB
+  }
+
+  const internalTxPositionA = (a.internalTxPosition !== undefined) ? a.internalTxPosition : -1
+  const internalTxPositionB = (b.internalTxPosition !== undefined) ? b.internalTxPosition : -1
+
+  return internalTxPositionA - internalTxPositionB
+}
+
+const ethTransferKey = (transfer: ETHTransfer) => `${transfer.blockNumber}-${transfer.transactionHash ?? ''}-${transfer.transactionPosition ?? ''}-${transfer.from}-${transfer.to}`
+
+export function assignInternalTransactionPosition(transfers: ETHTransfer[], groupByKey: (transfer: ETHTransfer) => string = ethTransferKey): ETHTransfer[] {
+  const grouped = groupBy(transfers, groupByKey)
+  const values: ETHTransfer[][] = Object.values(grouped)
+
+  return values.flatMap((transfersSameKey: ETHTransfer[]) => {
+    transfersSameKey.forEach((transfer: ETHTransfer, index: number) => {
+      transfer.internalTxPosition = index
+    })
+    return transfersSameKey
+  })
+}
+

src/test/eth/decode_transfers.spec.ts

@@ -115,6 +115,8 @@ describe('genesis transfers', function () {
     const result_expected = {
       'from': 'mining_block',
       'to': '0x2a65aca4d5fc5b5c859090a6c34d164135398226',
+      'transactionHash': '0x22f839c82ff455554ec8aa98ee2b9a03d0d5ed4707b46d4a0a217df7d58bda2c',
+      'transactionPosition': 10,
       'value': 5000000000000000000,
       'valueExactBase36': '11zk02pzlmwow',
       'blockNumber': 710093,