Rehauling BSC integration test

package.json

@@ -5,7 +5,7 @@
   "main": "index.js",
   "scripts": {
     "test": "TEST_ENV=1 BLOCKCHAIN=eth CONTRACT_MAPPING_FILE_PATH=test/erc20/contract_mapping/contract_mapping.json LOG_LEVEL=error CONTRACT_MODE=extract_exact_overwrite mocha --require source-map-support/register --recursive --reporter spec ./built/test/**/*.js",
-    "integration_test": "mocha --recursive --reporter spec integration_test",
+    "integration_test": "mocha --recursive --reporter spec ./built/integration_test",
     "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
     "start": "npm run build && node ./built/index.js",
     "build": "rm -r -f ./built && tsc && ./copy_assets.sh",

src/blockchains/erc20/erc20_worker.js

@@ -139,7 +139,7 @@ class ERC20Worker extends BaseWorker {
     }
 
     if (events.length > 0) {
-      extendEventsWithPrimaryKey(events, overwritten_events);
+      extendEventsWithPrimaryKey(events, this.settings.PRIMARY_KEY_MULTIPLIER, overwritten_events);
       logger.info(`Setting primary keys ${events.length} messages for blocks ${interval.fromBlock}:${interval.toBlock}`);
       this.lastPrimaryKey = events[events.length - 1].primaryKey;
     }

src/blockchains/erc20/lib/extend_events_key.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const { stableSort } = require('./util');
-const constants = require('./constants');
 const { logger } = require('../../../lib/logger');
 
 function transactionOrder(a, b) {
@@ -14,26 +13,26 @@ function transactionOrder(a, b) {
   }
 }
 
-function extendEventsWithPrimaryKey(events, overwritten_events = []) {
+function extendEventsWithPrimaryKey(events, primaryKeyMultiplier, overwritten_events = []) {
   stableSort(events, transactionOrder);
   const lastEvent = events[events.length - 1];
-  if (lastEvent.logIndex + overwritten_events.length >= constants.PRIMARY_KEY_MULTIPLIER) {
+  if (lastEvent.logIndex + overwritten_events.length >= primaryKeyMultiplier) {
     logger.error(`An event with log index ${lastEvent.logIndex} is breaking the primaryKey generation logic at block `
       + `${lastEvent.blockNumber}. There are ${overwritten_events.length} overwritten events.`);
   }
 
   // Store the last log index of the original events per block
   let lastLogIndexPerBlock = {};
   events.forEach(function (event) {
-    event.primaryKey = event.blockNumber * constants.PRIMARY_KEY_MULTIPLIER + event.logIndex;
+    event.primaryKey = event.blockNumber * primaryKeyMultiplier + event.logIndex;
     // We depend on the events being sorted by log index above
     lastLogIndexPerBlock[event.blockNumber] = event.logIndex;
   });
   // As the overwritten events are copies of the main events, they would have the same logIndex. To generate unique primary keys,
   // the primary keys of overwritten events start after the biggest primary key of the main events and increase by 1.
   overwritten_events.forEach(function (event) {
     lastLogIndexPerBlock[event.blockNumber] += 1;
-    event.primaryKey = event.blockNumber * constants.PRIMARY_KEY_MULTIPLIER + lastLogIndexPerBlock[event.blockNumber];
+    event.primaryKey = event.blockNumber * primaryKeyMultiplier + lastLogIndexPerBlock[event.blockNumber];
   });
 }
 

src/blockchains/eth/eth_worker.js

@@ -21,7 +21,11 @@ class ETHWorker extends BaseWorker {
     logger.info(`Applying the following settings: ${JSON.stringify(settings)}`);
     this.web3Wrapper = new Web3Wrapper(new Web3(new Web3.providers.HttpProvider(settings.NODE_URL)));
     this.ethClient = constructRPCClient(settings.NODE_URL);
-    this.feesDecoder = new FeesDecoder(this.web3Wrapper);
+    this.feesDecoder = new FeesDecoder(
+      this.web3Wrapper,
+      this.settings.BURN_ADDRESS,
+      this.settings.IS_ETH,
+      this.settings.LONDON_FORK_BLOCK);
     this.withdrawalsDecoder = new WithdrawalsDecoder(this.web3Wrapper);
   }
 
@@ -146,7 +150,11 @@ class ETHWorker extends BaseWorker {
       const decoded_transactions = this.feesDecoder.getFeesFromTransactionsInBlock(block, blockNumber, receipts);
       if (this.settings.IS_ETH && blockNumber >= this.settings.SHANGHAI_FORK_BLOCK) {
         const blockTimestamp = this.web3Wrapper.parseHexToNumber(block.timestamp);
-        decoded_transactions.push(...this.withdrawalsDecoder.getBeaconChainWithdrawals(block.withdrawals, blockNumber, blockTimestamp));
+        decoded_transactions.push(...this.withdrawalsDecoder.getBeaconChainWithdrawals(
+          block.withdrawals,
+          blockNumber,
+          blockTimestamp,
+          this.settings.ETH_WITHDRAWAL));
       }
       result.push(...decoded_transactions);
     }

src/blockchains/eth/lib/fees_decoder.js

@@ -1,8 +1,9 @@
-const constants = require('./constants');
-
 class FeesDecoder {
-  constructor(web3Wrapper) {
+  constructor(web3Wrapper, burnAddress, isEth, londonForkBlock) {
     this.web3Wrapper = web3Wrapper;
+    this.BURN_ADDRESS = burnAddress;
+    this.IS_ETH = isEth;
+    this.LONDON_FORK_BLOCK = londonForkBlock;
   }
 
   getPreLondonForkFees(transaction, block, receipts) {
@@ -23,7 +24,7 @@ class FeesDecoder {
     const gasExpense = BigInt(this.web3Wrapper.parseHexToNumber(block.baseFeePerGas)) * BigInt(this.web3Wrapper.parseHexToNumber(receipts[transaction.hash].gasUsed));
     result.push({
       from: transaction.from,
-      to: constants.BURN_ADDRESS,
+      to: this.BURN_ADDRESS,
       value: Number(gasExpense),
       valueExactBase36: gasExpense.toString(36),
       blockNumber: this.web3Wrapper.parseHexToNumber(transaction.blockNumber),
@@ -72,7 +73,7 @@ class FeesDecoder {
     const result = [];
     block.transactions.forEach((transaction) => {
       const feeTransfers =
-        constants.IS_ETH && blockNumber >= constants.LONDON_FORK_BLOCK ?
+        this.IS_ETH && blockNumber >= this.LONDON_FORK_BLOCK ?
           this.getPostLondonForkFees(transaction, block, receipts) :
           this.getPreLondonForkFees(transaction, block, receipts);
 

src/blockchains/eth/lib/withdrawals_decoder.js

@@ -1,15 +1,13 @@
-const constants = require('./constants');
-
 class WithdrawalsDecoder {
   constructor(web3Wrapper) {
     this.web3Wrapper = web3Wrapper;
   }
 
-  getBeaconChainWithdrawals(withdrawals, blockNumber, blockTimestamp) {
+  getBeaconChainWithdrawals(withdrawals, blockNumber, blockTimestamp, ethWithdrawal) {
     return withdrawals.map((withdrawal) => {
       const gweiAmount = BigInt(this.web3Wrapper.gweiToWei(withdrawal.amount));
       return {
-        from: constants.ETH_WITHDRAWAL,
+        from: ethWithdrawal,
         to: withdrawal.address,
         value: Number(gweiAmount),
         valueExactBase36: gweiAmount.toString(36),

src/blockchains/matic/matic_worker.js

@@ -36,7 +36,7 @@ class MaticWorker extends BaseWorker {
     const events = await getPastEvents(this.ethClient, this.web3Wrapper, result.fromBlock, result.toBlock);
 
     if (events.length > 0) {
-      extendEventsWithPrimaryKey(events);
+      extendEventsWithPrimaryKey(events, this.settings.PRIMARY_KEY_MULTIPLIER);
       logger.info(`Setting primary keys ${events.length} messages for blocks ${result.fromBlock}:${result.toBlock}`);
       this.lastPrimaryKey = events[events.length - 1].primaryKey;
     }

src/blockchains/utxo/utxo_worker.ts

@@ -85,7 +85,7 @@ class UtxoWorker extends BaseWorker {
     return await this.sendRequestWithRetry('getblock', [blockHash, 2]);
   }
 
-  async work() {
+  async work(): Promise<any> {
     if (this.lastConfirmedBlock === this.lastExportedBlock) {
       this.sleepTimeMsec = this.LOOP_INTERVAL_CURRENT_MODE_SEC * 1000;
 

src/index.js

@@ -5,7 +5,6 @@ const metrics = require('./lib/metrics');
 const { logger } = require('./lib/logger');
 const TaskManager = require('./lib/task_manager');
 const { Exporter } = require('./lib/kafka_storage');
-const { BLOCKCHAIN, EXPORT_TIMEOUT_MLS } = require('./lib/constants');
 const EXPORTER_NAME = process.env.EXPORTER_NAME || 'san-chain-exporter';
 const { BLOCKCHAIN, EXPORT_TIMEOUT_MLS, MAX_CONCURRENT_REQUESTS } = require('./lib/constants');
 const worker = require(`./blockchains/${BLOCKCHAIN}/${BLOCKCHAIN}_worker`);
@@ -158,12 +157,12 @@ class Main {
       if (workerContext !== NO_WORK_SLEEP) {
         const intervals = this.generateIntervals();
         this.pushTasks(intervals);
-        
+
         this.worker.lastRequestStartTime = new Date();
         this.worker.lastExportTime = Date.now();
 
         const [lastExportedBlock, buffer] = this.taskManager.retrieveCompleted();
-        this.worker.setLastExportedBlock(lastExportedBlock);
+        if (lastExportedBlock) this.worker.setLastExportedBlock(lastExportedBlock);
         this.worker.decorateWithPrimaryKeys(buffer);
         await this.waitOnStoreEvents(buffer);
 

src/integration_test/src/binance-native-token.spec.js

@@ -0,0 +1,54 @@
+const assert = require('assert');
+const TaskManager = require('../../lib/task_manager');
+const worker = require('../../blockchains/eth/eth_worker');
+const { nextIntervalCalculator } = require('../../blockchains/eth/lib/next_interval_calculator');
+
+
+describe('BSC worker test', function () {
+  it('BSC worker should extract blocks from 20000000 to 20000249 including', async function () {
+    this.timeout(120000);
+    const expectedData = require('../testdata/binance_native_token_block_20000000_to_20000999.json');
+
+    // Make sure we've read the comparison data correctly
+    assert(expectedData.length === 50735);
+
+    const settings = {
+      NODE_URL: 'https://binance.santiment.net',
+      CONFIRMATIONS: 3,
+      EXPORT_BLOCKS_LIST: false,
+      BLOCK_INTERVAL: 50,
+      RECEIPTS_API_METHOD: 'eth_getBlockReceipts',
+      MAX_CONCURRENT_REQUESTS: 5,
+      IS_ETH: 0,
+    };
+    const bscWorker = new worker.worker(settings);
+    await bscWorker.init();
+    const taskManager = await TaskManager.create(settings.MAX_CONCURRENT_REQUESTS);
+    bscWorker.lastQueuedBlock = 19999999;
+
+    const intervals = [];
+    for (let i = 0; i < 5; ++i) {
+      const interval = nextIntervalCalculator(
+        bscWorker.lastQueuedBlock,
+        bscWorker.lastConfirmedBlock,
+        settings.BLOCK_INTERVAL);
+      bscWorker.lastQueuedBlock = interval.toBlock;
+      intervals.push(interval);
+    }
+    for (let i = 0; i < 5; ++i) {
+      const metadata = {
+        interval: intervals[i],
+        lambda: (interval) => bscWorker.work(interval)
+      };
+      taskManager.pushToQueue(metadata);
+    }
+
+    let expectedDataPosition = 0;
+    await taskManager.queue.onIdle();
+    const events = taskManager.retrieveCompleted()[1];
+    for (const event of events) {
+      assert.deepEqual(event, expectedData[expectedDataPosition]);
+      ++expectedDataPosition;
+    }
+  });
+});

integration_test/src/binance-native-token.spec.js → src/integration_test/src/ethereum.spec.js

@@ -2,7 +2,7 @@ const assert = require('assert');
 const worker = require('../../blockchains/eth/eth_worker');
 
 
-describe('BEP20 worker test', function () {
+describe('ETH worker test', function () {
   it('BEP20 worker should extract blocks from 20000000 to 20000999 including', async function () {
     this.timeout(20000);
     const expectedData = require('../testdata/binance_native_token_block_20000000_to_20000999.json');
@@ -29,8 +29,5 @@ describe('BEP20 worker test', function () {
         ++expectedDataPosition;
       }
     }
-
-
   });
-
 });