feat: erc20 reindex too big contract

ci/config.json.ci

@@ -404,7 +404,10 @@
     "blocksPerCall": 100,
     "millisecondRepeatJob": 5000,
     "chunkSizeInsert": 1000,
-    "wrapExtensionContract": ["0xe974cc14c93fc6077b0d65f98832b846c5454a0b"]
+    "wrapExtensionContract": ["0xe974cc14c93fc6077b0d65f98832b846c5454a0b"],
+    "reindex": {
+      "limitRecordGet": 2000
+    }
   },
   "erc721": {
     "key": "erc721",

config.json

@@ -402,7 +402,10 @@
     "blocksPerCall": 100,
     "millisecondRepeatJob": 2000,
     "chunkSizeInsert": 1000,
-    "wrapExtensionContract": ["0xe974cc14c93fc6077b0d65f98832b846c5454a0b"]
+    "wrapExtensionContract": ["0xe974cc14c93fc6077b0d65f98832b846c5454a0b"],
+    "reindex": {
+      "limitRecordGet": 2000
+    }
   },
   "erc721": {
     "key": "erc721",

src/services/evm/erc20.service.ts

@@ -90,13 +90,12 @@ export default class Erc20Service extends BullableService {
           ],
           config.erc20.key
         );
-      const erc20Activities: Erc20Activity[] =
-        await Erc20Handler.buildErc20Activities(
-          startBlock,
-          endBlock,
-          trx,
-          this.logger
-        );
+      const { erc20Activities } = await Erc20Handler.buildErc20Activities(
+        startBlock,
+        endBlock,
+        trx,
+        this.logger
+      );
       await this.handleMissingErc20Contract(erc20Activities, trx);
       if (erc20Activities.length > 0) {
         this.logger.info(

src/services/evm/erc20_handler.ts

@@ -170,7 +170,12 @@ export class Erc20Handler {
     endBlock: number,
     trx: Knex.Transaction,
     logger: Moleculer.LoggerInstance,
-    addresses?: string[]
+    addresses?: string[],
+    page?: {
+      prevEvmEventId?: number;
+      limitRecordGet: number;
+      prevCosmosEventId?: number;
+    }
   ) {
     const erc20Activities: Erc20Activity[] = [];
     const erc20Events = await EvmEvent.query()
@@ -185,6 +190,11 @@ export class Erc20Handler {
         if (addresses) {
           builder.whereIn('evm_event.address', addresses);
         }
+        if (page && page.prevEvmEventId !== undefined) {
+          builder
+            .andWhere('evm_event.id', '>', page.prevEvmEventId)
+            .limit(page.limitRecordGet);
+        }
       })
       .where('evm_event.block_height', '>', startBlock)
       .andWhere('evm_event.block_height', '<=', endBlock)
@@ -213,6 +223,11 @@ export class Erc20Handler {
               .where('attributes.key', EventAttribute.ATTRIBUTE_KEY.ERC20_TOKEN)
               .whereIn(knex.raw('lower("value")'), addresses);
           }
+          if (page && page.prevCosmosEventId !== undefined) {
+            builder
+              .andWhere('event.id', '>', page.prevCosmosEventId)
+              .limit(page.limitRecordGet);
+          }
         })
         .withGraphFetched('[transaction, attributes]')
         .orderBy('event.id', 'asc');
@@ -244,14 +259,19 @@ export class Erc20Handler {
         erc20Activities.push(activity);
       }
     });
-    return _.sortBy(erc20Activities, 'cosmos_tx_id');
+    return {
+      erc20Activities: _.sortBy(erc20Activities, 'cosmos_tx_id'),
+      prevEvmEventId: erc20Events[erc20Events.length - 1]?.id,
+      prevCosmosEventId: erc20CosmosEvents[erc20CosmosEvents.length - 1]?.id,
+    };
   }
 
   static async getErc20Activities(
     startBlock: number,
     endBlock: number,
     trx?: Knex.Transaction,
-    addresses?: string[]
+    addresses?: string[],
+    page?: { prevId: number; limitRecordGet: number }
   ): Promise<Erc20Activity[]> {
     return Erc20Activity.query()
       .modify((builder) => {
@@ -261,6 +281,11 @@ export class Erc20Handler {
         if (trx) {
           builder.transacting(trx);
         }
+        if (page) {
+          builder
+            .andWhere('erc20_activity.id', '>', page.prevId)
+            .limit(page.limitRecordGet);
+        }
       })
       .leftJoin(
         'account as from_account',

src/services/evm/erc20_reindex.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-await-in-loop */
 import Moleculer from 'moleculer';
 import { getContract, PublicClient } from 'viem';
 import config from '../../../config.json' assert { type: 'json' };
@@ -74,49 +75,90 @@ export class Erc20Reindexer {
           })
         )
         .transacting(trx);
-      const erc20Activities = await Erc20Handler.buildErc20Activities(
-        0,
-        Number(blockHeight),
-        trx,
-        this.logger,
-        [address]
-      );
-      if (erc20Activities.length > 0) {
-        await knex
-          .batchInsert(
-            'erc20_activity',
-            erc20Activities,
-            config.erc20.chunkSizeInsert
-          )
-          .transacting(trx);
+      const {limitRecordGet} = config.erc20.reindex;
+      let prevEvmEventId = 0;
+      let prevCosmosEventId = '0';
+      let numActivities = 0;
+      while (true) {
+        const resultBuildErc20Activities =
+          await Erc20Handler.buildErc20Activities(
+            0,
+            Number(blockHeight),
+            trx,
+            this.logger,
+            [address],
+            {
+              prevEvmEventId,
+              limitRecordGet,
+            }
+          );
+        const {erc20Activities} = resultBuildErc20Activities;
+        if (erc20Activities.length > 0) {
+          const resultInsert: any = await knex
+            .batchInsert(
+              'erc20_activity',
+              erc20Activities,
+              config.erc20.chunkSizeInsert
+            )
+            .transacting(trx);
+          numActivities += resultInsert[0].rowCount;
+        }
+        prevEvmEventId = resultBuildErc20Activities.prevEvmEventId;
+        prevCosmosEventId = resultBuildErc20Activities.prevCosmosEventId;
+        if (prevEvmEventId === undefined && prevCosmosEventId === undefined) {
+          break;
+        }
       }
-      const erc20ActivitiesInDb = await Erc20Handler.getErc20Activities(
-        0,
-        Number(blockHeight),
-        trx,
-        [address]
-      );
-      // get missing Account
-      const missingAccountsAddress = Array.from(
-        new Set(
-          [
-            ...erc20ActivitiesInDb
-              .filter((e) => !e.from_account_id)
-              .map((e) => e.from),
-            ...erc20ActivitiesInDb
-              .filter((e) => !e.to_account_id)
-              .map((e) => e.to),
-          ].map((e) =>
-            convertEthAddressToBech32Address(config.networkPrefixAddress, e)
-          ) as string[]
-        )
-      );
-      if (missingAccountsAddress.length > 0) {
-        throw new Error(
-          `Missing accounts ${missingAccountsAddress}. You should reindex them`
+      this.logger.info(`Reindex erc20 activities ${address} done.`);
+      let prevId = 0;
+      let numChunk = 1;
+      while (true) {
+        const erc20ActivitiesInDb = await Erc20Handler.getErc20Activities(
+          0,
+          Number(blockHeight),
+          trx,
+          [address],
+          {
+            prevId,
+            limitRecordGet,
+          }
+        );
+        // get missing Account
+        const missingAccountsAddress = Array.from(
+          new Set(
+            [
+              ...erc20ActivitiesInDb
+                .filter((e) => !e.from_account_id)
+                .map((e) => e.from),
+              ...erc20ActivitiesInDb
+                .filter((e) => !e.to_account_id)
+                .map((e) => e.to),
+            ].map((e) =>
+              convertEthAddressToBech32Address(config.networkPrefixAddress, e)
+            ) as string[]
+          )
         );
+        if (missingAccountsAddress.length > 0) {
+          throw new Error(
+            `Missing accounts ${missingAccountsAddress}. You should reindex them`
+          );
+        }
+        if (erc20ActivitiesInDb.length > 0) {
+          await Erc20Handler.updateErc20AccountsBalance(
+            erc20ActivitiesInDb,
+            trx
+          );
+          prevId = erc20ActivitiesInDb[erc20ActivitiesInDb.length - 1].id;
+          this.logger.info(
+            `Reindex erc20 contract ${address}: Chunk ${numChunk}/${Math.floor(
+              numActivities / limitRecordGet + 1
+            )} done`
+          );
+          numChunk += 1;
+        } else {
+          break;
+        }
       }
-      await Erc20Handler.updateErc20AccountsBalance(erc20ActivitiesInDb, trx);
     });
   }
 }

test/unit/services/evm/erc20_handler.spec.ts

@@ -187,15 +187,15 @@ export default class Erc20HandlerTest {
     ];
     await EvmEvent.query().insert(evmEvents);
     await knex.transaction(async (trx) => {
-      const erc20Activitites = await Erc20Handler.buildErc20Activities(
+      const { erc20Activities } = await Erc20Handler.buildErc20Activities(
         blockHeight - 1,
         blockHeight,
         trx,
         this.broker.logger
       );
-      expect(erc20Activitites.length).toEqual(evmEvents.length - 1);
+      expect(erc20Activities.length).toEqual(evmEvents.length - 1);
       // test build transfer activity
-      const transferActivity = erc20Activitites[0];
+      const transferActivity = erc20Activities[0];
       expect(transferActivity).toMatchObject({
         action: ERC20_ACTION.TRANSFER,
         erc20_contract_address: erc20Contract.address,
@@ -204,7 +204,7 @@ export default class Erc20HandlerTest {
         amount,
       });
       // test build approve activity
-      const approvalActivity = erc20Activitites[1];
+      const approvalActivity = erc20Activities[1];
       expect(approvalActivity).toMatchObject({
         action: ERC20_ACTION.APPROVAL,
         erc20_contract_address: erc20Contract.address,
@@ -269,15 +269,15 @@ export default class Erc20HandlerTest {
     ];
     await EvmEvent.query().insert(evmEvents);
     await knex.transaction(async (trx) => {
-      const erc20Activitites = await Erc20Handler.buildErc20Activities(
+      const { erc20Activities } = await Erc20Handler.buildErc20Activities(
         blockHeight - 1,
         blockHeight,
         trx,
         this.broker.logger
       );
-      expect(erc20Activitites.length).toEqual(evmEvents.length);
+      expect(erc20Activities.length).toEqual(evmEvents.length);
       // test build deposit activity
-      const depositActivity = erc20Activitites[0];
+      const depositActivity = erc20Activities[0];
       expect(depositActivity).toMatchObject({
         action: ERC20_ACTION.DEPOSIT,
         erc20_contract_address: erc20WrapContract.address,
@@ -286,7 +286,7 @@ export default class Erc20HandlerTest {
         amount,
       });
       // test build withdrawal activity
-      const withdrawalActivity = erc20Activitites[1];
+      const withdrawalActivity = erc20Activities[1];
       expect(withdrawalActivity).toMatchObject({
         action: ERC20_ACTION.WITHDRAWAL,
         erc20_contract_address: erc20WrapContract.address,
@@ -470,14 +470,14 @@ export default class Erc20HandlerTest {
     ];
     await EvmEvent.query().insert(evmEvents);
     await knex.transaction(async (trx) => {
-      const erc20Activitites = await Erc20Handler.buildErc20Activities(
+      const { erc20Activities } = await Erc20Handler.buildErc20Activities(
         blockHeight - 2,
         blockHeight + 1,
         trx,
         this.broker.logger
       );
       // test convert coin activity
-      const convertCoinActivity = erc20Activitites[2];
+      const convertCoinActivity = erc20Activities[2];
       expect(convertCoinActivity).toMatchObject({
         from: ZERO_ADDRESS,
         to,
@@ -487,7 +487,7 @@ export default class Erc20HandlerTest {
         cosmos_event_id: transaction.events[0].id,
       });
       // test convert erc20 activity
-      const convertErc20Activity = erc20Activitites[1];
+      const convertErc20Activity = erc20Activities[1];
       expect(convertErc20Activity).toMatchObject({
         from,
         to: ZERO_ADDRESS,
@@ -497,7 +497,7 @@ export default class Erc20HandlerTest {
         cosmos_event_id: transaction.events[1].id,
       });
       // test sort order
-      const transferActivity1 = erc20Activitites[0];
+      const transferActivity1 = erc20Activities[0];
       expect(transferActivity1).toMatchObject({
         action: ERC20_ACTION.TRANSFER,
         erc20_contract_address: erc20Contract.address,
@@ -507,7 +507,7 @@ export default class Erc20HandlerTest {
         evm_tx_id: evmEvents[0].evm_tx_id,
         cosmos_tx_id: evmEvents[0].tx_id,
       });
-      const transferActivity2 = erc20Activitites[3];
+      const transferActivity2 = erc20Activities[3];
       expect(transferActivity2).toMatchObject({
         action: ERC20_ACTION.TRANSFER,
         erc20_contract_address: erc20Contract.address,