Feat/erc20 reindexing

src/models/account_balance.ts

@@ -4,6 +4,8 @@ import BaseModel from './base';
 import { Account } from './account';
 
 export class AccountBalance extends BaseModel {
+  static softDelete = false;
+
   account!: Account;
 
   id!: number;

src/services/api-gateways/api_gateway.service.ts

@@ -48,6 +48,7 @@ import config from '../../../config.json' assert { type: 'json' };
           'v1.job.insert-verify-by-codehash',
           'v1.erc721-admin.*',
           'v2.evm-statistics.syncPrevDateStatsByChainId',
+          'v1.erc20-admin.*',
         ],
       },
       {

src/services/api-gateways/erc20_admin.service.ts

@@ -0,0 +1,49 @@
+import { Post, Service } from '@ourparentcenter/moleculer-decorators-extended';
+import { Context, ServiceBroker } from 'moleculer';
+import networks from '../../../network.json' assert { type: 'json' };
+import BaseService from '../../base/base.service';
+
+@Service({
+  name: 'erc20-admin',
+  version: 1,
+})
+export default class Erc20AdminService extends BaseService {
+  public constructor(public broker: ServiceBroker) {
+    super(broker);
+  }
+
+  @Post('/erc20-reindexing', {
+    name: 'erc20Reindexing',
+    params: {
+      chainid: {
+        type: 'string',
+        optional: false,
+        enum: networks.map((network) => network.chainId),
+      },
+      addresses: {
+        type: 'array',
+        optional: false,
+        items: 'string',
+      },
+    },
+  })
+  async erc20Reindexing(
+    ctx: Context<
+      {
+        chainid: string;
+        addresses: string[];
+      },
+      Record<string, unknown>
+    >
+  ) {
+    const selectedChain = networks.find(
+      (network) => network.chainId === ctx.params.chainid
+    );
+    return this.broker.call(
+      `v1.Erc20.reindexing@${selectedChain?.moleculerNamespace}`,
+      {
+        addresses: ctx.params.addresses,
+      }
+    );
+  }
+}

src/services/evm/constant.ts

@@ -113,6 +113,10 @@ export const SERVICE = {
         key: 'insertNewErc20Contracts',
         path: 'v1.Erc20.insertNewErc20Contracts',
       },
+      reindexing: {
+        key: 'reindexing',
+        path: 'v1.Erc20.reindexing',
+      },
     },
     Erc721: {
       key: 'Erc721',
@@ -222,6 +226,7 @@ export const BULL_JOB_NAME = {
   SYNC_SOURCIFY: 'sync:sourcify',
   INSERT_VERIFY_BY_CODEHASH: 'job:insert-verify-by-codehash',
   HANDLE_SELF_DESTRUCT: 'handle:self-destruct',
+  REINDEX_ERC20: 'reindex:erc20',
 };
 
 export const MSG_TYPE = {

src/services/evm/erc20.service.ts

@@ -12,17 +12,12 @@ import BullableService, { QueueHandler } from '../../base/bullable.service';
 import { SERVICE as COSMOS_SERVICE, Config, getLcdClient } from '../../common';
 import knex from '../../common/utils/db_connection';
 import { getViemClient } from '../../common/utils/etherjs_client';
-import {
-  BlockCheckpoint,
-  EVMSmartContract,
-  Event,
-  EvmEvent,
-} from '../../models';
-import { AccountBalance } from '../../models/account_balance';
+import { BlockCheckpoint, EVMSmartContract } from '../../models';
 import { Erc20Activity } from '../../models/erc20_activity';
 import { Erc20Contract } from '../../models/erc20_contract';
 import { BULL_JOB_NAME, SERVICE as EVM_SERVICE, SERVICE } from './constant';
-import { ERC20_EVENT_TOPIC0, Erc20Handler } from './erc20_handler';
+import { Erc20Handler } from './erc20_handler';
+import { Erc20Reindexer } from './erc20_reindex';
 import { convertEthAddressToBech32Address } from './utils';
 
 const { NODE_ENV } = Config;
@@ -98,82 +93,14 @@ export default class Erc20Service extends BullableService {
           ],
           config.erc20.key
         );
-      // TODO: handle track erc20 contract only
-      const erc20Events = await EvmEvent.query()
-        .joinRelated('[evm_smart_contract,evm_transaction]')
-        .innerJoin(
-          'erc20_contract',
-          'evm_event.address',
-          'erc20_contract.address'
-        )
-        .where('evm_event.block_height', '>', startBlock)
-        .andWhere('evm_event.block_height', '<=', endBlock)
-        .orderBy('evm_event.id', 'asc')
-        .select(
-          'evm_event.*',
-          'evm_transaction.from as sender',
-          'evm_smart_contract.id as evm_smart_contract_id',
-          'evm_transaction.id as evm_tx_id'
-        );
-      let erc20CosmosEvents: Event[] = [];
-      if (config.evmOnly === false) {
-        if (!this.erc20ModuleAccount) {
-          const lcdClient = await getLcdClient();
-          const erc20Account: QueryModuleAccountByNameResponseSDKType =
-            await lcdClient.provider.cosmos.auth.v1beta1.moduleAccountByName({
-              name: 'erc20',
-            });
-          // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-          // @ts-ignore
-          this.erc20ModuleAccount = erc20Account.account.base_account.address;
-        }
-        erc20CosmosEvents = await Event.query()
-          .where('block_height', '>', startBlock)
-          .andWhere('block_height', '<=', endBlock)
-          .andWhere((query) => {
-            query
-              .where('type', Event.EVENT_TYPE.CONVERT_COIN)
-              .orWhere('type', Event.EVENT_TYPE.CONVERT_ERC20);
-          })
-          .withGraphFetched('[transaction, attributes]');
-      }
-      await this.handleMissingErc20Contract(
-        erc20Events,
-        erc20CosmosEvents,
-        trx
-      );
-      const erc20Activities: Erc20Activity[] = [];
-      erc20Events.forEach((e) => {
-        if (e.topic0 === ERC20_EVENT_TOPIC0.TRANSFER) {
-          const activity = Erc20Handler.buildTransferActivity(e, this.logger);
-          if (activity) {
-            erc20Activities.push(activity);
-          }
-        } else if (e.topic0 === ERC20_EVENT_TOPIC0.APPROVAL) {
-          const activity = Erc20Handler.buildApprovalActivity(e, this.logger);
-          if (activity) {
-            erc20Activities.push(activity);
-          }
-        } else if (config.erc20.wrapExtensionContract.includes(e.address)) {
-          const wrapActivity = Erc20Handler.buildWrapExtensionActivity(
-            e,
-            this.logger
-          );
-          if (wrapActivity) {
-            erc20Activities.push(wrapActivity);
-          }
-        }
-      });
-      erc20CosmosEvents.forEach((event) => {
-        const activity = Erc20Handler.buildTransferActivityByCosmos(
-          event,
-          this.erc20ModuleAccount,
+      const erc20Activities: Erc20Activity[] =
+        await Erc20Handler.buildErc20Activities(
+          startBlock,
+          endBlock,
+          trx,
           this.logger
         );
-        if (activity) {
-          erc20Activities.push(activity);
-        }
-      });
+      await this.handleMissingErc20Contract(erc20Activities, trx);
       if (erc20Activities.length > 0) {
         this.logger.info(
           `Crawl Erc20 activity from block ${startBlock} to block ${endBlock}:\n ${JSON.stringify(
@@ -209,7 +136,10 @@ export default class Erc20Service extends BullableService {
         config.erc20.key
       );
     // get Erc20 activities
-    let erc20Activities = await this.getErc20Activities(startBlock, endBlock);
+    let erc20Activities = await Erc20Handler.getErc20Activities(
+      startBlock,
+      endBlock
+    );
     // get missing Account
     const missingAccountsAddress = Array.from(
       new Set(
@@ -233,40 +163,13 @@ export default class Erc20Service extends BullableService {
           addresses: missingAccountsAddress,
         }
       );
-      erc20Activities = await this.getErc20Activities(startBlock, endBlock);
+      erc20Activities = await Erc20Handler.getErc20Activities(
+        startBlock,
+        endBlock
+      );
     }
     await knex.transaction(async (trx) => {
-      if (erc20Activities.length > 0) {
-        const accountBalances = _.keyBy(
-          await AccountBalance.query()
-            .transacting(trx)
-            .joinRelated('account')
-            .whereIn(
-              ['account.evm_address', 'denom'],
-              [
-                ...erc20Activities.map((e) => [
-                  e.from,
-                  e.erc20_contract_address,
-                ]),
-                ...erc20Activities.map((e) => [e.to, e.erc20_contract_address]),
-              ]
-            ),
-          (o) => `${o.account_id}_${o.denom}`
-        );
-        // construct cw721 handler object
-        const erc20Handler = new Erc20Handler(accountBalances, erc20Activities);
-        erc20Handler.process();
-        const updatedAccountBalances = Object.values(
-          erc20Handler.accountBalances
-        );
-        if (updatedAccountBalances.length > 0) {
-          await AccountBalance.query()
-            .transacting(trx)
-            .insert(updatedAccountBalances)
-            .onConflict(['account_id', 'denom'])
-            .merge();
-        }
-      }
+      await Erc20Handler.updateErc20AccountsBalance(erc20Activities, trx);
       updateBlockCheckpoint.height = endBlock;
       await BlockCheckpoint.query()
         .insert(updateBlockCheckpoint)
@@ -276,6 +179,52 @@ export default class Erc20Service extends BullableService {
     });
   }
 
+  @QueueHandler({
+    queueName: BULL_JOB_NAME.REINDEX_ERC20,
+    jobName: BULL_JOB_NAME.REINDEX_ERC20,
+  })
+  async reindexErc20(_payload: { address: `0x${string}` }): Promise<void> {
+    const { address } = _payload;
+    const erc20Reindexer = new Erc20Reindexer(this.viemClient, this.logger);
+    await erc20Reindexer.reindex(address.toLowerCase() as `0x${string}`);
+    this.logger.info(`Reindex erc20 contract ${address} done.`);
+  }
+
+  @Action({
+    name: SERVICE.V1.Erc20.reindexing.key,
+    params: {
+      addresses: {
+        type: 'array',
+        items: 'string',
+        optional: false,
+      },
+    },
+  })
+  public async reindexing(
+    ctx: Context<{
+      addresses: `0x${string}`[];
+    }>
+  ) {
+    const { addresses } = ctx.params;
+    if (addresses.length > 0) {
+      await Promise.all(
+        addresses.map((address) =>
+          this.createJob(
+            BULL_JOB_NAME.REINDEX_ERC20,
+            BULL_JOB_NAME.REINDEX_ERC20,
+            {
+              address,
+            },
+            {
+              jobId: address,
+              removeOnComplete: true,
+            }
+          )
+        )
+      );
+    }
+  }
+
   @Action({
     name: SERVICE.V1.Erc20.insertNewErc20Contracts.key,
     params: {
@@ -311,52 +260,13 @@ export default class Erc20Service extends BullableService {
     }
   }
 
-  async getErc20Activities(
-    startBlock: number,
-    endBlock: number
-  ): Promise<Erc20Activity[]> {
-    return Erc20Activity.query()
-      .leftJoin(
-        'account as from_account',
-        'erc20_activity.from',
-        'from_account.evm_address'
-      )
-      .leftJoin(
-        'account as to_account',
-        'erc20_activity.to',
-        'to_account.evm_address'
-      )
-      .leftJoin(
-        'erc20_contract as erc20_contract',
-        'erc20_activity.erc20_contract_address',
-        'erc20_contract.address'
-      )
-      .where('erc20_activity.height', '>', startBlock)
-      .andWhere('erc20_activity.height', '<=', endBlock)
-      .andWhere('erc20_contract.track', true)
-      .select(
-        'erc20_activity.*',
-        'from_account.id as from_account_id',
-        'to_account.id as to_account_id'
-      )
-      .orderBy('erc20_activity.id');
-  }
-
   async handleMissingErc20Contract(
-    events: EvmEvent[],
-    cosmosEvents: Event[],
+    erc20Activities: Erc20Activity[],
     trx: Knex.Transaction
   ) {
-    const evmEventsUniqByAddress = _.keyBy(events, (e) => e.address);
-    const cosmosEventsUniqByAddress = _.keyBy(cosmosEvents, (e) =>
-      e.attributes.find((a) => a.key === 'erc20_token')?.value.toLowerCase()
+    const addresses = _.uniq(
+      erc20Activities.map((activity) => activity.erc20_contract_address)
     );
-
-    const addresses = [
-      ...Object.keys(evmEventsUniqByAddress),
-      ...Object.keys(cosmosEventsUniqByAddress),
-    ];
-
     const erc20ContractsByAddress = _.keyBy(
       await Erc20Contract.query().whereIn('address', addresses),
       (e) => e.address
@@ -442,6 +352,17 @@ export default class Erc20Service extends BullableService {
   public async _start(): Promise<void> {
     this.viemClient = getViemClient();
     if (NODE_ENV !== 'test') {
+      if (config.evmOnly === false) {
+        const lcdClient = await getLcdClient();
+        const erc20Account: QueryModuleAccountByNameResponseSDKType =
+          await lcdClient.provider.cosmos.auth.v1beta1.moduleAccountByName({
+            name: 'erc20',
+          });
+        Erc20Handler.erc20ModuleAccount =
+          // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+          // @ts-ignore
+          erc20Account.account.base_account.address;
+      }
       await this.createJob(
         BULL_JOB_NAME.HANDLE_ERC20_CONTRACT,
         BULL_JOB_NAME.HANDLE_ERC20_CONTRACT,