feat: material view erc721 holder statistic (#896)

* feat: material view erc721 holder statistic

* refactor: index

* refactor: review

ci/config.json.ci

@@ -421,7 +421,9 @@
     "chunkSizeInsert": 1000,
     "mediaPerBatch": 10,
     "concurrencyHandleTokenMedia": 10,
-    "timeRefreshErc721Stats": "1 * * * *"
+    "timeRefreshErc721Stats": "1 * * * *",
+    "timeRefreshMViewErc721HolderStats": "*/10 * * * *",
+    "statementTimeout": 600000
   },
   "jobRefreshMViewAccountBalanceStatistic": {
     "timeRefreshMViewAccountBalanceStatistic": "*/10 * * * *"

config.json

@@ -419,7 +419,9 @@
     "chunkSizeInsert": 500,
     "mediaPerBatch": 10,
     "concurrencyHandleTokenMedia": 10,
-    "timeRefreshErc721Stats": "1 * * * *"
+    "timeRefreshErc721Stats": "1 * * * *",
+    "timeRefreshMViewErc721HolderStats": "*/10 * * * *",
+    "statementTimeout": 600000
   },
   "crawlEvmProxyHistory": {
     "key": "crawlEvmProxyHistory",

migrations/evm/20240905021812_m_view_erc721_holder_statistic.ts

@@ -0,0 +1,21 @@
+import { Knex } from 'knex';
+import config from '../../config.json' assert { type: 'json' };
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw(`set statement_timeout to ${config.erc721.statementTimeout}`);
+  await knex.raw(`
+  CREATE MATERIALIZED VIEW m_view_erc721_holder_statistic AS
+    select count(*), erc721_token.owner, erc721_token.erc721_contract_address
+    from erc721_token 
+    group by erc721_token.owner, erc721_token.erc721_contract_address;
+  CREATE INDEX m_view_erc721_holder_statistic_owner_index
+    ON m_view_erc721_holder_statistic (owner);
+  CREATE INDEX m_view_erc721_holder_statistic_erc721_contract_address_count_index
+    ON m_view_erc721_holder_statistic (erc721_contract_address, count);
+`);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropMaterializedViewIfExists(
+    'm_view_erc721_holder_statistic'
+  );
+}

src/services/evm/constant.ts

@@ -293,6 +293,7 @@ export const BULL_JOB_NAME = {
   INSERT_VERIFY_BY_CODEHASH: 'job:insert-verify-by-codehash',
   HANDLE_SELF_DESTRUCT: 'handle:self-destruct',
   REINDEX_ERC20: 'reindex:erc20',
+  REFRESH_ERC721_HOLDER_STATISTIC: 'refresh:erc721-holder-statistic',
 };
 
 export const MSG_TYPE = {

src/services/evm/erc721.service.ts

@@ -278,6 +278,21 @@ export default class Erc721Service extends BullableService {
     this.logger.info(`Reindex erc721 contract ${address} done.`);
   }
 
+  @QueueHandler({
+    queueName: BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
+    jobName: BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
+  })
+  public async refreshErc721HolderStatistic(): Promise<void> {
+    await knex.transaction(async (trx) => {
+      await knex
+        .raw(`set statement_timeout to ${config.erc721.statementTimeout}`)
+        .transacting(trx);
+      await knex.schema
+        .refreshMaterializedView('m_view_erc721_holder_statistic')
+        .transacting(trx);
+    });
+  }
+
   @Action({
     name: SERVICE.V1.Erc721.insertNewErc721Contracts.key,
     params: {
@@ -535,6 +550,20 @@ export default class Erc721Service extends BullableService {
           },
         }
       );
+      await this.createJob(
+        BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
+        BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
+        {},
+        {
+          removeOnComplete: true,
+          removeOnFail: {
+            count: 3,
+          },
+          repeat: {
+            pattern: config.erc721.timeRefreshMViewErc721HolderStats,
+          },
+        }
+      );
     }
     return super._start();
   }