From 8a24294eb686522c10b2d869c243fde6d51e5f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ph=E1=BA=A1m=20Th=C3=A0nh=20Phong?= <49814372+phamphong9981@users.noreply.github.com> Date: Thu, 12 Sep 2024 11:29:40 +0700 Subject: [PATCH 01/11] feat: material view erc721 holder statistic (#896) * feat: material view erc721 holder statistic * refactor: index * refactor: review --- ci/config.json.ci | 4 ++- config.json | 4 ++- ...05021812_m_view_erc721_holder_statistic.ts | 21 ++++++++++++++ src/services/evm/constant.ts | 1 + src/services/evm/erc721.service.ts | 29 +++++++++++++++++++ 5 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 migrations/evm/20240905021812_m_view_erc721_holder_statistic.ts diff --git a/ci/config.json.ci b/ci/config.json.ci index c6972e44d..006513ba5 100644 --- a/ci/config.json.ci +++ b/ci/config.json.ci @@ -421,7 +421,9 @@ "chunkSizeInsert": 1000, "mediaPerBatch": 10, "concurrencyHandleTokenMedia": 10, - "timeRefreshErc721Stats": "1 * * * *" + "timeRefreshErc721Stats": "1 * * * *", + "timeRefreshMViewErc721HolderStats": "*/10 * * * *", + "statementTimeout": 600000 }, "jobRefreshMViewAccountBalanceStatistic": { "timeRefreshMViewAccountBalanceStatistic": "*/10 * * * *" diff --git a/config.json b/config.json index 86f99b299..e2e103e8c 100644 --- a/config.json +++ b/config.json @@ -419,7 +419,9 @@ "chunkSizeInsert": 500, "mediaPerBatch": 10, "concurrencyHandleTokenMedia": 10, - "timeRefreshErc721Stats": "1 * * * *" + "timeRefreshErc721Stats": "1 * * * *", + "timeRefreshMViewErc721HolderStats": "*/10 * * * *", + "statementTimeout": 600000 }, "crawlEvmProxyHistory": { "key": "crawlEvmProxyHistory", diff --git a/migrations/evm/20240905021812_m_view_erc721_holder_statistic.ts b/migrations/evm/20240905021812_m_view_erc721_holder_statistic.ts new file mode 100644 index 000000000..12496c414 --- /dev/null +++ b/migrations/evm/20240905021812_m_view_erc721_holder_statistic.ts @@ -0,0 +1,21 @@ +import { Knex } from 'knex'; +import config from '../../config.json' assert { type: 'json' }; +export async function up(knex: Knex): Promise { + await knex.raw(`set statement_timeout to ${config.erc721.statementTimeout}`); + await knex.raw(` + CREATE MATERIALIZED VIEW m_view_erc721_holder_statistic AS + select count(*), erc721_token.owner, erc721_token.erc721_contract_address + from erc721_token + group by erc721_token.owner, erc721_token.erc721_contract_address; + CREATE INDEX m_view_erc721_holder_statistic_owner_index + ON m_view_erc721_holder_statistic (owner); + CREATE INDEX m_view_erc721_holder_statistic_erc721_contract_address_count_index + ON m_view_erc721_holder_statistic (erc721_contract_address, count); +`); +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropMaterializedViewIfExists( + 'm_view_erc721_holder_statistic' + ); +} diff --git a/src/services/evm/constant.ts b/src/services/evm/constant.ts index e107b7768..0dc717a69 100644 --- a/src/services/evm/constant.ts +++ b/src/services/evm/constant.ts @@ -293,6 +293,7 @@ export const BULL_JOB_NAME = { INSERT_VERIFY_BY_CODEHASH: 'job:insert-verify-by-codehash', HANDLE_SELF_DESTRUCT: 'handle:self-destruct', REINDEX_ERC20: 'reindex:erc20', + REFRESH_ERC721_HOLDER_STATISTIC: 'refresh:erc721-holder-statistic', }; export const MSG_TYPE = { diff --git a/src/services/evm/erc721.service.ts b/src/services/evm/erc721.service.ts index d47c123cc..b561549a5 100644 --- a/src/services/evm/erc721.service.ts +++ b/src/services/evm/erc721.service.ts @@ -278,6 +278,21 @@ export default class Erc721Service extends BullableService { this.logger.info(`Reindex erc721 contract ${address} done.`); } + @QueueHandler({ + queueName: BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC, + jobName: BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC, + }) + public async refreshErc721HolderStatistic(): Promise { + await knex.transaction(async (trx) => { + await knex + .raw(`set statement_timeout to ${config.erc721.statementTimeout}`) + .transacting(trx); + await knex.schema + .refreshMaterializedView('m_view_erc721_holder_statistic') + .transacting(trx); + }); + } + @Action({ name: SERVICE.V1.Erc721.insertNewErc721Contracts.key, params: { @@ -535,6 +550,20 @@ export default class Erc721Service extends BullableService { }, } ); + await this.createJob( + BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC, + BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC, + {}, + { + removeOnComplete: true, + removeOnFail: { + count: 3, + }, + repeat: { + pattern: config.erc721.timeRefreshMViewErc721HolderStats, + }, + } + ); } return super._start(); } From 3c4472520f22b215b2b6454eb842ef9c6a99a82c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ph=E1=BA=A1m=20Th=C3=A0nh=20Phong?= <49814372+phamphong9981@users.noreply.github.com> Date: Thu, 12 Sep 2024 13:41:58 +0700 Subject: [PATCH 02/11] refactor: erc20 balance big query (#902) --- src/services/evm/erc20_handler.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/services/evm/erc20_handler.ts b/src/services/evm/erc20_handler.ts index 56e69f621..64bd9910f 100644 --- a/src/services/evm/erc20_handler.ts +++ b/src/services/evm/erc20_handler.ts @@ -342,10 +342,16 @@ export class Erc20Handler { .joinRelated('account') .whereIn( ['account.evm_address', 'denom'], - [ - ...erc20Activities.map((e) => [e.from, e.erc20_contract_address]), - ...erc20Activities.map((e) => [e.to, e.erc20_contract_address]), - ] + _.uniqWith( + [ + ...erc20Activities.map((e) => [ + e.from, + e.erc20_contract_address, + ]), + ...erc20Activities.map((e) => [e.to, e.erc20_contract_address]), + ], + _.isEqual + ) ), (o) => `${o.account_id}_${o.denom}` ); @@ -354,7 +360,7 @@ export class Erc20Handler { .transacting(trx) .whereIn( 'address', - erc20Activities.map((e) => e.erc20_contract_address) + _.uniq(erc20Activities.map((e) => e.erc20_contract_address)) ), 'address' ); From 0d8687f5877791614cf8263609bdefcd279bbef1 Mon Sep 17 00:00:00 2001 From: fibonacci998 Date: Fri, 13 Sep 2024 14:40:17 +0700 Subject: [PATCH 03/11] feat: add evm_address to validator story --- ci/config.json.ci | 3 +- config.json | 3 +- ...0913070037_add_evm_address_to_validator.ts | 13 ++++++ network.json | 17 +++++++- src/models/validator.ts | 2 + .../crawl_validator.service.ts | 41 ++++++++++++++----- 6 files changed, 64 insertions(+), 15 deletions(-) create mode 100644 migrations/20240913070037_add_evm_address_to_validator.ts diff --git a/ci/config.json.ci b/ci/config.json.ci index 006513ba5..5cc666cce 100644 --- a/ci/config.json.ci +++ b/ci/config.json.ci @@ -11,7 +11,8 @@ "crawlValidator": { "millisecondCrawl": null, "queryPageLimit": 100, - "patternCrawl": "30 */2 * * * *" + "patternCrawl": "30 */2 * * * *", + "chunkSizeInsert": 500 }, "cw721": { "key": "cw721", diff --git a/config.json b/config.json index e2e103e8c..3f82a3081 100644 --- a/config.json +++ b/config.json @@ -11,7 +11,8 @@ "crawlValidator": { "millisecondCrawl": null, "queryPageLimit": 100, - "patternCrawl": "30 */2 * * * *" + "patternCrawl": "30 */2 * * * *", + "chunkSizeInsert": 500 }, "cw721": { "key": "cw721", diff --git a/migrations/20240913070037_add_evm_address_to_validator.ts b/migrations/20240913070037_add_evm_address_to_validator.ts new file mode 100644 index 000000000..c668c5934 --- /dev/null +++ b/migrations/20240913070037_add_evm_address_to_validator.ts @@ -0,0 +1,13 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('validator', (table) => { + table.string('evm_address').index(); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('validator', (table) => { + table.dropColumn('evm_address'); + }); +} diff --git a/network.json b/network.json index 17a3c0dc9..f022ad8df 100644 --- a/network.json +++ b/network.json @@ -78,7 +78,7 @@ "redisDBNumber": 4, "moleculerNamespace": "namespace-ancient8", "EVMJSONRPC": ["https://rpc.ancient8.gg"], - "EVMchainId": 888888888 + "EVMchainId": 888888888 }, { "chainId": "ancient-8-testnet", @@ -88,7 +88,7 @@ "redisDBNumber": 4, "moleculerNamespace": "namespace-auratestnet", "EVMJSONRPC": ["https://rpcv2-testnet.ancient8.gg"], - "EVMchainId": 28122024 + "EVMchainId": 28122024 }, { "chainId": "ethereum", @@ -103,5 +103,18 @@ "RPC": [], "LCD": [], "EVMchainId": 11155111 + }, + { + "chainId": "1513", + "RPC": [], + "LCD": ["https://story-testnet-lcd.aura.network"], + "databaseName": "local_evm", + "redisDBNumber": 0, + "EVMJSONRPC": [ + "https://story-testnet.aura.network", + "https://testnet.storyrpc.io" + ], + "EVMchainId": 1513, + "moleculerNamespace": "erascope-dev-story-testnet" } ] diff --git a/src/models/validator.ts b/src/models/validator.ts index 2d7e037fa..8b1f3eccf 100644 --- a/src/models/validator.ts +++ b/src/models/validator.ts @@ -65,6 +65,8 @@ export class Validator extends BaseModel { image_url!: string; + evm_address!: string; + static get tableName() { return 'validator'; } diff --git a/src/services/evm/story/crawl-validator/crawl_validator.service.ts b/src/services/evm/story/crawl-validator/crawl_validator.service.ts index 9d772428a..2ff1b66f5 100644 --- a/src/services/evm/story/crawl-validator/crawl_validator.service.ts +++ b/src/services/evm/story/crawl-validator/crawl_validator.service.ts @@ -11,8 +11,10 @@ import { toBech32, toHex, } from '@cosmjs/encoding'; +import { Secp256k1 } from '@cosmjs/crypto'; import axios from 'axios'; import { pubkeyToRawAddress } from '@cosmjs/tendermint-rpc'; +import { keccak256 } from 'viem'; import { Validator } from '../../../../models/validator'; import BullableService, { QueueHandler, @@ -43,15 +45,20 @@ export default class CrawlValidatorService extends BullableService { public async handleCrawlAllValidator(_payload: object): Promise { this.logger.info('Crawl validator info in story protocol'); const updateValidators: Validator[] = await this.getFullInfoValidators(); - await Validator.query() - .insert(updateValidators) - .onConflict('operator_address') - .merge() - .returning('id') - .catch((error) => { - this.logger.error('Error insert or update validators'); - this.logger.error(error); - }); + if (updateValidators.length > 0) { + for (let i = 0; i < updateValidators.length; i += 500) { + const chunk = updateValidators.slice(i, i + 500); + await Validator.query() + .insert(chunk) + .onConflict('operator_address') + .merge() + .returning('id') + .catch((error) => { + this.logger.error('Error insert or update validators'); + this.logger.error(error); + }); + } + } } private async getFullInfoValidators(): Promise { @@ -124,8 +131,14 @@ export default class CrawlValidatorService extends BullableService { } else { // mark this offchain validator is mapped with onchain offchainMapped.set(validator.operator_address, true); - + const unCompressPubKey = Secp256k1.uncompressPubkey( + fromBase64(validator.consensus_pubkey.value) + ); + const evmAddress = `0x${keccak256(unCompressPubKey.slice(1)).slice( + -40 + )}`; validatorEntity = foundValidator; + validatorEntity.evm_address = evmAddress; validatorEntity.consensus_pubkey = validator.consensus_pubkey; validatorEntity.jailed = validator.jailed ?? false; validatorEntity.status = validator.status; @@ -193,15 +206,21 @@ export default class CrawlValidatorService extends BullableService { ); const consensusPubkey = { type: validator.consensus_pubkey.type, - key: validator.consensus_pubkey.value, + value: validator.consensus_pubkey.value, }; + const unCompressPubKey = Secp256k1.uncompressPubkey( + fromBase64(consensusPubkey.value) + ); + const evmAddress = `0x${keccak256(unCompressPubKey.slice(1)).slice(-40)}`; + const validatorEntity = Validator.fromJson({ operator_address: validator.operator_address, account_address: accountAddress, consensus_address: consensusAddress, consensus_hex_address: consensusHexAddress, consensus_pubkey: consensusPubkey, + evm_address: evmAddress, jailed: validator.jailed ?? false, status: validator.status, tokens: validator.tokens, From bed0734b5500293f4d2daa03247ab382ae15868c Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Fri, 13 Sep 2024 17:09:13 +0700 Subject: [PATCH 04/11] fix: init erc20 total supply --- src/services/evm/erc20.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/evm/erc20.service.ts b/src/services/evm/erc20.service.ts index c3fc06503..1f4d76f05 100644 --- a/src/services/evm/erc20.service.ts +++ b/src/services/evm/erc20.service.ts @@ -283,7 +283,7 @@ export default class Erc20Service extends BullableService { Erc20Contract.fromJson({ evm_smart_contract_id: evmSmartContracts[index].id, address: e.address, - total_supply: erc20ContractsInfo[index].totalSupply, + total_supply: '0', symbol: erc20ContractsInfo[index].symbol, decimal: erc20ContractsInfo[index].decimals, name: erc20ContractsInfo[index].name, From 4e9c36ef60c4a597dc8346cb51f6020903417706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ph=E1=BA=A1m=20Th=C3=A0nh=20Phong?= <49814372+phamphong9981@users.noreply.github.com> Date: Wed, 18 Sep 2024 15:25:01 +0700 Subject: [PATCH 05/11] refactor: account balance index denom amount (#906) --- ...0240918080156_account_balance_index_denom_amount.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 migrations/evm/20240918080156_account_balance_index_denom_amount.ts diff --git a/migrations/evm/20240918080156_account_balance_index_denom_amount.ts b/migrations/evm/20240918080156_account_balance_index_denom_amount.ts new file mode 100644 index 000000000..275d461dc --- /dev/null +++ b/migrations/evm/20240918080156_account_balance_index_denom_amount.ts @@ -0,0 +1,10 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.raw(`set statement_timeout to 0`); + await knex.raw( + 'CREATE INDEX account_balance_denom_amount_index ON account_balance(denom, amount);' + ); +} + +export async function down(knex: Knex): Promise {} From b61bddbebf6fafb189a2f97931db9ddbbfb7c457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ph=E1=BA=A1m=20Th=C3=A0nh=20Phong?= <49814372+phamphong9981@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:51:37 +0700 Subject: [PATCH 06/11] Feat: erc721 holder statistic + erc721 total supply (#905) * feat: erc721 holder statistic * feat: erc721 contract total supply * test: test * fix: migration * fix: lint * fix: lint * refactor: review * refactor: review --- .../20240912082032_erc721_holder_statistic.ts | 21 ++ ...40916032201_erc721_contract_totalSupply.ts | 32 +++ src/models/erc721_contract.ts | 2 + src/models/erc721_holder_statistic.ts | 29 +++ src/models/index.ts | 1 + src/services/evm/erc721.service.ts | 99 +++++---- src/services/evm/erc721_handler.ts | 81 ++++++- src/services/evm/erc721_reindex.ts | 30 ++- .../{erc721 => evm}/erc721-media.spec.ts | 0 .../services/{erc721 => evm}/erc721.spec.ts | 119 +++++------ .../{erc721 => evm}/erc721_handler.spec.ts | 197 +++++++++++++++--- 11 files changed, 475 insertions(+), 136 deletions(-) create mode 100644 migrations/evm/20240912082032_erc721_holder_statistic.ts create mode 100644 migrations/evm/20240916032201_erc721_contract_totalSupply.ts create mode 100644 src/models/erc721_holder_statistic.ts rename test/unit/services/{erc721 => evm}/erc721-media.spec.ts (100%) rename test/unit/services/{erc721 => evm}/erc721.spec.ts (66%) rename test/unit/services/{erc721 => evm}/erc721_handler.spec.ts (60%) diff --git a/migrations/evm/20240912082032_erc721_holder_statistic.ts b/migrations/evm/20240912082032_erc721_holder_statistic.ts new file mode 100644 index 000000000..bbecb92b3 --- /dev/null +++ b/migrations/evm/20240912082032_erc721_holder_statistic.ts @@ -0,0 +1,21 @@ +import { Knex } from 'knex'; +export async function up(knex: Knex): Promise { + await knex.raw(`set statement_timeout to 0`); + await knex.raw(` + CREATE TABLE erc721_holder_statistic AS + select count(*), erc721_token.owner, erc721_token.erc721_contract_address + from erc721_token + group by erc721_token.owner, erc721_token.erc721_contract_address; + ALTER TABLE erc721_holder_statistic ADD COLUMN id SERIAL PRIMARY KEY; + ALTER TABLE erc721_holder_statistic ADD COLUMN last_updated_height INTEGER; + CREATE INDEX erc721_holder_statistic_owner_index + ON erc721_holder_statistic (owner); + CREATE UNIQUE INDEX erc721_holder_statistic_erc721_contract_address_owner_index + ON erc721_holder_statistic (erc721_contract_address, owner); + CREATE INDEX erc721_holder_statistic_last_updated_height_index ON erc721_holder_statistic (last_updated_height) + `); +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists('erc721_holder_statistic'); +} diff --git a/migrations/evm/20240916032201_erc721_contract_totalSupply.ts b/migrations/evm/20240916032201_erc721_contract_totalSupply.ts new file mode 100644 index 000000000..fc4b0aaa3 --- /dev/null +++ b/migrations/evm/20240916032201_erc721_contract_totalSupply.ts @@ -0,0 +1,32 @@ +import { Knex } from 'knex'; +import { Erc721Token } from '../../src/models'; +import { ZERO_ADDRESS } from '../../src/services/evm/constant'; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('erc721_contract', (table) => { + table.bigInteger('total_supply').defaultTo(0).index(); + }); + await knex.raw(`set statement_timeout to 0`); + const totalSupplies = await Erc721Token.query(knex) + .select('erc721_token.erc721_contract_address') + .where('erc721_token.owner', '!=', ZERO_ADDRESS) + .count() + .groupBy('erc721_token.erc721_contract_address'); + if (totalSupplies.length > 0) { + const stringListUpdates = totalSupplies + .map( + (totalSuply) => + `('${totalSuply.erc721_contract_address}', ${totalSuply.count})` + ) + .join(','); + await knex.raw( + `UPDATE erc721_contract SET total_supply = temp.total_supply from (VALUES ${stringListUpdates}) as temp(address, total_supply) where temp.address = erc721_contract.address` + ); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('erc721_contract', (table) => { + table.dropColumn('total_supply'); + }); +} diff --git a/src/models/erc721_contract.ts b/src/models/erc721_contract.ts index 2a3b9fb13..4aaba11cd 100644 --- a/src/models/erc721_contract.ts +++ b/src/models/erc721_contract.ts @@ -25,6 +25,8 @@ export class Erc721Contract extends BaseModel { last_updated_height!: number; + total_supply!: string; + static get tableName() { return 'erc721_contract'; } diff --git a/src/models/erc721_holder_statistic.ts b/src/models/erc721_holder_statistic.ts new file mode 100644 index 000000000..1fd62b660 --- /dev/null +++ b/src/models/erc721_holder_statistic.ts @@ -0,0 +1,29 @@ +import BaseModel from './base'; + +export class Erc721HolderStatistic extends BaseModel { + static softDelete = false; + + erc721_contract_address!: string; + + owner!: string; + + count!: string; + + last_updated_height!: number; + + static get tableName() { + return 'erc721_holder_statistic'; + } + + static get jsonSchema() { + return { + type: 'object', + required: ['erc721_contract_address', 'owner', 'count'], + properties: { + erc721_contract_address: { type: 'string' }, + owner: { type: 'string' }, + count: { type: 'string' }, + }, + }; + } +} diff --git a/src/models/index.ts b/src/models/index.ts index 1d4e16dda..92cbbd82a 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -48,3 +48,4 @@ export * from './erc721_stats'; export * from './evm_block'; export * from './optimism_deposit'; export * from './optimism_withdrawal'; +export * from './erc721_holder_statistic'; diff --git a/src/services/evm/erc721.service.ts b/src/services/evm/erc721.service.ts index b561549a5..49a130b32 100644 --- a/src/services/evm/erc721.service.ts +++ b/src/services/evm/erc721.service.ts @@ -15,6 +15,7 @@ import { BlockCheckpoint, EVMSmartContract, Erc721Activity, + Erc721HolderStatistic, Erc721Stats, Erc721Token, } from '../../models'; @@ -103,7 +104,15 @@ export default class Erc721Service extends BullableService { if (erc721Activities.length > 0) { // create chunk array const listChunkErc721Activities = []; - const erc721TokensOnDB: Erc721Token[] = []; + const erc721Contracts = _.keyBy( + await Erc721Contract.query() + .whereIn( + 'address', + erc721Activities.map((e) => e.erc721_contract_address) + ) + .transacting(trx), + 'address' + ); for ( let i = 0; i < erc721Activities.length; @@ -116,30 +125,57 @@ export default class Erc721Service extends BullableService { listChunkErc721Activities.push(chunk); } // process chunk array - await Promise.all( - // eslint-disable-next-line array-callback-return - listChunkErc721Activities.map(async (chunk) => { - const erc721TokensInChunk = await Erc721Token.query().whereIn( - ['erc721_contract_address', 'token_id'], - chunk.map((e) => [ - e.erc721_contract_address, - // if token_id undefined (case approval_all), replace by null => not get any token (because token must have token_id) - e.token_id || null, - ]) - ); - erc721TokensOnDB.push(...erc721TokensInChunk); - }) - ); - + const erc721TokensOnDB: Erc721Token[] = ( + await Promise.all( + listChunkErc721Activities.map(async (chunk) => + Erc721Token.query().whereIn( + ['erc721_contract_address', 'token_id'], + chunk.map((e) => [ + e.erc721_contract_address, + // if token_id undefined (case approval_all), replace by null => not get any token (because token must have token_id) + e.token_id || null, + ]) + ) + ) + ) + ).flat(); + // process chunk array + const erc721HolderStatsOnDB: Erc721HolderStatistic[] = ( + await Promise.all( + listChunkErc721Activities.map(async (chunk) => + Erc721HolderStatistic.query().whereIn( + ['erc721_contract_address', 'owner'], + _.uniqWith( + [ + ...chunk.map((e) => [e.erc721_contract_address, e.from]), + ...chunk.map((e) => [e.erc721_contract_address, e.to]), + ], + _.isEqual + ) + ) + ) + ) + ).flat(); const erc721Tokens = _.keyBy( erc721TokensOnDB, (o) => `${o.erc721_contract_address}_${o.token_id}` ); - const erc721Handler = new Erc721Handler(erc721Tokens, erc721Activities); + const erc721HolderStats = _.keyBy( + erc721HolderStatsOnDB, + (o) => `${o.erc721_contract_address}_${o.owner}` + ); + const erc721Handler = new Erc721Handler( + erc721Contracts, + erc721Tokens, + erc721Activities, + erc721HolderStats + ); erc721Handler.process(); await Erc721Handler.updateErc721( + Object.values(erc721Handler.erc721Contracts), erc721Activities, Object.values(erc721Handler.erc721Tokens), + Object.values(erc721Handler.erc721HolderStats), trx ); } @@ -278,21 +314,6 @@ export default class Erc721Service extends BullableService { this.logger.info(`Reindex erc721 contract ${address} done.`); } - @QueueHandler({ - queueName: BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC, - jobName: BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC, - }) - public async refreshErc721HolderStatistic(): Promise { - await knex.transaction(async (trx) => { - await knex - .raw(`set statement_timeout to ${config.erc721.statementTimeout}`) - .transacting(trx); - await knex.schema - .refreshMaterializedView('m_view_erc721_holder_statistic') - .transacting(trx); - }); - } - @Action({ name: SERVICE.V1.Erc721.insertNewErc721Contracts.key, params: { @@ -550,20 +571,6 @@ export default class Erc721Service extends BullableService { }, } ); - await this.createJob( - BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC, - BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC, - {}, - { - removeOnComplete: true, - removeOnFail: { - count: 3, - }, - repeat: { - pattern: config.erc721.timeRefreshMViewErc721HolderStats, - }, - } - ); } return super._start(); } diff --git a/src/services/evm/erc721_handler.ts b/src/services/evm/erc721_handler.ts index 5c9274b51..e6de23a0e 100644 --- a/src/services/evm/erc721_handler.ts +++ b/src/services/evm/erc721_handler.ts @@ -10,6 +10,7 @@ import { EVMTransaction, Erc721Activity, Erc721Contract, + Erc721HolderStatistic, Erc721Token, EvmEvent, } from '../../models'; @@ -64,18 +65,26 @@ export const ERC721_ACTION = { APPROVAL_FOR_ALL: 'approval_for_all', }; export class Erc721Handler { + erc721Contracts: Dictionary; + // key: {contract_address}_{token_id} // value: erc721 token erc721Tokens: Dictionary; erc721Activities: Erc721Activity[]; + erc721HolderStats: Dictionary; + constructor( + erc721Contracts: Dictionary, erc721Tokens: Dictionary, - erc721Activities: Erc721Activity[] + erc721Activities: Erc721Activity[], + erc721HolderStats: Dictionary ) { + this.erc721Contracts = erc721Contracts; this.erc721Tokens = erc721Tokens; this.erc721Activities = erc721Activities; + this.erc721HolderStats = erc721HolderStats; } process() { @@ -87,6 +96,8 @@ export class Erc721Handler { } handlerErc721Transfer(erc721Activity: Erc721Activity) { + const erc721Contract = + this.erc721Contracts[`${erc721Activity.erc721_contract_address}`]; const token = this.erc721Tokens[ `${erc721Activity.erc721_contract_address}_${erc721Activity.token_id}` @@ -95,6 +106,11 @@ export class Erc721Handler { // update new owner and last updated height token.owner = erc721Activity.to; token.last_updated_height = erc721Activity.height; + if (erc721Activity.to === ZERO_ADDRESS) { + erc721Contract.total_supply = ( + BigInt(erc721Contract.total_supply) - BigInt(1) + ).toString(); + } } else if (erc721Activity.from === ZERO_ADDRESS) { // handle mint this.erc721Tokens[ @@ -106,9 +122,37 @@ export class Erc721Handler { last_updated_height: erc721Activity.height, burned: false, }); + erc721Contract.total_supply = ( + BigInt(erc721Contract.total_supply) + BigInt(1) + ).toString(); } else { throw new Error('Handle erc721 tranfer error'); } + // update erc721 holder statistics + const erc721HolderStatFrom = + this.erc721HolderStats[ + `${erc721Activity.erc721_contract_address}_${erc721Activity.from}` + ]; + this.erc721HolderStats[ + `${erc721Activity.erc721_contract_address}_${erc721Activity.from}` + ] = Erc721HolderStatistic.fromJson({ + erc721_contract_address: erc721Activity.erc721_contract_address, + owner: erc721Activity.from, + count: (BigInt(erc721HolderStatFrom?.count || 0) - BigInt(1)).toString(), + last_updated_height: erc721Activity.height, + }); + const erc721HolderStatTo = + this.erc721HolderStats[ + `${erc721Activity.erc721_contract_address}_${erc721Activity.to}` + ]; + this.erc721HolderStats[ + `${erc721Activity.erc721_contract_address}_${erc721Activity.to}` + ] = Erc721HolderStatistic.fromJson({ + erc721_contract_address: erc721Activity.erc721_contract_address, + owner: erc721Activity.to, + count: (BigInt(erc721HolderStatTo?.count || 0) + BigInt(1)).toString(), + last_updated_height: erc721Activity.height, + }); } static buildTransferActivity( @@ -266,11 +310,28 @@ export class Erc721Handler { } static async updateErc721( + erc721Contracts: Erc721Contract[], erc721Activities: Erc721Activity[], erc721Tokens: Erc721Token[], + erc721HolderStats: Erc721HolderStatistic[], trx: Knex.Transaction ) { + // update erc721 contract: total supply + if (erc721Contracts.length > 0) { + const stringListUpdates = erc721Contracts + .map( + (erc721Contract) => + `(${erc721Contract.id}, ${erc721Contract.total_supply})` + ) + .join(','); + await knex + .raw( + `UPDATE erc721_contract SET total_supply = temp.total_supply from (VALUES ${stringListUpdates}) as temp(id, total_supply) where temp.id = erc721_contract.id` + ) + .transacting(trx); + } let updatedTokens: Dictionary = {}; + // update erc721 token: new token & new holder if (erc721Tokens.length > 0) { updatedTokens = _.keyBy( await Erc721Token.query() @@ -290,6 +351,7 @@ export class Erc721Handler { (o) => `${o.erc721_contract_address}_${o.token_id}` ); } + // insert new erc721 activities if (erc721Activities.length > 0) { erc721Activities.forEach((activity) => { const token = @@ -309,6 +371,23 @@ export class Erc721Handler { ) .transacting(trx); } + // update erc721 holder statistic + if (erc721HolderStats.length > 0) { + await Erc721HolderStatistic.query() + .transacting(trx) + .insert( + erc721HolderStats.map((e) => + Erc721HolderStatistic.fromJson({ + erc721_contract_address: e.erc721_contract_address, + owner: e.owner, + count: e.count, + last_updated_height: e.last_updated_height, + }) + ) + ) + .onConflict(['erc721_contract_address', 'owner']) + .merge(); + } } static async calErc721Stats(addresses?: string[]): Promise { diff --git a/src/services/evm/erc721_reindex.ts b/src/services/evm/erc721_reindex.ts index 99d56be04..0301f256c 100644 --- a/src/services/evm/erc721_reindex.ts +++ b/src/services/evm/erc721_reindex.ts @@ -1,9 +1,11 @@ import Moleculer from 'moleculer'; import { PublicClient, getContract } from 'viem'; +import { Dictionary } from 'lodash'; import knex from '../../common/utils/db_connection'; import { Erc721Activity, Erc721Contract, + Erc721HolderStatistic, Erc721Stats, Erc721Token, } from '../../models'; @@ -74,6 +76,10 @@ export class Erc721Reindexer { ) .first() .throwIfNotFound(); + await Erc721HolderStatistic.query() + .delete() + .where('erc721_contract_address', address) + .transacting(trx); await Erc721Stats.query() .delete() .where('erc721_contract_id', erc721Contract.id) @@ -100,7 +106,8 @@ export class Erc721Reindexer { contract.read.name().catch(() => Promise.resolve(undefined)), contract.read.symbol().catch(() => Promise.resolve(undefined)), ]); - await Erc721Contract.query() + const [tokens, height] = await this.getCurrentTokens(address); + const newErc721Contract: Erc721Contract = await Erc721Contract.query() .insert( Erc721Contract.fromJson({ evm_smart_contract_id: erc721Contract.evm_smart_contract_id, @@ -109,10 +116,10 @@ export class Erc721Reindexer { name: contractInfo[0], track: true, last_updated_height: Number(blockHeight), + total_supply: tokens.length, }) ) .transacting(trx); - const [tokens, height] = await this.getCurrentTokens(address); const activities = await Erc721Handler.getErc721Activities( 0, height, @@ -120,7 +127,24 @@ export class Erc721Reindexer { [address], trx ); - await Erc721Handler.updateErc721(activities, tokens, trx); + const erc721HolderStats: Dictionary = {}; + tokens.forEach((token) => { + const { owner } = token; + const currentCount = erc721HolderStats[owner]?.count || '0'; + const newCount = (BigInt(currentCount) + BigInt(1)).toString(); + erc721HolderStats[owner] = Erc721HolderStatistic.fromJson({ + erc721_contract_address: address, + owner, + count: newCount, + }); + }); + await Erc721Handler.updateErc721( + [newErc721Contract], + activities, + tokens, + Object.values(erc721HolderStats), + trx + ); }); const erc721Stats = await Erc721Handler.calErc721Stats([address]); if (erc721Stats.length > 0) { diff --git a/test/unit/services/erc721/erc721-media.spec.ts b/test/unit/services/evm/erc721-media.spec.ts similarity index 100% rename from test/unit/services/erc721/erc721-media.spec.ts rename to test/unit/services/evm/erc721-media.spec.ts diff --git a/test/unit/services/erc721/erc721.spec.ts b/test/unit/services/evm/erc721.spec.ts similarity index 66% rename from test/unit/services/erc721/erc721.spec.ts rename to test/unit/services/evm/erc721.spec.ts index d2b8c30b0..ad01f9554 100644 --- a/test/unit/services/erc721/erc721.spec.ts +++ b/test/unit/services/evm/erc721.spec.ts @@ -6,6 +6,7 @@ import { Test, } from '@jest-decorated/core'; import { ServiceBroker } from 'moleculer'; +import _ from 'lodash'; import knex from '../../../../src/common/utils/db_connection'; import { BlockCheckpoint, @@ -13,11 +14,13 @@ import { EVMTransaction, Erc721Activity, Erc721Contract, + Erc721HolderStatistic, Erc721Token, EvmEvent, } from '../../../../src/models'; import Erc721Service from '../../../../src/services/evm/erc721.service'; import { ERC721_ACTION } from '../../../../src/services/evm/erc721_handler'; +import { BULL_JOB_NAME } from '../../../../src/services/evm/constant'; @Describe('Test erc721') export default class Erc721Test { @@ -27,22 +30,24 @@ export default class Erc721Test { evmSmartContract = EVMSmartContract.fromJson({ id: 555, - address: 'ghghdfgdsgre', - creator: 'dfgdfbvxcvxgfds', + address: '0x98605ae21dd3be686337a6d7a8f156d0d8baee92', + creator: '0xc7663c6a454fc9971C93235A170c8997e8c5E661', created_height: 100, - created_hash: 'cvxcvcxv', + created_hash: + '0x9ed3b06713baf17b7d7266294a82d51c36c84514a7d29f28585d85e586249525', type: EVMSmartContract.TYPES.ERC721, - code_hash: 'dfgdfghf', + code_hash: '0x623d4ffdfa5555aa1234366sdaff1a1sdfafa4dasdfas4r', }); evmSmartContract2 = EVMSmartContract.fromJson({ id: 666, - address: 'bcvbcvbcv', - creator: 'dfgdfbvxcvxgfds', + address: '0xc211C2CF383A38933f8352CBDd326B51b94574e6', + creator: '0xc7663c6a454fc9971C93235A170c8997e8c5E661', created_height: 100, - created_hash: 'xdasfsf', + created_hash: + '0x888d3b06713baf17b7d7266294a82d51c36c84514a7d29f28585d85e586252200', type: EVMSmartContract.TYPES.ERC721, - code_hash: 'xcsadf', + code_hash: '0x5fdfa5555aa1234366sdaff1a1sdfafa4dasdfas4r', }); evmTx = EVMTransaction.fromJson({ @@ -53,10 +58,11 @@ export default class Erc721Test { tx_id: 223, contract_address: '', index: 1, + from: '0x38828FA9766dE6eb49011fCC970ed1beFE15974a', }); evmEvent = EvmEvent.fromJson({ - id: 888, + id: 1, tx_id: 1234, evm_tx_id: this.evmTx.id, tx_hash: '', @@ -73,12 +79,23 @@ export default class Erc721Test { address: this.evmSmartContract.address, }); + blockCheckpoints = [ + BlockCheckpoint.fromJson({ + job_name: BULL_JOB_NAME.HANDLE_ERC721_ACTIVITY, + height: this.evmTx.height - 1, + }), + BlockCheckpoint.fromJson({ + job_name: BULL_JOB_NAME.HANDLE_ERC721_CONTRACT, + height: 400, + }), + ]; + @BeforeAll() async initSuite() { - this.erc721Service.getQueueManager().stopAll(); + await this.erc721Service.getQueueManager().stopAll(); await this.broker.start(); await knex.raw( - 'TRUNCATE TABLE erc721_contract, account, erc721_activity, evm_smart_contract, evm_event, evm_transaction, block_checkpoint RESTART IDENTITY CASCADE' + 'TRUNCATE TABLE erc721_contract, account, erc721_activity, evm_smart_contract, evm_event, evm_transaction, block_checkpoint, erc721_holder_statistic RESTART IDENTITY CASCADE' ); await EVMSmartContract.query().insert([ this.evmSmartContract, @@ -92,8 +109,9 @@ export default class Erc721Test { @BeforeEach() async beforeEach() { await knex.raw( - 'TRUNCATE TABLE erc721_activity, erc721_token RESTART IDENTITY CASCADE' + 'TRUNCATE TABLE erc721_activity, erc721_token, block_checkpoint, evm_event RESTART IDENTITY CASCADE' ); + await BlockCheckpoint.query().insert(this.blockCheckpoints); } @AfterAll() @@ -105,27 +123,19 @@ export default class Erc721Test { @Test('test handleErc721Activity') async testHandleErc721Activity() { - jest.spyOn(BlockCheckpoint, 'getCheckpoint').mockResolvedValue([ - this.evmSmartContract.created_height - 1, - this.evmSmartContract.created_height, - BlockCheckpoint.fromJson({ - job_name: 'dfdsfgsg', - height: this.evmSmartContract.created_height - 1, - }), - ]); + const holder1 = '0x1317df02a4e712265f5376a9d34156f73ebad640'; + const holder2 = '0xa3b6d252c1df2ce88f01fdb75b5479bcdc8f5007'; const erc721Events = [ EvmEvent.fromJson({ - id: this.evmEvent.id, block_hash: '0x6d70a03cda3fb815b54742fbd47c6141a7e754ff4d7426f10a73644ac44411d2', - block_height: 21937980, + block_height: this.blockCheckpoints[0].height + 1, data: null, topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', topic1: '0x0000000000000000000000000000000000000000000000000000000000000000', - topic2: - '0x0000000000000000000000001317df02a4e712265f5376a9d34156f73ebad640', + topic2: `0x000000000000000000000000${holder1.slice(2)}`, topic3: '0x0000000000000000000000000000000000000000000000000000000000000000', address: this.evmSmartContract.address, @@ -133,22 +143,16 @@ export default class Erc721Test { tx_id: 1234, tx_hash: this.evmTx.hash, tx_index: 1, - sender: 'fgfdg', - evm_smart_contract_id: this.evmSmartContract.id, - track: true, }), EvmEvent.fromJson({ - id: this.evmEvent.id, block_hash: '0xd39b1e6c35a7985db6ca367b1e061162b7a8610097e99cadaf98bea6b81a6096', - block_height: 21937981, + block_height: this.blockCheckpoints[0].height + 2, data: null, topic0: '0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925', - topic1: - '0x0000000000000000000000001317df02a4e712265f5376a9d34156f73ebad640', - topic2: - '0x000000000000000000000000a3b6d252c1df2ce88f01fdb75b5479bcdc8f5007', + topic1: `0x000000000000000000000000${holder1.slice(2)}`, + topic2: `0x000000000000000000000000${holder2.slice(2)}`, topic3: '0x0000000000000000000000000000000000000000000000000000000000000000', address: this.evmSmartContract.address, @@ -156,22 +160,16 @@ export default class Erc721Test { tx_id: 1234, tx_hash: this.evmTx.hash, tx_index: 1, - sender: 'fgfdg', - evm_smart_contract_id: this.evmSmartContract.id, - track: true, }), EvmEvent.fromJson({ - id: this.evmEvent.id, block_hash: '0x6d70a03cda3fb815b54742fbd47c6141a7e754ff4d7426f10a73644ac44411d2', - block_height: 21937982, + block_height: this.blockCheckpoints[0].height + 3, data: null, topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', - topic1: - '0x0000000000000000000000001317df02a4e712265f5376a9d34156f73ebad640', - topic2: - '0x000000000000000000000000e39633931ec4a1841e438b15005a6f141d30789e', + topic1: `0x000000000000000000000000${holder1.slice(2)}`, + topic2: `0x000000000000000000000000${holder2.slice(2)}`, topic3: '0x0000000000000000000000000000000000000000000000000000000000000000', address: this.evmSmartContract.address, @@ -179,27 +177,14 @@ export default class Erc721Test { tx_id: 1234, tx_hash: this.evmTx.hash, tx_index: 1, - sender: 'fgfdg', - evm_smart_contract_id: this.evmSmartContract.id, - track: true, }), ]; - const mockQueryEvents: any = { - select: () => erc721Events, - transacting: () => mockQueryEvents, - joinRelated: () => mockQueryEvents, - where: () => mockQueryEvents, - andWhere: () => mockQueryEvents, - orderBy: () => mockQueryEvents, - innerJoin: () => mockQueryEvents, - modify: () => mockQueryEvents, - }; - jest.spyOn(EvmEvent, 'query').mockImplementation(() => mockQueryEvents); + await EvmEvent.query().insert(erc721Events); await this.erc721Service.handleErc721Activity(); const erc721Token = await Erc721Token.query().first().throwIfNotFound(); expect(erc721Token).toMatchObject({ token_id: '0', - owner: '0xe39633931ec4a1841e438b15005a6f141d30789e', + owner: holder2, erc721_contract_address: this.evmSmartContract.address, }); const erc721Activities = await Erc721Activity.query().orderBy('height'); @@ -207,22 +192,32 @@ export default class Erc721Test { action: ERC721_ACTION.TRANSFER, erc721_contract_address: erc721Events[0].address, from: '0x0000000000000000000000000000000000000000', - to: '0x1317df02a4e712265f5376a9d34156f73ebad640', + to: holder1, erc721_token_id: erc721Token.id, }); expect(erc721Activities[1]).toMatchObject({ action: ERC721_ACTION.APPROVAL, erc721_contract_address: erc721Events[0].address, - from: '0x1317df02a4e712265f5376a9d34156f73ebad640', - to: '0xa3b6d252c1df2ce88f01fdb75b5479bcdc8f5007', + from: holder1, + to: holder2, erc721_token_id: erc721Token.id, }); expect(erc721Activities[2]).toMatchObject({ action: ERC721_ACTION.TRANSFER, erc721_contract_address: erc721Events[0].address, - from: '0x1317df02a4e712265f5376a9d34156f73ebad640', - to: '0xe39633931ec4a1841e438b15005a6f141d30789e', + from: holder1, + to: holder2, erc721_token_id: erc721Token.id, }); + const erc721Contract = await Erc721Contract.query() + .first() + .throwIfNotFound(); + expect(erc721Contract.total_supply).toEqual('1'); + const erc721HolderStats = _.keyBy( + await Erc721HolderStatistic.query(), + 'owner' + ); + expect(erc721HolderStats[holder1].count).toEqual('0'); + expect(erc721HolderStats[holder2].count).toEqual('1'); } } diff --git a/test/unit/services/erc721/erc721_handler.spec.ts b/test/unit/services/evm/erc721_handler.spec.ts similarity index 60% rename from test/unit/services/erc721/erc721_handler.spec.ts rename to test/unit/services/evm/erc721_handler.spec.ts index 6c5a752a6..802a05481 100644 --- a/test/unit/services/erc721/erc721_handler.spec.ts +++ b/test/unit/services/evm/erc721_handler.spec.ts @@ -1,6 +1,5 @@ -import { BeforeAll, Describe, Test } from '@jest-decorated/core'; +import { AfterAll, BeforeAll, Describe, Test } from '@jest-decorated/core'; import { ServiceBroker } from 'moleculer'; -import { bytesToHex, hexToBytes } from 'viem'; import config from '../../../../config.json' assert { type: 'json' }; import knex from '../../../../src/common/utils/db_connection'; import { @@ -22,35 +21,36 @@ export default class Erc721HandlerTest { broker = new ServiceBroker({ logger: false }); evmSmartContract = EVMSmartContract.fromJson({ - id: 555, - address: 'ghghdfgdsgre', - creator: 'dfgdfbvxcvxgfds', + id: 1, + address: '0xE974cC14c93FC6077B0d65F98832B846C5454A0B', + creator: '0x5606b4eA93F696Dd82Ca983BAF5723d00729f127', created_height: 100, - created_hash: 'cvxcvcxv', + created_hash: + '0x8c46cf6373f2f6e528b56becf0ce6b460b5d90cf9b0325a136b9b3a820e1e489', + code_hash: '0xdsf3335453454tsgfsdrtserf43645y4h4tAAvfgfgds', type: EVMSmartContract.TYPES.ERC721, - code_hash: 'dfgdfghf', }); evmSmartContract2 = EVMSmartContract.fromJson({ - id: 666, - address: 'bcvbcvbcv', - creator: 'dfgdfbvxcvxgfds', + id: 2, + address: '0x3CB367e7C920Ff15879Bd4CBd708b8c60eB0f537', + creator: '0xa9497CC4F95773A744D408b54dAC724626ee31d2', created_height: 100, - created_hash: 'xdasfsf', + created_hash: + '0x5bca9ee42c4c32941c58f2a510dae5ff5c6ed848d9a396a8e9e146a166b3a3fc', + code_hash: '0xdfskjgdsgfgweruwie4535t3tu34tjkewtgjwe', type: EVMSmartContract.TYPES.ERC721, - code_hash: 'xcsadf', }); evmTx = EVMTransaction.fromJson({ id: 11111, - hash: hexToBytes( - '0x3faac2ed3ca031892c04598177f7c36e9fdcdf2fb3b6c4a13c520590facb82ef' - ), + hash: '', height: 111, tx_msg_id: 222, tx_id: 223, + contract_address: '', index: 1, - from: hexToBytes('0x51aeade652867f342ddc012e15c27d0cd6220398'), + from: '0x38828FA9766dE6eb49011fCC970ed1beFE15974a', }); erc721Contract1 = Erc721Contract.fromJson({ @@ -83,6 +83,13 @@ export default class Erc721HandlerTest { await EVMTransaction.query().insert(this.evmTx); } + @AfterAll() + async tearDown() { + await this.broker.stop(); + jest.resetAllMocks(); + jest.restoreAllMocks(); + } + @Test('test getErc721Activities') async testGetErc721Activities() { await knex.transaction(async (trx) => { @@ -103,7 +110,7 @@ export default class Erc721HandlerTest { address: this.evmSmartContract.address, evm_tx_id: this.evmTx.id, tx_id: 1234, - tx_hash: bytesToHex(this.evmTx.hash), + tx_hash: this.evmTx.hash, tx_index: 1, }), EvmEvent.fromJson({ @@ -122,7 +129,7 @@ export default class Erc721HandlerTest { address: this.evmSmartContract.address, evm_tx_id: this.evmTx.id, tx_id: 1234, - tx_hash: bytesToHex(this.evmTx.hash), + tx_hash: this.evmTx.hash, tx_index: 1, }), EvmEvent.fromJson({ @@ -141,7 +148,7 @@ export default class Erc721HandlerTest { address: this.evmSmartContract2.address, evm_tx_id: this.evmTx.id, tx_id: 1234, - tx_hash: bytesToHex(this.evmTx.hash), + tx_hash: this.evmTx.hash, tx_index: 1, }), EvmEvent.fromJson({ @@ -150,17 +157,16 @@ export default class Erc721HandlerTest { block_height: 21937982, data: null, topic0: - '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + '0x17307eab39ab6107e8899845ad3d59bd9653f200f220920489ca2b5937696c31', topic1: '0x0000000000000000000000001317df02a4e712265f5376a9d34156f73ebad640', topic2: '0x000000000000000000000000e39633931ec4a1841e438b15005a6f141d30789e', - topic3: - '0x0000000000000000000000000000000000000000000000000000000000000000', + topic3: null, address: this.evmSmartContract.address, evm_tx_id: this.evmTx.id, tx_id: 1234, - tx_hash: bytesToHex(this.evmTx.hash), + tx_hash: this.evmTx.hash, tx_index: 1, }), ]; @@ -191,11 +197,12 @@ export default class Erc721HandlerTest { to: '0xa3b6d252c1df2ce88f01fdb75b5479bcdc8f5007', }); expect(erc721Activities[3]).toMatchObject({ - action: ERC721_ACTION.TRANSFER, + action: ERC721_ACTION.APPROVAL_FOR_ALL, erc721_contract_address: this.evmSmartContract.address, from: '0x1317df02a4e712265f5376a9d34156f73ebad640', to: '0xe39633931ec4a1841e438b15005a6f141d30789e', }); + // get a contract's activities const erc721ActivitiesByContract = await Erc721Handler.getErc721Activities( this.evmTx.height - 1, @@ -214,6 +221,148 @@ export default class Erc721HandlerTest { }); } + @Test('test process') + async testProcess() { + const holders = [ + '0x38828FA9766dE6eb49011fCC970ed1beFE15974a', + '0x27CF763feEE58f14C2c69663EA8De784FCF2Dbbb', + '0xE1dbb634e2aefbe63085CB0b60f4155f9ABCa48f', + ]; + const initErc721Contract = Erc721Contract.fromJson({ + evm_smart_contract_id: 1, + address: '0x4c5f56a2FF75a8617337E33f75EB459Db422916F', + symbol: 'ITCIPA', + name: 'Iliad Testnet Commemorative IP Asset', + track: true, + last_updated_height: 0, + total_supply: '3', + }); + const initErc721Tokens = {}; + const initErc721HolderStats = {}; + const erc721Activities = [ + // mint token 0 to holder0 + Erc721Activity.fromJson({ + action: ERC721_ACTION.TRANSFER, + erc721_contract_address: initErc721Contract.address, + from: '0x0000000000000000000000000000000000000000', + to: holders[0], + token_id: '0', + height: 1, + evm_event_id: 1, + }), + // mint token 1 to holder0 + Erc721Activity.fromJson({ + action: ERC721_ACTION.TRANSFER, + erc721_contract_address: initErc721Contract.address, + from: '0x0000000000000000000000000000000000000000', + to: holders[0], + token_id: '1', + height: 1, + evm_event_id: 1, + }), + // mint token 2 to holder1 + Erc721Activity.fromJson({ + action: ERC721_ACTION.TRANSFER, + erc721_contract_address: initErc721Contract.address, + from: '0x0000000000000000000000000000000000000000', + to: holders[1], + token_id: '2', + height: 2, + evm_event_id: 1, + }), + // mint token 3 to holder0 + Erc721Activity.fromJson({ + action: ERC721_ACTION.TRANSFER, + erc721_contract_address: initErc721Contract.address, + from: '0x0000000000000000000000000000000000000000', + to: holders[0], + token_id: '3', + height: 3, + evm_event_id: 1, + }), + // transfer token 0 from holder0 to holder2 + Erc721Activity.fromJson({ + action: ERC721_ACTION.TRANSFER, + erc721_contract_address: initErc721Contract.address, + from: holders[0], + to: holders[2], + token_id: '0', + height: 4, + evm_event_id: 1, + }), + // transfer token 1 from holder0 to holder1 + Erc721Activity.fromJson({ + action: ERC721_ACTION.TRANSFER, + erc721_contract_address: initErc721Contract.address, + from: holders[0], + to: holders[1], + token_id: '1', + height: 5, + evm_event_id: 1, + }), + // burn token 3 + Erc721Activity.fromJson({ + action: ERC721_ACTION.TRANSFER, + erc721_contract_address: initErc721Contract.address, + from: holders[0], + to: '0x0000000000000000000000000000000000000000', + token_id: '3', + height: 6, + evm_event_id: 1, + }), + ]; + const erc721Handler = new Erc721Handler( + { [initErc721Contract.address]: initErc721Contract }, + initErc721Tokens, + erc721Activities, + initErc721HolderStats + ); + erc721Handler.process(); + const updatedErc721Contract = + erc721Handler.erc721Contracts[initErc721Contract.address]; + + // check total supply + expect(updatedErc721Contract.total_supply).toEqual('6'); + // check new owner for each token + const updatedTokens = erc721Handler.erc721Tokens; + expect(updatedTokens[`${initErc721Contract.address}_0`].owner).toEqual( + holders[2] + ); + expect(updatedTokens[`${initErc721Contract.address}_1`].owner).toEqual( + holders[1] + ); + expect(updatedTokens[`${initErc721Contract.address}_2`].owner).toEqual( + holders[1] + ); + const updatedErc721HolderStats = erc721Handler.erc721HolderStats; + expect( + updatedErc721HolderStats[`${initErc721Contract.address}_${holders[0]}`] + ).toMatchObject({ + count: '0', + last_updated_height: erc721Activities[6].height, + }); + expect( + updatedErc721HolderStats[`${initErc721Contract.address}_${holders[1]}`] + ).toMatchObject({ + count: '2', + last_updated_height: erc721Activities[5].height, + }); + expect( + updatedErc721HolderStats[`${initErc721Contract.address}_${holders[2]}`] + ).toMatchObject({ + count: '1', + last_updated_height: erc721Activities[4].height, + }); + expect( + updatedErc721HolderStats[ + `${initErc721Contract.address}_0x0000000000000000000000000000000000000000` + ] + ).toMatchObject({ + count: '-3', + last_updated_height: erc721Activities[6].height, + }); + } + @Test('test calErc721Stats') async testCalErc721Stats() { const evmEvent = EvmEvent.fromJson({ From 9e6012b3477fdc8c92853b797f58a695c3c757c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ph=E1=BA=A1m=20Th=C3=A0nh=20Phong?= <49814372+phamphong9981@users.noreply.github.com> Date: Mon, 23 Sep 2024 11:25:15 +0700 Subject: [PATCH 07/11] feat: erc20 total holder (#907) * feat: erc20 total holder * feat: erc20 total holder --- ...0024049_erc20_contract_add_total_holder.ts | 29 + src/models/account_balance.ts | 2 + src/models/erc20_contract.ts | 2 + src/services/evm/erc20_handler.ts | 13 +- test/unit/services/evm/erc20_handler.spec.ts | 1076 ++++++++++++++--- 5 files changed, 965 insertions(+), 157 deletions(-) create mode 100644 migrations/evm/20240920024049_erc20_contract_add_total_holder.ts diff --git a/migrations/evm/20240920024049_erc20_contract_add_total_holder.ts b/migrations/evm/20240920024049_erc20_contract_add_total_holder.ts new file mode 100644 index 000000000..58c06f09f --- /dev/null +++ b/migrations/evm/20240920024049_erc20_contract_add_total_holder.ts @@ -0,0 +1,29 @@ +import { Knex } from 'knex'; +import { AccountBalance } from '../../src/models'; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('erc20_contract', (table) => { + table.integer('total_holder').defaultTo(0).index(); + }); + await knex.raw(`set statement_timeout to 0`); + const totalHolders = await AccountBalance.query(knex) + .select('account_balance.denom') + .where('account_balance.type', AccountBalance.TYPE.ERC20_TOKEN) + .andWhere('account_balance.amount', '>', 0) + .count() + .groupBy('account_balance.denom'); + if (totalHolders.length > 0) { + const stringListUpdates = totalHolders + .map((totalHolder) => `('${totalHolder.denom}', ${totalHolder.count})`) + .join(','); + await knex.raw( + `UPDATE erc20_contract SET total_holder = temp.total_holder from (VALUES ${stringListUpdates}) as temp(address, total_holder) where temp.address = erc20_contract.address` + ); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('erc20_contract', (table) => { + table.dropColumn('total_holder'); + }); +} diff --git a/src/models/account_balance.ts b/src/models/account_balance.ts index 4f78806cd..81d98e51d 100644 --- a/src/models/account_balance.ts +++ b/src/models/account_balance.ts @@ -4,6 +4,8 @@ import BaseModel from './base'; import { Account } from './account'; export class AccountBalance extends BaseModel { + [relation: string]: any; + static softDelete = false; account!: Account; diff --git a/src/models/erc20_contract.ts b/src/models/erc20_contract.ts index 7694b430c..cf07704a4 100644 --- a/src/models/erc20_contract.ts +++ b/src/models/erc20_contract.ts @@ -23,6 +23,8 @@ export class Erc20Contract extends BaseModel { track!: boolean; + total_holder!: number; + last_updated_height!: number; static get tableName() { diff --git a/src/services/evm/erc20_handler.ts b/src/services/evm/erc20_handler.ts index 64bd9910f..c51e46361 100644 --- a/src/services/evm/erc20_handler.ts +++ b/src/services/evm/erc20_handler.ts @@ -114,6 +114,9 @@ export class Erc20Handler { const amount = ( BigInt(fromAccountBalance?.amount || 0) - BigInt(erc20Activity.amount) ).toString(); + if (BigInt(amount) === BigInt(0)) { + erc20Contract.total_holder -= 1; + } // update object accountBalance this.accountBalances[key] = AccountBalance.fromJson({ denom: erc20Activity.erc20_contract_address, @@ -144,10 +147,14 @@ export class Erc20Handler { `Process erc20 balance: toAccountBalance ${erc20Activity.to} was updated` ); } + const initAmount = toAccountBalance?.amount || 0; // calculate new balance: increase balance of to account const amount = ( - BigInt(toAccountBalance?.amount || 0) + BigInt(erc20Activity.amount) + BigInt(initAmount) + BigInt(erc20Activity.amount) ).toString(); + if (BigInt(amount) > BigInt(0) && BigInt(initAmount) === BigInt(0)) { + erc20Contract.total_holder += 1; + } // update object accountBalance this.accountBalances[key] = AccountBalance.fromJson({ denom: erc20Activity.erc20_contract_address, @@ -545,7 +552,7 @@ export class Erc20Handler { ) as [string, bigint]; return Erc20Activity.fromJson({ evm_event_id: e.id, - sender: e.sender, + sender: bytesToHex(e.sender), action: ERC20_ACTION.DEPOSIT, erc20_contract_address: e.address, amount: amount.toString(), @@ -573,7 +580,7 @@ export class Erc20Handler { ) as [string, bigint]; return Erc20Activity.fromJson({ evm_event_id: e.id, - sender: e.sender, + sender: bytesToHex(e.sender), action: ERC20_ACTION.WITHDRAWAL, erc20_contract_address: e.address, amount: amount.toString(), diff --git a/test/unit/services/evm/erc20_handler.spec.ts b/test/unit/services/evm/erc20_handler.spec.ts index 9b28e5f93..cff30a050 100644 --- a/test/unit/services/evm/erc20_handler.spec.ts +++ b/test/unit/services/evm/erc20_handler.spec.ts @@ -1,80 +1,52 @@ -import { AfterAll, BeforeAll, Describe, Test } from '@jest-decorated/core'; -import _ from 'lodash'; +import { fromBase64 } from '@cosmjs/encoding'; +import { + AfterAll, + BeforeAll, + BeforeEach, + Describe, + Test, +} from '@jest-decorated/core'; +import { Dictionary } from 'lodash'; import { ServiceBroker } from 'moleculer'; -import { decodeAbiParameters, hexToBytes, toHex } from 'viem'; +import { + decodeAbiParameters, + encodeAbiParameters, + fromHex, + hexToBytes, + toHex, +} from 'viem'; +import config from '../../../../config.json' assert { type: 'json' }; import knex from '../../../../src/common/utils/db_connection'; -import { getViemClient } from '../../../../src/common/utils/etherjs_client'; import { - Account, - AccountBalance, + Block, Erc20Activity, Erc20Contract, + Event, + EventAttribute, EvmEvent, EVMSmartContract, EVMTransaction, + Transaction, } from '../../../../src/models'; -import { ABI_TRANSFER_PARAMS } from '../../../../src/services/evm/erc20_handler'; -import { Erc20Reindexer } from '../../../../src/services/evm/erc20_reindex'; +import { AccountBalance } from '../../../../src/models/account_balance'; +import { ZERO_ADDRESS } from '../../../../src/services/evm/constant'; +import { + ABI_APPROVAL_PARAMS, + ABI_TRANSFER_PARAMS, + ERC20_ACTION, + Erc20Handler, +} from '../../../../src/services/evm/erc20_handler'; -const accounts = [ - Account.fromJson({ - id: 116058, - address: 'aura1az8cmnr4ppfggj0vt4llzdun0ptlkc8jxha27w', - balances: [ - { - denom: 'uaura', - amount: '42981159', - }, - ], - code_hash: null, - evm_address: '0xe88f8dcc7508528449ec5d7ff137937857fb60f2', - pubkey: {}, - sequence: 11, - account_number: 5, - spendable_balances: [ - { - denom: 'uaura', - amount: '42981159', - }, - ], - type: '/cosmos.auth.v1beta1.BaseAccount', - account_balances: [ - { - denom: '0x80b5a32e4f032b2a058b4f29ec95eefeeb87adcd', - amount: '422142', - type: 'ERC20_TOKEN', - }, - { - denom: '0xde47a655a5d9904bd3f7e1a536d8323fbd99993a', - amount: '-10000000000000000000', - type: 'ERC20_TOKEN', - }, - ], - }), - Account.fromJson({ - id: 116059, - address: 'aura1xnjpzgtqztcy9l8vhafmr5zulfn7a8l8sjlhp3', - balances: [ - { - denom: 'uaura', - amount: '32218705', - }, - ], - code_hash: null, - evm_address: '0x34e411216012f042fcecbf53b1d05cfa67ee9fe7', - pubkey: {}, - sequence: 11, - account_number: 5, - spendable_balances: [ - { - denom: 'uaura', - amount: '42981159', - }, - ], - type: '/cosmos.auth.v1beta1.BaseAccount', - account_balances: [], - }), -]; +const evmTransaction = EVMTransaction.fromJson({ + id: 2931, + hash: '0xf15467ec2a25eeef95798d93c2fe9ed8e7c891578b8e1bcc3284105849656c9d', + height: 121114, + tx_id: 1612438, + tx_msg_id: 4752908, + contract_address: null, + index: 0, + from: 'evmos1fwgemqt4mw39m0mn8e7ulyjpafl9r9pmzyv3hv', +}); const evmSmartContract = EVMSmartContract.fromJson({ id: 1, address: '0xde47a655a5d9904bd3f7e1a536d8323fbd99993a', @@ -85,7 +57,6 @@ const evmSmartContract = EVMSmartContract.fromJson({ type: EVMSmartContract.TYPES.ERC20, code_hash: '0xdsf3335453454tsgfsdrtserf43645y4h4tAAvfgfgds', }); - const erc20Contract = Erc20Contract.fromJson({ id: 10, address: evmSmartContract.address, @@ -94,134 +65,931 @@ const erc20Contract = Erc20Contract.fromJson({ symbol: 'WAURA', total_supply: '0', track: true, - activities: [ - { - from: '0x93f8e7ec7e054b476d7de8e6bb096e56cd575beb', - erc20_contract_address: '0xde47a655a5d9904bd3f7e1a536d8323fbd99993a', - height: 7116660, - sender: '0x34e411216012f042fcecbf53b1d05cfa67ee9fe7', - to: '0x34e411216012f042fcecbf53b1d05cfa67ee9fe7', - tx_hash: - '0xf15467ec2a25eeef95798d93c2fe9ed8e7c891578b8e1bcc3284105849656c9d', - action: 'transfer', - amount: '258773093659445577552', - }, - ], evm_smart_contract_id: evmSmartContract.id, }); -const evmTransaction = EVMTransaction.fromJson({ - id: 2931, - hash: hexToBytes( - '0xf15467ec2a25eeef95798d93c2fe9ed8e7c891578b8e1bcc3284105849656c9d' - ), - height: 1, - tx_id: 1612438, - tx_msg_id: 4752908, - contract_address: null, - index: 0, - from: hexToBytes('0x51aeade652867f342ddc012e15c27d0cd6220398'), +const wrapSmartContract = EVMSmartContract.fromJson({ + id: 2, + address: config.erc20.wrapExtensionContract[0], + creator: '0x5606b4eA93F696Dd82Ca983BAF5723d00729f127', + created_height: 100, + created_hash: + '0x8c46cf6373f2f6e528b56becf0ce6b460b5d90cf9b0325a136b9b3a820e1e489', + type: EVMSmartContract.TYPES.ERC20, + code_hash: '0xdsf3335453454tsgfsdrtserf43645y4h4tAAvfgfgds', }); - -@Describe('Test erc20 reindex') -export default class Erc20ReindexTest { +const erc20WrapContract = Erc20Contract.fromJson({ + id: 20, + address: wrapSmartContract.address, + decimal: '18', + name: 'Wrapped Aura', + symbol: 'WAURA', + total_supply: '0', + track: true, + evm_smart_contract_id: wrapSmartContract.id, +}); +const erc20ModuleAccount = 'aura1glht96kr2rseywuvhhay894qw7ekuc4q3jyctl'; +@Describe('Test erc20 handler') +export default class Erc20HandlerTest { broker = new ServiceBroker({ logger: false }); @BeforeAll() async initSuite() { await this.broker.start(); await knex.raw( - 'TRUNCATE TABLE evm_transaction, evm_smart_contract, erc20_contract, account, erc20_activity, account_balance, evm_event RESTART IDENTITY CASCADE' + 'TRUNCATE TABLE evm_transaction, evm_smart_contract, erc20_contract RESTART IDENTITY CASCADE' ); await EVMTransaction.query().insert(evmTransaction); - await EVMSmartContract.query().insert(evmSmartContract); - await Erc20Contract.query().insertGraph(erc20Contract); - await Account.query().insertGraph(accounts); + await EVMSmartContract.query().insert([ + evmSmartContract, + wrapSmartContract, + ]); + await Erc20Contract.query().insert([erc20Contract, erc20WrapContract]); } @AfterAll() async tearDown() { await this.broker.stop(); - jest.resetAllMocks(); - jest.restoreAllMocks(); } - @Test('test reindex') - async testReindex() { - const viemClient = getViemClient(); - jest.spyOn(viemClient, 'getBlockNumber').mockResolvedValue(BigInt(123456)); - // Instantiate Erc20Reindexer with the mock - const reindexer = new Erc20Reindexer(viemClient, this.broker.logger); - const event = EvmEvent.fromJson({ - id: 4227, + @BeforeEach() + async beforeEach() { + await knex.raw( + 'TRUNCATE TABLE evm_event, transaction, event, event_attribute, block RESTART IDENTITY CASCADE' + ); + } + + @Test('test build erc20 activities (transfer, approval)') + async testBuildErc20Activities() { + const from = '0x400207c680a1c5d5a86f35f97111afc00f2f1826'; + const to = '0xea780c13a5450ac7c3e6ae4b17a0445998132b15'; + const amount = '45222000'; + const evmEvents = [ + // transfer event + EvmEvent.fromJson({ + id: 1, + address: erc20Contract.address, + block_hash: + '0xed6a2d3c3ac9a2868420c4fdd67240d2d96298fc4272cd31455cd0cdaabf9093', + block_height: evmTransaction.height, + data: fromHex( + encodeAbiParameters([ABI_TRANSFER_PARAMS.VALUE], [amount]), + 'bytes' + ), + evm_tx_id: evmTransaction.id, + topic0: + '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: encodeAbiParameters([ABI_TRANSFER_PARAMS.FROM], [from]), + topic2: encodeAbiParameters([ABI_TRANSFER_PARAMS.TO], [to]), + tx_hash: + '0x8a82a0c8848487d716f10a91f0aefb0526d35bd0f489166cc5141718a4d8aa64', + topic3: null, + tx_id: evmTransaction.id, + tx_index: 0, + }), + // approval event + EvmEvent.fromJson({ + id: 2, + address: erc20Contract.address, + block_hash: + '0x3ad4778e1b3a03f2c4cfe51d3dd6e75ccea2cb5081cb5eab2f83f60a6960d353', + block_height: evmTransaction.height, + data: fromHex( + encodeAbiParameters([ABI_TRANSFER_PARAMS.VALUE], [amount]), + 'bytes' + ), + evm_tx_id: evmTransaction.id, + topic0: + '0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925', + topic1: encodeAbiParameters([ABI_TRANSFER_PARAMS.FROM], [from]), + topic2: encodeAbiParameters([ABI_TRANSFER_PARAMS.TO], [to]), + tx_hash: + '0xfc1219232e1dfd380464d0ee843e0ee5b7a587521f2e02cb9957a74f7c57b05c', + topic3: null, + tx_id: evmTransaction.id, + tx_index: 0, + }), + // deposit event but not wrap + EvmEvent.fromJson({ + id: 3, + address: erc20Contract.address, + block_hash: + '0xed6a2d3c3ac9a2868420c4fdd67240d2d96298fc4272cd31455cd0cdaabf9093', + block_height: evmTransaction.height, + data: fromHex( + encodeAbiParameters([ABI_TRANSFER_PARAMS.VALUE], [amount]), + 'bytes' + ), + evm_tx_id: evmTransaction.id, + topic0: + '0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c', + topic1: encodeAbiParameters([ABI_TRANSFER_PARAMS.TO], [to]), + tx_hash: + '0x8a82a0c8848487d716f10a91f0aefb0526d35bd0f489166cc5141718a4d8aa64', + topic2: null, + topic3: null, + tx_id: evmTransaction.id, + tx_index: 0, + }), + // transfer event + EvmEvent.fromJson({ + id: 4, + address: erc20Contract.address, + block_hash: + '0xed6a2d3c3ac9a2868420c4fdd67240d2d96298fc4272cd31455cd0cdaabf9093', + block_height: evmTransaction.height, + data: null, + evm_tx_id: evmTransaction.id, + topic0: + '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: encodeAbiParameters([ABI_TRANSFER_PARAMS.FROM], [from]), + topic2: encodeAbiParameters([ABI_TRANSFER_PARAMS.TO], [to]), + topic3: encodeAbiParameters([ABI_TRANSFER_PARAMS.VALUE], [amount]), + tx_hash: + '0x8a82a0c8848487d716f10a91f0aefb0526d35bd0f489166cc5141718a4d8aa64', + tx_id: evmTransaction.id, + tx_index: 0, + }), + ]; + await EvmEvent.query().insert(evmEvents); + await knex.transaction(async (trx) => { + const { erc20Activities } = await Erc20Handler.buildErc20Activities( + evmTransaction.height - 1, + evmTransaction.height, + trx, + this.broker.logger + ); + expect(erc20Activities.length).toEqual(evmEvents.length - 1); + // test build transfer activity + const transferActivity = erc20Activities[0]; + expect(transferActivity).toMatchObject({ + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: erc20Contract.address, + from, + to, + amount, + }); + // test build approve activity + const approvalActivity = erc20Activities[1]; + expect(approvalActivity).toMatchObject({ + action: ERC20_ACTION.APPROVAL, + erc20_contract_address: erc20Contract.address, + from, + to, + amount, + }); + // test build transfer activity case amount in topic 3 + const transfer2Activity = erc20Activities[2]; + expect(transfer2Activity).toMatchObject({ + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: erc20Contract.address, + from, + to, + amount, + }); + }); + } + + @Test('test build wrap erc20 activities (deposit, withdrawl)') + async testBuildWrapErc20Activities() { + const from = '0x400207c680a1c5d5a86f35f97111afc00f2f1826'; + const to = '0xea780c13a5450ac7c3e6ae4b17a0445998132b15'; + const amount = '45222000'; + const evmEvents = [ + // deposit event + EvmEvent.fromJson({ + id: 1, + address: erc20WrapContract.address, + block_hash: + '0xed6a2d3c3ac9a2868420c4fdd67240d2d96298fc4272cd31455cd0cdaabf9093', + block_height: evmTransaction.height, + data: fromHex( + encodeAbiParameters([ABI_TRANSFER_PARAMS.VALUE], [amount]), + 'bytes' + ), + evm_tx_id: evmTransaction.id, + topic0: + '0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c', + topic1: encodeAbiParameters([ABI_TRANSFER_PARAMS.TO], [to]), + tx_hash: + '0x8a82a0c8848487d716f10a91f0aefb0526d35bd0f489166cc5141718a4d8aa64', + topic2: null, + topic3: null, + tx_id: evmTransaction.id, + tx_index: 0, + }), + // withdrawal event + EvmEvent.fromJson({ + id: 2, + address: erc20WrapContract.address, + block_hash: + '0xed6a2d3c3ac9a2868420c4fdd67240d2d96298fc4272cd31455cd0cdaabf9093', + block_height: evmTransaction.height, + data: fromHex( + encodeAbiParameters([ABI_TRANSFER_PARAMS.VALUE], [amount]), + 'bytes' + ), + evm_tx_id: evmTransaction.id, + topic0: + '0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65', + topic1: encodeAbiParameters([ABI_TRANSFER_PARAMS.FROM], [from]), + tx_hash: + '0x8a82a0c8848487d716f10a91f0aefb0526d35bd0f489166cc5141718a4d8aa64', + topic2: null, + topic3: null, + tx_id: evmTransaction.id, + tx_index: 0, + }), + ]; + await EvmEvent.query().insert(evmEvents); + await knex.transaction(async (trx) => { + const { erc20Activities } = await Erc20Handler.buildErc20Activities( + evmTransaction.height - 1, + evmTransaction.height, + trx, + this.broker.logger + ); + expect(erc20Activities.length).toEqual(evmEvents.length); + // test build deposit activity + const depositActivity = erc20Activities[0]; + expect(depositActivity).toMatchObject({ + action: ERC20_ACTION.DEPOSIT, + erc20_contract_address: erc20WrapContract.address, + from: ZERO_ADDRESS, + to, + amount, + }); + // test build withdrawal activity + const withdrawalActivity = erc20Activities[1]; + expect(withdrawalActivity).toMatchObject({ + action: ERC20_ACTION.WITHDRAWAL, + erc20_contract_address: erc20WrapContract.address, + from, + to: ZERO_ADDRESS, + amount, + }); + }); + } + + @Test('test build erc20 activities from cosmos (convertCoin, convertErc20)') + async testBuildConvertCoinErc20Activity() { + const from = '0x400207c680a1c5d5a86f35f97111afc00f2f1826'; + const to = '0xea780c13a5450ac7c3e6ae4b17a0445998132b15'; + const amount = '45222000'; + const block = Block.fromJson({ + data: JSON.stringify({ + linkS3: 'https://nft.aurascan.io/rawlog/aura/aura_6322-2/block/7424149', + }), + tx_count: 2, + hash: '152A5BDEE0768D2BAB1E65C726C4B94ACC28FF792E202F4676EFC888B3E22A79', + height: evmTransaction.height, + proposer_address: '39A5D22101441C1B1D93C4F3B72A64681D59B2A0', + time: '2024-07-26T01:00:21.319351+07:00', + }); + const transaction = Transaction.fromJson({ + code: 0, + codespace: '', + data: { + linkS3: + 'https://nft.aurascan.io/rawlog/aura/auradev_1236-2/transaction/20534362/1406F9DDCE529F0E6EB32E07A88E5BC4EE220D3A2AB6D57E89DD12EB1945CC19', + }, + fee: JSON.stringify([ + { + denom: 'uaura', + amount: '6141', + }, + ]), + gas_limit: '2456353', + gas_used: '1775234', + gas_wanted: '2456353', + hash: '1406F9DDCE529F0E6EB32E07A88E5BC4EE220D3A2AB6D57E89DD12EB1945CC19', + height: evmTransaction.height, + id: evmTransaction.id, + index: 0, + memo: 'memo', + timestamp: '2024-07-15T17:08:43.386+07:00', + events: [ + { + id: 3, + tx_msg_index: 0, + type: Event.EVENT_TYPE.CONVERT_COIN, + source: 'TX_EVENT', + block_height: evmTransaction.height, + attributes: [ + { + block_height: evmTransaction.height, + index: 4, + key: EventAttribute.ATTRIBUTE_KEY.ERC20_TOKEN, + tx_id: 505671, + value: erc20Contract.address, + event_id: '1', + }, + { + block_height: evmTransaction.height, + index: 0, + key: EventAttribute.ATTRIBUTE_KEY.SENDER, + tx_id: 505671, + value: erc20ModuleAccount, + event_id: '1', + }, + { + block_height: evmTransaction.height, + index: 1, + key: EventAttribute.ATTRIBUTE_KEY.RECEIVER, + tx_id: 505671, + value: to, + event_id: '1', + }, + { + block_height: evmTransaction.height, + index: 2, + key: EventAttribute.ATTRIBUTE_KEY.AMOUNT, + tx_id: 505671, + value: amount, + event_id: '1', + }, + ], + }, + { + id: 2, + tx_msg_index: 1, + type: Event.EVENT_TYPE.CONVERT_ERC20, + source: 'TX_EVENT', + block_height: evmTransaction.height, + attributes: [ + { + block_height: evmTransaction.height, + index: 0, + key: 'sender', + tx_id: 505657, + value: from, + event_id: '1', + }, + { + block_height: evmTransaction.height, + index: 1, + key: 'receiver', + tx_id: 505657, + value: erc20ModuleAccount, + event_id: '1', + }, + { + block_height: evmTransaction.height, + index: 2, + key: 'amount', + tx_id: 505657, + value: amount, + event_id: '1', + }, + { + block_height: evmTransaction.height, + index: 4, + key: 'erc20_token', + tx_id: 505657, + value: erc20Contract.address, + event_id: '1', + }, + ], + }, + ], + }); + await Block.query().insert(block); + await Transaction.query().insertGraph(transaction); + const evmEvents = [ + // transfer event + EvmEvent.fromJson({ + id: 1, + address: erc20Contract.address, + block_hash: + '0xed6a2d3c3ac9a2868420c4fdd67240d2d96298fc4272cd31455cd0cdaabf9093', + block_height: evmTransaction.height - 1, + data: fromHex( + encodeAbiParameters([ABI_TRANSFER_PARAMS.VALUE], [amount]), + 'bytes' + ), + evm_tx_id: evmTransaction.id, + topic0: + '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: encodeAbiParameters([ABI_TRANSFER_PARAMS.FROM], [from]), + topic2: encodeAbiParameters([ABI_TRANSFER_PARAMS.TO], [to]), + tx_hash: + '0x8a82a0c8848487d716f10a91f0aefb0526d35bd0f489166cc5141718a4d8aa64', + topic3: null, + tx_id: evmTransaction.id - 1, + tx_index: 0, + }), + // transfer event + EvmEvent.fromJson({ + id: 2, + address: erc20Contract.address, + block_hash: + '0xed6a2d3c3ac9a2868420c4fdd67240d2d96298fc4272cd31455cd0cdaabf9093', + block_height: evmTransaction.height + 1, + data: fromHex( + encodeAbiParameters([ABI_TRANSFER_PARAMS.VALUE], [amount]), + 'bytes' + ), + evm_tx_id: evmTransaction.id, + topic0: + '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + topic1: encodeAbiParameters([ABI_TRANSFER_PARAMS.FROM], [from]), + topic2: encodeAbiParameters([ABI_TRANSFER_PARAMS.TO], [to]), + tx_hash: + '0x8a82a0c8848487d716f10a91f0aefb0526d35bd0f489166cc5141718a4d8aa64', + topic3: null, + tx_id: evmTransaction.id + 1, + tx_index: 0, + }), + ]; + await EvmEvent.query().insert(evmEvents); + await knex.transaction(async (trx) => { + const { erc20Activities } = await Erc20Handler.buildErc20Activities( + evmTransaction.height - 2, + evmTransaction.height + 1, + trx, + this.broker.logger + ); + if (config.evmOnly === false) { + // test convert coin activity + const convertCoinActivity = erc20Activities[2]; + expect(convertCoinActivity).toMatchObject({ + from: ZERO_ADDRESS, + to, + amount, + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: erc20Contract.address, + cosmos_event_id: transaction.events[0].id, + }); + // test convert erc20 activity + const convertErc20Activity = erc20Activities[1]; + expect(convertErc20Activity).toMatchObject({ + from, + to: ZERO_ADDRESS, + amount, + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: erc20Contract.address, + cosmos_event_id: transaction.events[1].id, + }); + // test sort order + const transferActivity1 = erc20Activities[0]; + expect(transferActivity1).toMatchObject({ + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: erc20Contract.address, + from, + to, + amount, + evm_tx_id: evmEvents[0].evm_tx_id, + cosmos_tx_id: evmEvents[0].tx_id, + }); + const transferActivity2 = erc20Activities[3]; + expect(transferActivity2).toMatchObject({ + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: erc20Contract.address, + from, + to, + amount, + evm_tx_id: evmEvents[1].evm_tx_id, + cosmos_tx_id: evmEvents[1].tx_id, + }); + } + }); + } + + @Test('test build erc20 transfer activity') + async testBuildErc20TransferActivity() { + const evmEvent = { + id: 872436, + tx_id: 9377483, + evm_tx_id: 6789103, + address: '0xf4dcd1ba7a2d862077a12918b9cf1889568b1fc5', topic0: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', topic1: - '0x000000000000000000000000e88f8dcc7508528449ec5d7ff137937857fb60f2', + '0x00000000000000000000000089413d5a8601622a03fd63f8aab595a12e65b9c0', topic2: - '0x00000000000000000000000034e411216012f042fcecbf53b1d05cfa67ee9fe7', + '0x0000000000000000000000004b919d8175dba25dbf733e7dcf9241ea7e51943b', topic3: null, + block_height: 22024821, tx_hash: - '0xf15467ec2a25eeef95798d93c2fe9ed8e7c891578b8e1bcc3284105849656c9d', + '0x1d646b55ef69dc9cf5e6b025b783c947f36d51c9b4e164895bbfe9e2af8b6e22', tx_index: 0, - tx_id: 1612438, - data: Buffer.from( - '\\x00000000000000000000000000000000000000000000000e0732b5a508244750' - ), - block_height: 1, block_hash: - '0x16f84b38d58b8aedbba1e108990f858b09fe8373610659d4c37a67f5c87e09e1', - address: evmSmartContract.address, - evm_tx_id: evmTransaction.id, - }); - await EvmEvent.query().insert(event); - // Call the reindex method - await reindexer.reindex(erc20Contract.address as `0x${string}`); - // Test phase - const erc20Activity = await Erc20Activity.query().first().throwIfNotFound(); + '0x6daa455dda31eb9e09000087bee9540bee9622842d5a423baf82da5b7b534a38', + data: fromBase64('AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADVdLhb6wpEI='), + sender: hexToBytes('0x8812d810EA7CC4e1c3FB45cef19D6a7ECBf2D85D'), + }; const [from, to, amount] = decodeAbiParameters( [ ABI_TRANSFER_PARAMS.FROM, ABI_TRANSFER_PARAMS.TO, ABI_TRANSFER_PARAMS.VALUE, ], - (event.topic1 + - event.topic2.slice(2) + - toHex(event.data).slice(2)) as `0x${string}` + (evmEvent.topic1 + + evmEvent.topic2.slice(2) + + toHex(evmEvent.data).slice(2)) as `0x${string}` ) as [string, string, bigint]; - // Test new activity had been inserted - expect(erc20Activity).toMatchObject({ - action: 'transfer', - erc20_contract_address: erc20Contract.address, + const result = Erc20Handler.buildTransferActivity( + EvmEvent.fromJson(evmEvent), + this.broker.logger + ); + expect(result).toMatchObject({ + evm_event_id: evmEvent.id, + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: evmEvent.address, amount: amount.toString(), from: from.toLowerCase(), to: to.toLowerCase(), + height: evmEvent.block_height, + tx_hash: evmEvent.tx_hash, + evm_tx_id: evmEvent.evm_tx_id, }); - const accountBalances = _.keyBy( - await AccountBalance.query(), - (o) => `${o.account_id}_${o.denom}` + } + + /** + * @input evm event that be approval erc20 + * @result build erc20ApprovalActivity + */ + @Test('test build erc20 approval activity') + async testBuildErc20ApprovalActivity() { + const evmEvent = { + id: 881548, + tx_id: 9381778, + evm_tx_id: 6793335, + address: '0xf4dcd1ba7a2d862077a12918b9cf1889568b1fc5', + topic0: + '0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925', + topic1: + '0x000000000000000000000000e57c921f5f3f3a2aade9a462dab70b0cb97ded4d', + topic2: + '0x000000000000000000000000cbd61600b891a738150e68d5a58646321189cf6f', + topic3: null, + block_height: 22033598, + tx_hash: + '0x89dd0093c3c7633276c20be92fd5838f1eca99314a0c6375e9050e5cc82b51c3', + tx_index: 1, + block_hash: + '0x692c859d6254ef6c27fc7accf1131d55351c62a1357fe261d8517e3144cfbebe', + data: fromBase64('AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='), + sender: hexToBytes('0x8812d810EA7CC4e1c3FB45cef19D6a7ECBf2D85D'), + }; + const result = Erc20Handler.buildApprovalActivity( + EvmEvent.fromJson(evmEvent), + this.broker.logger ); - // from account balance had been reindexed - expect( - accountBalances[`${accounts[0].id}_${erc20Contract.address}`] - ).toMatchObject({ - denom: erc20Contract.address, - amount: `-${amount.toString()}`, + const [from, to, amount] = decodeAbiParameters( + [ + ABI_APPROVAL_PARAMS.OWNER, + ABI_APPROVAL_PARAMS.SPENDER, + ABI_APPROVAL_PARAMS.VALUE, + ], + (evmEvent.topic1 + + evmEvent.topic2.slice(2) + + toHex(evmEvent.data).slice(2)) as `0x${string}` + ) as [string, string, bigint]; + expect(result).toMatchObject({ + evm_event_id: evmEvent.id, + action: ERC20_ACTION.APPROVAL, + erc20_contract_address: evmEvent.address, + amount: amount.toString(), + from: from.toLowerCase(), + to: to.toLowerCase(), + height: evmEvent.block_height, + tx_hash: evmEvent.tx_hash, + evm_tx_id: evmEvent.evm_tx_id, + }); + } + + /** + * @input erc20 transfer activity + * @result from/to account balances be updated + */ + @Test('test handlerErc20Transfer') + async testHandlerErc20Transfer() { + const fromAmount = '4424242424'; + const toAmount = '1123342'; + const totalSupply = '123654'; + const erc20Address = '0x98605ae21dd3be686337a6d7a8f156d0d8baee92'; + const erc20Activity = Erc20Activity.fromJson({ + evm_event_id: 1, + sender: '0x7c756Cba10Ff2C65016494E8BA37C12a108572b5', + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: erc20Address, + amount: '998222', + from: '0x3E665ACfE64628774d3bA8E589Fa8683eD8706C9', + to: '0xD83E708D7FE0E769Af80d990f9241458734808Ac', + height: 10000, + tx_hash: + '0xb97228e533e3af1323d873c9c3e4c0a9b85d95ecd8e98110c8890c9453d2f077', + evm_tx_id: 1, + from_account_id: 123, + to_account_id: 234, + }); + const [fromKey, toKey] = [ + `${erc20Activity.from_account_id}_${erc20Address}`, + `${erc20Activity.to_account_id}_${erc20Address}`, + ]; + const accountBalances: Dictionary = { + [fromKey]: AccountBalance.fromJson({ + denom: erc20Address, + amount: fromAmount, + last_updated_height: 1, + }), + [toKey]: AccountBalance.fromJson({ + denom: erc20Address, + amount: toAmount, + last_updated_height: 1, + }), + }; + const erc20Contracts = { + [erc20Address]: Erc20Contract.fromJson({ + evm_smart_contract_id: 1, + total_supply: totalSupply, + symbol: 'ALPHA', + address: erc20Address, + decimal: '20', + name: 'Alpha Grand Wolf', + track: true, + last_updated_height: 1, + total_holder: 2, + }), + }; + const erc20Handler = new Erc20Handler(accountBalances, [], erc20Contracts); + erc20Handler.handlerErc20Transfer(erc20Activity); + expect(erc20Handler.accountBalances[fromKey]).toMatchObject({ + denom: erc20Address, + amount: (BigInt(fromAmount) - BigInt(erc20Activity.amount)).toString(), + }); + expect(erc20Handler.accountBalances[toKey]).toMatchObject({ + denom: erc20Address, + amount: (BigInt(erc20Activity.amount) + BigInt(toAmount)).toString(), + }); + expect(erc20Contracts[erc20Address].total_supply).toEqual(totalSupply); + } + + @Test.only('test handlerErc20Transfer update total holder') + async testHandlerErc20TransferUpdateTotalHolder() { + const fromAmount = '10'; + const totalSupply = '123654'; + const erc20Address = '0x98605ae21dd3be686337a6d7a8f156d0d8baee92'; + const erc20Activity = Erc20Activity.fromJson({ + evm_event_id: 1, + sender: '0x7c756Cba10Ff2C65016494E8BA37C12a108572b5', + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: erc20Address, + amount: '1', + from: '0x3E665ACfE64628774d3bA8E589Fa8683eD8706C9', + to: '0xD83E708D7FE0E769Af80d990f9241458734808Ac', + height: 10000, + tx_hash: + '0xb97228e533e3af1323d873c9c3e4c0a9b85d95ecd8e98110c8890c9453d2f077', + evm_tx_id: 1, + from_account_id: 123, + to_account_id: 234, + }); + const [fromKey] = [`${erc20Activity.from_account_id}_${erc20Address}`]; + const accountBalances: Dictionary = { + [fromKey]: AccountBalance.fromJson({ + denom: erc20Address, + amount: fromAmount, + last_updated_height: 1, + }), + }; + const erc20Contracts = { + [erc20Address]: Erc20Contract.fromJson({ + evm_smart_contract_id: 1, + total_supply: totalSupply, + symbol: 'ALPHA', + address: erc20Address, + decimal: '20', + name: 'Alpha Grand Wolf', + track: true, + last_updated_height: 1, + total_holder: 2, + }), + }; + const erc20Handler = new Erc20Handler(accountBalances, [], erc20Contracts); + erc20Handler.handlerErc20Transfer(erc20Activity); + expect(erc20Handler.erc20Contracts[erc20Address].total_holder).toEqual(3); + // test transfer all from balance + const erc20ActivityTransferAllFrom = Erc20Activity.fromJson({ + evm_event_id: 1, + sender: '0x7c756Cba10Ff2C65016494E8BA37C12a108572b5', + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: erc20Address, + amount: '9', + from: '0x3E665ACfE64628774d3bA8E589Fa8683eD8706C9', + to: '0xD83E708D7FE0E769Af80d990f9241458734808Ac', + height: 10000, + tx_hash: + '0xb97228e533e3af1323d873c9c3e4c0a9b85d95ecd8e98110c8890c9453d2f077', + evm_tx_id: 1, + from_account_id: 123, + to_account_id: 234, + }); + erc20Handler.handlerErc20Transfer(erc20ActivityTransferAllFrom); + expect(erc20Handler.erc20Contracts[erc20Address].total_holder).toEqual(2); + } + + @Test('test handlerErc20Transfer when from is zero') + async testHandlerErc20TransferWhenFromIsZero() { + const toAmount = '242423234'; + const totalSupply = '123654'; + const erc20Activity = Erc20Activity.fromJson({ + evm_event_id: 1, + sender: '0x7c756Cba10Ff2C65016494E8BA37C12a108572b5', + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: '0x98605ae21dd3be686337a6d7a8f156d0d8baee92', + amount: '12345222', + from: ZERO_ADDRESS, + to: '0xD83E708D7FE0E769Af80d990f9241458734808Ac', + height: 10000, + tx_hash: + '0xb97228e533e3af1323d873c9c3e4c0a9b85d95ecd8e98110c8890c9453d2f077', + evm_tx_id: 1, + from_account_id: 123, + to_account_id: 234, + }); + const erc20Contracts = { + [erc20Activity.erc20_contract_address]: Erc20Contract.fromJson({ + evm_smart_contract_id: 1, + total_supply: totalSupply, + symbol: 'ALPHA', + address: erc20Activity.erc20_contract_address, + decimal: '20', + name: 'Alpha Grand Wolf', + track: true, + last_updated_height: 1, + }), + }; + const [fromKey, toKey] = [ + `${erc20Activity.from_account_id}_${erc20Activity.erc20_contract_address}`, + `${erc20Activity.to_account_id}_${erc20Activity.erc20_contract_address}`, + ]; + const accountBalances: Dictionary = { + [toKey]: AccountBalance.fromJson({ + denom: erc20Activity.erc20_contract_address, + amount: toAmount, + last_updated_height: 1, + }), + }; + const erc20Handler = new Erc20Handler(accountBalances, [], erc20Contracts); + erc20Handler.handlerErc20Transfer(erc20Activity); + expect(erc20Handler.accountBalances[fromKey]).toBeUndefined(); + expect(erc20Handler.accountBalances[toKey]).toMatchObject({ + denom: erc20Activity.erc20_contract_address, + amount: (BigInt(erc20Activity.amount) + BigInt(toAmount)).toString(), }); - // to account balance had been reindexed expect( - accountBalances[`${accounts[1].id}_${erc20Contract.address}`] - ).toMatchObject({ - denom: erc20Contract.address, - amount: amount.toString(), + erc20Contracts[erc20Activity.erc20_contract_address].total_supply + ).toEqual((BigInt(totalSupply) + BigInt(erc20Activity.amount)).toString()); + } + + @Test('test handlerErc20Transfer when to is zero') + async testHandlerErc20TransferWhenToIsZero() { + const balance = '242423234'; + const totalHolder = 2; + const erc20Activity = Erc20Activity.fromJson({ + evm_event_id: 1, + sender: '0x7c756Cba10Ff2C65016494E8BA37C12a108572b5', + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: '0x98605ae21dd3be686337a6d7a8f156d0d8baee92', + amount: '12345222', + from: '0xD83E708D7FE0E769Af80d990f9241458734808Ac', + to: ZERO_ADDRESS, + height: 10000, + tx_hash: + '0xb97228e533e3af1323d873c9c3e4c0a9b85d95ecd8e98110c8890c9453d2f077', + evm_tx_id: 1, + from_account_id: 123, + to_account_id: 234, + }); + const [fromKey, toKey] = [ + `${erc20Activity.from_account_id}_${erc20Activity.erc20_contract_address}`, + `${erc20Activity.to_account_id}_${erc20Activity.erc20_contract_address}`, + ]; + const accountBalances: Dictionary = { + [fromKey]: AccountBalance.fromJson({ + denom: erc20Activity.erc20_contract_address, + amount: balance, + last_updated_height: 1, + }), + }; + const totalSupply = '123654'; + const erc20Contracts = { + [erc20Activity.erc20_contract_address]: Erc20Contract.fromJson({ + evm_smart_contract_id: 1, + total_supply: totalSupply, + symbol: 'ALPHA', + address: erc20Activity.erc20_contract_address, + decimal: '20', + name: 'Alpha Grand Wolf', + track: true, + last_updated_height: 1, + total_holder: totalHolder, + }), + }; + const erc20Handler = new Erc20Handler(accountBalances, [], erc20Contracts); + erc20Handler.handlerErc20Transfer(erc20Activity); + expect(erc20Handler.accountBalances[fromKey]).toMatchObject({ + denom: erc20Activity.erc20_contract_address, + amount: (BigInt(balance) - BigInt(erc20Activity.amount)).toString(), }); - // from account balance without erc20 had been orinal + expect(erc20Handler.accountBalances[toKey]).toBeUndefined(); + expect( + erc20Contracts[erc20Activity.erc20_contract_address].total_supply + ).toEqual((BigInt(totalSupply) - BigInt(erc20Activity.amount)).toString()); expect( - accountBalances[ - `${accounts[0].id}_${accounts[0].account_balances[0].denom}` - ] - ).toMatchObject({ - denom: accounts[0].account_balances[0].denom, - amount: accounts[0].account_balances[0].amount, + erc20Contracts[erc20Activity.erc20_contract_address].total_holder + ).toEqual(totalHolder); + } + + @Test('test handlerErc20Transfer when last_updated_height not suitable') + async testHandlerErc20TransferWhenNotHeight() { + const fromAmount = '23442423'; + const toAmount = '32323232'; + const totalSupply = '123456'; + const erc20Activity = Erc20Activity.fromJson({ + evm_event_id: 1, + sender: '0x7c756Cba10Ff2C65016494E8BA37C12a108572b5', + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: '0x98605ae21dd3be686337a6d7a8f156d0d8baee92', + amount: '12345222', + from: '0x3E665ACfE64628774d3bA8E589Fa8683eD8706C9', + to: '0xD83E708D7FE0E769Af80d990f9241458734808Ac', + height: 10000, + tx_hash: + '0xb97228e533e3af1323d873c9c3e4c0a9b85d95ecd8e98110c8890c9453d2f077', + evm_tx_id: 1, + from_account_id: 123, + to_account_id: 234, }); + const erc20Contracts = { + [erc20Activity.erc20_contract_address]: Erc20Contract.fromJson({ + evm_smart_contract_id: 1, + total_supply: totalSupply, + symbol: 'ALPHA', + address: erc20Activity.erc20_contract_address, + decimal: '20', + name: 'Alpha Grand Wolf', + track: true, + last_updated_height: 1, + }), + }; + const [fromKey, toKey] = [ + `${erc20Activity.from_account_id}_${erc20Activity.erc20_contract_address}`, + `${erc20Activity.to_account_id}_${erc20Activity.erc20_contract_address}`, + ]; + const accountBalances: Dictionary = { + [fromKey]: AccountBalance.fromJson({ + denom: erc20Activity.erc20_contract_address, + amount: fromAmount, + last_updated_height: 10001, + }), + [toKey]: AccountBalance.fromJson({ + denom: erc20Activity.erc20_contract_address, + amount: toAmount, + last_updated_height: 1, + }), + }; + const erc20Handler = new Erc20Handler(accountBalances, [], erc20Contracts); + expect(() => erc20Handler.handlerErc20Transfer(erc20Activity)).toThrow( + `Process erc20 balance: fromAccountBalance ${erc20Activity.from} was updated` + ); + } + + @Test('test handlerErc20Transfer when from/to is erc20 module account') + async testHandlerErc20TransferWhenToIsErc20ModuleAccount() { + const erc20Activity = Erc20Activity.fromJson({ + evm_event_id: 1, + sender: '0x7c756Cba10Ff2C65016494E8BA37C12a108572b5', + action: ERC20_ACTION.TRANSFER, + erc20_contract_address: '0x98605ae21dd3be686337a6d7a8f156d0d8baee92', + amount: '12345222', + from: '0xD83E708D7FE0E769Af80d990f9241458734808Ac', + to: ZERO_ADDRESS, + height: 10000, + tx_hash: + '0xb97228e533e3af1323d873c9c3e4c0a9b85d95ecd8e98110c8890c9453d2f077', + evm_tx_id: 1, + from_account_id: 123, + to_account_id: 234, + }); + const totalSupply = '123654'; + const erc20Contracts = { + [erc20Activity.erc20_contract_address]: Erc20Contract.fromJson({ + evm_smart_contract_id: 1, + total_supply: totalSupply, + symbol: 'ALPHA', + address: erc20Activity.erc20_contract_address, + decimal: '20', + name: 'Alpha Grand Wolf', + track: true, + last_updated_height: 1, + }), + }; + const erc20Handler = new Erc20Handler({}, [], erc20Contracts); + erc20Handler.handlerErc20Transfer(erc20Activity); + expect( + erc20Contracts[erc20Activity.erc20_contract_address].total_supply + ).toEqual((BigInt(totalSupply) - BigInt(erc20Activity.amount)).toString()); } } From 2a17c0046fdffbaee478d35d984155c5352ff6f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ph=E1=BA=A1m=20Th=C3=A0nh=20Phong?= <49814372+phamphong9981@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:06:08 +0700 Subject: [PATCH 08/11] refactor: index performance (#909) --- migrations/evm/20240924041138_index_performance.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 migrations/evm/20240924041138_index_performance.ts diff --git a/migrations/evm/20240924041138_index_performance.ts b/migrations/evm/20240924041138_index_performance.ts new file mode 100644 index 000000000..c2149d231 --- /dev/null +++ b/migrations/evm/20240924041138_index_performance.ts @@ -0,0 +1,13 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.raw('set statement_timeout to 0'); + await knex.raw( + 'CREATE INDEX IF NOT EXISTS erc721_token_erc721_contract_address_id_index ON erc721_token(erc721_contract_address, id)' + ); + await knex.raw( + 'CREATE INDEX IF NOT EXISTS erc721_holder_statistic_erc721_contract_address_count_index ON erc721_holder_statistic(erc721_contract_address, count)' + ); +} + +export async function down(knex: Knex): Promise {} From 9553e92825f68644f52035d8450aafb7fee24f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ph=E1=BA=A1m=20Th=C3=A0nh=20Phong?= <49814372+phamphong9981@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:16:21 +0700 Subject: [PATCH 09/11] feat: erc20 contract no action (#908) * feat: erc20 contract no action * refactor: review --- ...20240923064605_erc20_contract_no_action.ts | 72 +++++++++++++++++++ src/models/erc20_contract.ts | 2 + src/services/evm/erc20_handler.ts | 5 ++ test/unit/services/evm/erc20.spec.ts | 30 ++++++++ 4 files changed, 109 insertions(+) create mode 100644 migrations/evm/20240923064605_erc20_contract_no_action.ts diff --git a/migrations/evm/20240923064605_erc20_contract_no_action.ts b/migrations/evm/20240923064605_erc20_contract_no_action.ts new file mode 100644 index 000000000..7dc850de1 --- /dev/null +++ b/migrations/evm/20240923064605_erc20_contract_no_action.ts @@ -0,0 +1,72 @@ +import { Knex } from 'knex'; +import { Erc20Activity } from '../../src/models'; +import { ERC20_ACTION } from '../../src/services/evm/erc20_handler'; +import _ from 'lodash'; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('erc20_contract', (table) => { + table.jsonb('total_actions').defaultTo('{}'); + }); + await knex.raw(`set statement_timeout to 0`); + const [totalTransfer, totalApproval, totalDeposit, totalWithdrawal] = + await Promise.all([ + _.keyBy( + await Erc20Activity.query(knex) + .where('action', ERC20_ACTION.TRANSFER) + .groupBy('erc20_activity.erc20_contract_address') + .select('erc20_activity.erc20_contract_address') + .count('* as transfer'), + 'erc20_contract_address' + ), + _.keyBy( + await Erc20Activity.query(knex) + .where('action', ERC20_ACTION.APPROVAL) + .groupBy('erc20_activity.erc20_contract_address') + .select('erc20_activity.erc20_contract_address') + .count('* as approval'), + 'erc20_contract_address' + ), + _.keyBy( + await Erc20Activity.query(knex) + .where('action', ERC20_ACTION.DEPOSIT) + .groupBy('erc20_activity.erc20_contract_address') + .select('erc20_activity.erc20_contract_address') + .count('* as deposit'), + 'erc20_contract_address' + ), + _.keyBy( + await Erc20Activity.query(knex) + .where('action', ERC20_ACTION.WITHDRAWAL) + .groupBy('erc20_activity.erc20_contract_address') + .select('erc20_activity.erc20_contract_address') + .count('* as withdrawal'), + 'erc20_contract_address' + ), + ]); + const totalAction = _.merge( + totalTransfer, + totalApproval, + totalDeposit, + totalWithdrawal + ); + const erc20ContractsAddr = Object.keys(totalAction); + if (erc20ContractsAddr.length > 0) { + const stringListUpdates = erc20ContractsAddr + .map( + (addr) => + `('${addr}', '${JSON.stringify( + _.omit(totalAction[addr], 'erc20_contract_address') + )}'::jsonb)` + ) + .join(','); + await knex.raw( + `UPDATE erc20_contract SET total_actions = temp.total_actions from (VALUES ${stringListUpdates}) as temp(address, total_actions) where temp.address = erc20_contract.address` + ); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('erc20_contract', (table) => { + table.dropColumn('total_actions'); + }); +} diff --git a/src/models/erc20_contract.ts b/src/models/erc20_contract.ts index cf07704a4..afcab11b0 100644 --- a/src/models/erc20_contract.ts +++ b/src/models/erc20_contract.ts @@ -27,6 +27,8 @@ export class Erc20Contract extends BaseModel { last_updated_height!: number; + total_actions!: any; + static get tableName() { return 'erc20_contract'; } diff --git a/src/services/evm/erc20_handler.ts b/src/services/evm/erc20_handler.ts index c51e46361..9b550ea93 100644 --- a/src/services/evm/erc20_handler.ts +++ b/src/services/evm/erc20_handler.ts @@ -86,6 +86,11 @@ export class Erc20Handler { ) { this.handlerErc20Transfer(erc20Activity); } + this.erc20Contracts[erc20Activity.erc20_contract_address].total_actions[ + erc20Activity.action + ] = + (this.erc20Contracts[erc20Activity.erc20_contract_address] + .total_actions[erc20Activity.action] || 0) + 1; }); } diff --git a/test/unit/services/evm/erc20.spec.ts b/test/unit/services/evm/erc20.spec.ts index dbd332db0..2587e84bd 100644 --- a/test/unit/services/evm/erc20.spec.ts +++ b/test/unit/services/evm/erc20.spec.ts @@ -224,6 +224,20 @@ export class Erc20Test { '0x21f905f14a26c5b35e43e1bfbcf8ff395e453d6497a0ce0a0c3cd7814ec0ba03', evm_tx_id: this.evmTx.id, }, + { + id: 3, + evm_event_id: this.evmEvent.id, + sender: '0xDF587daaC47ae7B5586E34bCdb23d0b900b18a6C', + action: 'approval', + erc20_contract_address: this.evmSmartContract.address, + amount: '2211', + from: this.account2.evm_address, + to: this.account3.evm_address, + height: this.blockCheckpoints[0].height + 1, + tx_hash: + '0x21f905f14a26c5b35e43e1bfbcf8ff395e453d6497a0ce0a0c3cd7814ec0ba03', + evm_tx_id: this.evmTx.id, + }, ]; await Erc20Activity.query().insert(erc20Activities); await this.erc20Service.handleErc20Balance(); @@ -247,6 +261,14 @@ export class Erc20Test { accountBalances[`${this.account3.id}_${this.evmSmartContract.address}`] .amount ).toEqual(`${BigInt(erc20Activities[1].amount).toString()}`); + const erc20Contract = await Erc20Contract.query() + .where('address', this.evmSmartContract.address) + .first() + .throwIfNotFound(); + expect(erc20Contract.total_actions).toEqual({ + transfer: erc20Activities.length - 1, + approval: 1, + }); } @Test('handle erc20 balance with missing account') @@ -344,5 +366,13 @@ export class Erc20Test { accountBalances[`${this.account3.id}_${this.evmSmartContract.address}`] .amount ).toEqual(`${BigInt(erc20Activities[1].amount).toString()}`); + const erc20Contract = await Erc20Contract.query() + .where('address', this.evmSmartContract.address) + .first() + .throwIfNotFound(); + expect(erc20Contract.total_actions).toEqual({ + transfer: erc20Activities.length + 2, + approval: 1, + }); } } From e8d9903640533ffd0c80ad7481127a00c23af8ce Mon Sep 17 00:00:00 2001 From: Tuan Phan Anh <38557844+fibonacci998@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:00:52 +0700 Subject: [PATCH 10/11] feat: optimize query daily statistic (#910) --- ...4858_create_index_created_at_in_account.ts | 14 +++++ .../evm/statistic/daily_statistics.service.ts | 62 ++++++++++++------- 2 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 migrations/evm/20240924084858_create_index_created_at_in_account.ts diff --git a/migrations/evm/20240924084858_create_index_created_at_in_account.ts b/migrations/evm/20240924084858_create_index_created_at_in_account.ts new file mode 100644 index 000000000..99ac13595 --- /dev/null +++ b/migrations/evm/20240924084858_create_index_created_at_in_account.ts @@ -0,0 +1,14 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.raw('SET statement_timeout TO 0'); + await knex.schema.alterTable('account', (table) => { + table.index('created_at'); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('account', (table) => { + table.dropIndex('created_at'); + }); +} diff --git a/src/services/evm/statistic/daily_statistics.service.ts b/src/services/evm/statistic/daily_statistics.service.ts index 433e8f40c..d2cc0cbe2 100644 --- a/src/services/evm/statistic/daily_statistics.service.ts +++ b/src/services/evm/statistic/daily_statistics.service.ts @@ -12,6 +12,7 @@ import { EVMTransaction, } from '../../../models'; import { BULL_JOB_NAME, SERVICE } from '../constant'; +import knex from '../../../common/utils/db_connection'; import config from '../../../../config.json' assert { type: 'json' }; import BullableService, { QueueHandler } from '../../../base/bullable.service'; @@ -53,7 +54,6 @@ export default class DailyEVMStatisticsService extends BullableService { @QueueHandler({ queueName: BULL_JOB_NAME.CRAWL_DAILY_EVM_STATISTICS, jobName: BULL_JOB_NAME.CRAWL_DAILY_EVM_STATISTICS, - // prefix: `horoscope-v2-${config.chainId}`, }) public async handleJob(_payload: { date: string }): Promise { const endTime = dayjs.utc(_payload.date).startOf('day').toDate(); @@ -62,36 +62,56 @@ export default class DailyEVMStatisticsService extends BullableService { `Get daily statistic events for day ${new Date(startTime)}` ); - const [startBlock, endBlock] = await Promise.all([ - EVMBlock.query() - .select('height') - .where('timestamp', '>=', startTime) - .limit(1) - .orderBy('height'), - EVMBlock.query() - .select('height') - .where('timestamp', '<', endTime) - .limit(1) - .orderBy('height', 'desc'), - ]); + const [startBlock, endBlock, dailyStatisticPreviousDay] = await Promise.all( + [ + EVMBlock.query() + .select('height') + .where('timestamp', '>=', startTime) + .limit(1) + .orderBy('height'), + EVMBlock.query() + .select('height') + .where('timestamp', '<', endTime) + .limit(1) + .orderBy('height', 'desc'), + DailyStatistics.query().findOne( + 'date', + dayjs.utc(startTime).subtract(1, 'day').toDate().toISOString() + ), + ] + ); - const [dailyTxs, totalAddresses] = await Promise.all([ + const [todayAccounts, totalTxs, totalActiveAddress] = await Promise.all([ + Account.query() + .count('id') + .findOne( + knex.raw(`created_at >= '${startTime.toISOString()}'::timestamp`) + ) + .andWhere( + knex.raw(`created_at < '${endTime.toISOString()}'::timestamp`) + ), EVMTransaction.query() - .where('height', '>=', startBlock[0].height) + .count() + .findOne('height', '>=', startBlock[0].height) + .andWhere('height', '<=', endBlock[0].height), + EVMTransaction.query() + .count(knex.raw('distinct("from")')) + .findOne('height', '>=', startBlock[0].height) .andWhere('height', '<=', endBlock[0].height), - Account.query().count('id'), ]); - const activeAddrs = Array.from(new Set(dailyTxs.map((tx) => tx.from))); - const dailyStat = DailyStatistics.fromJson({ - daily_txs: dailyTxs.length, - daily_active_addresses: activeAddrs.length, - unique_addresses: Number(totalAddresses[0].count), + daily_txs: totalTxs?.count, + daily_active_addresses: totalActiveAddress?.count, + unique_addresses: + // eslint-disable-next-line no-unsafe-optional-chaining + Number(dailyStatisticPreviousDay?.unique_addresses) + + Number(todayAccounts?.count), date: startTime.toISOString(), }); this.logger.info(`Insert new daily statistic for date ${startTime}`); + this.logger.info(dailyStat); await DailyStatistics.query() .insert(dailyStat) .catch((error) => { From e5e05c2e7d207ad9bf86553605d4db2a6eb23577 Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Wed, 25 Sep 2024 11:25:19 +0700 Subject: [PATCH 11/11] fix: migration --- ...20240923064605_erc20_contract_no_action.ts | 56 ++++++++++++------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/migrations/evm/20240923064605_erc20_contract_no_action.ts b/migrations/evm/20240923064605_erc20_contract_no_action.ts index 7dc850de1..89fc9bdde 100644 --- a/migrations/evm/20240923064605_erc20_contract_no_action.ts +++ b/migrations/evm/20240923064605_erc20_contract_no_action.ts @@ -11,35 +11,51 @@ export async function up(knex: Knex): Promise { const [totalTransfer, totalApproval, totalDeposit, totalWithdrawal] = await Promise.all([ _.keyBy( - await Erc20Activity.query(knex) - .where('action', ERC20_ACTION.TRANSFER) - .groupBy('erc20_activity.erc20_contract_address') - .select('erc20_activity.erc20_contract_address') - .count('* as transfer'), + ( + await Erc20Activity.query(knex) + .where('action', ERC20_ACTION.TRANSFER) + .groupBy('erc20_activity.erc20_contract_address') + .select( + 'erc20_activity.erc20_contract_address', + knex.raw('count(*)::integer as transfer') + ) + ).map((e) => e.toJSON()), 'erc20_contract_address' ), _.keyBy( - await Erc20Activity.query(knex) - .where('action', ERC20_ACTION.APPROVAL) - .groupBy('erc20_activity.erc20_contract_address') - .select('erc20_activity.erc20_contract_address') - .count('* as approval'), + ( + await Erc20Activity.query(knex) + .where('action', ERC20_ACTION.APPROVAL) + .groupBy('erc20_activity.erc20_contract_address') + .select( + 'erc20_activity.erc20_contract_address', + knex.raw('count(*)::integer as approval') + ) + ).map((e) => e.toJSON()), 'erc20_contract_address' ), _.keyBy( - await Erc20Activity.query(knex) - .where('action', ERC20_ACTION.DEPOSIT) - .groupBy('erc20_activity.erc20_contract_address') - .select('erc20_activity.erc20_contract_address') - .count('* as deposit'), + ( + await Erc20Activity.query(knex) + .where('action', ERC20_ACTION.DEPOSIT) + .groupBy('erc20_activity.erc20_contract_address') + .select( + 'erc20_activity.erc20_contract_address', + knex.raw('count(*)::integer as deposit') + ) + ).map((e) => e.toJSON()), 'erc20_contract_address' ), _.keyBy( - await Erc20Activity.query(knex) - .where('action', ERC20_ACTION.WITHDRAWAL) - .groupBy('erc20_activity.erc20_contract_address') - .select('erc20_activity.erc20_contract_address') - .count('* as withdrawal'), + ( + await Erc20Activity.query(knex) + .where('action', ERC20_ACTION.WITHDRAWAL) + .groupBy('erc20_activity.erc20_contract_address') + .select( + 'erc20_activity.erc20_contract_address', + knex.raw('count(*)::integer as withdrawal') + ) + ).map((e) => e.toJSON()), 'erc20_contract_address' ), ]);