diff --git a/.vscode/launch.json b/.vscode/launch.json index 42955131a..0328c95f9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,7 +19,8 @@ "src/services/**/*.service.ts", "--config", "src/moleculer.config.ts" - ] + ], + "console": "integratedTerminal" }, { "type": "node", diff --git a/ci/config.json.ci b/ci/config.json.ci index c0fcb2df6..b61b3cc8b 100644 --- a/ci/config.json.ci +++ b/ci/config.json.ci @@ -148,7 +148,10 @@ }, "dashboardStatistics": { "millisecondCrawl": 10000, - "queryPageLimit": 100 + "queryPageLimit": 100, + "currentInflation": 0.05, + "currentCommunityTax": 0, + "denomSupply": "stake" }, "graphiqlApi": { "hasuraRole": "internal_service", @@ -470,7 +473,8 @@ }, "transport": { "batchSize": 10, - "waitMilisecond": 1000 + "waitMilisecond": 1000, + "timeout": 10000 } }, "crawlEvmAccount": { diff --git a/config.json b/config.json index f9aacc2ee..0b1ad6ca0 100644 --- a/config.json +++ b/config.json @@ -141,7 +141,10 @@ }, "dashboardStatistics": { "millisecondCrawl": 10000, - "queryPageLimit": 100 + "queryPageLimit": 100, + "currentInflation": 0.05, + "currentCommunityTax": 0, + "denomSupply": "stake" }, "feegrant": { "updateFeegrant": { @@ -468,7 +471,8 @@ }, "transport": { "batchSize": 10, - "waitMilisecond": 1000 + "waitMilisecond": 1000, + "timeout": 10000 } }, "crawlEvmAccount": { diff --git a/migrations/20240909092417_create_index_validator.ts b/migrations/20240909092417_create_index_validator.ts new file mode 100644 index 000000000..ed7e24230 --- /dev/null +++ b/migrations/20240909092417_create_index_validator.ts @@ -0,0 +1,15 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('validator', (table) => { + table.index('tokens'); + table.index('status'); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('validator', (table) => { + table.dropIndex('tokens'); + table.dropIndex('status'); + }); +} diff --git a/migrations/20240327075001_evm_internal_transaction.ts b/migrations/evm/20240327075001_evm_internal_transaction.ts similarity index 88% rename from migrations/20240327075001_evm_internal_transaction.ts rename to migrations/evm/20240327075001_evm_internal_transaction.ts index b0c5be09b..6898117af 100644 --- a/migrations/20240327075001_evm_internal_transaction.ts +++ b/migrations/evm/20240327075001_evm_internal_transaction.ts @@ -1,5 +1,5 @@ import { Knex } from 'knex'; -import { EvmInternalTransaction } from '../src/models/evm_internal_transaction'; +import { EvmInternalTransaction } from '../../src/models/evm_internal_transaction'; export async function up(knex: Knex): Promise { await knex.schema.createTable(EvmInternalTransaction.tableName, (table) => { diff --git a/migrations/evm/20240909025646_optimize_evm_tx_schema.ts b/migrations/evm/20240909025646_optimize_evm_tx_schema.ts new file mode 100644 index 000000000..960db07ae --- /dev/null +++ b/migrations/evm/20240909025646_optimize_evm_tx_schema.ts @@ -0,0 +1,61 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.raw('SET statement_timeout TO 0'); + await knex.schema.alterTable('evm_transaction', (table) => { + table.dropIndex('tx_id'); + table.dropIndex('tx_msg_id'); + table.dropIndex('from'); + table.dropIndex('to'); + table.dropIndex('hash'); + table.dropIndex('status'); + table.dropIndex('contract_address'); + }); + await knex.raw(` + alter table evm_transaction + alter column "from" set data type bytea USING decode(substring("from",3), 'hex'), + alter column "to" set data type bytea USING decode(substring("to",3), 'hex'), + alter column "data" set data type bytea USING decode("data", 'hex'), + alter column "hash" set data type bytea USING decode(substring("hash",3), 'hex'), + alter column "contract_address" set data type bytea USING decode(substring("contract_address",3), 'hex')`); + await knex.schema.alterTable('evm_transaction', (table) => { + table.index('from', 'evm_transaction_from_index', 'hash'); + table.index('to', 'evm_transaction_to_index', 'hash'); + table.index( + 'contract_address', + 'evm_transaction_contract_address_index', + 'hash' + ); + table.index('hash', 'evm_transaction_hash_index', 'hash'); + }); +} + +export async function down(knex: Knex): Promise { + await knex.raw('SET statement_timeout TO 0'); + await knex.schema.alterTable('evm_transaction', (table) => { + table.dropIndex('from'); + table.dropIndex('to'); + table.dropIndex('hash'); + table.dropIndex('contract_address'); + }); + await knex.raw(` + alter table evm_transaction + alter column "from" set data type character varying(255), + alter column "to" set data type character varying(255), + alter column "hash" set data type character varying(255), + alter column "data" set data type text, + alter column "contract_address" set data type character varying(255)`); + await knex.schema.alterTable('evm_transaction', (table) => { + table.index('tx_id', 'evm_transaction_tx_id_index', 'btree'); + table.index('tx_msg_id', 'evm_transaction_tx_msg_id_index', 'btree'); + table.index('status', 'evm_transaction_status_index', 'btree'); + table.index('from', 'evm_transaction_from_index', 'btree'); + table.index('to', 'evm_transaction_to_index', 'btree'); + table.index( + 'contract_address', + 'evm_transaction_contract_address_index', + 'btree' + ); + table.index('hash', 'evm_transaction_hash_index', 'btree'); + }); +} diff --git a/migrations/evm/20240909092542_create_index_miner_in_evm_block.ts b/migrations/evm/20240909092542_create_index_miner_in_evm_block.ts new file mode 100644 index 000000000..18865d2af --- /dev/null +++ b/migrations/evm/20240909092542_create_index_miner_in_evm_block.ts @@ -0,0 +1,13 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('evm_block', (table) => { + table.index('miner'); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('evm_block', (table) => { + table.dropIndex('miner'); + }); +} diff --git a/src/common/utils/etherjs_client.ts b/src/common/utils/etherjs_client.ts index 3207457ad..94dac2ac8 100644 --- a/src/common/utils/etherjs_client.ts +++ b/src/common/utils/etherjs_client.ts @@ -27,6 +27,7 @@ export function getViemClient(chainId?: string): PublicClient { batchSize: config.viemConfig.transport.batchSize, wait: config.viemConfig.transport.waitMilisecond, }, + timeout: config.viemConfig.transport.timeout, }), }).extend(publicActionsL1()); viemClientMapping.set(chainId ?? config.chainId, viemClient); diff --git a/src/models/evm_transaction.ts b/src/models/evm_transaction.ts index b4ae24149..7970b852c 100644 --- a/src/models/evm_transaction.ts +++ b/src/models/evm_transaction.ts @@ -12,13 +12,13 @@ export class EVMTransaction extends BaseModel { id!: number; - hash!: string; + hash!: Buffer; height!: number; - from!: string; + from!: Buffer; - to!: string; + to!: Buffer; size!: string; @@ -30,7 +30,7 @@ export class EVMTransaction extends BaseModel { gas_tip_cap!: bigint; - data!: string; + data!: Buffer; nonce!: bigint; @@ -38,7 +38,7 @@ export class EVMTransaction extends BaseModel { tx_id!: number; - contract_address!: string; + contract_address!: Buffer; index!: number; @@ -65,9 +65,8 @@ export class EVMTransaction extends BaseModel { static get jsonSchema() { return { type: 'object', - required: ['hash', 'height'], + required: ['height'], properties: { - hash: { type: 'string' }, height: { type: 'number' }, }, }; diff --git a/src/services/evm/constant.ts b/src/services/evm/constant.ts index 282a1a8e5..0dc717a69 100644 --- a/src/services/evm/constant.ts +++ b/src/services/evm/constant.ts @@ -33,7 +33,7 @@ export const ABI_CHECK_INTERFACE_ERC_1155 = [ export const EVM_CONTRACT_METHOD_HEX_PREFIX = { // https://ethereum.stackexchange.com/questions/124906/how-to-tell-if-a-transaction-is-contract-creation - CREATE_CONTRACT: '60806040', + CREATE_CONTRACT: '0x60806040', ABI_INTERFACE_ERC20: ABI_CHECK_INTERFACE_ERC_20.map((method) => keccak256(toBytes(method)).slice(2, 10) ), diff --git a/src/services/evm/crawl_contract_evm.service.ts b/src/services/evm/crawl_contract_evm.service.ts index d98376d27..3b2cc5171 100644 --- a/src/services/evm/crawl_contract_evm.service.ts +++ b/src/services/evm/crawl_contract_evm.service.ts @@ -2,7 +2,12 @@ import { Service } from '@ourparentcenter/moleculer-decorators-extended'; import { whatsabi } from '@shazow/whatsabi'; import _, { Dictionary } from 'lodash'; import { ServiceBroker } from 'moleculer'; -import { GetBytecodeReturnType, PublicClient, keccak256 } from 'viem'; +import { + GetBytecodeReturnType, + PublicClient, + bytesToHex, + keccak256, +} from 'viem'; import config from '../../../config.json' assert { type: 'json' }; import BullableService, { QueueHandler } from '../../base/bullable.service'; import knex from '../../common/utils/db_connection'; @@ -112,6 +117,12 @@ export default class CrawlSmartContractEVMService extends BullableService { let addresses: string[] = []; const txCreationWithAdressses: Dictionary = {}; evmTxs.forEach((evmTx: any) => { + ['hash', 'from', 'to', 'data', 'contractAddress'].forEach((key) => { + if (evmTx[key]) { + // eslint-disable-next-line no-param-reassign + evmTx[key] = bytesToHex(evmTx[key]); + } + }); const { data, contractAddress } = evmTx; let currentAddresses: string[] = []; ['from', 'to', 'contractAddress'].forEach((key) => { diff --git a/src/services/evm/crawl_evm_account.service.ts b/src/services/evm/crawl_evm_account.service.ts index 5dc0dcdc2..2cb8b5b8b 100644 --- a/src/services/evm/crawl_evm_account.service.ts +++ b/src/services/evm/crawl_evm_account.service.ts @@ -6,7 +6,7 @@ import { import { Knex } from 'knex'; import _, { Dictionary } from 'lodash'; import { Context } from 'moleculer'; -import { PublicClient } from 'viem'; +import { bytesToHex, PublicClient } from 'viem'; import config from '../../../config.json' assert { type: 'json' }; import '../../../fetch-polyfill.js'; import BullableService, { QueueHandler } from '../../base/bullable.service'; @@ -70,7 +70,7 @@ export default class CrawlEvmAccountService extends BullableService { BULL_JOB_NAME.CRAWL_EVM_TRANSACTION, BULL_JOB_NAME.EVM_CRAWL_INTERNAL_TX, ], - config.evmCrawlInternalTx.key + config.crawlEvmAccount.key ); this.logger.info( `Crawl evm_account from block ${startBlock} to ${endBlock}` @@ -106,15 +106,21 @@ export default class CrawlEvmAccountService extends BullableService { .andWhere('evm_tx_id', '<=', toTx.id) .andWhere('evm_internal_transaction.error', null) .select('evm_internal_transaction.from', 'evm_internal_transaction.to'); - participants.forEach((partcicipant) => { - if (partcicipant.from !== ZERO_ADDRESS) { - accountsAddress.add(partcicipant.from); + participants.forEach((participant) => { + ['from', 'to', 'contractAddress'].forEach((key) => { + if (participant[key]) { + // eslint-disable-next-line no-param-reassign + participant[key] = bytesToHex(participant[key]); + } + }); + if (String(participant.from) !== ZERO_ADDRESS) { + accountsAddress.add(String(participant.from)); } - if (partcicipant.to && partcicipant.to !== ZERO_ADDRESS) { - accountsAddress.add(partcicipant.to); + if (participant.to && String(participant.to) !== ZERO_ADDRESS) { + accountsAddress.add(String(participant.to)); } - if (partcicipant.contractAddress) { - accountsAddress.add(partcicipant.contractAddress); + if (participant.contractAddress) { + accountsAddress.add(participant.contractAddress); } }); participantsFromInternal.forEach((participant) => { diff --git a/src/services/evm/crawl_evm_proxy_history.service.ts b/src/services/evm/crawl_evm_proxy_history.service.ts index bdee65458..d9e258794 100644 --- a/src/services/evm/crawl_evm_proxy_history.service.ts +++ b/src/services/evm/crawl_evm_proxy_history.service.ts @@ -75,15 +75,17 @@ export default class CrawlProxyContractEVMService extends BullableService { .where('block_height', '>', startBlock) .andWhere('block_height', '<=', endBlock) .select('address', 'topic0', 'topic1', 'block_height', 'tx_hash'); + const distictAddresses = _.uniq(evmEvents.map((e) => e.address)); const proxyContractDb = await EVMSmartContract.query() .whereIn('type', [ EVMSmartContract.TYPES.PROXY_EIP_1967, EVMSmartContract.TYPES.PROXY_EIP_1822, EVMSmartContract.TYPES.PROXY_OPEN_ZEPPELIN_IMPLEMENTATION, EVMSmartContract.TYPES.PROXY_EIP_1167, + EVMSmartContract.TYPES.PROXY_EIP_1967_BEACON, ]) + .andWhere('address', 'in', distictAddresses) .select('address'); - const distictAddresses = _.uniq(evmEvents.map((e) => e.address)); const batchReqs: any[] = []; distictAddresses.forEach((address) => { batchReqs.push( @@ -114,7 +116,7 @@ export default class CrawlProxyContractEVMService extends BullableService { }) as EVMSmartContract; const firstTimeCatchProxyEvent = proxyContractDb.find((proxy) => proxy.address === evmEvent.address) && - anyProxyHistoryByAddress[evmEvent.address]; + !anyProxyHistoryByAddress[evmEvent.address]; const newJSONProxy: Dictionary = {}; switch (evmEvent.topic0) { @@ -139,22 +141,29 @@ export default class CrawlProxyContractEVMService extends BullableService { // break; default: if (firstTimeCatchProxyEvent) { - implementationAddress = await this.contractHelper.isContractProxy( - evmEvent.address, - _.find( - EIPProxyContractSupportByteCode, - (value, __) => value.TYPE === evmEventProxy.type - )?.SLOT, - undefined, - bytecodes[evmEvent.address] - ); + implementationAddress = ( + await this.contractHelper.isContractProxy( + evmEvent.address, + _.find( + EIPProxyContractSupportByteCode, + (value, __) => value.TYPE === evmEventProxy.type + )?.SLOT, + undefined, + bytecodes[evmEvent.address] + ) + )?.logicContractAddress; } break; } newJSONProxy.proxy_contract = _.toLower(evmEvent.address); - newJSONProxy.implementation_contract = - _.toLower(implementationAddress as string) || null; + if (implementationAddress) { + newJSONProxy.implementation_contract = _.toLower( + implementationAddress as string + ); + } else { + newJSONProxy.implementation_contract = null; + } newJSONProxy.block_height = evmEvent.block_height; newJSONProxy.tx_hash = evmEvent.tx_hash; newJSONProxy.last_updated_height = lastUpdatedHeight; @@ -179,7 +188,7 @@ export default class CrawlProxyContractEVMService extends BullableService { ) { newProxyContractsToSave.push(proxyHistory); } else { - this.logger.warn( + this.logger.debug( `This contract address ${proxyHistory.proxy_contract} is not proxy, at tx hash ${proxyHistory.tx_hash}` ); } diff --git a/src/services/evm/crawl_evm_transaction.service.ts b/src/services/evm/crawl_evm_transaction.service.ts index fc9ddf9e0..d0a4cfe7c 100644 --- a/src/services/evm/crawl_evm_transaction.service.ts +++ b/src/services/evm/crawl_evm_transaction.service.ts @@ -1,6 +1,11 @@ import { Service } from '@ourparentcenter/moleculer-decorators-extended'; import { fromHex } from '@cosmjs/encoding'; -import { PublicClient, FormattedTransaction } from 'viem'; +import { + PublicClient, + FormattedTransaction, + hexToBytes, + bytesToHex, +} from 'viem'; import _ from 'lodash'; import { OpStackTransactionReceipt } from 'viem/chains'; import { BlockCheckpoint, EVMBlock } from '../../models'; @@ -45,9 +50,7 @@ export default class CrawlEvmTransactionService extends BullableService { .where('height', '>', startBlock) .andWhere('height', '<=', endBlock) .orderBy('height', 'asc'); - const { evmTxs, evmEvents } = await this.getEVMTxsFromBlocks(blocks); - await knex.transaction(async (trx) => { if (evmTxs.length > 0) { const insertedTxByHash = _.keyBy( @@ -59,22 +62,21 @@ export default class CrawlEvmTransactionService extends BullableService { ) .returning(['id', 'hash']) .transacting(trx), - 'hash' + (e) => bytesToHex(e.hash) ); if (evmEvents.length > 0) { - evmEvents.forEach((evmEvent) => { + // eslint-disable-next-line array-callback-return + evmEvents.map((evmEvent) => { // eslint-disable-next-line no-param-reassign evmEvent.evm_tx_id = insertedTxByHash[evmEvent.tx_hash].id; }); - const resultInsert = await knex + await knex .batchInsert( 'evm_event', evmEvents, config.crawlEvmTransaction.chunkSize ) - .returning('id') .transacting(trx); - this.logger.debug('result insert evmEvents: ', resultInsert); } } if (blockCheckpoint) { @@ -145,10 +147,10 @@ export default class CrawlEvmTransactionService extends BullableService { ); evmTxs.push({ - from: offchainTx.from.toLowerCase(), - to: offchainTx.to?.toLowerCase(), - hash: offchainTx.hash, - data: offchainTx.input ? offchainTx.input.substring(2) : null, + from: offchainTx.from ? hexToBytes(offchainTx.from) : null, + to: offchainTx.to ? hexToBytes(offchainTx.to) : null, + hash: offchainTx.hash ? hexToBytes(offchainTx.hash) : null, + data: offchainTx.input ? hexToBytes(offchainTx.input) : null, nonce: offchainTx.nonce, height: offchainTx.blockNumber, index: offchainTx.transactionIndex, @@ -157,7 +159,9 @@ export default class CrawlEvmTransactionService extends BullableService { gas: offchainTx.gas, type: offchainTx.type, status: Number(receiptTx.status), - contract_address: receiptTx.contractAddress, + contract_address: receiptTx.contractAddress + ? hexToBytes(receiptTx.contractAddress) + : null, value: offchainTx.value, timestamp: offchainTx.timestamp, additional_data: config.crawlEvmTransaction.additionalData.optimism diff --git a/src/services/evm/erc20_handler.ts b/src/services/evm/erc20_handler.ts index 9866eb895..56e69f621 100644 --- a/src/services/evm/erc20_handler.ts +++ b/src/services/evm/erc20_handler.ts @@ -1,7 +1,7 @@ import { Knex } from 'knex'; import _, { Dictionary } from 'lodash'; import Moleculer from 'moleculer'; -import { decodeAbiParameters, keccak256, toHex } from 'viem'; +import { bytesToHex, decodeAbiParameters, keccak256, toHex } from 'viem'; import config from '../../../config.json' assert { type: 'json' }; import knex from '../../common/utils/db_connection'; import { @@ -402,7 +402,7 @@ export class Erc20Handler { ) as [string, string, bigint]; return Erc20Activity.fromJson({ evm_event_id: e.id, - sender: e.sender, + sender: bytesToHex(e.sender), action: ERC20_ACTION.TRANSFER, erc20_contract_address: e.address, amount: amount.toString(), @@ -496,7 +496,7 @@ export class Erc20Handler { ) as [string, string, bigint]; return Erc20Activity.fromJson({ evm_event_id: e.id, - sender: e.sender, + sender: bytesToHex(e.sender), action: ERC20_ACTION.APPROVAL, erc20_contract_address: e.address, amount: amount.toString(), diff --git a/src/services/evm/erc721.service.ts b/src/services/evm/erc721.service.ts index 5b002fffc..ab238e3e1 100644 --- a/src/services/evm/erc721.service.ts +++ b/src/services/evm/erc721.service.ts @@ -190,7 +190,15 @@ export default class Erc721Service extends BullableService { tokenMedia.onchain.token_uri ); } catch (error) { - this.logger.error(error); + await Erc721Token.query() + .where('id', tokenMedia.erc721_token_id) + .patch({ + media_info: { + onchain: { + token_uri: tokenMedia.onchain.token_uri, + }, + }, + }); throw error; } } @@ -499,9 +507,6 @@ export default class Erc721Service extends BullableService { {}, { removeOnComplete: true, - removeOnFail: { - count: 3, - }, repeat: { every: config.erc721.millisecondRepeatJob, }, diff --git a/src/services/evm/erc721_handler.ts b/src/services/evm/erc721_handler.ts index 287e33617..5c9274b51 100644 --- a/src/services/evm/erc721_handler.ts +++ b/src/services/evm/erc721_handler.ts @@ -1,7 +1,7 @@ import { Knex } from 'knex'; import _, { Dictionary } from 'lodash'; import Moleculer from 'moleculer'; -import { decodeAbiParameters, keccak256, toHex } from 'viem'; +import { bytesToHex, decodeAbiParameters, keccak256, toHex } from 'viem'; import config from '../../../config.json' assert { type: 'json' }; import knex from '../../common/utils/db_connection'; import { @@ -122,7 +122,7 @@ export class Erc721Handler { ) as [string, string, number]; return Erc721Activity.fromJson({ evm_event_id: e.id, - sender: e.sender, + sender: bytesToHex(e.sender), action: ERC721_ACTION.TRANSFER, erc721_contract_address: e.address, from: from.toLowerCase(), @@ -149,7 +149,7 @@ export class Erc721Handler { ) as [string, string, number]; return Erc721Activity.fromJson({ evm_event_id: e.id, - sender: e.sender, + sender: bytesToHex(e.sender), action: ERC721_ACTION.APPROVAL, erc721_contract_address: e.address, from: from.toLowerCase(), @@ -176,7 +176,7 @@ export class Erc721Handler { ) as [string, string]; return Erc721Activity.fromJson({ evm_event_id: e.id, - sender: e.sender, + sender: bytesToHex(e.sender), action: ERC721_ACTION.APPROVAL_FOR_ALL, erc721_contract_address: e.address, from: from.toLowerCase(), diff --git a/src/services/evm/erc721_media_handler.ts b/src/services/evm/erc721_media_handler.ts index 5128e2d66..da9d250ac 100644 --- a/src/services/evm/erc721_media_handler.ts +++ b/src/services/evm/erc721_media_handler.ts @@ -1,13 +1,17 @@ /* eslint-disable no-param-reassign */ +import { fromBase64, fromUtf8 } from '@cosmjs/encoding'; import { AWSError } from 'aws-sdk'; import axios, { AxiosError } from 'axios'; import * as FileType from 'file-type'; import * as isIPFS from 'is-ipfs'; -import parse from 'parse-uri'; import Moleculer from 'moleculer'; +import parse from 'parse-uri'; import { Config } from '../../common'; import { S3Service } from '../../common/utils/s3'; +const SUPPORT_DECODED_TOKEN_URI = { + BASE64: 'data:application/json;base64', +}; export interface ITokenMediaInfo { erc721_token_id: number; address: string; @@ -168,7 +172,10 @@ export async function getMetadata(token_uri: string): Promise<{ image?: string; animation_url?: string; }> { - console.log(parseIPFSUri(token_uri)); + if (token_uri.split(',')[0] === SUPPORT_DECODED_TOKEN_URI.BASE64) { + const base64Metadata = token_uri.split(',')[1]; + return JSON.parse(fromUtf8(fromBase64(base64Metadata))); + } const metadata = await downloadAttachment(parseIPFSUri(token_uri)); return JSON.parse(metadata.toString()); } @@ -181,8 +188,9 @@ export async function downloadAttachment(url: string) { maxContentLength: parseInt(MAX_CONTENT_LENGTH_BYTE, 10), maxBodyLength: parseInt(MAX_BODY_LENGTH_BYTE, 10), }); - - return axiosClient.get(url).then((response: any) => { + const fromGithub = url.includes('//github.com'); + const formatedUrl = fromGithub ? `${url}?raw=true` : url; + return axiosClient.get(formatedUrl).then((response: any) => { const buffer = Buffer.from(response.data, 'base64'); return buffer; }); @@ -206,6 +214,9 @@ export function parseFilename(media_uri: string) { } return parsed.path.substring(1); // http://ipfs.io/ipfs/QmWov9DpE1vYZtTH7JLKXb7b8bJycN91rEPJEmXRXdmh2G/nerd_access_pass.gif } + if (media_uri.includes('//github.com')) { + return parsed.path.substring(1); // https://github.com/storyprotocol/protocol-core/blob/main/assets/license-image.gif + } } if (media_uri.startsWith('/ipfs/')) { return media_uri.substring(1); // /ipfs/QmPAGifcMvxDBgYr1XmEz9gZiC3DEkfYeinFdVSe364uQp/689.png diff --git a/src/services/evm/evm_crawl_internal_tx.service.ts b/src/services/evm/evm_crawl_internal_tx.service.ts index a1094f4b3..9994fd7b7 100644 --- a/src/services/evm/evm_crawl_internal_tx.service.ts +++ b/src/services/evm/evm_crawl_internal_tx.service.ts @@ -1,6 +1,7 @@ import { Service } from '@ourparentcenter/moleculer-decorators-extended'; import { ServiceBroker } from 'moleculer'; -import { PublicClient } from 'viem'; +import { bytesToHex, PublicClient } from 'viem'; +import _ from 'lodash'; import { getViemClient } from '../../common/utils/etherjs_client'; import { BlockCheckpoint, @@ -62,7 +63,7 @@ export default class EvmCrawlInternalTxService extends BullableService { .where('height', '>', startBlock) .andWhere('height', '<=', endBlock), ]); - + const evmTxsByHash = _.keyBy(evmTxs, (e) => bytesToHex(e.hash)); if (evmBlocks.length === 0) { this.logger.info(`No evm block found from ${startBlock} to ${endBlock}`); blockCheckpoint.height = endBlock; @@ -85,29 +86,6 @@ export default class EvmCrawlInternalTxService extends BullableService { ], }) ); - - // const requests = evmBlocks.map((evmBlock) => - // axios({ - // url: 'https://testnet.storyrpc.io', - // method: 'POST', - // headers: { - // Accept: 'application/json', - // 'Content-Type': 'application/json', - // }, - // data: { - // jsonrpc: '2.0', - // id: evmBlock.height, - // method: 'debug_traceBlockByNumber', - // params: [ - // `0x${evmBlock.height.toString(16)}`, - // { - // tracer: 'callTracer', - // timeout: config.evmCrawlInternalTx.timeoutJSONRPC, - // }, - // ], - // }, - // }) - // ); const responseBlocks = await Promise.all(requests); const internalTxSave: EvmInternalTransaction[] = []; @@ -116,7 +94,7 @@ export default class EvmCrawlInternalTxService extends BullableService { responseBlock.forEach((responseTx: any) => { if (responseTx.result?.calls) { const { txHash } = responseTx; - const evmTxDB = evmTxs.find((tx) => tx.hash === txHash); + const evmTxDB = evmTxsByHash[txHash]; if (!evmTxDB) { throw Error('Cannot found this evm_tx_id'); } diff --git a/src/services/evm/handle_tx_evm.service.ts b/src/services/evm/handle_tx_evm.service.ts index ab31bd1ee..79c996ba9 100644 --- a/src/services/evm/handle_tx_evm.service.ts +++ b/src/services/evm/handle_tx_evm.service.ts @@ -1,7 +1,7 @@ import { fromBase64, toHex } from '@cosmjs/encoding'; import { Service } from '@ourparentcenter/moleculer-decorators-extended'; import { ServiceBroker } from 'moleculer'; -import { PublicClient } from 'viem'; +import { bytesToHex, hexToBytes, PublicClient } from 'viem'; import config from '../../../config.json' assert { type: 'json' }; import BullableService, { QueueHandler } from '../../base/bullable.service'; import { BULL_JOB_NAME as COSMOS_BULL_JOB_NAME } from '../../common'; @@ -121,11 +121,13 @@ export default class HandleTransactionEVMService extends BullableService { .filter((evmTx) => !evmTx.to) .map(async (evmTx) => { const txReceipt = await this.viemClient.getTransactionReceipt({ - hash: evmTx.hash as `0x${string}`, + hash: bytesToHex(evmTx.hash) as `0x${string}`, }); if (txReceipt && txReceipt.contractAddress) { // eslint-disable-next-line no-param-reassign - evmTx.contract_address = txReceipt?.contractAddress.toLowerCase(); + evmTx.contract_address = Buffer.from( + hexToBytes(txReceipt?.contractAddress) + ); } }) ); diff --git a/src/services/evm/statistic/account_statistics.service.ts b/src/services/evm/statistic/account_statistics.service.ts index 1bc536a2d..2b601c01e 100644 --- a/src/services/evm/statistic/account_statistics.service.ts +++ b/src/services/evm/statistic/account_statistics.service.ts @@ -10,6 +10,7 @@ import BigNumber from 'bignumber.js'; import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; import _ from 'lodash'; +import { bytesToHex } from 'viem'; import { AccountStatistics, EVMBlock, @@ -197,13 +198,23 @@ export default class EVMAccountStatisticsService extends BullableService { .andWhere('evm_tx_id', '<=', toTx.id); }); dailyTxs.forEach((tx) => { - if (!accountStats[tx.from]) { - accountStats[tx.from] = AccountStatistics.newAccountStat(tx.from, date); + ['from'].forEach((key) => { + if (tx[key]) { + // eslint-disable-next-line no-param-reassign + tx[key] = bytesToHex(tx[key]); + } + }); + + if (!accountStats[String(tx.from)]) { + accountStats[String(tx.from)] = AccountStatistics.newAccountStat( + String(tx.from), + date + ); } - accountStats[tx.from].tx_sent += 1; + accountStats[String(tx.from)].tx_sent += 1; - accountStats[tx.from].gas_used = ( - BigInt(accountStats[tx.from].gas_used) + BigInt(tx.gas_used) + accountStats[String(tx.from)].gas_used = ( + BigInt(accountStats[String(tx.from)].gas_used) + BigInt(tx.gas_used) ).toString(); tx.evm_internal_transactions.forEach( diff --git a/src/services/evm/statistic/dashboard_statistics.service.ts b/src/services/evm/statistic/dashboard_statistics.service.ts index a1ecc9cfa..e7b2a3e95 100644 --- a/src/services/evm/statistic/dashboard_statistics.service.ts +++ b/src/services/evm/statistic/dashboard_statistics.service.ts @@ -2,6 +2,8 @@ /* eslint-disable import/no-extraneous-dependencies */ import { Service } from '@ourparentcenter/moleculer-decorators-extended'; import { ServiceBroker } from 'moleculer'; +import BigNumber from 'bignumber.js'; +import axios from 'axios'; import { BlockCheckpoint, DailyStatistics, @@ -13,12 +15,19 @@ import { import { BULL_JOB_NAME, REDIS_KEY, SERVICE } from '../constant'; import config from '../../../../config.json' assert { type: 'json' }; import BullableService, { QueueHandler } from '../../../base/bullable.service'; +import networks from '../../../../network.json' assert { type: 'json' }; @Service({ name: SERVICE.V1.DashboardEVMStatisticsService.key, version: 1, }) export default class DashboardEVMStatisticsService extends BullableService { + private selectedChain: any = networks.find( + (network) => network.chainId === config.chainId + ); + + private _lcd = this.selectedChain.LCD[0]; + public constructor(public broker: ServiceBroker) { super(broker); } @@ -109,18 +118,26 @@ export default class DashboardEVMStatisticsService extends BullableService { public async handleJob(_payload: object): Promise { this.logger.info('Update EVM dashboard statistics'); - const [totalBlocks, totalTxs, avgBlockTime, latestDailyStat] = - await Promise.all([ - BlockCheckpoint.query().findOne( - 'job_name', - config.evmOnly - ? BULL_JOB_NAME.CRAWL_EVM_TRANSACTION - : BULL_JOB_NAME.HANDLE_TRANSACTION_EVM - ), - this.statisticTotalTransaction(), - this.avgBlockTime(), - DailyStatistics.query().orderBy('date', 'desc').first(), - ]); + const [ + totalBlocks, + totalTxs, + avgBlockTime, + latestDailyStat, + totalSupply, + bondedTokens, + ] = await Promise.all([ + BlockCheckpoint.query().findOne( + 'job_name', + config.evmOnly + ? BULL_JOB_NAME.CRAWL_EVM_TRANSACTION + : BULL_JOB_NAME.HANDLE_TRANSACTION_EVM + ), + this.statisticTotalTransaction(), + this.avgBlockTime(), + DailyStatistics.query().orderBy('date', 'desc').first(), + this.getTotalSupplyByDenom(), + this.getBondedTokenInStakingPool(), + ]); const dashboardStatistics = { total_blocks: totalBlocks?.height, @@ -128,6 +145,14 @@ export default class DashboardEVMStatisticsService extends BullableService { avg_block_time: avgBlockTime, addresses: latestDailyStat ? latestDailyStat.unique_addresses : 0, daily_transaction: latestDailyStat ? latestDailyStat.daily_txs : 0, + total_supply: totalSupply, + staking_apr: BigNumber(config.dashboardStatistics.currentInflation) + .multipliedBy( + BigNumber(1 - config.dashboardStatistics.currentCommunityTax) + ) + .multipliedBy(BigNumber(totalSupply)) + .dividedBy(bondedTokens) + .multipliedBy(100), }; await this.broker.cacher?.set( @@ -136,6 +161,38 @@ export default class DashboardEVMStatisticsService extends BullableService { ); } + async getBondedTokenInStakingPool() { + if (!this._lcd) { + this.logger.error('Config network not have LCD'); + return 0; + } + const resultCallApi = await axios({ + baseURL: this._lcd, + url: 'staking/pool', + method: 'GET', + params: { + denom: config.dashboardStatistics.denomSupply, + }, + }); + return resultCallApi.data.msg.pool.bonded_tokens; + } + + async getTotalSupplyByDenom() { + if (!this._lcd) { + this.logger.error('Config network not have LCD'); + return 0; + } + const resultCallApi = await axios({ + baseURL: this._lcd, + url: '/bank/supply/by_denom', + method: 'GET', + params: { + denom: config.dashboardStatistics.denomSupply, + }, + }); + return resultCallApi.data.msg.amount.amount; + } + public async _start() { this.createJob( BULL_JOB_NAME.HANDLE_DASHBOARD_EVM_STATISTICS, diff --git a/src/services/evm/story/crawl-validator/crawl_validator.service.ts b/src/services/evm/story/crawl-validator/crawl_validator.service.ts index 761519465..9d772428a 100644 --- a/src/services/evm/story/crawl-validator/crawl_validator.service.ts +++ b/src/services/evm/story/crawl-validator/crawl_validator.service.ts @@ -4,16 +4,21 @@ import { Service } from '@ourparentcenter/moleculer-decorators-extended'; import { ServiceBroker } from 'moleculer'; import Long from 'long'; -import { fromBase64, fromBech32, toBech32, toHex } from '@cosmjs/encoding'; +import { + fromBase64, + fromBech32, + toBase64, + toBech32, + toHex, +} from '@cosmjs/encoding'; import axios from 'axios'; import { pubkeyToRawAddress } from '@cosmjs/tendermint-rpc'; import { Validator } from '../../../../models/validator'; import BullableService, { QueueHandler, } from '../../../../base/bullable.service'; -import knex from '../../../../common/utils/db_connection'; import config from '../../../../../config.json' assert { type: 'json' }; -import { BULL_JOB_NAME, IPagination, SERVICE } from '../../../../common'; +import { BULL_JOB_NAME, SERVICE } from '../../../../common'; import networks from '../../../../../network.json' assert { type: 'json' }; @Service({ @@ -55,8 +60,9 @@ export default class CrawlValidatorService extends BullableService { let resultCallApi: any; let done = false; - const pagination: IPagination = { + const pagination: any = { limit: Long.fromInt(config.crawlValidator.queryPageLimit), + count_total: true, }; const stakingPoolCallApi: any = await axios({ @@ -64,76 +70,86 @@ export default class CrawlValidatorService extends BullableService { url: '/staking/pool', method: 'GET', }); - + let validatorCount = 0; while (!done) { resultCallApi = await axios({ baseURL: this._lcd, url: '/staking/validators', method: 'GET', params: { + // status: 'BOND_STATUS_BONDED', 'pagination.limit': pagination.limit, 'pagination.key': pagination.key, + 'pagination.count_total': pagination.count_total, }, }); - + if ( + pagination.key && + resultCallApi.data.msg.pagination.next_key === toBase64(pagination.key) + ) { + break; + } validators.push(...resultCallApi.data.msg.validators); - if (!resultCallApi.data.msg.pagination.next_key) { + if ( + !resultCallApi.data.msg.pagination.next_key || + (validators.length >= validatorCount && validatorCount > 0) + ) { done = true; } else { + if (validatorCount === 0) { + validatorCount = Number(resultCallApi.data.msg.pagination.total); + } pagination.key = fromBase64(resultCallApi.data.msg.pagination.next_key); + pagination.count_total = false; } } let validatorInDB: Validator[] = []; - await knex.transaction(async (trx) => { - validatorInDB = await Validator.query() - .select('*') - .whereNot('status', Validator.STATUS.UNRECOGNIZED) - .forUpdate() - .transacting(trx); - }); + + validatorInDB = await Validator.query() + .select('*') + .whereIn('status', ['1', '2', '3']); const offchainMapped: Map = new Map(); - await Promise.all( - validators.map(async (validator) => { - const foundValidator = validatorInDB.find( - (val: Validator) => - val.operator_address === validator.operator_address - ); - let validatorEntity: Validator; - if (!foundValidator) { - validatorEntity = this.createNewValidator(validator); - } else { - // mark this offchain validator is mapped with onchain - offchainMapped.set(validator.operator_address, true); - - validatorEntity = foundValidator; - validatorEntity.consensus_pubkey = validator.consensus_pubkey; - validatorEntity.jailed = validator.jailed === undefined; - validatorEntity.status = validator.status; - validatorEntity.tokens = validator.tokens; - validatorEntity.percent_voting_power = - Number( - (BigInt(validator.tokens) * BigInt(100000000)) / - BigInt(stakingPoolCallApi.data.msg.pool.bonded_tokens) - ) / 1000000; - validatorEntity.delegator_shares = validator.delegator_shares; - validatorEntity.description = validator.description; - validatorEntity.unbonding_height = Number.parseInt( - validator.unbonding_height ?? 0, - 10 - ); - validatorEntity.unbonding_time = validator.unbonding_time; - validatorEntity.commission = validator.commission; - validatorEntity.jailed_until = ( - foundValidator.jailed_until as unknown as Date - ).toISOString(); - } + // eslint-disable-next-line array-callback-return + validators.map((validator) => { + const foundValidator = validatorInDB.find( + (val: Validator) => val.operator_address === validator.operator_address + ); - updateValidators.push(validatorEntity); - }) - ); + let validatorEntity: Validator; + if (!foundValidator) { + validatorEntity = this.createNewValidator(validator); + } else { + // mark this offchain validator is mapped with onchain + offchainMapped.set(validator.operator_address, true); + + validatorEntity = foundValidator; + validatorEntity.consensus_pubkey = validator.consensus_pubkey; + validatorEntity.jailed = validator.jailed ?? false; + validatorEntity.status = validator.status; + validatorEntity.tokens = validator.tokens; + validatorEntity.percent_voting_power = + Number( + (BigInt(validator.tokens) * BigInt(100000000)) / + BigInt(stakingPoolCallApi.data.msg.pool.bonded_tokens) + ) / 1000000; + validatorEntity.delegator_shares = validator.delegator_shares; + validatorEntity.description = validator.description; + validatorEntity.unbonding_height = Number.parseInt( + validator.unbonding_height ?? 0, + 10 + ); + validatorEntity.unbonding_time = validator.unbonding_time; + validatorEntity.commission = validator.commission; + validatorEntity.jailed_until = ( + foundValidator.jailed_until as unknown as Date + )?.toISOString(); + } + + updateValidators.push(validatorEntity); + }); // loop all validator not found onchain, update status is UNRECOGNIZED validatorInDB @@ -149,9 +165,9 @@ export default class CrawlValidatorService extends BullableService { validatorNotOnchain.percent_voting_power = 0; validatorNotOnchain.jailed_until = - validatorNotOnchain.jailed_until.toISOString(); + validatorNotOnchain.jailed_until?.toISOString(); validatorNotOnchain.unbonding_time = - validatorNotOnchain.unbonding_time.toISOString(); + validatorNotOnchain.unbonding_time?.toISOString(); updateValidators.push(validatorNotOnchain); }); return updateValidators; @@ -186,7 +202,7 @@ export default class CrawlValidatorService extends BullableService { consensus_address: consensusAddress, consensus_hex_address: consensusHexAddress, consensus_pubkey: consensusPubkey, - jailed: validator.jailed !== undefined, + jailed: validator.jailed ?? false, status: validator.status, tokens: validator.tokens, delegator_shares: Number.parseInt(validator.delegator_shares, 10), diff --git a/test/unit/services/erc721/erc721_handler.spec.ts b/test/unit/services/erc721/erc721_handler.spec.ts index f02644e7a..6c5a752a6 100644 --- a/test/unit/services/erc721/erc721_handler.spec.ts +++ b/test/unit/services/erc721/erc721_handler.spec.ts @@ -1,5 +1,6 @@ import { BeforeAll, Describe, Test } from '@jest-decorated/core'; import { ServiceBroker } from 'moleculer'; +import { bytesToHex, hexToBytes } from 'viem'; import config from '../../../../config.json' assert { type: 'json' }; import knex from '../../../../src/common/utils/db_connection'; import { @@ -42,12 +43,14 @@ export default class Erc721HandlerTest { evmTx = EVMTransaction.fromJson({ id: 11111, - hash: '', + hash: hexToBytes( + '0x3faac2ed3ca031892c04598177f7c36e9fdcdf2fb3b6c4a13c520590facb82ef' + ), height: 111, tx_msg_id: 222, tx_id: 223, - contract_address: '', index: 1, + from: hexToBytes('0x51aeade652867f342ddc012e15c27d0cd6220398'), }); erc721Contract1 = Erc721Contract.fromJson({ @@ -100,7 +103,7 @@ export default class Erc721HandlerTest { address: this.evmSmartContract.address, evm_tx_id: this.evmTx.id, tx_id: 1234, - tx_hash: this.evmTx.hash, + tx_hash: bytesToHex(this.evmTx.hash), tx_index: 1, }), EvmEvent.fromJson({ @@ -119,7 +122,7 @@ export default class Erc721HandlerTest { address: this.evmSmartContract.address, evm_tx_id: this.evmTx.id, tx_id: 1234, - tx_hash: this.evmTx.hash, + tx_hash: bytesToHex(this.evmTx.hash), tx_index: 1, }), EvmEvent.fromJson({ @@ -138,7 +141,7 @@ export default class Erc721HandlerTest { address: this.evmSmartContract2.address, evm_tx_id: this.evmTx.id, tx_id: 1234, - tx_hash: this.evmTx.hash, + tx_hash: bytesToHex(this.evmTx.hash), tx_index: 1, }), EvmEvent.fromJson({ @@ -157,7 +160,7 @@ export default class Erc721HandlerTest { address: this.evmSmartContract.address, evm_tx_id: this.evmTx.id, tx_id: 1234, - tx_hash: this.evmTx.hash, + tx_hash: bytesToHex(this.evmTx.hash), tx_index: 1, }), ]; diff --git a/test/unit/services/evm/erc20_handler.spec.ts b/test/unit/services/evm/erc20_handler.spec.ts index ef92dbf97..9b28e5f93 100644 --- a/test/unit/services/evm/erc20_handler.spec.ts +++ b/test/unit/services/evm/erc20_handler.spec.ts @@ -1,7 +1,7 @@ import { AfterAll, BeforeAll, Describe, Test } from '@jest-decorated/core'; import _ from 'lodash'; import { ServiceBroker } from 'moleculer'; -import { decodeAbiParameters, toHex } from 'viem'; +import { decodeAbiParameters, hexToBytes, toHex } from 'viem'; import knex from '../../../../src/common/utils/db_connection'; import { getViemClient } from '../../../../src/common/utils/etherjs_client'; import { @@ -111,12 +111,15 @@ const erc20Contract = Erc20Contract.fromJson({ }); const evmTransaction = EVMTransaction.fromJson({ id: 2931, - hash: '0xf15467ec2a25eeef95798d93c2fe9ed8e7c891578b8e1bcc3284105849656c9d', + hash: hexToBytes( + '0xf15467ec2a25eeef95798d93c2fe9ed8e7c891578b8e1bcc3284105849656c9d' + ), height: 1, tx_id: 1612438, tx_msg_id: 4752908, contract_address: null, index: 0, + from: hexToBytes('0x51aeade652867f342ddc012e15c27d0cd6220398'), }); @Describe('Test erc20 reindex') diff --git a/test/unit/services/evm/erc20_reindex.spec.ts b/test/unit/services/evm/erc20_reindex.spec.ts index ef92dbf97..9b28e5f93 100644 --- a/test/unit/services/evm/erc20_reindex.spec.ts +++ b/test/unit/services/evm/erc20_reindex.spec.ts @@ -1,7 +1,7 @@ import { AfterAll, BeforeAll, Describe, Test } from '@jest-decorated/core'; import _ from 'lodash'; import { ServiceBroker } from 'moleculer'; -import { decodeAbiParameters, toHex } from 'viem'; +import { decodeAbiParameters, hexToBytes, toHex } from 'viem'; import knex from '../../../../src/common/utils/db_connection'; import { getViemClient } from '../../../../src/common/utils/etherjs_client'; import { @@ -111,12 +111,15 @@ const erc20Contract = Erc20Contract.fromJson({ }); const evmTransaction = EVMTransaction.fromJson({ id: 2931, - hash: '0xf15467ec2a25eeef95798d93c2fe9ed8e7c891578b8e1bcc3284105849656c9d', + hash: hexToBytes( + '0xf15467ec2a25eeef95798d93c2fe9ed8e7c891578b8e1bcc3284105849656c9d' + ), height: 1, tx_id: 1612438, tx_msg_id: 4752908, contract_address: null, index: 0, + from: hexToBytes('0x51aeade652867f342ddc012e15c27d0cd6220398'), }); @Describe('Test erc20 reindex') diff --git a/test/unit/services/job/create_constraint_evm_transaction_partition.spec.ts b/test/unit/services/job/create_constraint_evm_transaction_partition.spec.ts index 4ac9c3eeb..0f5f3d01c 100644 --- a/test/unit/services/job/create_constraint_evm_transaction_partition.spec.ts +++ b/test/unit/services/job/create_constraint_evm_transaction_partition.spec.ts @@ -1,5 +1,6 @@ import { BeforeEach, Describe, Test } from '@jest-decorated/core'; import { ServiceBroker } from 'moleculer'; +import { hexToBytes } from 'viem'; import knex from '../../../../src/common/utils/db_connection'; import CreateConstraintInEVMTransactionPartitionJob from '../../../../src/services/evm/job/create_constraint_in_evm_transaction_partition.service'; import { EVMTransaction } from '../../../../src/models'; @@ -15,7 +16,7 @@ export default class CreateEVMTransactionConstraintPartitionSpec { newEVMTx.id = desiredId; newEVMTx.tx_id = 1; newEVMTx.tx_msg_id = 2; - newEVMTx.hash = 'hash'; + newEVMTx.hash = Buffer.from(hexToBytes('0x1234')); newEVMTx.height = 123; await EVMTransaction.query().insert(newEVMTx); }