Skip to content

Commit

Permalink
Feat/erc20 reindexing (#851) TG-231 #ready-to-test
Browse files Browse the repository at this point in the history 
* feat: erc20 reindex

* test: test

* fix: ci

* fix: test

* test: test

* test: run with an erc20 contract

* refactor: instantiate erc20 module account

* refactor: instantiate erc20 module account
  • Loading branch information
@phamphong9981
phamphong9981 authored Jul 23, 2024
1 parent bfafaed commit d32da37
Show file tree
Hide file tree
Showing 9 changed files with 1,079 additions and 169 deletions.
2 changes: 2 additions & 0 deletions src/models/account_balance.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import BaseModel from './base';
import { Account } from './account';

export class AccountBalance extends BaseModel {
static softDelete = false;

account!: Account;

id!: number;
Expand Down
1 change: 1 addition & 0 deletions src/services/api-gateways/api_gateway.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import config from '../../../config.json' assert { type: 'json' };
'v1.job.insert-verify-by-codehash',
'v1.erc721-admin.*',
'v2.evm-statistics.syncPrevDateStatsByChainId',
'v1.erc20-admin.*',
],
},
{
Expand Down
49 changes: 49 additions & 0 deletions src/services/api-gateways/erc20_admin.service.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import { Post, Service } from '@ourparentcenter/moleculer-decorators-extended';
import { Context, ServiceBroker } from 'moleculer';
import networks from '../../../network.json' assert { type: 'json' };
import BaseService from '../../base/base.service';

@Service({
name: 'erc20-admin',
version: 1,
})
export default class Erc20AdminService extends BaseService {
public constructor(public broker: ServiceBroker) {
super(broker);
}

@Post('/erc20-reindexing', {
name: 'erc20Reindexing',
params: {
chainid: {
type: 'string',
optional: false,
enum: networks.map((network) => network.chainId),
},
addresses: {
type: 'array',
optional: false,
items: 'string',
},
},
})
async erc20Reindexing(
ctx: Context<
{
chainid: string;
addresses: string[];
},
Record<string, unknown>
>
) {
const selectedChain = networks.find(
(network) => network.chainId === ctx.params.chainid
);
return this.broker.call(
`v1.Erc20.reindexing@${selectedChain?.moleculerNamespace}`,
{
addresses: ctx.params.addresses,
}
);
}
}
5 changes: 5 additions & 0 deletions src/services/evm/constant.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,10 @@ export const SERVICE = {
key: 'insertNewErc20Contracts',
path: 'v1.Erc20.insertNewErc20Contracts',
},
reindexing: {
key: 'reindexing',
path: 'v1.Erc20.reindexing',
},
},
Erc721: {
key: 'Erc721',
Expand Down Expand Up @@ -222,6 +226,7 @@ export const BULL_JOB_NAME = {
SYNC_SOURCIFY: 'sync:sourcify',
INSERT_VERIFY_BY_CODEHASH: 'job:insert-verify-by-codehash',
HANDLE_SELF_DESTRUCT: 'handle:self-destruct',
REINDEX_ERC20: 'reindex:erc20',
};

export const MSG_TYPE = {
Expand Down
235 changes: 78 additions & 157 deletions src/services/evm/erc20.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,12 @@ import BullableService, { QueueHandler } from '../../base/bullable.service';
import { SERVICE as COSMOS_SERVICE, Config, getLcdClient } from '../../common';
import knex from '../../common/utils/db_connection';
import { getViemClient } from '../../common/utils/etherjs_client';
import {
BlockCheckpoint,
EVMSmartContract,
Event,
EvmEvent,
} from '../../models';
import { AccountBalance } from '../../models/account_balance';
import { BlockCheckpoint, EVMSmartContract } from '../../models';
import { Erc20Activity } from '../../models/erc20_activity';
import { Erc20Contract } from '../../models/erc20_contract';
import { BULL_JOB_NAME, SERVICE as EVM_SERVICE, SERVICE } from './constant';
import { ERC20_EVENT_TOPIC0, Erc20Handler } from './erc20_handler';
import { Erc20Handler } from './erc20_handler';
import { Erc20Reindexer } from './erc20_reindex';
import { convertEthAddressToBech32Address } from './utils';

const { NODE_ENV } = Config;
Expand Down Expand Up @@ -98,82 +93,14 @@ export default class Erc20Service extends BullableService {
],
config.erc20.key
);
// TODO: handle track erc20 contract only
const erc20Events = await EvmEvent.query()
.joinRelated('[evm_smart_contract,evm_transaction]')
.innerJoin(
'erc20_contract',
'evm_event.address',
'erc20_contract.address'
)
.where('evm_event.block_height', '>', startBlock)
.andWhere('evm_event.block_height', '<=', endBlock)
.orderBy('evm_event.id', 'asc')
.select(
'evm_event.*',
'evm_transaction.from as sender',
'evm_smart_contract.id as evm_smart_contract_id',
'evm_transaction.id as evm_tx_id'
);
let erc20CosmosEvents: Event[] = [];
if (config.evmOnly === false) {
if (!this.erc20ModuleAccount) {
const lcdClient = await getLcdClient();
const erc20Account: QueryModuleAccountByNameResponseSDKType =
await lcdClient.provider.cosmos.auth.v1beta1.moduleAccountByName({
name: 'erc20',
});
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
this.erc20ModuleAccount = erc20Account.account.base_account.address;
}
erc20CosmosEvents = await Event.query()
.where('block_height', '>', startBlock)
.andWhere('block_height', '<=', endBlock)
.andWhere((query) => {
query
.where('type', Event.EVENT_TYPE.CONVERT_COIN)
.orWhere('type', Event.EVENT_TYPE.CONVERT_ERC20);
})
.withGraphFetched('[transaction, attributes]');
}
await this.handleMissingErc20Contract(
erc20Events,
erc20CosmosEvents,
trx
);
const erc20Activities: Erc20Activity[] = [];
erc20Events.forEach((e) => {
if (e.topic0 === ERC20_EVENT_TOPIC0.TRANSFER) {
const activity = Erc20Handler.buildTransferActivity(e, this.logger);
if (activity) {
erc20Activities.push(activity);
}
} else if (e.topic0 === ERC20_EVENT_TOPIC0.APPROVAL) {
const activity = Erc20Handler.buildApprovalActivity(e, this.logger);
if (activity) {
erc20Activities.push(activity);
}
} else if (config.erc20.wrapExtensionContract.includes(e.address)) {
const wrapActivity = Erc20Handler.buildWrapExtensionActivity(
e,
this.logger
);
if (wrapActivity) {
erc20Activities.push(wrapActivity);
}
}
});
erc20CosmosEvents.forEach((event) => {
const activity = Erc20Handler.buildTransferActivityByCosmos(
event,
this.erc20ModuleAccount,
const erc20Activities: Erc20Activity[] =
await Erc20Handler.buildErc20Activities(
startBlock,
endBlock,
trx,
this.logger
);
if (activity) {
erc20Activities.push(activity);
}
});
await this.handleMissingErc20Contract(erc20Activities, trx);
if (erc20Activities.length > 0) {
this.logger.info(
`Crawl Erc20 activity from block ${startBlock} to block ${endBlock}:\n ${JSON.stringify(
Expand Down Expand Up @@ -209,7 +136,10 @@ export default class Erc20Service extends BullableService {
config.erc20.key
);
// get Erc20 activities
let erc20Activities = await this.getErc20Activities(startBlock, endBlock);
let erc20Activities = await Erc20Handler.getErc20Activities(
startBlock,
endBlock
);
// get missing Account
const missingAccountsAddress = Array.from(
new Set(
Expand All @@ -233,40 +163,13 @@ export default class Erc20Service extends BullableService {
addresses: missingAccountsAddress,
}
);
erc20Activities = await this.getErc20Activities(startBlock, endBlock);
erc20Activities = await Erc20Handler.getErc20Activities(
startBlock,
endBlock
);
}
await knex.transaction(async (trx) => {
if (erc20Activities.length > 0) {
const accountBalances = _.keyBy(
await AccountBalance.query()
.transacting(trx)
.joinRelated('account')
.whereIn(
['account.evm_address', 'denom'],
[
...erc20Activities.map((e) => [
e.from,
e.erc20_contract_address,
]),
...erc20Activities.map((e) => [e.to, e.erc20_contract_address]),
]
),
(o) => `${o.account_id}_${o.denom}`
);
// construct cw721 handler object
const erc20Handler = new Erc20Handler(accountBalances, erc20Activities);
erc20Handler.process();
const updatedAccountBalances = Object.values(
erc20Handler.accountBalances
);
if (updatedAccountBalances.length > 0) {
await AccountBalance.query()
.transacting(trx)
.insert(updatedAccountBalances)
.onConflict(['account_id', 'denom'])
.merge();
}
}
await Erc20Handler.updateErc20AccountsBalance(erc20Activities, trx);
updateBlockCheckpoint.height = endBlock;
await BlockCheckpoint.query()
.insert(updateBlockCheckpoint)
Expand All @@ -276,6 +179,52 @@ export default class Erc20Service extends BullableService {
});
}

@QueueHandler({
queueName: BULL_JOB_NAME.REINDEX_ERC20,
jobName: BULL_JOB_NAME.REINDEX_ERC20,
})
async reindexErc20(_payload: { address: `0x${string}` }): Promise<void> {
const { address } = _payload;
const erc20Reindexer = new Erc20Reindexer(this.viemClient, this.logger);
await erc20Reindexer.reindex(address.toLowerCase() as `0x${string}`);
this.logger.info(`Reindex erc20 contract ${address} done.`);
}

@Action({
name: SERVICE.V1.Erc20.reindexing.key,
params: {
addresses: {
type: 'array',
items: 'string',
optional: false,
},
},
})
public async reindexing(
ctx: Context<{
addresses: `0x${string}`[];
}>
) {
const { addresses } = ctx.params;
if (addresses.length > 0) {
await Promise.all(
addresses.map((address) =>
this.createJob(
BULL_JOB_NAME.REINDEX_ERC20,
BULL_JOB_NAME.REINDEX_ERC20,
{
address,
},
{
jobId: address,
removeOnComplete: true,
}
)
)
);
}
}

@Action({
name: SERVICE.V1.Erc20.insertNewErc20Contracts.key,
params: {
Expand Down Expand Up @@ -311,52 +260,13 @@ export default class Erc20Service extends BullableService {
}
}

async getErc20Activities(
startBlock: number,
endBlock: number
): Promise<Erc20Activity[]> {
return Erc20Activity.query()
.leftJoin(
'account as from_account',
'erc20_activity.from',
'from_account.evm_address'
)
.leftJoin(
'account as to_account',
'erc20_activity.to',
'to_account.evm_address'
)
.leftJoin(
'erc20_contract as erc20_contract',
'erc20_activity.erc20_contract_address',
'erc20_contract.address'
)
.where('erc20_activity.height', '>', startBlock)
.andWhere('erc20_activity.height', '<=', endBlock)
.andWhere('erc20_contract.track', true)
.select(
'erc20_activity.*',
'from_account.id as from_account_id',
'to_account.id as to_account_id'
)
.orderBy('erc20_activity.id');
}

async handleMissingErc20Contract(
events: EvmEvent[],
cosmosEvents: Event[],
erc20Activities: Erc20Activity[],
trx: Knex.Transaction
) {
const evmEventsUniqByAddress = _.keyBy(events, (e) => e.address);
const cosmosEventsUniqByAddress = _.keyBy(cosmosEvents, (e) =>
e.attributes.find((a) => a.key === 'erc20_token')?.value.toLowerCase()
const addresses = _.uniq(
erc20Activities.map((activity) => activity.erc20_contract_address)
);

const addresses = [
...Object.keys(evmEventsUniqByAddress),
...Object.keys(cosmosEventsUniqByAddress),
];

const erc20ContractsByAddress = _.keyBy(
await Erc20Contract.query().whereIn('address', addresses),
(e) => e.address
Expand Down Expand Up @@ -442,6 +352,17 @@ export default class Erc20Service extends BullableService {
public async _start(): Promise<void> {
this.viemClient = getViemClient();
if (NODE_ENV !== 'test') {
if (config.evmOnly === false) {
const lcdClient = await getLcdClient();
const erc20Account: QueryModuleAccountByNameResponseSDKType =
await lcdClient.provider.cosmos.auth.v1beta1.moduleAccountByName({
name: 'erc20',
});
Erc20Handler.erc20ModuleAccount =
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
erc20Account.account.base_account.address;
}
await this.createJob(
BULL_JOB_NAME.HANDLE_ERC20_CONTRACT,
BULL_JOB_NAME.HANDLE_ERC20_CONTRACT,
Expand Down
Loading

0 comments on commit d32da37

Please sign in to comment.