From b72577af42732c419beb38dbf9ad2508d7c8b311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ph=E1=BA=A1m=20Th=C3=A0nh=20Phong?= <49814372+phamphong9981@users.noreply.github.com> Date: Fri, 29 Sep 2023 08:56:00 +0700 Subject: [PATCH] Hotfix: cw721 refactor migration handle too much activities ( develop ) (#390) * docs: cw721 * docs: cw20 * fix: update validator as UNSPECIFIED when not found onchain * fix: remove token and delegator_shares when validator is UNRECOGNIZED * feat: add test for unrecognized validator * feat: add migration to add index time to block, tx * fix: allow whitelist query can have depth more than config (#312) * feat: reindex cw20 service ( main ) (#277) * feat: reindex cw20 service * test: test cw20 reindexing * fix: idiot * refactor: code * feat: api admin * refactor: cw20/cw721 index and add relation ( main ) (#306) * refactor: cw20 index and add relation * refactor: code * fix: duplicate latest migrate contract (#304) * Feat/statistics (#272) * feat: move statistic feature from v1 to v2 (not tested yet) * feat: change job params so it can aggregate data at a specific date * fix: query fields from exact table * fix: query exact fields * feat: add logger when done * feat: finish statistic jobs (not tested) * feat: create a separate interval job that gets the current date and create statistics jobs with date * feat: update logic using dayjs and lodash lib * feat: update cron jobs logic, move api actions to its service folder * fix: query event from db and group it based on event_id and tx_id * fix: just use a single job to query all daily data * fix: move all queries in daily_stats job into a single Promise all * fix: move dayjs.extend to after import statements * fix: remove _start * fix: only count native token in account_stats job * feat: add api to sync stats from prev dates * fix: support case when user just pass startDate to api * fix: fix whitelist graphql * fix: update column in account_statistic table, update bignum top_tx_sent * fix: remove drop index in modify account statistic table --------- Co-authored-by: AnDQK Co-authored-by: Phan Anh Tuan * refactor: migrate cw721 activity missing smart contract event ( main ) (#322) * refactor: migrate cw721 activity missing smart contract event * feat: api * test: test params * test: test params * test: test params * test: test params * test: test params * test: test params * refactor: code * refactor: code * refactor: code * refactor: code * refactor: code * fix: ci * Fix/account stat with feegrant (#337) * fix: get use_feegrant_grantee if has, other get tx_fee_payer * fix: remove code not used * Update api_gateway.service.ts (#336) * fix: cw721 activity missing from/to ( main ) (#294) * fix: cw721 activity missing from/to * fix: fill from/to cw721 activity * fix: fill from/to cw721 activity * fix: fill from/to cw721 activity * fix: fill from/to cw721 activity * refactor: code * fix: code * refactor: code --------- Co-authored-by: Vu Ngoc Quang * feat: crawl ibc tao ( main ) (#278) * feat: crawl ibc tao * feat: crawl genesis ibc tao * fix: code * fix: code * Fix/lint json file (#359) * fix: display json on multiple lines * fix: update launch.json to debug jest in 1 file; use moleculer.config.ts when debug service * refactor: cw721 * Feat/ibc app ( main ) (#321) * feat: crawl ibc tao * feat: crawl genesis ibc tao * fix: code * fix: code * feat: ibc app * feat: ibc app * test: ibc app * fix: review * test: full test * feat: migration * fix: lint * fix: test * fix: test * fix: smart contract missing label ( main ) (#366) * fix: smart contract missing label * fix: test * fix: test * fix: test * fix: test * refactor: test * refactor: code * refactor: test * fix: test * test: code * test: code * fix: update msg_index by order in event and log (#327) * fix: update msg_index by order in event and log * feat: add test mapping event and log * feat: add test mapping by authz tx * feat: add test mapping by authz tx * fix: refactor mapping event to log, update unit test * fix: comment set index msg with order, use count attribute to compare * feat: add checkMappingEventToLog after mapped event * fix: check map event log by flatten array * fix: update checking map event log * fix: add null value handle in checkMappingEventToLog * feat: init job re assign msg_index to event * fix: use transaction when update event * fix: use forEach loop in job * fix: sort graph when query db * feat: add unittest for generateListUpdateMsgIndex function * fix: update msg_index in 1 query per tx for each value * fix: fix null when compare value attribute value * fix: fix if tx fail, then no need set msg index * feat: health check job ( main ) (#345) * feat: health check job * feat: validator * refactor: code * refactor: performance cw721 media info ( main ) (#369) * refactor: cw721 media info * refactor: code * refactor: code * refactor: code * refactor: review * refactor: review * fix: fix lint and remove resilient, protobuf, upgrade graphql (#380) * fix: fix lint and remove resilient, protobuf, upgrade graphql * fix: add package-lock.json * refactor: clean redundance get DB field * refactor: cw721 * test: full test * feat: migration * fix: lint * fix: test * fix: test * refactor: test * refactor: code * refactor: test * fix: test * test: code * test: code * refactor: code * refactor: review * refactor: review * refactor: clean redundance get DB field * fix: handle too much data * fix: migration * fix: migration * fix: migration * fix: code --------- Co-authored-by: Tuan Phan Anh <38557844+fibonacci998@users.noreply.github.com> Co-authored-by: Phan Anh Tuan Co-authored-by: Vu Ngoc Quang Co-authored-by: AnDQK --- ...39_cw721_activity_update_owner_for_each.ts | 65 +++++++++++-------- src/services/cw721/cw721.service.ts | 2 +- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/migrations/20230915032739_cw721_activity_update_owner_for_each.ts b/migrations/20230915032739_cw721_activity_update_owner_for_each.ts index 8743bc36d..d453a61af 100644 --- a/migrations/20230915032739_cw721_activity_update_owner_for_each.ts +++ b/migrations/20230915032739_cw721_activity_update_owner_for_each.ts @@ -8,37 +8,46 @@ export async function up(knex: Knex): Promise { sender: knex.ref('from'), }); await knex.transaction(async (trx) => { - const activities = await CW721Activity.query() - .joinRelated('event') - .orderBy('event.id', 'asc') - .transacting(trx); + let currentId = 0; const latestOwners: Dictionary = {}; - activities.forEach((activity) => { - const latestOwner = - latestOwners[ - activity.cw721_contract_id + '_' + activity.cw721_token_id - ]; - if (latestOwner) { - activity.from = latestOwner; + while (true) { + const activities = await CW721Activity.query() + .transacting(trx) + .joinRelated('event') + .where('event.id', '>', currentId) + .orderBy('event.id', 'asc') + .limit(1000) + .select('cw721_activity.*', 'event.id as event_id'); + activities.forEach((activity) => { + const latestOwner = + latestOwners[ + activity.cw721_contract_id + '_' + activity.cw721_token_id + ]; + if (latestOwner) { + activity.from = latestOwner; + } else { + activity.from = null; + } + if ( + activity.action === CW721_ACTION.MINT || + activity.action === CW721_ACTION.TRANSFER || + activity.action === CW721_ACTION.SEND_NFT + ) { + latestOwners[ + activity.cw721_contract_id + '_' + activity.cw721_token_id + ] = activity.to; + } + }); + if (activities.length > 0) { + await CW721Activity.query() + .insert(activities.map((activity) => _.omit(activity, 'event_id'))) + .onConflict(['id']) + .merge() + .transacting(trx); + currentId = activities[activities.length - 1].event_id; } else { - activity.from = null; + break; } - if ( - activity.action === CW721_ACTION.MINT || - activity.action === CW721_ACTION.TRANSFER || - activity.action === CW721_ACTION.SEND_NFT - ) { - latestOwners[ - activity.cw721_contract_id + '_' + activity.cw721_token_id - ] = activity.to; - } - }); - if (activities.length > 0) { - await CW721Activity.query() - .insert(activities) - .onConflict(['id']) - .merge() - .transacting(trx); } }); } diff --git a/src/services/cw721/cw721.service.ts b/src/services/cw721/cw721.service.ts index 0dd28ee89..681f71bee 100644 --- a/src/services/cw721/cw721.service.ts +++ b/src/services/cw721/cw721.service.ts @@ -188,7 +188,7 @@ export default class Cw721HandlerService extends BullableService { getAttributeFrom( msg.attributes, EventAttribute.ATTRIBUTE_KEY.TOKEN_ID - ), + ) || null, ]) ), (o) => `${o.contract.smart_contract.address}_${o.token_id}`