From 45fa9b49e4c5d4213e326418ed8870827b0a2911 Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Wed, 5 Jul 2023 14:55:59 +0700 Subject: [PATCH 01/17] docs: cw721 --- docs/services/cw721/cw721-media.md | 53 ++++++++++++++++++++++++++++++ docs/services/cw721/cw721.md | 38 +++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 docs/services/cw721/cw721-media.md create mode 100644 docs/services/cw721/cw721.md diff --git a/docs/services/cw721/cw721-media.md b/docs/services/cw721/cw721-media.md new file mode 100644 index 000000000..22e00e852 --- /dev/null +++ b/docs/services/cw721/cw721-media.md @@ -0,0 +1,53 @@ +## Cw721 Media Service + +```mermaid + sequenceDiagram + autonumber + participant A as Cw721MediaService + participant B as DB + participant C as Network + participant D as S3 + + loop Interval + A->>B: Get top 10 token which media_info = null (unprocessed) + activate B + B-->>A: token instances + deactivate B + + A->>C: get token info (token_uri, extension) + activate C + C-->>A: token info + deactivate C + + alt token_uri!=null + A->>C: call ipfs link from token_uri to get metadata + else token_uri==null + A->>A: metadata = extension + end + + A->>A: get image url from metadata + alt image_url!=null + A->>D: get image then upload image to S3 + activate D + D-->>A: S3 link + deactivate D + else image_url==null + A->>A: empty image offchain + else error + A->>A: empty image offchain + end + + alt animation_url!=null + A->>D: get animation then upload image to S3 + activate D + D-->>A: S3 link + deactivate D + else animation_url==null + A->>A: empty animation offchain + else error + A->>A: empty animation offchain + end + + A-->>B: update media_info (animation offchain, image offchain) + end +``` diff --git a/docs/services/cw721/cw721.md b/docs/services/cw721/cw721.md new file mode 100644 index 000000000..20798a40c --- /dev/null +++ b/docs/services/cw721/cw721.md @@ -0,0 +1,38 @@ +## Cw721 Service + +```mermaid + sequenceDiagram + autonumber + participant A as Cw721Service + participant B as DB + + loop Interval + A->>B: Get BlockCheckpoint for Cw721Service + activate B + B-->>A: Return BlockCheckpoint + deactivate B + alt not found BlockCheckpoint + A->>A: Set checkpoint = startBlock config + end + A->>A: endBlock = startBlock + config.BlocksPerCall + + A->>B: Get cw721 smart contract event from startBlock to endBlock + activate B + B-->>A: return list cw721 events + deactivate B + A->>A: handle each cw721 event type + + alt instantiate + A->>B: Upsert new cw721 contract instance + else mint + A->>B: Insert new cw721 token instance + else transfer + A->>B: Update new owner + else burn + A->>B: Update burn status + end + + A->>B: insert new cw721 event instances + A->>B: Update checkpoint = endBlock + end +``` From b27aa4c5f9700a5e8171ceb30209f917b24afd4d Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Fri, 7 Jul 2023 14:25:31 +0700 Subject: [PATCH 02/17] docs: cw20 --- docs/services/cw20/cw20.md | 33 +++++++++++++++++++ docs/services/cw20/cw20_update_by_contract.md | 23 +++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 docs/services/cw20/cw20.md create mode 100644 docs/services/cw20/cw20_update_by_contract.md diff --git a/docs/services/cw20/cw20.md b/docs/services/cw20/cw20.md new file mode 100644 index 000000000..05d447c4a --- /dev/null +++ b/docs/services/cw20/cw20.md @@ -0,0 +1,33 @@ +## Cw20 Service + +```mermaid + sequenceDiagram + autonumber + participant A as Cw20Service + participant B as DB + participant C as Network + participant D as Cw20UpdateByContractService + loop Interval + A->>B: Get BlockCheckpoint for Cw20Service + activate B + B-->>A: Return BlockCheckpoint + deactivate B + alt not found BlockCheckpoint + A->>A: Set checkpoint = startBlock config + end + A->>A: endBlock = startBlock + config.BlocksPerCall + A->>B: Get cw20 smart contract event from startBlock to endBlock + activate B + B-->>A: return list cw20 events + deactivate B + A->>A: handle each cw20 instantiate event + A->>C: Get all current holders and their balance + activate C + C->>A: Return results + deactivate C + A->>B: Insert into cw20_contract and cw20_holder + A->>B: insert new cw20 event instances + A->>B: Update checkpoint = endBlock + A->>D: Call action update cw20 by contract {contract, startBlock, endBlock} + end +``` diff --git a/docs/services/cw20/cw20_update_by_contract.md b/docs/services/cw20/cw20_update_by_contract.md new file mode 100644 index 000000000..d4b665ee4 --- /dev/null +++ b/docs/services/cw20/cw20_update_by_contract.md @@ -0,0 +1,23 @@ +## Cw20 Update By Contract Service + +```mermaid + sequenceDiagram + autonumber + participant A as Cw20UpdateByContract + participant B as DB + loop Interval + A->>B: Get cw20 smart contract event from startBlock to endBlock for specified contract + activate B + B-->>A: return list cw20 events + deactivate B + A->>A: handle each cw20 event type + alt mint + A->>B: Insert new cw721_holder instance or add balance if he already has balance + else transfer + A->>B: Add balance to recipient and sub balance to sender + else burn + A->>B: Sub balance to owner + end + A->>B: Update total_supply + end +``` From 5e50079af69f8d0aaab0ed734184b55bf9f8d182 Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Mon, 31 Jul 2023 14:16:24 +0700 Subject: [PATCH 03/17] fix: update validator as UNSPECIFIED when not found onchain --- src/models/validator.ts | 3 +++ .../crawl_validator.service.ts | 22 ++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/models/validator.ts b/src/models/validator.ts index 71c58a81b..4e9ea8758 100644 --- a/src/models/validator.ts +++ b/src/models/validator.ts @@ -73,6 +73,9 @@ export class Validator extends BaseModel { return { BONDED: 'BOND_STATUS_BONDED', UNBONDED: 'BOND_STATUS_UNBONDED', + UNSPECIFIED: 'BOND_STATUS_UNSPECIFIED', + UNBONDING: 'BOND_STATUS_UNBONDING', + UNRECOGNIZED: 'UNRECOGNIZED', }; } diff --git a/src/services/crawl-validator/crawl_validator.service.ts b/src/services/crawl-validator/crawl_validator.service.ts index d724269ab..af0aa6ab6 100644 --- a/src/services/crawl-validator/crawl_validator.service.ts +++ b/src/services/crawl-validator/crawl_validator.service.ts @@ -111,7 +111,7 @@ export default class CrawlValidatorService extends BullableService { } const validatorInDB: Validator[] = await knex('validator').select('*'); - + const offchainMapped: Map = new Map(); await Promise.all( validators.map(async (validator) => { const foundValidator = validatorInDB.find( @@ -123,6 +123,9 @@ export default class CrawlValidatorService extends BullableService { if (!foundValidator) { validatorEntity = Validator.createNewValidator(validator); } else { + // mark this offchain validator is mapped with onchain + offchainMapped.set(validator.operator_address, true); + validatorEntity = foundValidator; validatorEntity.jailed = validator.jailed; validatorEntity.status = validator.status; @@ -146,6 +149,23 @@ export default class CrawlValidatorService extends BullableService { updateValidators = await this.loadCustomInfo(updateValidators); + // loop all validator not found onchain, update status is UNSPECIFIED + validatorInDB + .filter((val: any) => !offchainMapped.get(val.operator_address)) + .forEach(async (validatorNotOnchain: any) => { + this.logger.debug( + 'Account not found onchain: ', + validatorNotOnchain.operator_address + ); + validatorNotOnchain.status = Validator.STATUS.UNSPECIFIED; + + validatorNotOnchain.jailed_until = + validatorNotOnchain.jailed_until.toISOString(); + validatorNotOnchain.unbonding_time = + validatorNotOnchain.unbonding_time.toISOString(); + updateValidators.push(validatorNotOnchain); + }); + await Validator.query() .insert(updateValidators) .onConflict('operator_address') From 147fb9bd058277412ce5adaf4b4c277759f0317c Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Tue, 1 Aug 2023 14:30:04 +0700 Subject: [PATCH 04/17] fix: remove token and delegator_shares when validator is UNRECOGNIZED --- src/services/crawl-validator/crawl_validator.service.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/services/crawl-validator/crawl_validator.service.ts b/src/services/crawl-validator/crawl_validator.service.ts index af0aa6ab6..6e0cbe132 100644 --- a/src/services/crawl-validator/crawl_validator.service.ts +++ b/src/services/crawl-validator/crawl_validator.service.ts @@ -157,7 +157,9 @@ export default class CrawlValidatorService extends BullableService { 'Account not found onchain: ', validatorNotOnchain.operator_address ); - validatorNotOnchain.status = Validator.STATUS.UNSPECIFIED; + validatorNotOnchain.status = Validator.STATUS.UNRECOGNIZED; + validatorNotOnchain.tokens = 0; + validatorNotOnchain.delegator_shares = 0; validatorNotOnchain.jailed_until = validatorNotOnchain.jailed_until.toISOString(); From 6fd8005890c3f92d55d9714bc50d05a2325e4388 Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Thu, 3 Aug 2023 16:55:08 +0700 Subject: [PATCH 05/17] feat: add test for unrecognized validator --- .../crawl_validator.service.ts | 8 ++-- .../crawl-validator/crawl_validator.spec.ts | 47 +++++++++++++++++-- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/services/crawl-validator/crawl_validator.service.ts b/src/services/crawl-validator/crawl_validator.service.ts index 6e0cbe132..1a8412ef8 100644 --- a/src/services/crawl-validator/crawl_validator.service.ts +++ b/src/services/crawl-validator/crawl_validator.service.ts @@ -70,7 +70,6 @@ export default class CrawlValidatorService extends BullableService { .select('value') .limit(1) .offset(0); - await knex.transaction(async (trx) => { if (resultTx.length > 0) { await this.updateValidators(trx); @@ -110,7 +109,9 @@ export default class CrawlValidatorService extends BullableService { } } - const validatorInDB: Validator[] = await knex('validator').select('*'); + const validatorInDB: Validator[] = await knex('validator') + .select('*') + .whereNot('status', Validator.STATUS.UNRECOGNIZED); const offchainMapped: Map = new Map(); await Promise.all( validators.map(async (validator) => { @@ -149,7 +150,7 @@ export default class CrawlValidatorService extends BullableService { updateValidators = await this.loadCustomInfo(updateValidators); - // loop all validator not found onchain, update status is UNSPECIFIED + // loop all validator not found onchain, update status is UNRECOGNIZED validatorInDB .filter((val: any) => !offchainMapped.get(val.operator_address)) .forEach(async (validatorNotOnchain: any) => { @@ -160,6 +161,7 @@ export default class CrawlValidatorService extends BullableService { validatorNotOnchain.status = Validator.STATUS.UNRECOGNIZED; validatorNotOnchain.tokens = 0; validatorNotOnchain.delegator_shares = 0; + validatorNotOnchain.percent_voting_power = 0; validatorNotOnchain.jailed_until = validatorNotOnchain.jailed_until.toISOString(); diff --git a/test/unit/services/crawl-validator/crawl_validator.spec.ts b/test/unit/services/crawl-validator/crawl_validator.spec.ts index fccb4be8f..ebc87dd91 100644 --- a/test/unit/services/crawl-validator/crawl_validator.spec.ts +++ b/test/unit/services/crawl-validator/crawl_validator.spec.ts @@ -1,4 +1,4 @@ -import { AfterAll, BeforeAll, Describe, Test } from '@jest-decorated/core'; +import { AfterEach, BeforeEach, Describe, Test } from '@jest-decorated/core'; import { ServiceBroker } from 'moleculer'; import { BULL_JOB_NAME } from '../../../../src/common'; import { @@ -75,7 +75,7 @@ export default class CrawlValidatorTest { crawlSigningInfoService?: CrawlSigningInfoService; - @BeforeAll() + @BeforeEach() async initSuite() { await this.broker.start(); this.crawlSigningInfoService = this.broker.createService( @@ -97,7 +97,7 @@ export default class CrawlValidatorTest { await BlockCheckpoint.query().insert(this.blockCheckpoint); } - @AfterAll() + @AfterEach() async tearDown() { await Promise.all([ Validator.query().delete(true), @@ -128,4 +128,45 @@ export default class CrawlValidatorTest { )?.consensus_address ).toEqual('auravalcons1rvq6km74pua3pt9g7u5svm4r6mrw8z08walfep'); } + + @Test('Set validator not found onchain is UNRECOGNIZED') + public async testCrawlValidatorNotFoundOnchain() { + await Validator.query().insert( + Validator.fromJson({ + operator_address: 'xxx', + account_address: 'xxx', + consensus_address: 'xxx', + consensus_hex_address: 'xxx', + consensus_pubkey: {}, + jailed: false, + status: Validator.STATUS.UNBONDED, + tokens: 100, + delegator_shares: 100, + description: {}, + unbonding_height: 0, + unbonding_time: '1970-01-01 00:00:00+00', + commission: {}, + min_self_delegation: 0, + uptime: 0, + self_delegation_balance: 0, + percent_voting_power: 100, + start_height: 0, + index_offset: 0, + jailed_until: '1970-01-01 00:00:00+00', + tombstoned: false, + missed_blocks_counter: 0, + delegators_count: 0, + delegators_last_height: 0, + image_url: 'xxx', + }) + ); + + await this.crawlValidatorService?.handleCrawlAllValidator({}); + + const validator = await Validator.query().findOne({ + operator_address: 'xxx', + }); + expect(validator?.status).toEqual(Validator.STATUS.UNRECOGNIZED); + expect(validator?.tokens).toEqual('0'); + } } From d8eb803a649473c64c91f882815094e2c96d0bf7 Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Mon, 7 Aug 2023 09:10:58 +0700 Subject: [PATCH 06/17] feat: add migration to add index time to block, tx --- .../20230807020810_add_time_index_block_tx.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 migrations/20230807020810_add_time_index_block_tx.ts diff --git a/migrations/20230807020810_add_time_index_block_tx.ts b/migrations/20230807020810_add_time_index_block_tx.ts new file mode 100644 index 000000000..b197a6741 --- /dev/null +++ b/migrations/20230807020810_add_time_index_block_tx.ts @@ -0,0 +1,19 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('block', (table) => { + table.index('time'); + }); + await knex.schema.alterTable('transaction', (table) => { + table.index('timestamp'); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('block', (table) => { + table.dropIndex('time'); + }); + await knex.schema.alterTable('transaction', (table) => { + table.dropIndex('timestamp'); + }); +} From 164ab7789a19799fb42f6a899fd4e810bcf79e78 Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Wed, 16 Aug 2023 17:16:56 +0700 Subject: [PATCH 07/17] fix: update msg_index by order in event and log --- src/services/crawl-tx/crawl_tx.service.ts | 135 +++++++++++++++------- 1 file changed, 94 insertions(+), 41 deletions(-) diff --git a/src/services/crawl-tx/crawl_tx.service.ts b/src/services/crawl-tx/crawl_tx.service.ts index 156fa3b97..326418bb9 100644 --- a/src/services/crawl-tx/crawl_tx.service.ts +++ b/src/services/crawl-tx/crawl_tx.service.ts @@ -333,56 +333,109 @@ export default class CrawlTxService extends BullableService { this.logger.debug('result insert tx', resultInsertGraph); } - private setMsgIndexToEvent(tx: any) { - const mapEventMsgIdx: Map = new Map(); - - // set index_msg from log to mapEventMsgIdx - tx.tx_response.logs?.forEach((log: any, index: number) => { - log.events.forEach((event: any) => { - const { type } = event; - event.attributes.forEach((attribute: any) => { - const keyInMap = `${type}_${attribute.key}_${attribute.value}`; - if (mapEventMsgIdx.has(keyInMap)) { - const listIndex = mapEventMsgIdx.get(keyInMap); - listIndex?.push(index); - } else { - mapEventMsgIdx.set(keyInMap, [index]); - } - }); + public mappingEventToLog(eventInLog: any, eventInTx: any, index: number) { + const arrSrc = eventInTx; + const arrFlat: any[] = []; + arrSrc.forEach((item: any, index: number) => { + item.attributes.forEach((attr: any) => { + arrFlat.push({ ...attr, indexEvent: index }); }); }); + // eslint-disable-next-line no-inner-declarations + function checkMapping(arr: any, arrCompare: any) { + const indexes = _.uniq( + arr + .filter((item: any) => item.indexMapped !== undefined) + .map((e: any) => e.indexEvent) + ); + // all event must be included in log + return indexes.every((index: any) => { + const isMapped = arrSrc[index].attributes.every((item: any) => { + const key = item.key ? fromUtf8(fromBase64(item.key)) : null; + const value = item.value ? fromUtf8(fromBase64(item.value)) : null; + // check if key and value found + return arrCompare.some( + (att: any) => att.key === key && att.value === value + ); + }); + return isMapped; + }); + } - // set index_msg from mapEventMsgIdx to event - tx.tx_response.events.forEach((event: any) => { - const { type } = event; - event.attributes.forEach((attribute: any) => { - const key = attribute?.key - ? fromUtf8(fromBase64(attribute?.key)) - : null; - const value = attribute?.value - ? fromUtf8(fromBase64(attribute?.value)) + // eslint-disable-next-line no-inner-declarations + function recursive( + arrSrc: any[], + arrDest: any[], + indexSrc: number, + indexDest: number, + indexMsg: number + ): boolean { + if (indexDest === arrDest.length) { + const isMapped = checkMapping(arrSrc, arrDest); + return isMapped; + } + if (indexSrc === arrSrc.length || indexDest === arrDest.length) { + return false; + } + let result = false; + for (let i = indexSrc; i < arrSrc.length; i += 1) { + const key = arrSrc[i].key ? fromUtf8(fromBase64(arrSrc[i].key)) : null; + const value = arrSrc[i].value + ? fromUtf8(fromBase64(arrSrc[i].value)) : null; - const keyInMap = `${type}_${key}_${value}`; - - const listIndex = mapEventMsgIdx.get(keyInMap); - // get first index with this key - const firstIndex = listIndex?.shift(); + if ( + key === arrDest[indexDest].key && + value === arrDest[indexDest].value + ) { + // eslint-disable-next-line no-param-reassign + arrSrc[i].indexMapped = indexMsg; + result = recursive(arrSrc, arrDest, i + 1, indexDest + 1, indexMsg); + if (result) { + break; + } + // eslint-disable-next-line no-param-reassign + delete arrSrc[i].indexMapped; + } + } + return result; + } - if (firstIndex != null) { - if (event.msg_index && event.msg_index !== firstIndex) { + const result = recursive(arrFlat, eventInLog.attributes, 0, 0, index); + if (result === false) { + this.logger.warn('something wrong: mapping event to log failed'); + } else { + arrFlat + .filter((item) => item.indexMapped !== undefined) + .forEach((item) => { + if ( + arrSrc[item.indexEvent].msg_index !== undefined && + arrSrc[item.indexEvent].msg_index !== item.indexMapped + ) { this.logger.warn( - `something wrong: setting index ${firstIndex} to existed index ${event.msg_index}` + `something wrong: setting index ${ + item.indexMapped + } to existed index ${arrSrc[item.indexEvent].msg_index}` ); - } else { - // eslint-disable-next-line no-param-reassign - event.msg_index = firstIndex; } - } + arrSrc[item.indexEvent].msg_index = item.indexMapped; + }); + } + } - // delete key in map if value is empty - if (listIndex?.length === 0) { - mapEventMsgIdx.delete(keyInMap); - } + private setMsgIndexToEvent(tx: any) { + // loop logs (has order for each messages) + tx.tx_response.logs.forEach((log: any, index: number) => { + // loop each event in log to compare with event encoded + log.events.forEach((eventInLog: any) => { + // filter list event has type equal eventInLog.type and not be setted index msg + const filtedEventByType = tx.tx_response.events.filter( + (eventEncoded: any) => + eventEncoded.type === eventInLog.type && + eventEncoded.msg_index === undefined + ); + + // mapping between log and event + this.mappingEventToLog(eventInLog, filtedEventByType, index); }); }); } From 70d87c0d80f178d98d93d641133ba65607137bdf Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Wed, 16 Aug 2023 17:16:56 +0700 Subject: [PATCH 08/17] fix: update msg_index by order in event and log --- src/services/crawl-tx/crawl_tx.service.ts | 135 +++++++++++++++------- 1 file changed, 94 insertions(+), 41 deletions(-) diff --git a/src/services/crawl-tx/crawl_tx.service.ts b/src/services/crawl-tx/crawl_tx.service.ts index 156fa3b97..326418bb9 100644 --- a/src/services/crawl-tx/crawl_tx.service.ts +++ b/src/services/crawl-tx/crawl_tx.service.ts @@ -333,56 +333,109 @@ export default class CrawlTxService extends BullableService { this.logger.debug('result insert tx', resultInsertGraph); } - private setMsgIndexToEvent(tx: any) { - const mapEventMsgIdx: Map = new Map(); - - // set index_msg from log to mapEventMsgIdx - tx.tx_response.logs?.forEach((log: any, index: number) => { - log.events.forEach((event: any) => { - const { type } = event; - event.attributes.forEach((attribute: any) => { - const keyInMap = `${type}_${attribute.key}_${attribute.value}`; - if (mapEventMsgIdx.has(keyInMap)) { - const listIndex = mapEventMsgIdx.get(keyInMap); - listIndex?.push(index); - } else { - mapEventMsgIdx.set(keyInMap, [index]); - } - }); + public mappingEventToLog(eventInLog: any, eventInTx: any, index: number) { + const arrSrc = eventInTx; + const arrFlat: any[] = []; + arrSrc.forEach((item: any, index: number) => { + item.attributes.forEach((attr: any) => { + arrFlat.push({ ...attr, indexEvent: index }); }); }); + // eslint-disable-next-line no-inner-declarations + function checkMapping(arr: any, arrCompare: any) { + const indexes = _.uniq( + arr + .filter((item: any) => item.indexMapped !== undefined) + .map((e: any) => e.indexEvent) + ); + // all event must be included in log + return indexes.every((index: any) => { + const isMapped = arrSrc[index].attributes.every((item: any) => { + const key = item.key ? fromUtf8(fromBase64(item.key)) : null; + const value = item.value ? fromUtf8(fromBase64(item.value)) : null; + // check if key and value found + return arrCompare.some( + (att: any) => att.key === key && att.value === value + ); + }); + return isMapped; + }); + } - // set index_msg from mapEventMsgIdx to event - tx.tx_response.events.forEach((event: any) => { - const { type } = event; - event.attributes.forEach((attribute: any) => { - const key = attribute?.key - ? fromUtf8(fromBase64(attribute?.key)) - : null; - const value = attribute?.value - ? fromUtf8(fromBase64(attribute?.value)) + // eslint-disable-next-line no-inner-declarations + function recursive( + arrSrc: any[], + arrDest: any[], + indexSrc: number, + indexDest: number, + indexMsg: number + ): boolean { + if (indexDest === arrDest.length) { + const isMapped = checkMapping(arrSrc, arrDest); + return isMapped; + } + if (indexSrc === arrSrc.length || indexDest === arrDest.length) { + return false; + } + let result = false; + for (let i = indexSrc; i < arrSrc.length; i += 1) { + const key = arrSrc[i].key ? fromUtf8(fromBase64(arrSrc[i].key)) : null; + const value = arrSrc[i].value + ? fromUtf8(fromBase64(arrSrc[i].value)) : null; - const keyInMap = `${type}_${key}_${value}`; - - const listIndex = mapEventMsgIdx.get(keyInMap); - // get first index with this key - const firstIndex = listIndex?.shift(); + if ( + key === arrDest[indexDest].key && + value === arrDest[indexDest].value + ) { + // eslint-disable-next-line no-param-reassign + arrSrc[i].indexMapped = indexMsg; + result = recursive(arrSrc, arrDest, i + 1, indexDest + 1, indexMsg); + if (result) { + break; + } + // eslint-disable-next-line no-param-reassign + delete arrSrc[i].indexMapped; + } + } + return result; + } - if (firstIndex != null) { - if (event.msg_index && event.msg_index !== firstIndex) { + const result = recursive(arrFlat, eventInLog.attributes, 0, 0, index); + if (result === false) { + this.logger.warn('something wrong: mapping event to log failed'); + } else { + arrFlat + .filter((item) => item.indexMapped !== undefined) + .forEach((item) => { + if ( + arrSrc[item.indexEvent].msg_index !== undefined && + arrSrc[item.indexEvent].msg_index !== item.indexMapped + ) { this.logger.warn( - `something wrong: setting index ${firstIndex} to existed index ${event.msg_index}` + `something wrong: setting index ${ + item.indexMapped + } to existed index ${arrSrc[item.indexEvent].msg_index}` ); - } else { - // eslint-disable-next-line no-param-reassign - event.msg_index = firstIndex; } - } + arrSrc[item.indexEvent].msg_index = item.indexMapped; + }); + } + } - // delete key in map if value is empty - if (listIndex?.length === 0) { - mapEventMsgIdx.delete(keyInMap); - } + private setMsgIndexToEvent(tx: any) { + // loop logs (has order for each messages) + tx.tx_response.logs.forEach((log: any, index: number) => { + // loop each event in log to compare with event encoded + log.events.forEach((eventInLog: any) => { + // filter list event has type equal eventInLog.type and not be setted index msg + const filtedEventByType = tx.tx_response.events.filter( + (eventEncoded: any) => + eventEncoded.type === eventInLog.type && + eventEncoded.msg_index === undefined + ); + + // mapping between log and event + this.mappingEventToLog(eventInLog, filtedEventByType, index); }); }); } From 68d247fcd01f96d9bda985ee8e4543ee4b2f9088 Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Thu, 17 Aug 2023 10:52:47 +0700 Subject: [PATCH 09/17] feat: add test mapping event and log --- .../parse_transaction.spec.ts | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/test/unit/services/crawl-transaction/parse_transaction.spec.ts b/test/unit/services/crawl-transaction/parse_transaction.spec.ts index c72cbe542..b9abf2159 100644 --- a/test/unit/services/crawl-transaction/parse_transaction.spec.ts +++ b/test/unit/services/crawl-transaction/parse_transaction.spec.ts @@ -154,6 +154,85 @@ export default class CrawlTransactionTest { // ); } + @Test('Mapping event and log') + public async testMappingEventToLog() { + const arrDest = { + index: 3, + type: 'message', + attributes: [ + { + key: 'action', + value: '/cosmos.staking.v1beta1.MsgDelegate', + }, + { + key: 'sender', + value: 'aura1jv65s3grqf6v6jl3dp4t6c9t9rk99cd8ufn7tx', + }, + { + key: 'module', + value: 'staking', + }, + { + key: 'sender', + value: 'aura15x4v36r6rl73nhn9h0954mwp42sawrc25f0rnx', + }, + ], + }; + const arrSrc: any[] = [ + { + // this event is not in log (arrDest), so it doesn't has checkedIndex + type: 'message', + attributes: [ + { + key: 'c2VuZGVy', + value: + 'YXVyYTE1eDR2MzZyNnJsNzNuaG45aDA5NTRtd3A0MnNhd3JjMjVmMHJueA==', + }, + ], + }, + { + checkedIndex: 3, + type: 'message', + attributes: [ + { + key: 'YWN0aW9u', + value: 'L2Nvc21vcy5zdGFraW5nLnYxYmV0YTEuTXNnRGVsZWdhdGU=', + }, + ], + }, + { + checkedIndex: 3, + type: 'message', + attributes: [ + { + key: 'c2VuZGVy', + value: + 'YXVyYTFqdjY1czNncnFmNnY2amwzZHA0dDZjOXQ5cms5OWNkOHVmbjd0eA==', + }, + ], + }, + { + checkedIndex: 3, + type: 'message', + attributes: [ + { + key: 'bW9kdWxl', + value: 'c3Rha2luZw==', + }, + { + key: 'c2VuZGVy', + value: + 'YXVyYTE1eDR2MzZyNnJsNzNuaG45aDA5NTRtd3A0MnNhd3JjMjVmMHJueA==', + }, + ], + }, + ]; + this.crawlTxService?.mappingEventToLog(arrDest, arrSrc, arrDest.index); + arrSrc.forEach((item) => { + expect(item.checkedIndex).toEqual(item.msg_index); + }); + } + @AfterEach() async tearDown() { this.crawlTxService?.getQueueManager().stopAll(); From 200c00c80bea92651c387d3b0a9c2da8a5803403 Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Thu, 17 Aug 2023 11:25:05 +0700 Subject: [PATCH 10/17] feat: add test mapping by authz tx --- .../parse_transaction.spec.ts | 443 +++++++++++++++--- 1 file changed, 378 insertions(+), 65 deletions(-) diff --git a/test/unit/services/crawl-transaction/parse_transaction.spec.ts b/test/unit/services/crawl-transaction/parse_transaction.spec.ts index b9abf2159..5f8608704 100644 --- a/test/unit/services/crawl-transaction/parse_transaction.spec.ts +++ b/test/unit/services/crawl-transaction/parse_transaction.spec.ts @@ -154,81 +154,394 @@ export default class CrawlTransactionTest { // ); } - @Test('Mapping event and log') - public async testMappingEventToLog() { - const arrDest = { - index: 3, - type: 'message', + arrDest = { + index: 3, + type: 'coin_received', + attributes: [ + { key: 'receiver', value: 'aura162x2llsxzxmavtyuxjesceewmy4wvrp79ndcrw' }, + { key: 'amount', value: '1341uaura' }, + { key: 'authz_msg_index', value: '0' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '909uaura' }, + { key: 'authz_msg_index', value: '0' }, + { key: 'receiver', value: 'aura1ujv2gmfwrwzj504ntggqld0q5euafp76vgx5lj' }, + { key: 'amount', value: '154uaura' }, + { key: 'authz_msg_index', value: '1' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '104uaura' }, + { key: 'authz_msg_index', value: '1' }, + { key: 'receiver', value: 'aura177cgzmjve5m0je6yjukcj4mmmwj8p4dkqekglz' }, + { key: 'amount', value: '4511uaura' }, + { key: 'authz_msg_index', value: '2' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '3056uaura' }, + { key: 'authz_msg_index', value: '2' }, + { key: 'receiver', value: 'aura1rqll2d4wyylvl03ht6mhglswj46gkcr3ksvkm7' }, + { key: 'amount', value: '3617uaura' }, + { key: 'authz_msg_index', value: '3' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '2451uaura' }, + { key: 'authz_msg_index', value: '3' }, + { key: 'receiver', value: 'aura1a6x0znjhztz73tq07gjvzt9ru99866jm665w9p' }, + { key: 'amount', value: '5417uaura' }, + { key: 'authz_msg_index', value: '4' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '3670uaura' }, + { key: 'authz_msg_index', value: '4' }, + { key: 'receiver', value: 'aura1q93xkwtfv7nut0eqjjws377wkjk97265zsxlx6' }, + { key: 'amount', value: '30uaura' }, + { key: 'authz_msg_index', value: '5' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '20uaura' }, + { key: 'authz_msg_index', value: '5' }, + { key: 'receiver', value: 'aura1xk6nnn0gen9n9fduz0t3twyzt8c2uzedy2545a' }, + { key: 'amount', value: '2797uaura' }, + { key: 'authz_msg_index', value: '6' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '1895uaura' }, + { key: 'authz_msg_index', value: '6' }, + { key: 'receiver', value: 'aura1f3qxww8pnx0xrea7e03ffxnlau0fk5ufs3v0zj' }, + { key: 'amount', value: '2457uaura' }, + { key: 'authz_msg_index', value: '7' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '1665uaura' }, + { key: 'authz_msg_index', value: '7' }, + { key: 'receiver', value: 'aura1fwtkqe4yp652svrj5lzdu9lnykysh947msc4xq' }, + { key: 'amount', value: '2589uaura' }, + { key: 'authz_msg_index', value: '8' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '1754uaura' }, + { key: 'authz_msg_index', value: '8' }, + { key: 'receiver', value: 'aura1j6kwc05lw0p8e08ce3r5z5qlygjzu0aq095scc' }, + { key: 'amount', value: '212uaura' }, + { key: 'authz_msg_index', value: '9' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '143uaura' }, + { key: 'authz_msg_index', value: '9' }, + { key: 'receiver', value: 'aura15f6wn3nymdnhnh5ddlqletuptjag09tryrtpq5' }, + { key: 'amount', value: '1689uaura' }, + { key: 'authz_msg_index', value: '10' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '1144uaura' }, + { key: 'authz_msg_index', value: '10' }, + { key: 'receiver', value: 'aura1efq5q4uzn583nh5mauzc5cmgms53w9l6vs5dxz' }, + { key: 'amount', value: '105uaura' }, + { key: 'authz_msg_index', value: '11' }, + { key: 'receiver', value: 'aura1fl48vsnmsdzcv85q5d2q4z5ajdha8yu3wd7dmw' }, + { key: 'amount', value: '71uaura' }, + { key: 'authz_msg_index', value: '11' }, + ], + }; + + arrSrc: any[] = [ + { + type: 'coin_received', attributes: [ { - key: 'action', - value: '/cosmos.staking.v1beta1.MsgDelegate', + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTE3eHBmdmFrbTJhbWc5NjJ5bHM2Zjg0ejNrZWxsOGM1bHQwNXpmeQ==', }, + { key: 'YW1vdW50', value: 'ODk2dWF1cmE=' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ { - key: 'sender', - value: 'aura1jv65s3grqf6v6jl3dp4t6c9t9rk99cd8ufn7tx', + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTE2MngybGxzeHp4bWF2dHl1eGplc2NlZXdteTR3dnJwNzluZGNydw==', }, + { key: 'YW1vdW50', value: 'MTM0MXVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'MA==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ { - key: 'module', - value: 'staking', + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', }, + { key: 'YW1vdW50', value: 'OTA5dWF1cmE=' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'MA==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ { - key: 'sender', - value: 'aura15x4v36r6rl73nhn9h0954mwp42sawrc25f0rnx', + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTF1anYyZ21md3J3emo1MDRudGdncWxkMHE1ZXVhZnA3NnZneDVsag==', }, + { key: 'YW1vdW50', value: 'MTU0dWF1cmE=' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'MQ==' }, ], - }; - const arrSrc: any[] = [ - { - // this event is not in log (arrDest), so it doesn't has checkedIndex - type: 'message', - attributes: [ - { - key: 'c2VuZGVy', - value: - 'YXVyYTE1eDR2MzZyNnJsNzNuaG45aDA5NTRtd3A0MnNhd3JjMjVmMHJueA==', - }, - ], - }, - { - checkedIndex: 3, - type: 'message', - attributes: [ - { - key: 'YWN0aW9u', - value: 'L2Nvc21vcy5zdGFraW5nLnYxYmV0YTEuTXNnRGVsZWdhdGU=', - }, - ], - }, - { - checkedIndex: 3, - type: 'message', - attributes: [ - { - key: 'c2VuZGVy', - value: - 'YXVyYTFqdjY1czNncnFmNnY2amwzZHA0dDZjOXQ5cms5OWNkOHVmbjd0eA==', - }, - ], - }, - { - checkedIndex: 3, - type: 'message', - attributes: [ - { - key: 'bW9kdWxl', - value: 'c3Rha2luZw==', - }, - { - key: 'c2VuZGVy', - value: - 'YXVyYTE1eDR2MzZyNnJsNzNuaG45aDA5NTRtd3A0MnNhd3JjMjVmMHJueA==', - }, - ], - }, - ]; - this.crawlTxService?.mappingEventToLog(arrDest, arrSrc, arrDest.index); - arrSrc.forEach((item) => { + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'MTA0dWF1cmE=' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'MQ==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTE3N2Nnem1qdmU1bTBqZTZ5anVrY2o0bW1td2o4cDRka3Fla2dseg==', + }, + { key: 'YW1vdW50', value: 'NDUxMXVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'Mg==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'MzA1NnVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'Mg==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFycWxsMmQ0d3l5bHZsMDNodDZtaGdsc3dqNDZna2NyM2tzdmttNw==', + }, + { key: 'YW1vdW50', value: 'MzYxN3VhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'Mw==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'MjQ1MXVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'Mw==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFhNngwem5qaHp0ejczdHEwN2dqdnp0OXJ1OTk4NjZqbTY2NXc5cA==', + }, + { key: 'YW1vdW50', value: 'NTQxN3VhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'NA==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'MzY3MHVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'NA==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFxOTN4a3d0ZnY3bnV0MGVxamp3czM3N3drams5NzI2NXpzeGx4Ng==', + }, + { key: 'YW1vdW50', value: 'MzB1YXVyYQ==' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'NQ==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'MjB1YXVyYQ==' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'NQ==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTF4azZubm4wZ2VuOW45ZmR1ejB0M3R3eXp0OGMydXplZHkyNTQ1YQ==', + }, + { key: 'YW1vdW50', value: 'Mjc5N3VhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'Ng==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'MTg5NXVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'Ng==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmM3F4d3c4cG54MHhyZWE3ZTAzZmZ4bmxhdTBmazV1ZnMzdjB6ag==', + }, + { key: 'YW1vdW50', value: 'MjQ1N3VhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'Nw==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'MTY2NXVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'Nw==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmd3RrcWU0eXA2NTJzdnJqNWx6ZHU5bG55a3lzaDk0N21zYzR4cQ==', + }, + { key: 'YW1vdW50', value: 'MjU4OXVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'OA==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'MTc1NHVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'OA==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFqNmt3YzA1bHcwcDhlMDhjZTNyNXo1cWx5Z2p6dTBhcTA5NXNjYw==', + }, + { key: 'YW1vdW50', value: 'MjEydWF1cmE=' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'OQ==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'MTQzdWF1cmE=' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'OQ==' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTE1ZjZ3bjNueW1kbmhuaDVkZGxxbGV0dXB0amFnMDl0cnlydHBxNQ==', + }, + { key: 'YW1vdW50', value: 'MTY4OXVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'MTA=' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'MTE0NHVhdXJh' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'MTA=' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFlZnE1cTR1em41ODNuaDVtYXV6YzVjbWdtczUzdzlsNnZzNWR4eg==', + }, + { key: 'YW1vdW50', value: 'MTA1dWF1cmE=' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'MTE=' }, + ], + }, + { + type: 'coin_received', + checkedIndex: 3, + attributes: [ + { + key: 'cmVjZWl2ZXI=', + value: 'YXVyYTFmbDQ4dnNubXNkemN2ODVxNWQycTR6NWFqZGhhOHl1M3dkN2Rtdw==', + }, + { key: 'YW1vdW50', value: 'NzF1YXVyYQ==' }, + { key: 'YXV0aHpfbXNnX2luZGV4', value: 'MTE=' }, + ], + }, + ]; + + @Test('Mapping event and log') + public async testMappingEventToLog() { + this.crawlTxService?.mappingEventToLog( + this.arrDest, + this.arrSrc, + this.arrDest.index + ); + this.arrSrc.forEach((item) => { expect(item.checkedIndex).toEqual(item.msg_index); }); } From 67663de85f48aa127b9bf1dcf5c9e34b072e1cfe Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Thu, 17 Aug 2023 11:26:08 +0700 Subject: [PATCH 11/17] feat: add test mapping by authz tx --- test/unit/services/crawl-transaction/parse_transaction.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/services/crawl-transaction/parse_transaction.spec.ts b/test/unit/services/crawl-transaction/parse_transaction.spec.ts index 5f8608704..e0a633b0b 100644 --- a/test/unit/services/crawl-transaction/parse_transaction.spec.ts +++ b/test/unit/services/crawl-transaction/parse_transaction.spec.ts @@ -235,6 +235,7 @@ export default class CrawlTransactionTest { arrSrc: any[] = [ { + // this event not has checkedIndex because its belong to tx event type: 'coin_received', attributes: [ { From 336c953e6a8da301ed796ef71a8b45fd96a4136d Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Fri, 18 Aug 2023 14:42:35 +0700 Subject: [PATCH 12/17] fix: refactor mapping event to log, update unit test --- src/services/crawl-tx/crawl_tx.service.ts | 206 +++++++++++------- .../parse_transaction.spec.ts | 30 ++- 2 files changed, 151 insertions(+), 85 deletions(-) diff --git a/src/services/crawl-tx/crawl_tx.service.ts b/src/services/crawl-tx/crawl_tx.service.ts index 326418bb9..deeb23fde 100644 --- a/src/services/crawl-tx/crawl_tx.service.ts +++ b/src/services/crawl-tx/crawl_tx.service.ts @@ -333,111 +333,155 @@ export default class CrawlTxService extends BullableService { this.logger.debug('result insert tx', resultInsertGraph); } - public mappingEventToLog(eventInLog: any, eventInTx: any, index: number) { - const arrSrc = eventInTx; - const arrFlat: any[] = []; - arrSrc.forEach((item: any, index: number) => { - item.attributes.forEach((attr: any) => { - arrFlat.push({ ...attr, indexEvent: index }); - }); - }); - // eslint-disable-next-line no-inner-declarations - function checkMapping(arr: any, arrCompare: any) { - const indexes = _.uniq( - arr - .filter((item: any) => item.indexMapped !== undefined) - .map((e: any) => e.indexEvent) - ); - // all event must be included in log - return indexes.every((index: any) => { - const isMapped = arrSrc[index].attributes.every((item: any) => { - const key = item.key ? fromUtf8(fromBase64(item.key)) : null; - const value = item.value ? fromUtf8(fromBase64(item.value)) : null; - // check if key and value found - return arrCompare.some( - (att: any) => att.key === key && att.value === value - ); - }); - return isMapped; - }); - } + public mappingFlatEventToLog( + eventHasIndex: any, + eventEncodedFlat: any, + indexMsg: number + ) { + const attributesInEvent = eventHasIndex.attributes; + // i, j are index of eventHasIndex and eventEncodedFlat + let i = 0; + let j = 0; + while (i < attributesInEvent.length && j < eventEncodedFlat.length) { + // find last element with same eventIndex + let lastIndexSameEvent = j; + while ( + lastIndexSameEvent < eventEncodedFlat.length && + eventEncodedFlat[lastIndexSameEvent].indexEvent === + eventEncodedFlat[j].indexEvent + ) { + lastIndexSameEvent += 1; + } - // eslint-disable-next-line no-inner-declarations - function recursive( - arrSrc: any[], - arrDest: any[], - indexSrc: number, - indexDest: number, - indexMsg: number - ): boolean { - if (indexDest === arrDest.length) { - const isMapped = checkMapping(arrSrc, arrDest); - return isMapped; + if ( + attributesInEvent[i].key === eventEncodedFlat[j].key && + attributesInEvent[i].value === eventEncodedFlat[j].value + ) { + const isEventMapped = eventEncodedFlat + .slice(j, lastIndexSameEvent) + // eslint-disable-next-line @typescript-eslint/no-loop-func, no-inner-declarations + .every((item: any, index: number) => { + if ( + attributesInEvent[i + index].key === item.key && + attributesInEvent[i + index].value === item.value + ) { + return true; + } + return false; + }); + if (isEventMapped) { + // mark event encoded to be mapped + for (let k = j; k < lastIndexSameEvent; k += 1) { + // eslint-disable-next-line no-param-reassign + eventEncodedFlat[k].indexMapped = indexMsg; + } + i += lastIndexSameEvent - j; + } } - if (indexSrc === arrSrc.length || indexDest === arrDest.length) { - return false; + j = lastIndexSameEvent; + } + } + + // self check msg index by counting event + private selfCheckByAnotherWay(tx: any) { + // count total attribute for each message, countAttributeInEvent[i] = x mean message i has x attributes + const countAttributeInEvent: number[] = []; + tx.tx_response.logs.forEach((log: any) => { + const countAttribute = log.events.reduce( + (acc: number, curr: any) => acc + curr.attributes.length, + 0 + ); + countAttributeInEvent.push(countAttribute); + }); + + let reachLastEventTypeTx = false; + let countCurrentAttribute = 0; + let currentCompareEventId = 0; + for (let i = 0; i < tx.tx_response.events.length; i += 1) { + if (tx.tx_response.events[i].type === 'tx') { + reachLastEventTypeTx = true; } - let result = false; - for (let i = indexSrc; i < arrSrc.length; i += 1) { - const key = arrSrc[i].key ? fromUtf8(fromBase64(arrSrc[i].key)) : null; - const value = arrSrc[i].value - ? fromUtf8(fromBase64(arrSrc[i].value)) - : null; + if (reachLastEventTypeTx && tx.tx_response.events[i].type !== 'tx') { if ( - key === arrDest[indexDest].key && - value === arrDest[indexDest].value + countCurrentAttribute < countAttributeInEvent[currentCompareEventId] ) { - // eslint-disable-next-line no-param-reassign - arrSrc[i].indexMapped = indexMsg; - result = recursive(arrSrc, arrDest, i + 1, indexDest + 1, indexMsg); - if (result) { - break; - } - // eslint-disable-next-line no-param-reassign - delete arrSrc[i].indexMapped; + countCurrentAttribute += tx.tx_response.events[i].attributes.length; } - } - return result; - } - const result = recursive(arrFlat, eventInLog.attributes, 0, 0, index); - if (result === false) { - this.logger.warn('something wrong: mapping event to log failed'); - } else { - arrFlat - .filter((item) => item.indexMapped !== undefined) - .forEach((item) => { - if ( - arrSrc[item.indexEvent].msg_index !== undefined && - arrSrc[item.indexEvent].msg_index !== item.indexMapped - ) { - this.logger.warn( - `something wrong: setting index ${ - item.indexMapped - } to existed index ${arrSrc[item.indexEvent].msg_index}` - ); - } - arrSrc[item.indexEvent].msg_index = item.indexMapped; - }); + // after count, check if count is equal countAttributeInEvent[currentCompareEventId] or not + if ( + countCurrentAttribute === countAttributeInEvent[currentCompareEventId] + ) { + // if true, count success, then next currentCompareEventId and reset count = 0 + currentCompareEventId += 1; + countCurrentAttribute = 0; + } else if ( + countCurrentAttribute > countAttributeInEvent[currentCompareEventId] + ) { + this.logger.warn('Count event in log is not equal event encoded'); + return false; + } + } } + return true; } private setMsgIndexToEvent(tx: any) { + // flatten event and decode key value + const eventEncodedFlats: any[] = []; + tx.tx_response.events.forEach((event: any, index: number) => { + event.attributes.forEach((attr: any) => { + eventEncodedFlats.push({ + ...{ + key: attr.key ? fromUtf8(fromBase64(attr.key)) : null, + value: attr.value ? fromUtf8(fromBase64(attr.value)) : null, + }, + indexEvent: index, + type: event.type, + }); + }); + }); + // loop logs (has order for each messages) tx.tx_response.logs.forEach((log: any, index: number) => { // loop each event in log to compare with event encoded log.events.forEach((eventInLog: any) => { // filter list event has type equal eventInLog.type and not be setted index msg - const filtedEventByType = tx.tx_response.events.filter( + const filtedEventByTypeFlat = eventEncodedFlats.filter( (eventEncoded: any) => eventEncoded.type === eventInLog.type && eventEncoded.msg_index === undefined ); // mapping between log and event - this.mappingEventToLog(eventInLog, filtedEventByType, index); + this.mappingFlatEventToLog(eventInLog, filtedEventByTypeFlat, index); }); }); + // set index msg to event encoded + eventEncodedFlats + .filter((item: any) => item.indexMapped !== undefined) + .forEach((item: any) => { + if ( + tx.tx_response.events[item.indexEvent].msg_index !== undefined && + tx.tx_response.events[item.indexEvent].msg_index !== item.indexMapped + ) { + this.logger.warn( + `something wrong: setting index ${ + item.indexMapped + } to existed index ${ + tx.tx_response.eventstx.tx_response.events[item.indexEvent] + .msg_index + }` + ); + } + // eslint-disable-next-line no-param-reassign + tx.tx_response.events[item.indexEvent].msg_index = item.indexMapped; + }); + // self check msg index by counting event + const selfCheck = this.selfCheckByAnotherWay(tx); + if (!selfCheck) { + this.logger.warn('selfcheck fail'); + } } private _findAttribute( diff --git a/test/unit/services/crawl-transaction/parse_transaction.spec.ts b/test/unit/services/crawl-transaction/parse_transaction.spec.ts index e0a633b0b..ff785aa85 100644 --- a/test/unit/services/crawl-transaction/parse_transaction.spec.ts +++ b/test/unit/services/crawl-transaction/parse_transaction.spec.ts @@ -4,6 +4,7 @@ import { AfterEach, BeforeEach, Describe, Test } from '@jest-decorated/core'; import { ServiceBroker } from 'moleculer'; import { Log } from '@cosmjs/stargate/build/logs'; import { Attribute, Event } from '@cosmjs/stargate/build/events'; +import { fromBase64, fromUtf8 } from '@cosmjs/encoding'; import { Transaction, Event as EventModel, @@ -537,14 +538,35 @@ export default class CrawlTransactionTest { @Test('Mapping event and log') public async testMappingEventToLog() { - this.crawlTxService?.mappingEventToLog( + const eventEncodedFlats: any[] = []; + this.arrSrc.forEach((event: any, index: number) => { + event.attributes.forEach((attr: any) => { + eventEncodedFlats.push({ + ...{ + key: attr.key ? fromUtf8(fromBase64(attr.key)) : null, + value: attr.value ? fromUtf8(fromBase64(attr.value)) : null, + }, + indexEvent: index, + type: event.type, + }); + }); + }); + + this.crawlTxService?.mappingFlatEventToLog( this.arrDest, this.arrSrc, this.arrDest.index ); - this.arrSrc.forEach((item) => { - expect(item.checkedIndex).toEqual(item.msg_index); - }); + + eventEncodedFlats + .filter((item: any) => item.indexMapped !== undefined) + .forEach((item: any) => { + // eslint-disable-next-line no-param-reassign + this.arrSrc[item.indexEvent].msg_index = item.indexMapped; + expect(this.arrSrc[item.indexEvent].msg_index).toEqual( + item.checkedIndex + ); + }); } @AfterEach() From 9cea3a343708b21a8c3ce0432a20fd2a2d8f4672 Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Fri, 18 Aug 2023 17:07:50 +0700 Subject: [PATCH 13/17] fix: comment set index msg with order, use count attribute to compare --- src/services/crawl-tx/crawl_tx.service.ts | 144 ++++++++++++++-------- 1 file changed, 96 insertions(+), 48 deletions(-) diff --git a/src/services/crawl-tx/crawl_tx.service.ts b/src/services/crawl-tx/crawl_tx.service.ts index deeb23fde..ad10bb807 100644 --- a/src/services/crawl-tx/crawl_tx.service.ts +++ b/src/services/crawl-tx/crawl_tx.service.ts @@ -427,60 +427,108 @@ export default class CrawlTxService extends BullableService { } private setMsgIndexToEvent(tx: any) { - // flatten event and decode key value - const eventEncodedFlats: any[] = []; - tx.tx_response.events.forEach((event: any, index: number) => { - event.attributes.forEach((attr: any) => { - eventEncodedFlats.push({ - ...{ - key: attr.key ? fromUtf8(fromBase64(attr.key)) : null, - value: attr.value ? fromUtf8(fromBase64(attr.value)) : null, - }, - indexEvent: index, - type: event.type, - }); - }); + /*------ + DO NOT USE CURRENTLY + MAPPING BY ORDER IN EVENT AND LOG + THIS CASE BASED ON ORDER NOT CHANGED BETWEEN EVENT AND LOG + --------*/ + // // flatten event and decode key value + // const eventEncodedFlats: any[] = []; + // tx.tx_response.events.forEach((event: any, index: number) => { + // event.attributes.forEach((attr: any) => { + // eventEncodedFlats.push({ + // ...{ + // key: attr.key ? fromUtf8(fromBase64(attr.key)) : null, + // value: attr.value ? fromUtf8(fromBase64(attr.value)) : null, + // }, + // indexEvent: index, + // type: event.type, + // }); + // }); + // }); + // // loop logs (has order for each messages) + // tx.tx_response.logs.forEach((log: any, index: number) => { + // // loop each event in log to compare with event encoded + // log.events.forEach((eventInLog: any) => { + // // filter list event has type equal eventInLog.type and not be setted index msg + // const filtedEventByTypeFlat = eventEncodedFlats.filter( + // (eventEncoded: any) => + // eventEncoded.type === eventInLog.type && + // eventEncoded.msg_index === undefined + // ); + // // mapping between log and event + // this.mappingFlatEventToLog(eventInLog, filtedEventByTypeFlat, index); + // }); + // }); + // // set index msg to event encoded + // eventEncodedFlats + // .filter((item: any) => item.indexMapped !== undefined) + // .forEach((item: any) => { + // if ( + // tx.tx_response.events[item.indexEvent].msg_index !== undefined && + // tx.tx_response.events[item.indexEvent].msg_index !== item.indexMapped + // ) { + // this.logger.warn( + // `something wrong: setting index ${ + // item.indexMapped + // } to existed index ${ + // tx.tx_response.eventstx.tx_response.events[item.indexEvent] + // .msg_index + // }` + // ); + // } + // // eslint-disable-next-line no-param-reassign + // tx.tx_response.events[item.indexEvent].msg_index = item.indexMapped; + // }); + // // self check msg index by counting event + // const selfCheck = this.selfCheckByAnotherWay(tx); + // if (!selfCheck) { + // this.logger.warn('selfcheck fail'); + // } + + /*--------- + TESTING + MAPPING EVENT BY COUNT EACH LOG AND EVENT MUST BE SAME + -----------*/ + // count total attribute for each message, countAttributeInEvent[i] = x mean message i has x attributes + const countAttributeInEvent: number[] = []; + tx.tx_response.logs.forEach((log: any) => { + const countAttribute = log.events.reduce( + (acc: number, curr: any) => acc + curr.attributes.length, + 0 + ); + countAttributeInEvent.push(countAttribute); }); - // loop logs (has order for each messages) - tx.tx_response.logs.forEach((log: any, index: number) => { - // loop each event in log to compare with event encoded - log.events.forEach((eventInLog: any) => { - // filter list event has type equal eventInLog.type and not be setted index msg - const filtedEventByTypeFlat = eventEncodedFlats.filter( - (eventEncoded: any) => - eventEncoded.type === eventInLog.type && - eventEncoded.msg_index === undefined - ); + let reachLastEventTypeTx = false; + let countCurrentAttribute = 0; + let currentCompareEventId = 0; + for (let i = 0; i < tx.tx_response.events.length; i += 1) { + if (tx.tx_response.events[i].type === 'tx') { + reachLastEventTypeTx = true; + } + if (reachLastEventTypeTx && tx.tx_response.events[i].type !== 'tx') { + if ( + countCurrentAttribute < countAttributeInEvent[currentCompareEventId] + ) { + countCurrentAttribute += tx.tx_response.events[i].attributes.length; + // eslint-disable-next-line no-param-reassign + tx.tx_response.events[i].msg_index = currentCompareEventId; + } - // mapping between log and event - this.mappingFlatEventToLog(eventInLog, filtedEventByTypeFlat, index); - }); - }); - // set index msg to event encoded - eventEncodedFlats - .filter((item: any) => item.indexMapped !== undefined) - .forEach((item: any) => { + // after count, check if count is equal countAttributeInEvent[currentCompareEventId] or not if ( - tx.tx_response.events[item.indexEvent].msg_index !== undefined && - tx.tx_response.events[item.indexEvent].msg_index !== item.indexMapped + countCurrentAttribute === countAttributeInEvent[currentCompareEventId] ) { - this.logger.warn( - `something wrong: setting index ${ - item.indexMapped - } to existed index ${ - tx.tx_response.eventstx.tx_response.events[item.indexEvent] - .msg_index - }` - ); + // if true, count success, then next currentCompareEventId and reset count = 0 + currentCompareEventId += 1; + countCurrentAttribute = 0; + } else if ( + countCurrentAttribute > countAttributeInEvent[currentCompareEventId] + ) { + this.logger.warn('Count event in log is not equal event encoded'); } - // eslint-disable-next-line no-param-reassign - tx.tx_response.events[item.indexEvent].msg_index = item.indexMapped; - }); - // self check msg index by counting event - const selfCheck = this.selfCheckByAnotherWay(tx); - if (!selfCheck) { - this.logger.warn('selfcheck fail'); + } } } From 97ddec263ec147c928d8b9e98ff1ea718d07554c Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Fri, 18 Aug 2023 17:07:50 +0700 Subject: [PATCH 14/17] fix: comment set index msg with order, use count attribute to compare --- src/services/crawl-tx/crawl_tx.service.ts | 114 +++++++++++++++++++--- 1 file changed, 101 insertions(+), 13 deletions(-) diff --git a/src/services/crawl-tx/crawl_tx.service.ts b/src/services/crawl-tx/crawl_tx.service.ts index 326418bb9..343a20ee9 100644 --- a/src/services/crawl-tx/crawl_tx.service.ts +++ b/src/services/crawl-tx/crawl_tx.service.ts @@ -423,21 +423,109 @@ export default class CrawlTxService extends BullableService { } private setMsgIndexToEvent(tx: any) { - // loop logs (has order for each messages) - tx.tx_response.logs.forEach((log: any, index: number) => { - // loop each event in log to compare with event encoded - log.events.forEach((eventInLog: any) => { - // filter list event has type equal eventInLog.type and not be setted index msg - const filtedEventByType = tx.tx_response.events.filter( - (eventEncoded: any) => - eventEncoded.type === eventInLog.type && - eventEncoded.msg_index === undefined - ); + /*------ + DO NOT USE CURRENTLY + MAPPING BY ORDER IN EVENT AND LOG + THIS CASE BASED ON ORDER NOT CHANGED BETWEEN EVENT AND LOG + --------*/ + // // flatten event and decode key value + // const eventEncodedFlats: any[] = []; + // tx.tx_response.events.forEach((event: any, index: number) => { + // event.attributes.forEach((attr: any) => { + // eventEncodedFlats.push({ + // ...{ + // key: attr.key ? fromUtf8(fromBase64(attr.key)) : null, + // value: attr.value ? fromUtf8(fromBase64(attr.value)) : null, + // }, + // indexEvent: index, + // type: event.type, + // }); + // }); + // }); + // // loop logs (has order for each messages) + // tx.tx_response.logs.forEach((log: any, index: number) => { + // // loop each event in log to compare with event encoded + // log.events.forEach((eventInLog: any) => { + // // filter list event has type equal eventInLog.type and not be setted index msg + // const filtedEventByTypeFlat = eventEncodedFlats.filter( + // (eventEncoded: any) => + // eventEncoded.type === eventInLog.type && + // eventEncoded.msg_index === undefined + // ); + // // mapping between log and event + // this.mappingFlatEventToLog(eventInLog, filtedEventByTypeFlat, index); + // }); + // }); + // // set index msg to event encoded + // eventEncodedFlats + // .filter((item: any) => item.indexMapped !== undefined) + // .forEach((item: any) => { + // if ( + // tx.tx_response.events[item.indexEvent].msg_index !== undefined && + // tx.tx_response.events[item.indexEvent].msg_index !== item.indexMapped + // ) { + // this.logger.warn( + // `something wrong: setting index ${ + // item.indexMapped + // } to existed index ${ + // tx.tx_response.eventstx.tx_response.events[item.indexEvent] + // .msg_index + // }` + // ); + // } + // // eslint-disable-next-line no-param-reassign + // tx.tx_response.events[item.indexEvent].msg_index = item.indexMapped; + // }); + // // self check msg index by counting event + // const selfCheck = this.selfCheckByAnotherWay(tx); + // if (!selfCheck) { + // this.logger.warn('selfcheck fail'); + // } - // mapping between log and event - this.mappingEventToLog(eventInLog, filtedEventByType, index); - }); + /*--------- + TESTING + MAPPING EVENT BY COUNT EACH LOG AND EVENT MUST BE SAME + -----------*/ + // count total attribute for each message, countAttributeInEvent[i] = x mean message i has x attributes + const countAttributeInEvent: number[] = []; + tx.tx_response.logs.forEach((log: any) => { + const countAttribute = log.events.reduce( + (acc: number, curr: any) => acc + curr.attributes.length, + 0 + ); + countAttributeInEvent.push(countAttribute); }); + + let reachLastEventTypeTx = false; + let countCurrentAttribute = 0; + let currentCompareEventId = 0; + for (let i = 0; i < tx.tx_response.events.length; i += 1) { + if (tx.tx_response.events[i].type === 'tx') { + reachLastEventTypeTx = true; + } + if (reachLastEventTypeTx && tx.tx_response.events[i].type !== 'tx') { + if ( + countCurrentAttribute < countAttributeInEvent[currentCompareEventId] + ) { + countCurrentAttribute += tx.tx_response.events[i].attributes.length; + // eslint-disable-next-line no-param-reassign + tx.tx_response.events[i].msg_index = currentCompareEventId; + } + + // after count, check if count is equal countAttributeInEvent[currentCompareEventId] or not + if ( + countCurrentAttribute === countAttributeInEvent[currentCompareEventId] + ) { + // if true, count success, then next currentCompareEventId and reset count = 0 + currentCompareEventId += 1; + countCurrentAttribute = 0; + } else if ( + countCurrentAttribute > countAttributeInEvent[currentCompareEventId] + ) { + this.logger.warn('Count event in log is not equal event encoded'); + } + } + } } private _findAttribute( From 8c0d454717e4de6f5c0ca372c61e1d9db6d86069 Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Mon, 21 Aug 2023 15:32:35 +0700 Subject: [PATCH 15/17] feat: add checkMappingEventToLog after mapped event --- src/services/crawl-tx/crawl_tx.service.ts | 37 +++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/src/services/crawl-tx/crawl_tx.service.ts b/src/services/crawl-tx/crawl_tx.service.ts index ad10bb807..2a355d7b7 100644 --- a/src/services/crawl-tx/crawl_tx.service.ts +++ b/src/services/crawl-tx/crawl_tx.service.ts @@ -426,6 +426,38 @@ export default class CrawlTxService extends BullableService { return true; } + private checkMappingEventToLog(tx: any) { + const checkResult = tx?.tx_response?.logs?.every( + (log: any, index: number) => + log?.events?.every((event: any) => + event?.attributes?.every((attr: any) => { + const filtedEvent = tx?.tx_response?.events?.filter( + (item: any) => + item.type === event.type && item.msg_index === index + ); + + return filtedEvent.some((event: any) => + event.attributes.some((attrEncoded: any) => { + const key = attrEncoded.key + ? fromUtf8(fromBase64(attrEncoded.key)) + : null; + const value = attrEncoded.value + ? fromUtf8(fromBase64(attrEncoded.value)) + : null; + if (key === attr.key && value === attr.value) { + return true; + } + return false; + }) + ); + }) + ) + ); + if (checkResult === false) { + this.logger.warn('Mapping event to log is wrong'); + } + } + private setMsgIndexToEvent(tx: any) { /*------ DO NOT USE CURRENTLY @@ -492,7 +524,7 @@ export default class CrawlTxService extends BullableService { -----------*/ // count total attribute for each message, countAttributeInEvent[i] = x mean message i has x attributes const countAttributeInEvent: number[] = []; - tx.tx_response.logs.forEach((log: any) => { + tx?.tx_response?.logs?.forEach((log: any) => { const countAttribute = log.events.reduce( (acc: number, curr: any) => acc + curr.attributes.length, 0 @@ -503,7 +535,7 @@ export default class CrawlTxService extends BullableService { let reachLastEventTypeTx = false; let countCurrentAttribute = 0; let currentCompareEventId = 0; - for (let i = 0; i < tx.tx_response.events.length; i += 1) { + for (let i = 0; i < tx?.tx_response?.events?.length; i += 1) { if (tx.tx_response.events[i].type === 'tx') { reachLastEventTypeTx = true; } @@ -530,6 +562,7 @@ export default class CrawlTxService extends BullableService { } } } + this.checkMappingEventToLog(tx); } private _findAttribute( From 31346fff4139b1c2e0c9ca68c52aaf18fd1e0f86 Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Fri, 25 Aug 2023 09:16:14 +0700 Subject: [PATCH 16/17] fix: check map event log by flatten array --- src/services/crawl-tx/crawl_tx.service.ts | 74 +++++++++++++---------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/src/services/crawl-tx/crawl_tx.service.ts b/src/services/crawl-tx/crawl_tx.service.ts index 2a355d7b7..d8bbe566c 100644 --- a/src/services/crawl-tx/crawl_tx.service.ts +++ b/src/services/crawl-tx/crawl_tx.service.ts @@ -427,32 +427,41 @@ export default class CrawlTxService extends BullableService { } private checkMappingEventToLog(tx: any) { - const checkResult = tx?.tx_response?.logs?.every( - (log: any, index: number) => - log?.events?.every((event: any) => - event?.attributes?.every((attr: any) => { - const filtedEvent = tx?.tx_response?.events?.filter( - (item: any) => - item.type === event.type && item.msg_index === index - ); + this.logger.info('checking mapping log in tx :', tx.tx_response.txhash); + const flattenLog: string[] = []; + const flattenEventEncoded: string[] = []; + + tx?.tx_response?.logs.forEach((log: any, index: number) => { + log.events.forEach((event: any) => { + event.attributes.forEach((attr: any) => { + flattenLog.push(`${index}-${event.type}-${attr.key}-${attr.value}`); + }); + }); + }); - return filtedEvent.some((event: any) => - event.attributes.some((attrEncoded: any) => { - const key = attrEncoded.key - ? fromUtf8(fromBase64(attrEncoded.key)) - : null; - const value = attrEncoded.value - ? fromUtf8(fromBase64(attrEncoded.value)) - : null; - if (key === attr.key && value === attr.value) { - return true; - } - return false; - }) - ); - }) - ) - ); + tx?.tx_response?.events?.forEach((event: any) => { + event.attributes.forEach((attr: any) => { + if (event.msg_index !== undefined) { + const key = attr.key ? fromUtf8(fromBase64(attr.key)) : null; + const value = attr.value ? fromUtf8(fromBase64(attr.value)) : null; + flattenEventEncoded.push( + `${event.msg_index}-${event.type}-${key}-${value}` + ); + } + }); + }); + // compare 2 array + if (flattenLog.length !== flattenEventEncoded.length) { + this.logger.warn('Length between 2 flatten array is not equal'); + } + const checkResult = flattenLog.every((item: any) => { + const foundIndex = flattenEventEncoded.findIndex((e) => e === item); + if (foundIndex !== -1) { + flattenEventEncoded[foundIndex] = 'null'; + return true; + } + return false; + }); if (checkResult === false) { this.logger.warn('Mapping event to log is wrong'); } @@ -523,14 +532,13 @@ export default class CrawlTxService extends BullableService { MAPPING EVENT BY COUNT EACH LOG AND EVENT MUST BE SAME -----------*/ // count total attribute for each message, countAttributeInEvent[i] = x mean message i has x attributes - const countAttributeInEvent: number[] = []; - tx?.tx_response?.logs?.forEach((log: any) => { - const countAttribute = log.events.reduce( - (acc: number, curr: any) => acc + curr.attributes.length, - 0 - ); - countAttributeInEvent.push(countAttribute); - }); + const countAttributeInEvent: number[] = tx?.tx_response?.logs?.map( + (log: any) => + log.events.reduce( + (acc: number, curr: any) => acc + curr.attributes.length, + 0 + ) + ); let reachLastEventTypeTx = false; let countCurrentAttribute = 0; From c6a91dd642797d5ddd82f2552faa7e9a75ab2bc9 Mon Sep 17 00:00:00 2001 From: Phan Anh Tuan Date: Fri, 25 Aug 2023 14:20:34 +0700 Subject: [PATCH 17/17] fix: update checking map event log --- src/services/crawl-tx/crawl_tx.service.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/services/crawl-tx/crawl_tx.service.ts b/src/services/crawl-tx/crawl_tx.service.ts index d8bbe566c..318171a1a 100644 --- a/src/services/crawl-tx/crawl_tx.service.ts +++ b/src/services/crawl-tx/crawl_tx.service.ts @@ -428,8 +428,8 @@ export default class CrawlTxService extends BullableService { private checkMappingEventToLog(tx: any) { this.logger.info('checking mapping log in tx :', tx.tx_response.txhash); - const flattenLog: string[] = []; - const flattenEventEncoded: string[] = []; + let flattenLog: string[] = []; + let flattenEventEncoded: string[] = []; tx?.tx_response?.logs.forEach((log: any, index: number) => { log.events.forEach((event: any) => { @@ -454,14 +454,11 @@ export default class CrawlTxService extends BullableService { if (flattenLog.length !== flattenEventEncoded.length) { this.logger.warn('Length between 2 flatten array is not equal'); } - const checkResult = flattenLog.every((item: any) => { - const foundIndex = flattenEventEncoded.findIndex((e) => e === item); - if (foundIndex !== -1) { - flattenEventEncoded[foundIndex] = 'null'; - return true; - } - return false; - }); + flattenLog = flattenLog.sort(); + flattenEventEncoded = flattenEventEncoded.sort(); + const checkResult = flattenLog.every( + (item: string, index: number) => item === flattenEventEncoded[index] + ); if (checkResult === false) { this.logger.warn('Mapping event to log is wrong'); }