Feat/ibc app ics20 add timestamp ( develop ) (#385)

* docs: cw721

* docs: cw20

* fix: update validator as UNSPECIFIED when not found onchain

* fix: remove token and delegator_shares when validator is UNRECOGNIZED

* feat: crawl ibc tao

* feat: add test for unrecognized validator

* feat: crawl genesis ibc tao

* feat: add migration to add index time to block, tx

* fix: code

* fix: code

* feat: ibc app

* feat: ibc app

* fix: allow whitelist query can have depth more than config (#312)

* feat: reindex cw20 service ( main ) (#277)

* feat: reindex cw20 service

* test: test cw20 reindexing

* fix: idiot

* refactor: code

* feat: api admin

* refactor: cw20/cw721 index and add relation ( main ) (#306)

* refactor: cw20 index and add relation

* refactor: code

* fix: duplicate latest migrate contract (#304)

* test: ibc app

* fix: review

* Feat/statistics (#272)

* feat: move statistic feature from v1 to v2 (not tested yet)

* feat: change job params so it can aggregate data at a specific date

* fix: query fields from exact table

* fix: query exact fields

* feat: add logger when done

* feat: finish statistic jobs (not tested)

* feat: create a separate interval job that gets the current date and create statistics jobs with date

* feat: update logic using dayjs and lodash lib

* feat: update cron jobs logic, move api actions to its service folder

* fix: query event from db and group it based on event_id and tx_id

* fix: just use a single job to query all daily data

* fix: move all queries in daily_stats job into a single Promise all

* fix: move dayjs.extend to after import statements

* fix: remove _start

* fix: only count native token in account_stats job

* feat: add api to sync stats from prev dates

* fix: support case when user just pass startDate to api

* fix: fix whitelist graphql

* fix: update column in account_statistic table, update bignum top_tx_sent

* fix: remove drop index in modify account statistic table

---------

Co-authored-by: AnDQK <[email protected]>
Co-authored-by: Phan Anh Tuan <[email protected]>

* refactor: migrate cw721 activity missing smart contract event ( main ) (#322)

* refactor: migrate cw721 activity missing smart contract event

* feat: api

* test: test params

* test: test params

* test: test params

* test: test params

* test: test params

* test: test params

* refactor: code

* refactor: code

* refactor: code

* refactor: code

* refactor: code

* fix: ci

* feat: ibc app ics20

* Fix/account stat with feegrant (#337)

* fix: get use_feegrant_grantee if has, other get tx_fee_payer

* fix: remove code not used

* Update api_gateway.service.ts (#336)

* feat: ibc ics20 send

* feat: ibc ics20 send

* feat: ibc ics20 recv

* feat: ibc ics20 recv

* feat: handle ics20 ack

* feat: handle ics20 timeout

* test: test

* fix: code

* fix: code

* feat: ibc app ics20

* feat: ibc ics20 send

* feat: ibc ics20 send

* feat: ibc ics20 recv

* feat: ibc ics20 recv

* feat: handle ics20 ack

* feat: handle ics20 timeout

* test: test

* fix: code

* fix: code

* fix: cw721 activity missing from/to ( main ) (#294)

* fix: cw721 activity missing from/to

* fix: fill from/to cw721 activity

* fix: fill from/to cw721 activity

* fix: fill from/to cw721 activity

* fix: fill from/to cw721 activity

* refactor: code

* fix: code

* refactor: code

---------

Co-authored-by: Vu Ngoc Quang <[email protected]>

* feat: crawl ibc tao ( main ) (#278)

* feat: crawl ibc tao

* feat: crawl genesis ibc tao

* fix: code

* fix: code

* refactor: code

* fix: test

* refactor: code

* fix: merge code

* refactor: review code

* refactor: review code

* Fix/lint json file (#359)

* fix: display json on multiple lines

* fix: update launch.json to debug jest in 1 file; use moleculer.config.ts when debug service

* Feat/ibc app ( main ) (#321)

* feat: crawl ibc tao

* feat: crawl genesis ibc tao

* fix: code

* fix: code

* feat: ibc app

* feat: ibc app

* test: ibc app

* fix: review

* refactor: update status

* fix: smart contract missing label ( main ) (#366)

* fix: smart contract missing label

* fix: test

* fix: test

* fix: test

* fix: test

* refactor: code

* fix: excessive data

* fix: update msg_index by order in event and log (#327)

* fix: update msg_index by order in event and log

* feat: add test mapping event and log

* feat: add test mapping by authz tx

* feat: add test mapping by authz tx

* fix: refactor mapping event to log, update unit test

* fix: comment set index msg with order, use count attribute to compare

* feat: add checkMappingEventToLog after mapped event

* fix: check map event log by flatten array

* fix: update checking map event log

* fix: add null value handle in checkMappingEventToLog

* feat: init job re assign msg_index to event

* fix: use transaction when update event

* fix: use forEach loop in job

* fix: sort graph when query db

* feat: add unittest for generateListUpdateMsgIndex function

* fix: update msg_index in 1 query per tx for each value

* fix: fix null when compare value attribute value

* fix: fix if tx fail, then no need set msg index

* feat: add memo column

* feat: health check job ( main ) (#345)

* feat: health check job

* feat: validator

* refactor: code

* refactor: performance cw721 media info ( main ) (#369)

* refactor: cw721 media info

* refactor: code

* refactor: code

* fix: fix lint and remove resilient, protobuf, upgrade graphql (#380)

* fix: fix lint and remove resilient, protobuf, upgrade graphql

* fix: add package-lock.json

* feat: add timestamp columns

* refactor: add start time and finish time

* refactor: migration

---------

Co-authored-by: Tuan Phan Anh <[email protected]>
Co-authored-by: Phan Anh Tuan <[email protected]>
Co-authored-by: Vu Ngoc Quang <[email protected]>
Co-authored-by: AnDQK <[email protected]>

migrations/20230926042146_ibc_ics20_add_timestamp.ts

@@ -0,0 +1,53 @@
+import { Knex } from 'knex';
+import { IbcIcs20, IbcMessage } from '../src/models';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('ibc_ics20', (table) => {
+    table.timestamp('start_time');
+    table.timestamp('finish_time');
+  });
+  await knex.transaction(async (trx) => {
+    const ibcIcs20s = await IbcIcs20.query().transacting(trx);
+    const ibcMsgs = await IbcMessage.query()
+      .transacting(trx)
+      .joinRelated('message.transaction')
+      .whereIn(
+        'sequence_key',
+        ibcIcs20s.map((ibcIcs20) => ibcIcs20.sequence_key)
+      )
+      .select(
+        'message:transaction.timestamp',
+        'ibc_message.sequence_key',
+        'ibc_message.type'
+      );
+    if (ibcIcs20s.length > 0) {
+      ibcIcs20s.forEach((ibcIcs20) => {
+        ibcIcs20.start_time = ibcMsgs.find(
+          (e) =>
+            e.sequence_key === ibcIcs20.sequence_key &&
+            e.type === IbcMessage.EVENT_TYPE.SEND_PACKET
+        )?.timestamp;
+        ibcIcs20.finish_time = ibcMsgs.find(
+          (e) =>
+            e.sequence_key === ibcIcs20.sequence_key &&
+            [
+              IbcMessage.EVENT_TYPE.RECV_PACKET,
+              IbcMessage.EVENT_TYPE.TIMEOUT_PACKET,
+              IbcMessage.EVENT_TYPE.ACKNOWLEDGE_PACKET,
+            ].includes(e.type)
+        )?.timestamp;
+      });
+      await IbcIcs20.query()
+        .transacting(trx)
+        .insert(ibcIcs20s)
+        .onConflict('id')
+        .merge();
+    }
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('ibc_ics20', (table) => {
+    table.dropColumns('start_time', 'finish_time');
+  });
+}

src/models/ibc_ics20.ts

@@ -29,6 +29,10 @@ export class IbcIcs20 extends BaseModel {
 
   memo!: string;
 
+  start_time!: Date;
+
+  finish_time!: Date;
+
   static get tableName() {
     return 'ibc_ics20';
   }

src/models/ibc_message.ts

@@ -6,6 +6,8 @@ import { TransactionMessage } from './transaction_message';
 import config from '../../config.json' assert { type: 'json' };
 
 export class IbcMessage extends BaseModel {
+  [relation: string]: any;
+
   id!: number;
 
   transaction_message_id!: number;

src/models/transaction.ts

@@ -29,7 +29,7 @@ export class Transaction extends BaseModel {
 
   index!: number;
 
-  timstamp!: Date;
+  timestamp!: Date;
 
   data!: any;
 

src/services/ibc/crawl_ibc_ics20.service.ts

@@ -62,6 +62,14 @@ export default class CrawlIBCIcs20Service extends BullableService {
       .andWhere('ibc_message.type', IbcMessage.EVENT_TYPE.SEND_PACKET)
       .andWhere('message:transaction.height', '>', startHeight)
       .andWhere('message:transaction.height', '<=', endHeight)
+      .select(
+        'message:transaction.timestamp',
+        'ibc_message.sequence_key',
+        'ibc_message.id',
+        'ibc_message.src_channel_id',
+        'ibc_message.type',
+        'ibc_message.data'
+      )
       .orderBy('message.id')
       .transacting(trx);
     if (ics20Sends.length > 0) {
@@ -79,6 +87,7 @@ export default class CrawlIBCIcs20Service extends BullableService {
           status: IbcIcs20.STATUS_TYPE.ONGOING,
           sequence_key: msg.sequence_key,
           type: msg.type,
+          start_time: msg.timestamp,
         })
       );
       await IbcIcs20.query().insert(ibcIcs20s).transacting(trx);
@@ -104,6 +113,14 @@ export default class CrawlIBCIcs20Service extends BullableService {
       .andWhere('ibc_message.type', IbcMessage.EVENT_TYPE.RECV_PACKET)
       .andWhere('message:transaction.height', '>', startHeight)
       .andWhere('message:transaction.height', '<=', endHeight)
+      .select(
+        'message:transaction.timestamp',
+        'ibc_message.sequence_key',
+        'ibc_message.id',
+        'ibc_message.dst_port_id',
+        'ibc_message.dst_channel_id',
+        'ibc_message.type'
+      )
       .orderBy('message.id')
       .transacting(trx);
     if (ics20Recvs.length > 0) {
@@ -150,6 +167,7 @@ export default class CrawlIBCIcs20Service extends BullableService {
           sequence_key: msg.sequence_key,
           type: msg.type,
           memo,
+          finish_time: msg.timestamp,
         });
       });
       await IbcIcs20.query().insert(ibcIcs20s).transacting(trx);
@@ -174,6 +192,11 @@ export default class CrawlIBCIcs20Service extends BullableService {
       .andWhere('message:transaction.height', '>', startHeight)
       .andWhere('message:transaction.height', '<=', endHeight)
       .orderBy('message.id')
+      .select(
+        'message:transaction.timestamp',
+        'ibc_message.sequence_key',
+        'ibc_message.id'
+      )
       .transacting(trx);
     if (ics20Acks.length > 0) {
       // update success ack status for origin send ics20
@@ -229,6 +252,7 @@ export default class CrawlIBCIcs20Service extends BullableService {
       .andWhere('message:transaction.height', '>', startHeight)
       .andWhere('message:transaction.height', '<=', endHeight)
       .orderBy('message.id')
+      .select('message:transaction.timestamp', 'ibc_message.sequence_key')
       .transacting(trx);
     if (ics20Timeouts.length > 0) {
       await this.updateOriginSendStatus(
@@ -257,16 +281,26 @@ export default class CrawlIBCIcs20Service extends BullableService {
     type: string,
     trx: Knex.Transaction
   ) {
-    await IbcIcs20.query()
-      .transacting(trx)
-      .patch({
-        status: type,
-      })
-      .whereIn(
-        'sequence_key',
-        msgs.map((msg) => msg.sequence_key)
-      )
-      .andWhere('type', IbcMessage.EVENT_TYPE.SEND_PACKET);
+    if (msgs.length > 0) {
+      const ibcIcs20sKeyBy = _.keyBy(
+        await IbcIcs20.query()
+          .transacting(trx)
+          .whereIn(
+            'sequence_key',
+            msgs.map((msg) => msg.sequence_key)
+          )
+          .andWhere('type', IbcMessage.EVENT_TYPE.SEND_PACKET),
+        'sequence_key'
+      );
+      msgs.forEach((msg) => {
+        ibcIcs20sKeyBy[msg.sequence_key].finish_time = msg.timestamp;
+        ibcIcs20sKeyBy[msg.sequence_key].status = type;
+      });
+      await IbcIcs20.query()
+        .insert(Object.values(ibcIcs20sKeyBy))
+        .onConflict('id')
+        .merge();
+    }
   }
 
   async _start(): Promise<void> {

test/unit/services/ibc/crawl_ibc_ics20.spec.ts

@@ -116,6 +116,8 @@ export default class CrawlIbcIcs20Test {
       expect(result?.sequence_key).toEqual(ibcMessage.sequence_key);
       expect(result?.type).toEqual(ibcMessage.type);
       expect(result?.channel_id).toEqual(ibcMessage.src_channel_id);
+      expect(result?.finish_time).toBeNull();
+      expect(result?.start_time).toEqual(new Date(this.transaction.timestamp));
     });
   }
 
@@ -245,6 +247,8 @@ export default class CrawlIbcIcs20Test {
       expect(result?.sequence_key).toEqual(ibcMessage.sequence_key);
       expect(result?.type).toEqual(ibcMessage.type);
       expect(result?.channel_id).toEqual(ibcMessage.dst_channel_id);
+      expect(result?.start_time).toBeNull();
+      expect(result?.finish_time).toEqual(new Date(this.transaction.timestamp));
       await trx.rollback();
     });
   }
@@ -344,6 +348,8 @@ export default class CrawlIbcIcs20Test {
       expect(result?.sequence_key).toEqual(ibcMessage.sequence_key);
       expect(result?.type).toEqual(ibcMessage.type);
       expect(result?.channel_id).toEqual(ibcMessage.dst_channel_id);
+      expect(result?.start_time).toBeNull();
+      expect(result?.finish_time).toEqual(new Date(this.transaction.timestamp));
       await trx.rollback();
     });
   }
@@ -449,6 +455,9 @@ export default class CrawlIbcIcs20Test {
         .first()
         .transacting(trx);
       expect(originSend?.status).toEqual(IbcIcs20.STATUS_TYPE.ACK_ERROR);
+      expect(originSend?.finish_time).toEqual(
+        new Date(this.transaction.timestamp)
+      );
       await trx.rollback();
     });
   }
@@ -554,6 +563,9 @@ export default class CrawlIbcIcs20Test {
         .first()
         .transacting(trx);
       expect(originSend?.status).toEqual(IbcIcs20.STATUS_TYPE.ACK_SUCCESS);
+      expect(originSend?.finish_time).toEqual(
+        new Date(this.transaction.timestamp)
+      );
       await trx.rollback();
     });
   }
@@ -632,6 +644,9 @@ export default class CrawlIbcIcs20Test {
         .first()
         .transacting(trx);
       expect(originSend?.status).toEqual(IbcIcs20.STATUS_TYPE.TIMEOUT);
+      expect(originSend?.finish_time).toEqual(
+        new Date(this.transaction.timestamp)
+      );
       await trx.rollback();
     });
   }