Merge branch 'evm' of github.com:aura-nw/horoscope-v2 into feat/suppo…

…rt-json-token-uri

ci/config.json.ci

@@ -11,7 +11,8 @@
   "crawlValidator": {
     "millisecondCrawl": null,
     "queryPageLimit": 100,
-    "patternCrawl": "30 */2 * * * *"
+    "patternCrawl": "30 */2 * * * *",
+    "chunkSizeInsert": 500
   },
   "cw721": {
     "key": "cw721",
@@ -421,7 +422,9 @@
     "chunkSizeInsert": 1000,
     "mediaPerBatch": 10,
     "concurrencyHandleTokenMedia": 10,
-    "timeRefreshErc721Stats": "1 * * * *"
+    "timeRefreshErc721Stats": "1 * * * *",
+    "timeRefreshMViewErc721HolderStats": "*/10 * * * *",
+    "statementTimeout": 600000
   },
   "jobRefreshMViewAccountBalanceStatistic": {
     "timeRefreshMViewAccountBalanceStatistic": "*/10 * * * *"

config.json

@@ -11,7 +11,8 @@
   "crawlValidator": {
     "millisecondCrawl": null,
     "queryPageLimit": 100,
-    "patternCrawl": "30 */2 * * * *"
+    "patternCrawl": "30 */2 * * * *",
+    "chunkSizeInsert": 500
   },
   "cw721": {
     "key": "cw721",
@@ -419,7 +420,9 @@
     "chunkSizeInsert": 500,
     "mediaPerBatch": 10,
     "concurrencyHandleTokenMedia": 10,
-    "timeRefreshErc721Stats": "1 * * * *"
+    "timeRefreshErc721Stats": "1 * * * *",
+    "timeRefreshMViewErc721HolderStats": "*/10 * * * *",
+    "statementTimeout": 600000
   },
   "crawlEvmProxyHistory": {
     "key": "crawlEvmProxyHistory",

migrations/20240913070037_add_evm_address_to_validator.ts

@@ -0,0 +1,13 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('validator', (table) => {
+    table.string('evm_address').index();
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('validator', (table) => {
+    table.dropColumn('evm_address');
+  });
+}

migrations/evm/20240905021812_m_view_erc721_holder_statistic.ts

@@ -0,0 +1,21 @@
+import { Knex } from 'knex';
+import config from '../../config.json' assert { type: 'json' };
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw(`set statement_timeout to ${config.erc721.statementTimeout}`);
+  await knex.raw(`
+  CREATE MATERIALIZED VIEW m_view_erc721_holder_statistic AS
+    select count(*), erc721_token.owner, erc721_token.erc721_contract_address
+    from erc721_token 
+    group by erc721_token.owner, erc721_token.erc721_contract_address;
+  CREATE INDEX m_view_erc721_holder_statistic_owner_index
+    ON m_view_erc721_holder_statistic (owner);
+  CREATE INDEX m_view_erc721_holder_statistic_erc721_contract_address_count_index
+    ON m_view_erc721_holder_statistic (erc721_contract_address, count);
+`);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropMaterializedViewIfExists(
+    'm_view_erc721_holder_statistic'
+  );
+}

migrations/evm/20240912082032_erc721_holder_statistic.ts

@@ -0,0 +1,21 @@
+import { Knex } from 'knex';
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw(`set statement_timeout to 0`);
+  await knex.raw(`
+    CREATE TABLE erc721_holder_statistic AS
+      select count(*), erc721_token.owner, erc721_token.erc721_contract_address
+      from erc721_token 
+      group by erc721_token.owner, erc721_token.erc721_contract_address;
+    ALTER TABLE erc721_holder_statistic ADD COLUMN id SERIAL PRIMARY KEY;
+    ALTER TABLE erc721_holder_statistic ADD COLUMN last_updated_height INTEGER;
+    CREATE INDEX erc721_holder_statistic_owner_index
+      ON erc721_holder_statistic (owner);
+    CREATE UNIQUE INDEX erc721_holder_statistic_erc721_contract_address_owner_index
+      ON erc721_holder_statistic (erc721_contract_address, owner);
+    CREATE INDEX erc721_holder_statistic_last_updated_height_index ON erc721_holder_statistic (last_updated_height)
+  `);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists('erc721_holder_statistic');
+}

migrations/evm/20240916032201_erc721_contract_totalSupply.ts

@@ -0,0 +1,32 @@
+import { Knex } from 'knex';
+import { Erc721Token } from '../../src/models';
+import { ZERO_ADDRESS } from '../../src/services/evm/constant';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('erc721_contract', (table) => {
+    table.bigInteger('total_supply').defaultTo(0).index();
+  });
+  await knex.raw(`set statement_timeout to 0`);
+  const totalSupplies = await Erc721Token.query(knex)
+    .select('erc721_token.erc721_contract_address')
+    .where('erc721_token.owner', '!=', ZERO_ADDRESS)
+    .count()
+    .groupBy('erc721_token.erc721_contract_address');
+  if (totalSupplies.length > 0) {
+    const stringListUpdates = totalSupplies
+      .map(
+        (totalSuply) =>
+          `('${totalSuply.erc721_contract_address}', ${totalSuply.count})`
+      )
+      .join(',');
+    await knex.raw(
+      `UPDATE erc721_contract SET total_supply = temp.total_supply from (VALUES ${stringListUpdates}) as temp(address, total_supply) where temp.address = erc721_contract.address`
+    );
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('erc721_contract', (table) => {
+    table.dropColumn('total_supply');
+  });
+}

migrations/evm/20240918080156_account_balance_index_denom_amount.ts

@@ -0,0 +1,10 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw(`set statement_timeout to 0`);
+  await knex.raw(
+    'CREATE INDEX account_balance_denom_amount_index ON account_balance(denom, amount);'
+  );
+}
+
+export async function down(knex: Knex): Promise<void> {}

migrations/evm/20240920024049_erc20_contract_add_total_holder.ts

@@ -0,0 +1,29 @@
+import { Knex } from 'knex';
+import { AccountBalance } from '../../src/models';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('erc20_contract', (table) => {
+    table.integer('total_holder').defaultTo(0).index();
+  });
+  await knex.raw(`set statement_timeout to 0`);
+  const totalHolders = await AccountBalance.query(knex)
+    .select('account_balance.denom')
+    .where('account_balance.type', AccountBalance.TYPE.ERC20_TOKEN)
+    .andWhere('account_balance.amount', '>', 0)
+    .count()
+    .groupBy('account_balance.denom');
+  if (totalHolders.length > 0) {
+    const stringListUpdates = totalHolders
+      .map((totalHolder) => `('${totalHolder.denom}', ${totalHolder.count})`)
+      .join(',');
+    await knex.raw(
+      `UPDATE erc20_contract SET total_holder = temp.total_holder from (VALUES ${stringListUpdates}) as temp(address, total_holder) where temp.address = erc20_contract.address`
+    );
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('erc20_contract', (table) => {
+    table.dropColumn('total_holder');
+  });
+}

migrations/evm/20240923064605_erc20_contract_no_action.ts

@@ -0,0 +1,88 @@
+import { Knex } from 'knex';
+import { Erc20Activity } from '../../src/models';
+import { ERC20_ACTION } from '../../src/services/evm/erc20_handler';
+import _ from 'lodash';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('erc20_contract', (table) => {
+    table.jsonb('total_actions').defaultTo('{}');
+  });
+  await knex.raw(`set statement_timeout to 0`);
+  const [totalTransfer, totalApproval, totalDeposit, totalWithdrawal] =
+    await Promise.all([
+      _.keyBy(
+        (
+          await Erc20Activity.query(knex)
+            .where('action', ERC20_ACTION.TRANSFER)
+            .groupBy('erc20_activity.erc20_contract_address')
+            .select(
+              'erc20_activity.erc20_contract_address',
+              knex.raw('count(*)::integer as transfer')
+            )
+        ).map((e) => e.toJSON()),
+        'erc20_contract_address'
+      ),
+      _.keyBy(
+        (
+          await Erc20Activity.query(knex)
+            .where('action', ERC20_ACTION.APPROVAL)
+            .groupBy('erc20_activity.erc20_contract_address')
+            .select(
+              'erc20_activity.erc20_contract_address',
+              knex.raw('count(*)::integer as approval')
+            )
+        ).map((e) => e.toJSON()),
+        'erc20_contract_address'
+      ),
+      _.keyBy(
+        (
+          await Erc20Activity.query(knex)
+            .where('action', ERC20_ACTION.DEPOSIT)
+            .groupBy('erc20_activity.erc20_contract_address')
+            .select(
+              'erc20_activity.erc20_contract_address',
+              knex.raw('count(*)::integer as deposit')
+            )
+        ).map((e) => e.toJSON()),
+        'erc20_contract_address'
+      ),
+      _.keyBy(
+        (
+          await Erc20Activity.query(knex)
+            .where('action', ERC20_ACTION.WITHDRAWAL)
+            .groupBy('erc20_activity.erc20_contract_address')
+            .select(
+              'erc20_activity.erc20_contract_address',
+              knex.raw('count(*)::integer as withdrawal')
+            )
+        ).map((e) => e.toJSON()),
+        'erc20_contract_address'
+      ),
+    ]);
+  const totalAction = _.merge(
+    totalTransfer,
+    totalApproval,
+    totalDeposit,
+    totalWithdrawal
+  );
+  const erc20ContractsAddr = Object.keys(totalAction);
+  if (erc20ContractsAddr.length > 0) {
+    const stringListUpdates = erc20ContractsAddr
+      .map(
+        (addr) =>
+          `('${addr}', '${JSON.stringify(
+            _.omit(totalAction[addr], 'erc20_contract_address')
+          )}'::jsonb)`
+      )
+      .join(',');
+    await knex.raw(
+      `UPDATE erc20_contract SET total_actions = temp.total_actions from (VALUES ${stringListUpdates}) as temp(address, total_actions) where temp.address = erc20_contract.address`
+    );
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('erc20_contract', (table) => {
+    table.dropColumn('total_actions');
+  });
+}

migrations/evm/20240924041138_index_performance.ts

@@ -0,0 +1,13 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw('set statement_timeout to 0');
+  await knex.raw(
+    'CREATE INDEX IF NOT EXISTS erc721_token_erc721_contract_address_id_index ON erc721_token(erc721_contract_address, id)'
+  );
+  await knex.raw(
+    'CREATE INDEX IF NOT EXISTS erc721_holder_statistic_erc721_contract_address_count_index ON erc721_holder_statistic(erc721_contract_address, count)'
+  );
+}
+
+export async function down(knex: Knex): Promise<void> {}

migrations/evm/20240924084858_create_index_created_at_in_account.ts

@@ -0,0 +1,14 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw('SET statement_timeout TO 0');
+  await knex.schema.alterTable('account', (table) => {
+    table.index('created_at');
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('account', (table) => {
+    table.dropIndex('created_at');
+  });
+}

network.json

@@ -78,7 +78,7 @@
     "redisDBNumber": 4,
     "moleculerNamespace": "namespace-ancient8",
     "EVMJSONRPC": ["https://rpc.ancient8.gg"],
-    "EVMchainId": 888888888 
+    "EVMchainId": 888888888
   },
   {
     "chainId": "ancient-8-testnet",
@@ -88,7 +88,7 @@
     "redisDBNumber": 4,
     "moleculerNamespace": "namespace-auratestnet",
     "EVMJSONRPC": ["https://rpcv2-testnet.ancient8.gg"],
-    "EVMchainId": 28122024 
+    "EVMchainId": 28122024
   },
   {
     "chainId": "ethereum",
@@ -103,5 +103,18 @@
     "RPC": [],
     "LCD": [],
     "EVMchainId": 11155111
+  },
+  {
+    "chainId": "1513",
+    "RPC": [],
+    "LCD": ["https://story-testnet-lcd.aura.network"],
+    "databaseName": "local_evm",
+    "redisDBNumber": 0,
+    "EVMJSONRPC": [
+      "https://story-testnet.aura.network",
+      "https://testnet.storyrpc.io"
+    ],
+    "EVMchainId": 1513,
+    "moleculerNamespace": "erascope-dev-story-testnet"
   }
 ]

src/models/account_balance.ts

@@ -4,6 +4,8 @@ import BaseModel from './base';
 import { Account } from './account';
 
 export class AccountBalance extends BaseModel {
+  [relation: string]: any;
+
   static softDelete = false;
 
   account!: Account;

src/models/erc20_contract.ts

@@ -23,8 +23,12 @@ export class Erc20Contract extends BaseModel {
 
   track!: boolean;
 
+  total_holder!: number;
+
   last_updated_height!: number;
 
+  total_actions!: any;
+
   static get tableName() {
     return 'erc20_contract';
   }

src/models/erc721_contract.ts

@@ -25,6 +25,8 @@ export class Erc721Contract extends BaseModel {
 
   last_updated_height!: number;
 
+  total_supply!: string;
+
   static get tableName() {
     return 'erc721_contract';
   }

src/models/erc721_holder_statistic.ts

@@ -0,0 +1,29 @@
+import BaseModel from './base';
+
+export class Erc721HolderStatistic extends BaseModel {
+  static softDelete = false;
+
+  erc721_contract_address!: string;
+
+  owner!: string;
+
+  count!: string;
+
+  last_updated_height!: number;
+
+  static get tableName() {
+    return 'erc721_holder_statistic';
+  }
+
+  static get jsonSchema() {
+    return {
+      type: 'object',
+      required: ['erc721_contract_address', 'owner', 'count'],
+      properties: {
+        erc721_contract_address: { type: 'string' },
+        owner: { type: 'string' },
+        count: { type: 'string' },
+      },
+    };
+  }
+}

src/models/index.ts

@@ -48,3 +48,4 @@ export * from './erc721_stats';
 export * from './evm_block';
 export * from './optimism_deposit';
 export * from './optimism_withdrawal';
+export * from './erc721_holder_statistic';