From a550c444f896cc6c42ca5c3a5d5f471f575691c2 Mon Sep 17 00:00:00 2001 From: Mikhail Reenko Date: Thu, 19 Nov 2020 15:44:24 +0700 Subject: [PATCH] VUL-120 storage watching: mapping (address=> uint) * VUL-120 Storage watching init commit * VUL-120 Remove web3.js * VUL-120 Create table for state * VUL-120 Sava state to db * VUL-120 Get contract by address hash * VUL-126 Watch for fixed lenth slots (sync old data) * VUL-115 Preimage table in contract watcher * VUL-115 Preimage table in contract watcher * Rework test (#29) * remove old path for tests * add mock for store * add tests for dataService * add full tests for processEvent and processState Co-authored-by: Ilnur Galiev * VUL-120 Save data to db * add contract parser * add functions toStructure and toFields Co-authored-by: Ilnur Galiev --- .env.example | 1 + __tests__/dataServicePgTypeTest .ts | 40 -- __tests__/dataServiceTest.ts | 140 ------- __tests__/zeroTest.ts | 3 - config.example.toml | 5 +- package-lock.json | 5 + package.json | 1 + src/__mocks__/store.ts | 83 ++++ src/env.ts | 3 + src/migrations/1604551153589-StateConfig.ts | 21 + src/migrations/1604987750715-StateProgress.ts | 31 ++ src/migrations/1605072677007-Addresses.ts | 25 ++ src/models/contract/contract.ts | 4 + src/models/contract/state.ts | 15 + src/models/data/address.ts | 15 + src/models/data/stateProgress.ts | 18 + src/repositories/contract/stateRepository.ts | 10 + .../__mocks__/addressIdSlotIdRepository.ts | 3 + .../data/addressIdSlotIdRepository.ts | 64 +++ src/repositories/data/addressRepository.ts | 19 + .../data/stateProgressRepository.ts | 52 +++ src/repositories/graphqlRepository.ts | 82 +++- src/server.ts | 6 + src/services/contractService.ts | 14 + src/services/dataService.test.ts | 386 ++++++++++++++++++ src/services/dataService.ts | 307 +++++++++++++- src/services/dataTypeParser.test.ts | 245 +++++++++++ src/services/dataTypeParser.ts | 150 +++++++ src/services/graphqlService.ts | 18 +- src/store.ts | 66 ++- src/syncDaemon.ts | 34 ++ 31 files changed, 1651 insertions(+), 215 deletions(-) delete mode 100644 __tests__/dataServicePgTypeTest .ts delete mode 100644 __tests__/dataServiceTest.ts delete mode 100644 __tests__/zeroTest.ts create mode 100644 src/__mocks__/store.ts create mode 100644 src/migrations/1604551153589-StateConfig.ts create mode 100644 src/migrations/1604987750715-StateProgress.ts create mode 100644 src/migrations/1605072677007-Addresses.ts create mode 100644 src/models/contract/state.ts create mode 100644 src/models/data/address.ts create mode 100644 src/models/data/stateProgress.ts create mode 100644 src/repositories/contract/stateRepository.ts create mode 100644 src/repositories/data/__mocks__/addressIdSlotIdRepository.ts create mode 100644 src/repositories/data/addressIdSlotIdRepository.ts create mode 100644 src/repositories/data/addressRepository.ts create mode 100644 src/repositories/data/stateProgressRepository.ts create mode 100644 src/services/dataService.test.ts create mode 100644 src/services/dataTypeParser.test.ts create mode 100644 src/services/dataTypeParser.ts diff --git a/.env.example b/.env.example index f95e230..3a29f39 100644 --- a/.env.example +++ b/.env.example @@ -19,3 +19,4 @@ GRAPHQL_URI=http://localhost:5000 ENABLE_EVENT_WATCHER=true ENABLE_HEADER_WATCHER=true +ENABLE_STORAGE_WATCHER=true diff --git a/__tests__/dataServicePgTypeTest .ts b/__tests__/dataServicePgTypeTest .ts deleted file mode 100644 index 0947f25..0000000 --- a/__tests__/dataServicePgTypeTest .ts +++ /dev/null @@ -1,40 +0,0 @@ -/* eslint-disable @typescript-eslint/ban-ts-ignore */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import DataService from '../src/services/dataService'; - -test('_getPgType, bool', async () => { - // @ts-ignore - expect(await DataService._getPgType('bool')).toEqual('boolean'); -}); - -test('_getPgType, uint8', async () => { - // @ts-ignore - expect(await DataService._getPgType('uint8')).toEqual('numeric'); -}); - -test('_getPgType, uint256', async () => { - // @ts-ignore - expect(await DataService._getPgType('uint256')).toEqual('numeric'); -}); - -test('_getPgType, int8', async () => { - // @ts-ignore - expect(await DataService._getPgType('int8')).toEqual('numeric'); -}); - -test('_getPgType, int256', async () => { - // @ts-ignore - expect(await DataService._getPgType('int256')).toEqual('numeric'); -}); - -test('_getPgType, address', async () => { - // @ts-ignore - expect(await DataService._getPgType('address')).toEqual('character varying(66)'); -}); - -test('_getPgType, bytes', async () => { - // @ts-ignore - expect(await DataService._getPgType('bytes')).toEqual('bytea'); -}); diff --git a/__tests__/dataServiceTest.ts b/__tests__/dataServiceTest.ts deleted file mode 100644 index 0f52d00..0000000 --- a/__tests__/dataServiceTest.ts +++ /dev/null @@ -1,140 +0,0 @@ -/* eslint-disable @typescript-eslint/ban-ts-ignore */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import DataService from '../src/services/dataService'; -import Event from '../src/models/contract/event'; -import Contract from '../src/models/contract/contract'; - -const mockEvent1 = { eventId: 1, name: 'TestEvent1' } as Event; -const mockEvent2 = { eventId: 2, name: 'MessageChanged' } as Event; - -const mockContract1 = { - contractId: 1, - name: 'Contract1', - startingBlock: 0, - abi: [{"anonymous":false,"inputs":[{"indexed":false,"internalType":"string","name":"message","type":"string"}],"name":"MessageChanged","type":"event"},{"inputs":[],"name":"getMessage","outputs":[{"internalType":"string","name":"_message","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"message","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"string","name":"_message","type":"string"}],"name":"setMessage","outputs":[],"stateMutability":"nonpayable","type":"function"}], -} as Contract; - -const mockGraphqlService = { - ethHeaderCidByBlockNumber: () => ({ - ethHeaderCidByBlockNumber: { - nodes: [{ - ethTransactionCidsByHeaderId: { - nodes: [{ - id: 1, - }] - } - }] - } - }), -} as any; - -const mockDataService = { - processEvent: () => Promise.resolve(), -} as any; - -test('syncEventForContractPage Test Full', async () => { - - const mockProgressRepository = { - findSyncedBlocks: jest.fn().mockResolvedValueOnce([2,3,4,5].map((blockNumber) => ({ blockNumber }))) - } as any; - - const startBlock = 0; - const maxBlock = 9; - const limit = 10; - const page = 1; - - // @ts-ignore - const blocks = await DataService._syncEventForContractPage({ - graphqlService: mockGraphqlService, - dataService: mockDataService, - progressRepository: mockProgressRepository, - }, mockEvent1, mockContract1, startBlock, maxBlock, page, limit); - - expect(blocks).toEqual([0,1,6,7,8,9]); -}); - -test('syncEventForContractPage Test without synced blocks', async () => { - - const mockProgressRepository = { - findSyncedBlocks: jest.fn().mockResolvedValueOnce([]) - } as any; - - const startBlock = 0; - const maxBlock = 9; - const limit = 10; - const page = 1; - - // @ts-ignore - const blocks = await DataService._syncEventForContractPage({ - graphqlService: mockGraphqlService, - dataService: mockDataService, - progressRepository: mockProgressRepository, - }, mockEvent1, mockContract1, startBlock, maxBlock, page, limit); - - expect(blocks).toEqual([0,1,2,3,4,5,6,7,8,9]); -}); - -test('syncEventForContractPage Test with all synced blocks', async () => { - - const mockProgressRepository = { - findSyncedBlocks: jest.fn().mockResolvedValueOnce([0,1,2].map((blockNumber) => ({ blockNumber }))) - } as any; - - const startBlock = 0; - const maxBlock = 2; - const limit = 10; - const page = 1; - - // @ts-ignore - const blocks = await DataService._syncEventForContractPage({ - graphqlService: mockGraphqlService, - dataService: mockDataService, - progressRepository: mockProgressRepository, - }, mockEvent1, mockContract1, startBlock, maxBlock, page, limit); - - expect(blocks).toEqual([]); -}); - -test('_getTableOptions, test 1', async () => { - // @ts-ignore - const tableName = await DataService._getTableName(mockContract1.contractId, mockEvent1.eventId); - expect(tableName).toEqual('data.contract_id_1_event_id_1'); -}); - -test('_getTableOptions, test 2', async () => { - // @ts-ignore - const tableName = await DataService._getTableName(mockContract1.contractId, mockEvent2.eventId); - expect(tableName).toEqual('data.contract_id_1_event_id_2'); -}); - -test('_getTableOptions', async () => { - - // @ts-ignore - const tableOptions = await DataService._getTableOptions(mockContract1, mockEvent2); - - expect(tableOptions).toEqual({ - columns: [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "name": "event_id", - "type": "integer", - }, { - "name": "contract_id", - "type": "integer", - }, { - "name": "mh_key", - "type": "text", - }, { - "isNullable": true, - "name": "data_message", - "type": "text", - }], - "name": "data.contract_id_1_event_id_2", - }); -}); \ No newline at end of file diff --git a/__tests__/zeroTest.ts b/__tests__/zeroTest.ts deleted file mode 100644 index 853787b..0000000 --- a/__tests__/zeroTest.ts +++ /dev/null @@ -1,3 +0,0 @@ -test('adds 1 + 2 to equal 3', () => { - expect(1 + 2).toBe(3); -}); diff --git a/config.example.toml b/config.example.toml index 671040f..3e001bb 100644 --- a/config.example.toml +++ b/config.example.toml @@ -19,5 +19,6 @@ uri = "http://localhost:5000" [watcher] - event = true - header =true + event = true + header = true + storage = true diff --git a/package-lock.json b/package-lock.json index 02ace1e..b757506 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9643,6 +9643,11 @@ } } }, + "solidity-parser-diligence": { + "version": "0.4.18", + "resolved": "https://registry.npmjs.org/solidity-parser-diligence/-/solidity-parser-diligence-0.4.18.tgz", + "integrity": "sha512-mauO/qG2v59W9sOn5TYV2dS7+fvFKqIHwiku+TH82e1Yca4H8s6EDG12ZpXO2cmgLlCKX3FOqqC73aYLB8WwNg==" + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/package.json b/package.json index afe03d1..b19acd1 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "node-fetch": "^2.6.1", "postgraphile": "^4.9.0", "react": "^16.13.1", + "solidity-parser-diligence": "^0.4.18", "subscriptions-transport-ws": "^0.9.18", "ts-node": "^8.10.2", "typeorm": "^0.2.26", diff --git a/src/__mocks__/store.ts b/src/__mocks__/store.ts new file mode 100644 index 0000000..1deef6a --- /dev/null +++ b/src/__mocks__/store.ts @@ -0,0 +1,83 @@ +import Address from 'models/data/address'; +import Contract from '../models/contract/contract'; + +export const contractsByAddrHash = { + "emptyStateLeafKey": null, + "someStateLeafKey": { + contractId: 1, + address: "0xAddress", + } +}; + +export const eventsByContractId = { + 1: [ + ], + 3: [{ + name: 'ename' + }], +}; + +export const statesByContractId = { + 1: [{ + slot: "slot1", + type: "uint", + }, { + slot: "slot2", + type: "uint", + }] +}; + +export const mockGetContractByAddressHash = jest.fn().mockImplementation(function (addrHash: string) { + return contractsByAddrHash[addrHash]; +}); + +export const mockGetStatesByContractId = jest.fn().mockImplementation(function (contractId: number) { + return statesByContractId[contractId]; +}); + +export const mockGetContracts = jest.fn().mockImplementation(function (): Contract[] { + return [ + { address: 'address1' } as Contract, + { address: 'address2' } as Contract, + { + address: 'address3', + contractId: 3, + events: [1], + abi: [{ + name: 'ename', + inputs: [], + }], + } as Contract, + ] +}); + +export const mockGetEventsByContractId = jest.fn().mockImplementation(function (contractId: number) { + return eventsByContractId[contractId]; +}); + +export const mockGetAddressById = jest.fn().mockImplementation(function (addressId: number): Address { + return null; +}); + +export const mockGetAddress = jest.fn().mockImplementation(function (addressString: string): Address { + return { + addressId: 0, + address: addressString, + hash: '' + }; +}); + +export const mockGetStore = jest.fn().mockImplementation(() => { + return { + getContractByAddressHash: mockGetContractByAddressHash, + getStatesByContractId: mockGetStatesByContractId, + getContracts: mockGetContracts, + getEventsByContractId: mockGetEventsByContractId, + getAddressById: mockGetAddressById, + getAddress: mockGetAddress + } +}); + +export default { + getStore: mockGetStore +}; \ No newline at end of file diff --git a/src/env.ts b/src/env.ts index 6127971..4658ee4 100644 --- a/src/env.ts +++ b/src/env.ts @@ -74,6 +74,9 @@ export default envalid.cleanEnv( ENABLE_HEADER_WATCHER: envalid.bool({ default: tomlConfig?.watcher?.header }), + ENABLE_STORAGE_WATCHER: envalid.bool({ + default: tomlConfig?.watcher?.storage + }), }, { strict: true } diff --git a/src/migrations/1604551153589-StateConfig.ts b/src/migrations/1604551153589-StateConfig.ts new file mode 100644 index 0000000..4196d4c --- /dev/null +++ b/src/migrations/1604551153589-StateConfig.ts @@ -0,0 +1,21 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class StateConfig1604551153589 implements MigrationInterface { + name = 'StateConfig1604551153589' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "contract"."state" ( + "state_id" SERIAL NOT NULL, + "slot" integer NOT NULL, + "type" character varying NOT NULL, + CONSTRAINT "states_pk" PRIMARY KEY ("state_id") + )`); + await queryRunner.query(`ALTER TABLE "contract"."contracts" ADD "states" integer array`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "contract"."contracts" DROP COLUMN "states"`); + await queryRunner.query(`DROP TABLE "contract"."state"`); + } + +} diff --git a/src/migrations/1604987750715-StateProgress.ts b/src/migrations/1604987750715-StateProgress.ts new file mode 100644 index 0000000..63d5757 --- /dev/null +++ b/src/migrations/1604987750715-StateProgress.ts @@ -0,0 +1,31 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class StateProgress1604987750715 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + create table if not exists data.state_progress + ( + state_progress_id serial not null + constraint state_progress_pk + primary key, + state_id integer not null + constraint state_progress_states_state_id_fk + references contract.state, + contract_id integer not null + constraint state_progress_contracts_contract_id_fk + references contract.contracts, + block_number integer not null + ); + comment on table data.state_progress is 'Sync state progress'; + comment on column data.state_progress.state_id is 'State ID'; + comment on column data.state_progress.contract_id is 'Contract ID'; + comment on column data.state_progress.block_number is 'Number of Block'; + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "data"."state_progress"`); + } + +} diff --git a/src/migrations/1605072677007-Addresses.ts b/src/migrations/1605072677007-Addresses.ts new file mode 100644 index 0000000..1d78ee2 --- /dev/null +++ b/src/migrations/1605072677007-Addresses.ts @@ -0,0 +1,25 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class Addresses1605072677007 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + create table if not exists data.addresses + ( + address_id serial not null + constraint addresses_pk + primary key, + address varchar not null, + hash varchar not null + ); + comment on column data.addresses.address_id is 'PK'; + comment on column data.addresses.address is 'Contract address'; + comment on column data.addresses.hash is 'Keccak hash'; + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable('data.addresses'); + } + +} diff --git a/src/models/contract/contract.ts b/src/models/contract/contract.ts index 15474b2..a256027 100644 --- a/src/models/contract/contract.ts +++ b/src/models/contract/contract.ts @@ -22,6 +22,10 @@ export default class Contract { @Column("int4", { name: "methods", nullable: true, array: true }) methods: number[] | null; + @Column("int4", { name: "states", nullable: true, array: true }) + states: number[] | null; + @Column("integer", { name: "starting_block" }) startingBlock: number; + } diff --git a/src/models/contract/state.ts b/src/models/contract/state.ts new file mode 100644 index 0000000..fff4c60 --- /dev/null +++ b/src/models/contract/state.ts @@ -0,0 +1,15 @@ +import { Column, Entity, Index, PrimaryGeneratedColumn } from "typeorm"; + +@Index("states_pk", ["stateId"], { unique: true }) +@Entity("state", { schema: "contract" }) + +export default class State { + @PrimaryGeneratedColumn({ type: "integer", name: "state_id" }) + stateId: number; + + @Column("integer", { name: "slot" }) + slot: number; + + @Column("character varying", { name: "type" }) + type: string; +} diff --git a/src/models/data/address.ts b/src/models/data/address.ts new file mode 100644 index 0000000..27ec381 --- /dev/null +++ b/src/models/data/address.ts @@ -0,0 +1,15 @@ +import { Column, Entity, Index, PrimaryGeneratedColumn } from "typeorm"; + +@Index("addresses_pk", ["addressId"], { unique: true }) +@Entity("addresses", { schema: "data" }) + +export default class Address { + @PrimaryGeneratedColumn({ type: "integer", name: "address_id" }) + addressId: number; + + @Column("character varying", { name: "address" }) + address: string; + + @Column("character varying", { name: "hash" }) + hash: string; +} diff --git a/src/models/data/stateProgress.ts b/src/models/data/stateProgress.ts new file mode 100644 index 0000000..58e4e28 --- /dev/null +++ b/src/models/data/stateProgress.ts @@ -0,0 +1,18 @@ +import { Column, Entity, Index, PrimaryGeneratedColumn } from "typeorm"; + +@Index("state_progress_pk", ["stateProgressId"], { unique: true }) +@Entity("state_progress", { schema: "data" }) + +export default class StateProgress { + @PrimaryGeneratedColumn({ type: "integer", name: "state_progress_id" }) + stateProgressId: number; + + @Column("integer", { name: "contract_id" }) + contractId: number; + + @Column("integer", { name: "state_id" }) + stateId: number; + + @Column("integer", { name: "block_number" }) + blockNumber: number +} diff --git a/src/repositories/contract/stateRepository.ts b/src/repositories/contract/stateRepository.ts new file mode 100644 index 0000000..f1bdc71 --- /dev/null +++ b/src/repositories/contract/stateRepository.ts @@ -0,0 +1,10 @@ +import {EntityRepository, Repository} from 'typeorm'; +import State from '../../models/contract/state'; + +@EntityRepository(State) +export default class StateRepository extends Repository { + + public findAll(): Promise { + return this.find(); + } +} diff --git a/src/repositories/data/__mocks__/addressIdSlotIdRepository.ts b/src/repositories/data/__mocks__/addressIdSlotIdRepository.ts new file mode 100644 index 0000000..4d68b18 --- /dev/null +++ b/src/repositories/data/__mocks__/addressIdSlotIdRepository.ts @@ -0,0 +1,3 @@ +export default class AddressIdSlotIdRepository { + +} \ No newline at end of file diff --git a/src/repositories/data/addressIdSlotIdRepository.ts b/src/repositories/data/addressIdSlotIdRepository.ts new file mode 100644 index 0000000..97bec4f --- /dev/null +++ b/src/repositories/data/addressIdSlotIdRepository.ts @@ -0,0 +1,64 @@ +import {EntityRepository, QueryRunner, Table} from 'typeorm'; +import { TableOptions } from 'typeorm/schema-builder/options/TableOptions'; + +@EntityRepository() +export default class AddressIdSlotIdRepository { + private queryRunner: QueryRunner; + + constructor(queryRunner: QueryRunner) { + this.queryRunner = queryRunner; + } + + public async createTable(addressId, slotId): Promise { + const tableName = `data.address_id_${addressId}_slot_id_${slotId}`; + const table = await this.queryRunner.getTable(tableName); + + if (table) { + console.log(`Table ${tableName} already exists`); + return; + } + + const tableOptions: TableOptions = { + name: tableName, + columns: [ + { + name: 'id', + type: 'integer', + isPrimary: true, + isGenerated: true, + generationStrategy: 'increment' + }, { + name: 'address_id', + type: 'integer', + }, { + name: 'hash', + type: 'character varying', + }, + ] + }; + + await this.queryRunner.createTable(new Table(tableOptions), true); + console.log('create new table', tableName); + } + + public async add(cotractAddressId: number, addressId, slotId: number, hash: string): Promise { + const tableName = `data.address_id_${cotractAddressId}_slot_id_${slotId}`; + const sql = `INSERT INTO ${tableName} (address_id, hash) VALUES (${addressId}, '${hash}');`; + console.log(sql); + + return this.queryRunner.query(sql); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public async getAddressIdByHash(cotractAddressId: number, slotId: number, hash: string): Promise { + const tableName = `data.address_id_${cotractAddressId}_slot_id_${slotId}`; + const sql = `SELECT * FROM ${tableName} WHERE hash='${hash}';`; + + const data = await this.queryRunner.query(sql); + if (!data) { + return null; + } + + return data[0]?.address_id; + } +} diff --git a/src/repositories/data/addressRepository.ts b/src/repositories/data/addressRepository.ts new file mode 100644 index 0000000..dfbcf24 --- /dev/null +++ b/src/repositories/data/addressRepository.ts @@ -0,0 +1,19 @@ +import {EntityRepository, Repository} from 'typeorm'; +import Address from '../../models/data/address'; +import { keccak256 } from 'ethereumjs-util'; + +@EntityRepository(Address) +export default class AddressRepository extends Repository
{ + + public findAll(): Promise { + return this.find(); + } + + public async add(address: string): Promise
{ + const hash = '0x' + keccak256(Buffer.from(address.replace('0x', ''), 'hex')).toString('hex'); + return this.save({ + address, + hash, + }); + } +} diff --git a/src/repositories/data/stateProgressRepository.ts b/src/repositories/data/stateProgressRepository.ts new file mode 100644 index 0000000..136198a --- /dev/null +++ b/src/repositories/data/stateProgressRepository.ts @@ -0,0 +1,52 @@ +import {EntityRepository, Repository} from 'typeorm'; +import StateProgress from '../../models/data/stateProgress'; + +@EntityRepository(StateProgress) +export default class StateProgressRepository extends Repository { + + public async add(contractId: number, stateId: number, blockNumber: number): Promise { + return this.save({ + contractId, + stateId, + blockNumber, + }); + } + + public async isSync(contractId: number, stateId: number, blockNumber: number): Promise { + const item = await this.findOne({ + contractId, + stateId, + blockNumber, + }); + + if (item) { + return true; + } + + return false; + } + + public async findSyncedBlocks(contractId: number, stateId: number, offset = 0, limit = 1000): Promise { + const query = this.createQueryBuilder('state_progress') + .where(`contract_id=${contractId}`) + .andWhere(`state_id=${stateId}`) + .orderBy({ + 'state_progress.state_progress_id': 'ASC', + }) + .take(limit) + .offset(offset); + + return query.getMany(); + } + + public async getMaxBlockNumber(contractId: number, stateId: number): Promise { + const query = this.createQueryBuilder('state_progress') + .where(`contract_id=${contractId}`) + .andWhere(`state_id=${stateId}`) + .select("MAX(state_progress.block_number)", "max"); + + const result = await query.getRawOne(); + + return result?.max || 0; + } +} diff --git a/src/repositories/graphqlRepository.ts b/src/repositories/graphqlRepository.ts index 55db9aa..48e7926 100644 --- a/src/repositories/graphqlRepository.ts +++ b/src/repositories/graphqlRepository.ts @@ -18,7 +18,7 @@ export default class GraphqlRepository { this.graphqlClient = new GraphqlClient(); } - public ethHeaderCidByBlockNumber(blockNumber: string | number): Promise { + public ethHeaderCidWithTransactionByBlockNumber(blockNumber: string | number): Promise { return this.graphqlClient.query(` query MyQuery { ethHeaderCidByBlockNumber(n: "${blockNumber}") { @@ -54,6 +54,46 @@ export default class GraphqlRepository { `); } + public ethHeaderCidWithStateByBlockNumber(blockNumber: string | number): Promise { + return this.graphqlClient.query(` + query MyQuery { + ethHeaderCidByBlockNumber(n: "${blockNumber}") { + nodes { + stateCidsByHeaderId { + nodes { + id + blockByMhKey { + data + key + } + stateLeafKey + statePath + mhKey + headerId + storageCidsByStateId { + nodes { + storageLeafKey + storagePath + mhKey + id + stateId + blockByMhKey { + data + key + } + } + } + ethHeaderCidByHeaderId { + blockNumber + } + } + } + } + } + } + `); + } + public ethHeaderCidById(headerId: number): Promise { return this.graphqlClient.query(` query MyQuery { @@ -170,5 +210,45 @@ export default class GraphqlRepository { } `, onNext); } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public subscriptionStateCids(onNext: (value: any) => void): Promise { + return this.graphqlClient.subscribe(` + subscription MySubscription { + listen(topic: "state_cids") { + relatedNode { + ... on StateCid { + id + blockByMhKey { + data + key + } + stateLeafKey + statePath + mhKey + headerId + storageCidsByStateId { + nodes { + storageLeafKey + storagePath + mhKey + id + stateId + blockByMhKey { + data + key + } + } + } + ethHeaderCidByHeaderId { + blockNumber + } + } + } + } + } + `, onNext); + } + } diff --git a/src/server.ts b/src/server.ts index 4e72366..41e403b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -30,6 +30,12 @@ import GraphqlService from './services/graphqlService'; console.info('Header watcher is not enabled'); } + if (env.ENABLE_STORAGE_WATCHER) { + graphqlService.subscriptionStateCids(); // async + } else { + console.info('Storage watcher is not enabled'); + } + if (env.HTTP_ENABLE) { createServer(app.app).listen(env.HTTP_PORT, env.HTTP_ADDR,() => console.info(`Http server running on port ${env.HTTP_ADDR}:${env.HTTP_PORT}`) diff --git a/src/services/contractService.ts b/src/services/contractService.ts index b5d4657..f28ffcc 100644 --- a/src/services/contractService.ts +++ b/src/services/contractService.ts @@ -3,9 +3,13 @@ import { getConnection } from 'typeorm'; import Contract from '../models/contract/contract'; import Event from '../models/contract/event'; import Method from '../models/contract/method'; +import State from '../models/contract/state'; +import Address from '../models/data/address'; import ContractRepository from '../repositories/contract/contractRepository'; import EventRepository from '../repositories/contract/eventRepository'; import MethodRepository from '../repositories/contract/methodRepository'; +import StateRepository from '../repositories/contract/stateRepository'; +import AddressRepository from '../repositories/data/addressRepository'; export default class ContractService { @@ -30,4 +34,14 @@ export default class ContractService { return methods; } + public async loadStates (): Promise { + const stateRepository: StateRepository = getConnection().getCustomRepository(StateRepository); + return stateRepository.findAll(); + } + + public async loadAddresses (): Promise { + const addressRepository: AddressRepository = getConnection().getCustomRepository(AddressRepository); + return addressRepository.findAll(); + } + } diff --git a/src/services/dataService.test.ts b/src/services/dataService.test.ts new file mode 100644 index 0000000..08b6192 --- /dev/null +++ b/src/services/dataService.test.ts @@ -0,0 +1,386 @@ +jest.mock('../store'); +jest.mock('../repositories/data/addressIdSlotIdRepository'); + +import DataService from './dataService'; +import Event from '../models/contract/event'; +import Contract from '../models/contract/contract'; +import State from '../models/contract/state'; +import HeaderCids from '../models/eth/headerCids'; +import TransactionCids from '../models/eth/transactionCids'; + +//@ts-ignore +import { mockGetEventsByContractId, mockGetStore, mockGetContractByAddressHash, mockGetStatesByContractId, mockGetContracts } from '../store'; +import { rlp } from 'ethereumjs-util'; + +const mockGraphqlService = { + ethHeaderCidWithTransactionByBlockNumber: () => ({ + ethHeaderCidByBlockNumber: { + nodes: [{ + ethTransactionCidsByHeaderId: { + nodes: [{ + id: 1, + }], + }, + }], + }, + }), +} as any; + +const mockDataService = { + processEvent: () => Promise.resolve(), +} as any; + +const mockContract = { + contractId: 1, + name: 'Contract1', + startingBlock: 0, + abi: [{ + "anonymous": false, + "inputs": [{ + "indexed": false, + "internalType": "string", + "name": "message", + "type": "string" + }], + "name": "MessageChanged", + "type": "event" + }, { + "inputs": [], + "name": "getMessage", + "outputs": [{ + "internalType": "string", + "name": "_message", + "type": "string" + }], + "stateMutability": "view", + "type": "function" + }, { + "inputs": [], + "name": "message", + "outputs": [{ + "internalType": "string", + "name": "", + "type": "string" + }], + "stateMutability": "view", + "type": "function" + }, { + "inputs": [{ + "internalType": "string", + "name": "_message", + "type": "string" + }], + "name": "setMessage", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }], +} as Contract; + +test('_getPgType', async function () { + // @ts-ignore + expect(await DataService._getPgType('bool')).toEqual('boolean'); + // @ts-ignore + expect(await DataService._getPgType('uint8')).toEqual('numeric'); + // @ts-ignore + expect(await DataService._getPgType('uint256')).toEqual('numeric'); + // @ts-ignore + expect(await DataService._getPgType('int8')).toEqual('numeric'); + // @ts-ignore + expect(await DataService._getPgType('int256')).toEqual('numeric'); + // @ts-ignore + expect(await DataService._getPgType('address')).toEqual('character varying(66)'); + // @ts-ignore + expect(await DataService._getPgType('bytes')).toEqual('bytea'); +}); + +describe('_getTableOptions', function () { + test('_getTableName', async function () { + // @ts-ignore + const tableName1 = await DataService._getTableName({ id: 1, contractId: 1 }); + expect(tableName1).toEqual('data.contract_id_1_event_id_1'); + // @ts-ignore + const tableName2 = await DataService._getTableName({ id: 2, contractId: 1 }); + expect(tableName2).toEqual('data.contract_id_1_event_id_2'); + // @ts-ignore + const tableName3 = await DataService._getTableName({ id: 1, contractId: 2 }); + expect(tableName3).toEqual('data.contract_id_2_event_id_1'); + }); + + test('by event', async function () { + const event: Event = { eventId: 2, name: 'MessageChanged' }; + // @ts-ignore + const tableOptions = await DataService._getTableOptions(mockContract, { event }); + expect(tableOptions).toEqual({ + columns: [{ + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "name": "contract_id", + "type": "integer", + }, { + "name": "mh_key", + "type": "text", + }, { + "name": "event_id", + "type": "integer", + }, { + "isNullable": true, + "name": "data_message", + "type": "text", + }], + "name": "data.contract_id_1_event_id_2", + }); + }); + + test('by state', async function () { + const state: State = { + stateId: 1, + slot: 0, + type: 'uint' + }; + // @ts-ignore + const tableOptions = await DataService._getTableOptions(mockContract, { state }); + expect(tableOptions).toEqual({ + columns: [{ + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "name": "contract_id", + "type": "integer", + }, { + "name": "mh_key", + "type": "text", + }, { + "name": "state_id", + "type": "integer", + }, { + "isNullable": true, + "name": "slot_0", + "type": "numeric", + }], + "name": "data.contract_id_1_state_id_1", + }); + }); +}); + +describe('_syncEventForContractPage', function () { + const mockEvent: Event = { + eventId: 1, + name: 'TestEvent1' + }; + + test('full', async function () { + const mockProgressRepository = { + findSyncedBlocks: jest.fn().mockResolvedValueOnce( + [2, 3, 4, 5].map(blockNumber => ({ blockNumber })) + ) + } as any; + + const startBlock = 0; + const maxBlock = 9; + const limit = 10; + const page = 1; + + // @ts-ignore + const blocks = await DataService._syncEventForContractPage({ + graphqlService: mockGraphqlService, + dataService: mockDataService, + progressRepository: mockProgressRepository, + }, mockEvent, mockContract, startBlock, maxBlock, page, limit); + + expect(blocks).toEqual([0, 1, 6, 7, 8, 9,]); + }); + + test('without synced blocks', async function () { + const mockProgressRepository = { + findSyncedBlocks: jest.fn().mockResolvedValueOnce([]) + } as any; + + const startBlock = 0; + const maxBlock = 9; + const limit = 10; + const page = 1; + + // @ts-ignore + const blocks = await DataService._syncEventForContractPage({ + graphqlService: mockGraphqlService, + dataService: mockDataService, + progressRepository: mockProgressRepository, + }, mockEvent, mockContract, startBlock, maxBlock, page, limit); + + expect(blocks).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + }); + + test('with all synced blocks', async function () { + const mockProgressRepository = { + findSyncedBlocks: jest.fn().mockResolvedValueOnce( + [0, 1, 2].map((blockNumber) => ({ blockNumber })) + ) + } as any; + + const startBlock = 0; + const maxBlock = 2; + const limit = 10; + const page = 1; + + // @ts-ignore + const blocks = await DataService._syncEventForContractPage({ + graphqlService: mockGraphqlService, + dataService: mockDataService, + progressRepository: mockProgressRepository, + }, mockEvent, mockContract, startBlock, maxBlock, page, limit); + + expect(blocks).toEqual([]); + }); +}); + +describe('processState', function () { + let dataService: DataService + + beforeEach(async function () { + dataService = new DataService(); + }); + + test('empty args', async function () { + expect(await dataService.processState(undefined)).toEqual(undefined); + expect(await dataService.processState(null)).toEqual(undefined); + expect(await dataService.processState({})).toEqual(undefined); + }); + + test('no contracts', async function () { + const stateLeafKey = "emptyStateLeafKey"; + + await dataService.processState({ stateLeafKey }); + + expect(mockGetStore).toBeCalledTimes(1); + expect(mockGetContractByAddressHash).toBeCalledTimes(1); + expect(mockGetContractByAddressHash).toBeCalledWith(stateLeafKey); + expect(mockGetStatesByContractId).not.toBeCalled(); + }); + + test('check uint', async function () { + dataService.addState = jest.fn().mockImplementation(function (contractId: number, mhKey: string, state: State, value: any, blockNumber: number): Promise { + return null + }); + + const stateLeafKey = "someStateLeafKey"; + const relatedNode = { + stateLeafKey, + ethHeaderCidByHeaderId: { + blockNumber: 0, + }, + storageCidsByStateId: { + nodes: [{ + storageLeafKey: "0x471ccdcb79bddea38175f8cc115b52365f2c864200fbce48e994511bb9c6006f", + blockByMhKey: { + data: "c512c2345678", + }, + }] + } + } + + const stateCids = await dataService.processState(relatedNode); + + expect(mockGetStore).toBeCalledTimes(4); + expect(mockGetContractByAddressHash).toBeCalledTimes(2); + expect(mockGetContractByAddressHash).toBeCalledWith(stateLeafKey); + expect(mockGetStatesByContractId).toBeCalledTimes(1); + expect(dataService.addState).toBeCalledTimes(2); + }); +}); + +describe('processEvent', function () { + let dataService: DataService + + beforeEach(async function () { + dataService = new DataService(); + }); + + test('empty args', async function () { + expect(await dataService.processEvent(undefined)).toEqual(undefined); + expect(await dataService.processEvent(null)).toEqual(undefined); + }); + + test('no logContracts and no target', async function () { + dataService.processHeader = jest.fn().mockImplementation(async function (relatedNode: { td; blockHash; blockNumber; bloom; cid; mhKey; nodeId; ethNodeId; parentHash; receiptRoot; uncleRoot; stateRoot; txRoot; reward; timesValidated; timestamp }): Promise { + return { + id: 0, + } as HeaderCids; + }); + dataService.processTransaction = jest.fn().mockImplementation(function (ethTransaction, headerId: number): Promise { + return null + }); + + const relatedNode1 = { + ethTransactionCidByTxId: { + ethHeaderCidByHeaderId: "0xheaderCidByHeaderId", + } + }; + const resp1 = await dataService.processEvent(relatedNode1); + + expect(dataService.processHeader).toBeCalledTimes(1); + expect(dataService.processHeader).toBeCalledWith(relatedNode1.ethTransactionCidByTxId.ethHeaderCidByHeaderId) + expect(dataService.processTransaction).toBeCalledTimes(1); + expect(dataService.processTransaction).toBeCalledWith(relatedNode1.ethTransactionCidByTxId, 0); + expect(resp1).toEqual(undefined); + + const relatedNode2 = { + ethTransactionCidByTxId: { + ethHeaderCidByHeaderId: "0xheaderCidByHeaderId", + }, + logContracts: [ + "veryUniqueAddress" + ] + }; + const resp2 = await dataService.processEvent(relatedNode2); + expect(mockGetContracts).toBeCalledTimes(1); + expect(resp2).toEqual(undefined); + }); + + test('check cycle', async function () { + dataService.processHeader = jest.fn().mockImplementation(async function (relatedNode: { td; blockHash; blockNumber; bloom; cid; mhKey; nodeId; ethNodeId; parentHash; receiptRoot; uncleRoot; stateRoot; txRoot; reward; timesValidated; timestamp }): Promise { + return { + id: 0, + } as HeaderCids; + }); + dataService.processTransaction = jest.fn().mockImplementation(function (ethTransaction, headerId: number): Promise { + return null + }); + dataService.addEvent = jest.fn().mockImplementation(function (): Promise { + return null + }) + + await dataService.processEvent({ + ethTransactionCidByTxId: { + ethHeaderCidByHeaderId: "0xheaderCidByHeaderId", + }, + logContracts: [ + "address3" + ], + topic0S: [ + "0x3655aa002c9e821cc231138d4f8790003402c95dd1c0cfb6378146d16c7ea582" + ], + blockByMhKey: { + data: rlp.encode(["1", "2", "3", [[["1"], ["1"], []]]]).toString("hex"), + }, + }); + + expect(dataService.processHeader).toBeCalledTimes(1); + expect(dataService.processHeader).toBeCalledWith("0xheaderCidByHeaderId") + expect(dataService.processTransaction).toBeCalledTimes(1); + expect(dataService.processTransaction).toBeCalledWith({ ethHeaderCidByHeaderId: "0xheaderCidByHeaderId" }, 0); + expect(mockGetContracts).toBeCalledTimes(2); + + expect(mockGetEventsByContractId).toBeCalledTimes(1) + expect(mockGetEventsByContractId).toBeCalledWith(3) + + expect(dataService.addEvent).toBeCalledTimes(1); + }); +}); \ No newline at end of file diff --git a/src/services/dataService.ts b/src/services/dataService.ts index d392634..74ef7ae 100644 --- a/src/services/dataService.ts +++ b/src/services/dataService.ts @@ -3,7 +3,7 @@ import to from 'await-to-js'; import { getConnection, Table } from 'typeorm'; import { TableOptions } from 'typeorm/schema-builder/options/TableOptions'; import * as abi from 'ethereumjs-abi'; -import { keccak256, rlp } from 'ethereumjs-util' +import { keccak256, keccakFromHexString, rlp } from 'ethereumjs-util'; import Store from '../store'; import Event from '../models/contract/event'; import Contract from '../models/contract/contract'; @@ -13,8 +13,16 @@ import HeaderCids from '../models/eth/headerCids'; import TransactionCids from '../models/eth/transactionCids'; import TransactionCidsRepository from '../repositories/eth/transactionCidsRepository'; import HeaderCidsRepository from '../repositories/eth/headerCidsRepository'; +import StateCids from '../models/eth/stateCids'; +import State from '../models/contract/state'; +import ApplicationError from '../errors/applicationError'; +import StateProgressRepository from '../repositories/data/stateProgressRepository'; +import Address from '../models/data/address'; +import AddressRepository from '../repositories/data/addressRepository'; +import AddressIdSlotIdRepository from '../repositories/data/addressIdSlotIdRepository'; const LIMIT = 1000; +const zero64 = '0000000000000000000000000000000000000000000000000000000000000000'; type ABIInput = { name: string; @@ -40,7 +48,12 @@ export default class DataService { for (const contract of contracts) { const events: Event[] = Store.getStore().getEventsByContractId(contract.contractId); for (const event of events) { - await this._createTable(contract, event) + await this._createEventTable(contract, event) + } + + const states: State[] = Store.getStore().getStatesByContractId(contract.contractId); + for (const state of states) { + await this._createStateTable(contract, state) } } } @@ -48,15 +61,17 @@ export default class DataService { // eslint-disable-next-line @typescript-eslint/no-explicit-any public async addEvent (eventId: number, contractId: number, data: ABIInputData[], mhKey: string, blockNumber: number): Promise { - const tableName = DataService._getTableName(contractId, eventId); + const tableName = DataService._getTableName({ + contractId, + type: 'event', + id: eventId + }); if (!data) { return; } return getConnection().transaction(async (entityManager) => { - - const progressRepository: ProgressRepository = entityManager.getCustomRepository(ProgressRepository); const sql = `INSERT INTO ${tableName} (event_id, contract_id, mh_key, ${data.map((line) => 'data_' + line.name.toLowerCase().trim()).join(',')}) VALUES @@ -70,10 +85,39 @@ VALUES console.log(err); } + const progressRepository: ProgressRepository = entityManager.getCustomRepository(ProgressRepository); await progressRepository.add(contractId, eventId, blockNumber); }); } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public async addState (contractId: number, mhKey: string, state: State, value: any, blockNumber: number): Promise { + + const tableName = DataService._getTableName({ + contractId, + type: 'state', + id: state.stateId, + }); + + return getConnection().transaction(async (entityManager) => { + const sql = `INSERT INTO ${tableName} +(state_id, contract_id, mh_key, slot_${state.slot}) +VALUES +(${state.stateId}, ${contractId}, '${mhKey}', '${value}');`; + + console.log(sql); + + const [err] = await to(entityManager.queryRunner.query(sql)); + if (err) { + // TODO: throw err + console.log(err); + } + + const stateProgressRepository: StateProgressRepository = entityManager.getCustomRepository(StateProgressRepository); + await stateProgressRepository.add(contractId, state.stateId, blockNumber); + }); + } + private static _getPgType(abiType: string): string { let pgType = 'TEXT'; @@ -238,7 +282,7 @@ VALUES const notSyncedBlocks = allBlocks.filter(x => !syncedBlocks.includes(x)); for (const blockNumber of notSyncedBlocks) { - const header = await graphqlService.ethHeaderCidByBlockNumber(blockNumber); + const header = await graphqlService.ethHeaderCidWithTransactionByBlockNumber(blockNumber); if (!header) { console.warn(`No header for ${blockNumber} block`); @@ -282,6 +326,135 @@ VALUES }); } + public async processState(relatedNode): Promise { + + if (!relatedNode || !relatedNode.stateLeafKey) { + return; + } + + console.log(JSON.stringify(relatedNode, null, 2)); + + const contract = Store.getStore().getContractByAddressHash(relatedNode.stateLeafKey); + if (contract && relatedNode?.storageCidsByStateId?.nodes?.length) { + const address = Store.getStore().getAddress(contract.address); + const states = Store.getStore().getStatesByContractId(contract.contractId); + + for (const state of states) { + const slot = state.slot.toString(); + let storageLeafKey = null ; + if (state.type === 'mapping') { // TODO: mapping(address=>uint) + const addressIdSlotIdRepository: AddressIdSlotIdRepository = new AddressIdSlotIdRepository(getConnection().createQueryRunner()); + + for (const storage of relatedNode?.storageCidsByStateId?.nodes) { + console.log('storage.storageLeafKey', address.addressId, state.stateId, storage.storageLeafKey); + const addressId = await addressIdSlotIdRepository.getAddressIdByHash(address.addressId, state.stateId, storage.storageLeafKey); + + if (!addressId) { + continue; + } + + const buffer = Buffer.from(storage.blockByMhKey.data.replace('\\x',''), 'hex'); + const decoded: any = rlp.decode(buffer); // eslint-disable-line + const value = abi.rawDecode([ 'uint' ], rlp.decode(Buffer.from(decoded[1], 'hex')))[0]; + + console.log(decoded); + console.log(rlp.decode(Buffer.from(decoded[1], 'hex'))); + + console.log(decoded[0].toString('hex')); + console.log(value); + + await this.addState(contract.contractId, storage.blockByMhKey.key, state, value, relatedNode.ethHeaderCidByHeaderId.blockNumber); + + + } + } else if (state.type === 'uint') { + storageLeafKey = '0x' + keccak256(Buffer.from(zero64.substring(0, zero64.length - slot.length) + slot, 'hex')).toString('hex'); + console.log('storageLeafKey', storageLeafKey); + + const storage = relatedNode?.storageCidsByStateId?.nodes.find((s) => s.storageLeafKey === storageLeafKey); + console.log('storage', storage); + if (!storage) { + continue; + } + + const buffer = Buffer.from(storage.blockByMhKey.data.replace('\\x',''), 'hex'); + const decoded: any = rlp.decode(buffer); // eslint-disable-line + const value = abi.rawDecode([ state.type ], Buffer.from(decoded[1], 'hex'))[0]; + + console.log(decoded[0].toString('hex')); + console.log(value); + + await this.addState(contract.contractId, storage.blockByMhKey.key, state, value, relatedNode.ethHeaderCidByHeaderId.blockNumber); + } + } + } + } + + public static async syncStatesForContract({ + graphqlService, stateProgressRepository, dataService + }: { graphqlService: GraphqlService; dataService: DataService; stateProgressRepository: StateProgressRepository }, + state: State, + contract: Contract, + ): Promise { + const startingBlock = contract.startingBlock; + const maxBlock = await stateProgressRepository.getMaxBlockNumber(contract.contractId, state.stateId); + const maxPage = Math.ceil(maxBlock / LIMIT) || 1; + + for (let page = 1; page <= maxPage; page++) { + await DataService._syncStatesForContractPage( + { + graphqlService, + stateProgressRepository, + dataService + }, + state, + contract, + startingBlock, + maxBlock, + page, + ) + } + } + + private static async _syncStatesForContractPage({ + graphqlService, stateProgressRepository, dataService + }: { graphqlService: GraphqlService; dataService: DataService; stateProgressRepository: StateProgressRepository }, + state: State, + contract: Contract, + startingBlock: number, + maxBlock: number, + page: number, + limit: number = LIMIT, + ): Promise { + const progresses = await stateProgressRepository.findSyncedBlocks(contract.contractId, state.stateId, (page - 1) * limit, limit); + + const max = Math.min(maxBlock, page * limit); // max block for current page + const start = startingBlock + (page -1) * limit; // start block for current page + + const allBlocks = Array.from({ length: max - start + 1 }, (_, i) => i + start); + const syncedBlocks = progresses.map((p) => p.blockNumber); + const notSyncedBlocks = allBlocks.filter(x => !syncedBlocks.includes(x)); + + console.log('notSyncedBlocks', notSyncedBlocks); + + for (const blockNumber of notSyncedBlocks) { + const header = await graphqlService.ethHeaderCidWithStateByBlockNumber(blockNumber); + + if (!header) { + console.warn(`No header for ${blockNumber} block`); + continue; + } + + for (const ethHeader of header?.ethHeaderCidByBlockNumber?.nodes) { + for (const state of ethHeader.stateCidsByHeaderId.nodes) { + await dataService.processState(state); + } + } + } + + return notSyncedBlocks; + } + public static async syncHeaders({ graphqlService, headerCidsRepository, dataService }: { graphqlService: GraphqlService; dataService: DataService; headerCidsRepository: HeaderCidsRepository } @@ -324,18 +497,67 @@ VALUES for (const headerId of notSyncedIds) { const header = await graphqlService.ethHeaderCidById(headerId); await dataService.processHeader(header.ethHeaderCidById); - } return notSyncedIds; } - private static _getTableName(contractId: number, eventId: number): string { - return `data.contract_id_${contractId}_event_id_${eventId}`; + public async prepareAddresses(contracts: Contract[] = []): Promise { + const addressRepository: AddressRepository = getConnection().getCustomRepository(AddressRepository); + const addressIdSlotIdRepository: AddressIdSlotIdRepository = new AddressIdSlotIdRepository(getConnection().createQueryRunner()); + + for (const contract of contracts) { + let address: Address = Store.getStore().getAddress(contract.address); + if (!address) { + address = await addressRepository.add(contract.address); + Store.getStore().addAddress(address); + } + + + + const states = Store.getStore().getStatesByContractId(contract.contractId); + for (const state of states) { + if (state.type === 'mapping') { // TODO: mapping(address=>uint) + await addressIdSlotIdRepository.createTable(address.addressId, state.stateId); + console.log('contract.address', contract.address); + const addresses: Address[] = Store.getStore().getAddresses(); // 100 m + for (const adr of addresses) { + const adrStr = (zero64.substring(0, zero64.length - adr.address.length) + adr.address.replace('0x', '0')).toLowerCase(); + const slot = zero64.substring(0, zero64.length - state.slot.toString().length) + state.slot; + // TODO: !!! FIX DOULBE keccak !!! + const hash = '0x' + keccakFromHexString('0x' + keccakFromHexString('0x' + adrStr + slot).toString('hex')).toString('hex'); + + await addressIdSlotIdRepository.add(address.addressId, adr.addressId, state.stateId, hash); + } + } + } + } } - private static _getTableOptions(contract: Contract, event: Event): TableOptions { - const tableName = this._getTableName(contract.contractId, event.eventId); + private static _getTableName({ contractId, type = 'event', id}): string { + return `data.contract_id_${contractId}_${type}_id_${id}`; + } + + private static _getTableOptions(contract: Contract, { event, state }: { event?: Event; state?: State }): TableOptions { + let tableName; + + if (!event && !state) { + throw new ApplicationError('Bad params'); + } + + if (event) { + tableName = this._getTableName({ + contractId: contract.contractId, + type: 'event', + id: event.eventId, + }); + } else if (state) { + tableName = this._getTableName({ + contractId: contract.contractId, + type: 'state', + id: state.stateId, + }); + } const tableOptions: TableOptions = { name: tableName, @@ -346,9 +568,6 @@ VALUES isPrimary: true, isGenerated: true, generationStrategy: 'increment' - },{ - name: 'event_id', - type: 'integer', }, { name: 'contract_id', type: 'integer', @@ -359,21 +578,65 @@ VALUES ] }; - const data: ABIInput[] = (contract.abi as ABI)?.find((e) => e.name === event.name)?.inputs; - data.forEach((line) => { + if (event) { + tableOptions.columns.push({ + name: 'event_id', + type: 'integer', + }); + + const data: ABIInput[] = (contract.abi as ABI)?.find((e) => e.name === event.name)?.inputs; + data.forEach((line) => { + tableOptions.columns.push({ + name: `data_${line.name.toLowerCase().trim()}`, + type: this._getPgType(line.internalType), + isNullable: true, + }); + }); + } + + if (state) { + tableOptions.columns.push({ + name: 'state_id', + type: 'integer', + }); + tableOptions.columns.push({ - name: `data_${line.name.toLowerCase().trim()}`, - type: this._getPgType(line.internalType), + name: `slot_${state.slot}`, + type: this._getPgType(state.type), isNullable: true, }); - }); + } return tableOptions; } - private async _createTable(contract: Contract, event: Event): Promise { + private async _createEventTable(contract: Contract, event: Event): Promise { return getConnection().transaction(async (entityManager) => { - const tableName = DataService._getTableName(contract.contractId, event.eventId); + const tableName = DataService._getTableName({ + contractId: contract.contractId, + type: 'event', + id: event.eventId + }); + const table = await entityManager.queryRunner.getTable(tableName); + + if (table) { + console.log(`Table ${tableName} already exists`); + return; + } + + const tableOptions = DataService._getTableOptions(contract, { event }); + await entityManager.queryRunner.createTable(new Table(tableOptions), true); + console.log('create new table', tableName); + }); + } + + private async _createStateTable(contract: Contract, state: State): Promise { + return getConnection().transaction(async (entityManager) => { + const tableName = DataService._getTableName({ + contractId: contract.contractId, + type: 'state', + id: state.stateId, + }); const table = await entityManager.queryRunner.getTable(tableName); if (table) { @@ -381,7 +644,7 @@ VALUES return; } - const tableOptions = DataService._getTableOptions(contract, event); + const tableOptions = DataService._getTableOptions(contract, { state }); await entityManager.queryRunner.createTable(new Table(tableOptions), true); console.log('create new table', tableName); }); diff --git a/src/services/dataTypeParser.test.ts b/src/services/dataTypeParser.test.ts new file mode 100644 index 0000000..259cf9d --- /dev/null +++ b/src/services/dataTypeParser.test.ts @@ -0,0 +1,245 @@ +import { toStructure, toFields, } from './dataTypeParser'; + +describe('dataTypeParser', function () { + test('elementary types', function () { + const st1 = toStructure('string public constant name = "Uniswap";', 'name'); + expect(st1).toStrictEqual({ + "kind": "string", + "name": "name", + "type": "simple" + }); + expect(toFields(st1)).toStrictEqual([{ "name": "name", "type": "string" }]); + + const st2 = toStructure('uint8 public constant decimals = 18;', 'decimals'); + expect(st2).toStrictEqual({ + "kind": "uint8", + "name": "decimals", + "type": "simple" + }); + expect(toFields(st2)).toStrictEqual([{ "name": "decimals", "type": "uint8" }]); + }); + + test('array types', function () { + const st1 = toStructure('string[] public names = ["Uniswap"];', 'names'); + expect(st1).toStrictEqual({ + "kind": { + "kind": "string", + "name": "value0", + "type": "simple" + }, + "name": "names", + "type": "array" + }); + expect(toFields(st1)).toStrictEqual([ + { "name": "key0", "type": "uint" }, + { "name": "value0", "type": "string" }, + ]); + + const st2 = toStructure('string[][] public names;', 'names'); + expect(st2).toStrictEqual({ + "kind": { + "kind": { + "kind": "string", + "name": "value1", + "type": "simple" + }, + "name": + "value0", + "type": "array" + }, + "name": "names", + "type": "array" + }); + expect(toFields(st2)).toStrictEqual([ + { "name": "key0", "type": "uint" }, + { "name": "key1", "type": "uint" }, + { "name": "value1", "type": "string" }, + ]); + }); + + test('mapping types', function () { + const st1 = toStructure('mapping (address => uint96) internal balances;', 'balances'); + expect(st1).toStrictEqual({ + "key": "address", + "name": "balances", + "type": "mapping", + "value": { + "kind": "uint96", + "name": "value0", + "type": "simple" + } + }); + expect(toFields(st1)).toStrictEqual([ + { "name": "key0", "type": "address" }, + { "name": "value0", "type": "uint96" }, + ]); + + const st2 = toStructure('mapping (address => mapping (address => uint96)) internal allowances;', 'allowances'); + expect(st2).toStrictEqual({ + "key": "address", + "name": "allowances", + "type": "mapping", + "value": { + "key": "address", + "name": "value0", + "type": "mapping", + "value": { + "kind": + "uint96", + "name": "value1", + "type": "simple" + } + } + }); + expect(toFields(st2)).toStrictEqual([ + { "name": "key0", "type": "address" }, + { "name": "key1", "type": "address" }, + { "name": "value1", "type": "uint96" }, + ]); + }); + + test('user defined types', function () { + const vars1 = ` + Checkpoint checkpoint; + struct Checkpoint { + uint32 fromBlock; + uint96 votes; + } + `; + const st1 = toStructure(vars1, 'checkpoint'); + expect(st1).toStrictEqual({ + "fields": [ + { "kind": "uint32", "name": "fromBlock", "type": "simple" }, + { "kind": "uint96", "name": "votes", "type": "simple" } + ], + "name": "checkpoint", + "type": "struct" + }); + expect(toFields(st1)).toStrictEqual([ + { "name": "votes", "type": "uint96" }, + { "name": "fromBlock", "type": "uint32" } + ]); + + const vars2 = ` + Checkpoint[] public checkpoint; + struct Checkpoint { + uint32 fromBlock; + uint96 votes; + } + `; + const st2 = toStructure(vars2, 'checkpoint'); + expect(st2).toStrictEqual({ + "kind": { + "fields": [ + { "kind": "uint32", "name": "fromBlock", "type": "simple" }, + { "kind": "uint96", "name": "votes", "type": "simple" } + ], + "name": "value0", + "type": "struct" + }, + "name": "checkpoint", + "type": "array" + }); + expect(toFields(st2)).toStrictEqual([ + { "name": "key0", "type": "uint" }, + { "name": "votes", "type": "uint96" }, + { "name": "fromBlock", "type": "uint32" } + ]); + + const vars3 = ` + mapping(address => Checkpoint) public checkpoint; + struct Checkpoint { + uint32 fromBlock; + uint96 votes; + } + `; + const st3 = toStructure(vars3, 'checkpoint'); + expect(st3).toStrictEqual({ + "key": "address", + "name": "checkpoint", + "type": "mapping", + "value": { + "fields": [ + { "kind": "uint32", "name": "fromBlock", "type": "simple" }, + { "kind": "uint96", "name": "votes", "type": "simple" } + ], + "name": "value0", + "type": "struct" + } + }); + expect(toFields(st3)).toStrictEqual([ + { "name": "key0", "type": "address" }, + { "name": "votes", "type": "uint96" }, + { "name": "fromBlock", "type": "uint32" } + ]); + + const vars4 = ` + mapping(address => mapping(uint => Checkpoint)) public checkpoint; + struct Checkpoint { + uint32 fromBlock; + uint96 votes; + } + `; + const st4 = toStructure(vars4, 'checkpoint'); + expect(st4).toStrictEqual({ + "key": "address", + "name": "checkpoint", + "type": "mapping", + "value": { + "key": "uint", + "name": "value0", + "type": "mapping", + "value": { + "fields": [ + { "kind": "uint32", "name": "fromBlock", "type": "simple" }, + { "kind": "uint96", "name": "votes", "type": "simple" } + ], + "name": "value1", + "type": "struct" + } + } + }); + expect(toFields(st4)).toStrictEqual([ + { "name": "key0", "type": "address" }, + { "name": "key1", "type": "uint" }, + { "name": "votes", "type": "uint96" }, + { "name": "fromBlock", "type": "uint32" } + ]); + + const vars5 = ` + mapping(address => mapping(uint => Checkpoint)) public checkpoint; + struct Checkpoint { + uint32 fromBlock; + SomeVotes votes; + } + struct SomeVotes { + uint96 votes; + } + `; + const st5 = toStructure(vars5, 'checkpoint'); + expect(st5).toStrictEqual({ + "key": "address", + "name": "checkpoint", + "type": "mapping", + "value": { + "key": "uint", + "name": "value0", + "type": "mapping", + "value": { + "fields": [ + { "kind": "uint32", "name": "fromBlock", "type": "simple" }, + { "fields": [{ "kind": "uint96", "name": "votes", "type": "simple" }], "name": "votes", "type": "struct" } + ], + "name": "value1", + "type": "struct" + } + } + }); + expect(toFields(st5)).toStrictEqual([ + { "name": "key0", "type": "address" }, + { "name": "key1", "type": "uint" }, + { "name": "votes", "type": "uint96" }, + { "name": "fromBlock", "type": "uint32" } + ]); + }); +}) \ No newline at end of file diff --git a/src/services/dataTypeParser.ts b/src/services/dataTypeParser.ts new file mode 100644 index 0000000..387cc9a --- /dev/null +++ b/src/services/dataTypeParser.ts @@ -0,0 +1,150 @@ +import { parse, SourceUnit, ContractDefinition, StateVariableDeclaration, StructDefinition, TypeName } from 'solidity-parser-diligence'; + +export const errUnknownVariable = new Error('unknown variable'); + +export type Type = 'simple' | 'array' | 'mapping' | 'struct'; + +export type BaseStructure = { + name: string; + type: Type; +} + +export type SimpleStructure = BaseStructure & { + type: 'simple'; + kind: string; +} + +export type ArrayStructure = BaseStructure & { + type: 'array'; + kind: Structure; +} + +export type MappingStructure = BaseStructure & { + type: 'mapping'; + key: string; + value: Structure; +} + +export type CustomStructure = BaseStructure & { + type: 'struct'; + fields: Structure[]; +} + +export type Structure = SimpleStructure | ArrayStructure | MappingStructure | CustomStructure; + +export type Field = { + name: string; + type: string; +} + +function parseStructure(name: string, typeName: TypeName, structs: StructDefinition[], level: number = 0): Structure { + switch (typeName.type) { + case 'ElementaryTypeName': + return { name, type: 'simple', kind: typeName.name } as SimpleStructure; + case 'ArrayTypeName': + return { + name, + type: 'array', + kind: parseStructure(`value${level}`, typeName.baseTypeName, structs, level + 1), + } as ArrayStructure; + case 'Mapping': + return { + name, + type: 'mapping', + key: typeName.keyType.name, + value: parseStructure(`value${level}`, typeName.valueType, structs, level + 1), + } as MappingStructure; + case 'UserDefinedTypeName': + const members = structs.find(s => s.name == typeName.namePath)?.members; + if (!members) { + return null; + } + return { + name, + type: 'struct', + fields: members.map(m => parseStructure(m.name, m.typeName, structs, level + 1)) + } as CustomStructure + } +} + +/** + * Parse structure from list of variables and types + * ```typescript + * toStructure(` + * mapping(address => mapping(uint => Checkpoint)) public checkpoint; + * struct Checkpoint { + * uint32 fromBlock; + * SomeVotes votes; + * } + * struct SomeVotes { + * uint96 votes; + * } + * `,'checkpoint'); + * ``` + * @param vars list variables and types + * @param name variable name + */ +export function toStructure(vars: string, name: string): Structure { + const source = parse(`contract wrapper{ ${vars} }`, {}) as SourceUnit; + const anodes = (source.children[0] as ContractDefinition).subNodes; + + const states = anodes.filter(n => n.type == 'StateVariableDeclaration') as StateVariableDeclaration[]; + const structs = anodes.filter(n => n.type == 'StructDefinition') as StructDefinition[]; + + const variable = states.find(s => s?.variables?.some(v => v.name == name))?.variables[0]; + if (!variable) { + throw errUnknownVariable; + } + + return parseStructure(name, variable.typeName, structs); +} + +/** + * Сonverts structure to fields + * ```typescript + * const structure: SimpleStructure = { + * kind: 'string' + * name: 'someName' + * type: 'simple' + * } + * toFields(structure) + * ``` + * @param obj + */ +export function toFields(obj: Structure): Field[] { + const stack: Structure[] = [obj]; + const fields: Field[] = []; + let level: number = 0; + while (stack.length > 0) { + const obj = stack.pop(); + switch (obj.type) { + case 'simple': + fields.push({ + name: obj.name, + type: obj.kind, + }); + break; + case 'array': + fields.push({ + name: `key${level}`, + type: 'uint', + }); + stack.push(obj.kind) + break; + case 'mapping': + fields.push({ + name: `key${level}`, + type: obj.key, + }); + stack.push(obj.value); + break; + case 'struct': + for (const field of obj.fields) { + stack.push(field); + } + break; + } + level++; + } + return fields; +} \ No newline at end of file diff --git a/src/services/graphqlService.ts b/src/services/graphqlService.ts index cb13417..84eaef6 100644 --- a/src/services/graphqlService.ts +++ b/src/services/graphqlService.ts @@ -4,11 +4,18 @@ import DataService from './dataService'; export default class GraphqlService { // eslint-disable-next-line @typescript-eslint/no-explicit-any - public async ethHeaderCidByBlockNumber(blockNumber: string | number): Promise { + public async ethHeaderCidWithTransactionByBlockNumber(blockNumber: string | number): Promise { const graphqlRepository: GraphqlRepository = GraphqlRepository.getRepository(); - return graphqlRepository.ethHeaderCidByBlockNumber(blockNumber); + return graphqlRepository.ethHeaderCidWithTransactionByBlockNumber(blockNumber); } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public async ethHeaderCidWithStateByBlockNumber(blockNumber: string | number): Promise { + const graphqlRepository: GraphqlRepository = GraphqlRepository.getRepository(); + return graphqlRepository.ethHeaderCidWithStateByBlockNumber(blockNumber); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any public async ethHeaderCidById(headerId: number): Promise { const graphqlRepository: GraphqlRepository = GraphqlRepository.getRepository(); @@ -29,4 +36,11 @@ export default class GraphqlService { return graphqlRepository.subscriptionHeaderCids((data) => dataService.processHeader(data?.data?.listen?.relatedNode)) } + public async subscriptionStateCids(): Promise { + const dataService = new DataService(); + + const graphqlRepository: GraphqlRepository = GraphqlRepository.getRepository(); + return graphqlRepository.subscriptionStateCids((data) => dataService.processState(data?.data?.listen?.relatedNode)) + } + } diff --git a/src/store.ts b/src/store.ts index d14fb0d..84e1268 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,6 +1,8 @@ import Contract from "./models/contract/contract"; import Event from "./models/contract/event"; import Method from "./models/contract/method"; +import State from "./models/contract/state"; +import Address from "./models/data/address"; import ContractService from "./services/contractService"; import DataService from "./services/dataService"; import env from './env'; @@ -11,6 +13,8 @@ export default class Store { private contracts: Contract[]; private events: Event[]; private methods: Method[]; + private states: State[]; + private addresses: Address[]; // TODO: do not store addresses in memory private contractService: ContractService; private dataService: DataService; @@ -22,6 +26,8 @@ export default class Store { this.contracts = []; this.events = []; this.methods = []; + this.states = []; + this.addresses = []; } public static getStore(): Store { @@ -45,6 +51,16 @@ export default class Store { return this.contracts; } + // TODO: move to repository and make sql request + public getContractByAddressHash(hash: string): Contract { + const address = this.getAddressByHash(hash); + if (!address) { + return null; + } + + return (this.contracts || []).find((contract) => contract.address === address.address); + } + public getEvents(): Event[] { return this.events; } @@ -62,16 +78,56 @@ export default class Store { return this.methods; } + public getStates(): State[] { + return this.states; + } + + public getStatesByContractId(contractId: number): State[] { + const contract = (this.contracts || []).find((contract) => contract.contractId === contractId); + if (!contract || !contract.states) { + return []; + } + + return (this.states || []).filter((state) => contract.states.includes(state.stateId)); + } + + public getAddresses(): Address[] { + return this.addresses; + } + + public getAddressById(addressId: number): Address { + return (this.addresses || []).find((a) => a.addressId === addressId); + } + + public getAddress(addressString: string): Address { + return (this.addresses || []).find((a) => a.address === addressString); + } + + public getAddressByHash(hash: string): Address { + return (this.addresses || []).find((a) => a.hash === hash); + } + + public addAddress(address: Address): void { + this.addresses.push(address); + } + public async syncData(): Promise { - [this.contracts, this.events, this.methods] = await Promise.all([ - this.contractService.loadContracts(), - this.contractService.loadEvents(), - this.contractService.loadMethods() + [this.contracts, this.events, this.methods, this.states, this.addresses] = await Promise.all([ + this.contractService?.loadContracts(), + this.contractService?.loadEvents(), + this.contractService?.loadMethods(), + this.contractService?.loadStates(), + this.contractService?.loadAddresses(), ]) await this.dataService.createTables(this.contracts); + await this.dataService.prepareAddresses(this.contracts); - console.log(`Loaded ${this.contracts.length} contracts config and ${this.events.length} events and ${this.methods.length} methods`); + console.log(`Contracts: \t${this.contracts.length}`); + console.log(`Events: \t${this.events.length}`); + console.log(`Methods: \t${this.methods.length}`); + console.log(`States: \t${this.states.length}`); + console.log(`Addresses: \t${this.addresses.length}`); } } \ No newline at end of file diff --git a/src/syncDaemon.ts b/src/syncDaemon.ts index ef128ba..9a2ecef 100644 --- a/src/syncDaemon.ts +++ b/src/syncDaemon.ts @@ -4,9 +4,11 @@ dotenv.config(); import * as cron from 'node-cron'; import {createConnection, getConnection, getConnectionOptions} from 'typeorm'; import ProgressRepository from './repositories/data/progressRepository'; +import StateProgressRepository from './repositories/data/stateProgressRepository'; import HeaderCidsRepository from './repositories/eth/headerCidsRepository'; import Contract from './models/contract/contract'; import Event from './models/contract/event'; +import State from './models/contract/state'; import Store from './store'; import DataService from './services/dataService'; import GraphqlService from './services/graphqlService'; @@ -73,5 +75,37 @@ console.log('Cron daemon is started'); statusHeaderSync = 'waiting'; }); } + + if (env.ENABLE_STORAGE_WATCHER) { + let statusEventSync = 'waiting'; + cron.schedule('0 * * * * *', async () => { // every minute + if (statusEventSync !== 'waiting') { + console.log('Cron already running'); + return; + } + + statusEventSync = 'running'; + + // start Store without autoupdate data + const store = Store.getStore(); + await store.syncData(); + + const contracts: Contract[] = store.getContracts(); + + console.log('Contracts', contracts.length); + + const stateProgressRepository: StateProgressRepository = getConnection().getCustomRepository(StateProgressRepository); + for (const contract of contracts) { + const states: State[] = store.getStatesByContractId(contract.contractId); + for (const state of states) { + console.log('Contract', contract.contractId, 'Slot', state.slot); + + await DataService.syncStatesForContract({ graphqlService, dataService, stateProgressRepository }, state, contract); + } + } + + statusEventSync = 'waiting'; + }); + } }); })();