From 6b8c73d1a0562fdc38848da718992468da2ab1df Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Fri, 11 Feb 2022 17:41:47 +0530 Subject: [PATCH] Add indexing of token properties in erc20-watcher and fix smoke-tests (#4) * Add indexing of token properties in erc20-watcher * Fix uni-info-watcher smoke-test --- packages/erc20-watcher/src/database.ts | 47 +++++++++++ .../erc20-watcher/src/entity/Allowance.ts | 2 +- packages/erc20-watcher/src/entity/Balance.ts | 2 +- packages/erc20-watcher/src/entity/Decimals.ts | 27 ++++++ packages/erc20-watcher/src/entity/Name.ts | 27 ++++++ packages/erc20-watcher/src/entity/Symbol.ts | 27 ++++++ packages/erc20-watcher/src/indexer.ts | 45 +++++++++- packages/uni-info-watcher/package.json | 3 +- packages/uni-info-watcher/src/job-runner.ts | 4 +- packages/uni-info-watcher/src/smoke.test.ts | 69 ++++++++------- packages/uni-info-watcher/test/init.ts | 83 +++++++++++++++++++ 11 files changed, 301 insertions(+), 35 deletions(-) create mode 100644 packages/erc20-watcher/src/entity/Decimals.ts create mode 100644 packages/erc20-watcher/src/entity/Name.ts create mode 100644 packages/erc20-watcher/src/entity/Symbol.ts create mode 100644 packages/uni-info-watcher/test/init.ts diff --git a/packages/erc20-watcher/src/database.ts b/packages/erc20-watcher/src/database.ts index 87b5a0d0..ff503cda 100644 --- a/packages/erc20-watcher/src/database.ts +++ b/packages/erc20-watcher/src/database.ts @@ -10,6 +10,9 @@ import { Database as BaseDatabase, QueryOptions, Where } from '@vulcanize/util'; import { Allowance } from './entity/Allowance'; import { Balance } from './entity/Balance'; +import { Name } from './entity/Name'; +import { Symbol } from './entity/Symbol'; +import { Decimals } from './entity/Decimals'; import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; @@ -62,6 +65,31 @@ export class Database { .getOne(); } + async getName ({ blockHash, token }: { blockHash: string, token: string }): Promise { + return this._conn.getRepository(Name) + .findOne({ + blockHash, + token + }); + } + + // eslint-disable-next-line @typescript-eslint/ban-types + async getSymbol ({ blockHash, token }: { blockHash: string, token: string }): Promise { + return this._conn.getRepository(Symbol) + .findOne({ + blockHash, + token + }); + } + + async getDecimals ({ blockHash, token }: { blockHash: string, token: string }): Promise { + return this._conn.getRepository(Decimals) + .findOne({ + blockHash, + token + }); + } + async saveBalance ({ blockHash, blockNumber, token, owner, value, proof }: DeepPartial): Promise { const repo = this._conn.getRepository(Balance); const entity = repo.create({ blockHash, blockNumber, token, owner, value, proof }); @@ -74,6 +102,25 @@ export class Database { return repo.save(entity); } + async saveName ({ blockHash, blockNumber, token, value, proof }: DeepPartial): Promise { + const repo = this._conn.getRepository(Name); + const entity = repo.create({ blockHash, blockNumber, token, value, proof }); + return repo.save(entity); + } + + // eslint-disable-next-line @typescript-eslint/ban-types + async saveSymbol ({ blockHash, blockNumber, token, value, proof }: DeepPartial): Promise { + const repo = this._conn.getRepository(Symbol); + const entity = repo.create({ blockHash, blockNumber, token, value, proof }); + return repo.save(entity); + } + + async saveDecimals ({ blockHash, blockNumber, token, value, proof }: DeepPartial): Promise { + const repo = this._conn.getRepository(Decimals); + const entity = repo.create({ blockHash, blockNumber, token, value, proof }); + return repo.save(entity); + } + async getContracts (): Promise { const repo = this._conn.getRepository(Contract); diff --git a/packages/erc20-watcher/src/entity/Allowance.ts b/packages/erc20-watcher/src/entity/Allowance.ts index 76d9e92b..1f60b087 100644 --- a/packages/erc20-watcher/src/entity/Allowance.ts +++ b/packages/erc20-watcher/src/entity/Allowance.ts @@ -6,7 +6,7 @@ import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() -@Index(['blockHash', 'blockNumber', 'token', 'owner', 'spender'], { unique: true }) +@Index(['blockHash', 'token', 'owner', 'spender'], { unique: true }) export class Allowance { @PrimaryGeneratedColumn() id!: number; diff --git a/packages/erc20-watcher/src/entity/Balance.ts b/packages/erc20-watcher/src/entity/Balance.ts index 5d8d8588..c114fd40 100644 --- a/packages/erc20-watcher/src/entity/Balance.ts +++ b/packages/erc20-watcher/src/entity/Balance.ts @@ -6,7 +6,7 @@ import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() -@Index(['blockHash', 'blockNumber', 'token', 'owner'], { unique: true }) +@Index(['blockHash', 'token', 'owner'], { unique: true }) export class Balance { @PrimaryGeneratedColumn() id!: number; diff --git a/packages/erc20-watcher/src/entity/Decimals.ts b/packages/erc20-watcher/src/entity/Decimals.ts new file mode 100644 index 00000000..5016c362 --- /dev/null +++ b/packages/erc20-watcher/src/entity/Decimals.ts @@ -0,0 +1,27 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; + +@Entity() +@Index(['blockHash', 'token'], { unique: true }) +export class Decimals { + @PrimaryGeneratedColumn() + id!: number; + + @Column('varchar', { length: 66 }) + blockHash!: string; + + @Column('integer') + blockNumber!: number; + + @Column('varchar', { length: 42 }) + token!: string; + + @Column('integer') + value!: number; + + @Column('text', { nullable: true }) + proof!: string; +} diff --git a/packages/erc20-watcher/src/entity/Name.ts b/packages/erc20-watcher/src/entity/Name.ts new file mode 100644 index 00000000..4baefe79 --- /dev/null +++ b/packages/erc20-watcher/src/entity/Name.ts @@ -0,0 +1,27 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; + +@Entity() +@Index(['blockHash', 'token'], { unique: true }) +export class Name { + @PrimaryGeneratedColumn() + id!: number; + + @Column('varchar', { length: 66 }) + blockHash!: string; + + @Column('integer') + blockNumber!: number; + + @Column('varchar', { length: 42 }) + token!: string; + + @Column('varchar') + value!: string; + + @Column('text', { nullable: true }) + proof!: string; +} diff --git a/packages/erc20-watcher/src/entity/Symbol.ts b/packages/erc20-watcher/src/entity/Symbol.ts new file mode 100644 index 00000000..038ae2ce --- /dev/null +++ b/packages/erc20-watcher/src/entity/Symbol.ts @@ -0,0 +1,27 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; + +@Entity() +@Index(['blockHash', 'token'], { unique: true }) +export class Symbol { + @PrimaryGeneratedColumn() + id!: number; + + @Column('varchar', { length: 66 }) + blockHash!: string; + + @Column('integer') + blockNumber!: number; + + @Column('varchar', { length: 42 }) + token!: string; + + @Column('varchar') + value!: string; + + @Column('text', { nullable: true }) + proof!: string; +} diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts index 358ff1b0..b4e97639 100644 --- a/packages/erc20-watcher/src/indexer.ts +++ b/packages/erc20-watcher/src/indexer.ts @@ -180,8 +180,21 @@ export class Indexer { } async name (blockHash: string, token: string): Promise { + const entity = await this._db.getName({ blockHash, token }); + if (entity) { + log('name: db hit.'); + + return { + value: entity.value, + proof: JSON.parse(entity.proof) + }; + } + + log('name: db miss, fetching from upstream server'); let result: ValueResult; + const { block: { number: blockNumber } } = await this._ethClient.getBlockByHash(blockHash); + if (this._serverMode === ETH_CALL_MODE) { const value = await fetchTokenName(this._ethProvider, blockHash, token); @@ -190,14 +203,27 @@ export class Indexer { result = await this._baseIndexer.getStorageValue(this._storageLayout, blockHash, token, '_name'); } - // log(JSONbig.stringify(result, null, 2)); + await this._db.saveName({ blockHash, blockNumber, token, value: result.value, proof: JSONbig.stringify(result.proof) }); return result; } async symbol (blockHash: string, token: string): Promise { + const entity = await this._db.getSymbol({ blockHash, token }); + if (entity) { + log('symbol: db hit.'); + + return { + value: entity.value, + proof: JSON.parse(entity.proof) + }; + } + + log('symbol: db miss, fetching from upstream server'); let result: ValueResult; + const { block: { number: blockNumber } } = await this._ethClient.getBlockByHash(blockHash); + if (this._serverMode === ETH_CALL_MODE) { const value = await fetchTokenSymbol(this._ethProvider, blockHash, token); @@ -206,14 +232,27 @@ export class Indexer { result = await this._baseIndexer.getStorageValue(this._storageLayout, blockHash, token, '_symbol'); } - // log(JSONbig.stringify(result, null, 2)); + await this._db.saveSymbol({ blockHash, blockNumber, token, value: result.value, proof: JSONbig.stringify(result.proof) }); return result; } async decimals (blockHash: string, token: string): Promise { + const entity = await this._db.getDecimals({ blockHash, token }); + if (entity) { + log('decimals: db hit.'); + + return { + value: entity.value, + proof: JSON.parse(entity.proof) + }; + } + + log('decimals: db miss, fetching from upstream server'); let result: ValueResult; + const { block: { number: blockNumber } } = await this._ethClient.getBlockByHash(blockHash); + if (this._serverMode === ETH_CALL_MODE) { const value = await fetchTokenDecimals(this._ethProvider, blockHash, token); @@ -224,6 +263,8 @@ export class Indexer { throw new Error('Not implemented.'); } + await this._db.saveDecimals({ blockHash, blockNumber, token, value: result.value, proof: JSONbig.stringify(result.proof) }); + return result; } diff --git a/packages/uni-info-watcher/package.json b/packages/uni-info-watcher/package.json index 80d34a62..89c2a0a2 100644 --- a/packages/uni-info-watcher/package.json +++ b/packages/uni-info-watcher/package.json @@ -28,6 +28,7 @@ "test": "mocha src/**/*.test.ts", "test:gpev": "mocha src/get-prev-entity.test.ts", "test:server": "mocha src/server.test.ts", + "test:init": "ts-node test/init.ts", "build": "tsc", "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js", "server:prof": "DEBUG=vulcanize:* node --require pprof --enable-source-maps dist/server.js", @@ -36,7 +37,7 @@ "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js", "job-runner:prof": "DEBUG=vulcanize:* node --require pprof --enable-source-maps dist/job-runner.js", "job-runner:dev": "DEBUG=vulcanize:* nodemon --watch src src/job-runner.ts", - "smoke-test": "mocha src/smoke.test.ts", + "smoke-test": "yarn test:init && mocha src/smoke.test.ts", "fill": "DEBUG=vulcanize:* node --enable-source-maps dist/fill.js", "fill:prof": "DEBUG=vulcanize:* node --require pprof --enable-source-maps dist/fill.js", "fill:dev": "DEBUG=vulcanize:* ts-node src/fill.ts", diff --git a/packages/uni-info-watcher/src/job-runner.ts b/packages/uni-info-watcher/src/job-runner.ts index d3162b2f..08dbaf45 100644 --- a/packages/uni-info-watcher/src/job-runner.ts +++ b/packages/uni-info-watcher/src/job-runner.ts @@ -134,7 +134,9 @@ export const main = async (): Promise => { const indexer = new Indexer(db, uniClient, erc20Client, ethClient, postgraphileClient, ethProvider, jobQueue, mode); await indexer.init(); - await indexer.addContracts(); + if (mode !== 'demo') { + await indexer.addContracts(); + } const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue); await jobRunner.start(); diff --git a/packages/uni-info-watcher/src/smoke.test.ts b/packages/uni-info-watcher/src/smoke.test.ts index 2ab0d3bc..cf27089c 100644 --- a/packages/uni-info-watcher/src/smoke.test.ts +++ b/packages/uni-info-watcher/src/smoke.test.ts @@ -53,6 +53,7 @@ describe('uni-info-watcher', () => { let token1Address: string; let nfpm: Contract; + let nfpmTokenId: number; let tickLower: number; let tickUpper: number; let signer: Signer; @@ -85,35 +86,29 @@ describe('uni-info-watcher', () => { signer = provider.getSigner(); recipient = await signer.getAddress(); - // Deadline set to 2 days from current date. - const deadlineDate = new Date(); - deadlineDate.setDate(deadlineDate.getDate() + 2); - deadline = Math.floor(deadlineDate.getTime() / 1000); - }); + // Get the factory contract address. + const factoryContract = await uniClient.getContract('factory'); + expect(factoryContract).to.not.be.empty; - it('should have a Factory entity', async () => { - // Getting the Factory from uni-info-watcher graphQL endpoint. - const factories = await client.getFactories(1); - expect(factories).to.not.be.empty; + // Initialize the factory contract. + factory = new Contract(factoryContract.address, FACTORY_ABI, signer); - // Initializing the factory variable. - const factoryAddress = factories[0].id; - factory = new ethers.Contract(factoryAddress, FACTORY_ABI, signer); - expect(factory.address).to.not.be.empty; - }); + // Get the NFPM contract address. + const nfpmContract = await uniClient.getContract('nfpm'); + expect(nfpmContract).to.not.be.empty; - it('should have a Bundle entity', async () => { - // Getting the Bundle from uni-info-watcher graphQL endpoint. - const bundles = await client.getBundles(1); - expect(bundles).to.not.be.empty; + // Initialize the NFPM contract. + nfpm = new Contract(nfpmContract.address, NFPM_ABI, signer); - const bundleId = '1'; - expect(bundles[0].id).to.equal(bundleId); + // Deadline set to 2 days from current date. + const deadlineDate = new Date(); + deadlineDate.setDate(deadlineDate.getDate() + 2); + deadline = Math.floor(deadlineDate.getTime() / 1000); }); describe('PoolCreatedEvent', () => { // NOTE: Skipping checking entity updates that cannot be gotten/derived using queries. - // Checked entities: Token, Pool. + // Checked entities: Token, Factory, Bundle, Pool. const fee = 500; @@ -158,6 +153,21 @@ describe('uni-info-watcher', () => { expect(token1).to.not.be.null; }); + it('should create a Factory entity', async () => { + // Check that a Factory entity is present. + const factories = await client.getFactories(1); + expect(factories).to.not.be.empty; + }); + + it('should create a Bundle entity', async () => { + // Check that a Bundle entity is present. + const bundles = await client.getBundles(1); + expect(bundles).to.not.be.empty; + + const bundleId = '1'; + expect(bundles[0].id).to.equal(bundleId); + }); + it('should create a Pool entity', async () => { // Checked values: feeTier @@ -783,6 +793,9 @@ describe('uni-info-watcher', () => { transaction ]); + // Store tokenId for further usage. + nfpmTokenId = Number(eventValue.event.tokenId); + // Sleeping for 15 sec for the events to be processed. await wait(15000); }); @@ -833,7 +846,6 @@ describe('uni-info-watcher', () => { let oldPosition: any; let eventValue: any; - const tokenId = 1; const amount0Desired = 15; const amount1Desired = 15; const amount0Min = 0; @@ -841,14 +853,15 @@ describe('uni-info-watcher', () => { before(async () => { // Get initial entity values. - const positions = await client.getPositions({ id: Number(tokenId) }, 1); + const positions = await client.getPositions({ id: nfpmTokenId }, 1); + expect(positions).to.not.be.empty; oldPosition = positions[0]; }); it('should trigger IncreaseLiquidityEvent', async () => { // Position manger increase liquidity and wait for MintEvent. const transaction = nfpm.increaseLiquidity({ - tokenId, + tokenId: nfpmTokenId, amount0Desired, amount1Desired, amount0Min, @@ -905,21 +918,20 @@ describe('uni-info-watcher', () => { let oldPosition: any; let eventValue: any; - const tokenId = 1; const liquidity = 5; const amount0Min = 0; const amount1Min = 0; before(async () => { // Get initial entity values. - const positions = await client.getPositions({ id: Number(tokenId) }, 1); + const positions = await client.getPositions({ id: nfpmTokenId }, 1); oldPosition = positions[0]; }); it('should trigger DecreaseLiquidityEvent', async () => { // Position manger decrease liquidity and wait for BurnEvent. const transaction = nfpm.decreaseLiquidity({ - tokenId, + tokenId: nfpmTokenId, liquidity, amount0Min, amount1Min, @@ -973,14 +985,13 @@ describe('uni-info-watcher', () => { // Checked entities: Transaction. // Unchecked entities: Position. - const tokenId = 1; const amount0Max = 15; const amount1Max = 15; it('should trigger CollectEvent', async () => { // Position manger collect and wait for BurnEvent. const transaction = nfpm.collect({ - tokenId, + tokenId: nfpmTokenId, recipient, amount0Max, amount1Max diff --git a/packages/uni-info-watcher/test/init.ts b/packages/uni-info-watcher/test/init.ts new file mode 100644 index 00000000..72d77ce6 --- /dev/null +++ b/packages/uni-info-watcher/test/init.ts @@ -0,0 +1,83 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import assert from 'assert'; + +import { + getConfig, + getResetConfig, + JobQueue +} from '@vulcanize/util'; +import { Client as ERC20Client } from '@vulcanize/erc20-watcher'; +import { Client as UniClient } from '@vulcanize/uni-watcher'; + +import { Database } from '../src/database'; +import { Indexer } from '../src/indexer'; + +const CONFIG_FILE = './environments/test.toml'; + +const watchContract = async (indexer: Indexer, address: string, kind: string): Promise => { + const watchedContract = indexer.isWatchedContract(address); + + if (!watchedContract) { + await indexer.watchContract(address, kind, 100); + } +}; + +const main = async () => { + // Get config. + const config = await getConfig(CONFIG_FILE); + + const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: { mode } } = config; + + assert(dbConfig, 'Missing dbConfig.'); + // Initialize database. + const db = new Database(dbConfig); + await db.init(); + + // Initialize uniClient. + assert(upstream, 'Missing upstream config'); + const { uniWatcher: { gqlEndpoint, gqlSubscriptionEndpoint }, tokenWatcher } = upstream; + const uniClient = new UniClient({ + gqlEndpoint, + gqlSubscriptionEndpoint + }); + + const erc20Client = new ERC20Client(tokenWatcher); + + const { ethClient, postgraphileClient, ethProvider } = await getResetConfig(config); + + assert(jobQueueConfig, 'Missing job queue config'); + const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; + assert(dbConnectionString, 'Missing job queue db connection string'); + + const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); + await jobQueue.start(); + + const indexer = new Indexer(db, uniClient, erc20Client, ethClient, postgraphileClient, ethProvider, jobQueue, mode); + await indexer.init(); + + // Get the factory contract address. + const factoryContract = await uniClient.getContract('factory'); + assert(factoryContract !== null, 'Factory contract not available'); + + // Get the NFPM contract address. + const nfpmContract = await uniClient.getContract('nfpm'); + assert(nfpmContract !== null, 'NFPM contract not available'); + + // Watch factory contract. + await watchContract(indexer, factoryContract.address, 'factory'); + // Watch NFPM contract. + await watchContract(indexer, nfpmContract.address, 'nfpm'); + + // Closing the database. + await db.close(); +}; + +main() + .then(() => process.exit(0)) + .catch(error => { + console.error(error); + process.exit(1); + });