diff --git a/README.md b/README.md index df1e7c7d..84c7a210 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Install packages (Node.JS v16.13.1): ```bash yarn + +yarn build ``` ### Services @@ -15,9 +17,26 @@ yarn The default config files used by the watchers assume the following services are setup and running on localhost: * `vulcanize/go-ethereum` on port 8545 -* `vulcanize/ipld-eth-server` with native GQL API enabled, on port 8082 +* `vulcanize/ipld-eth-server` with native GQL API enabled on port 8082 and RPC API on port 8081 * `postgraphile` on the `vulcanize/ipld-eth-server` database, on port 5000 +To check whether the endpoints in watcher config are working, run: + +```bash +cd packages/util + +yarn check-config --config-file ../erc20-watcher/environments/local.toml + +# Check config file in other watcher. +yarn check-config --config-file ../uni-watcher/environments/local.toml +# vulcanize:check-config Checking ipld-eth-server GQL endpoint http://127.0.0.1:8082/graphql +0ms +# vulcanize:check-config ipld-eth-server GQL endpoint working +33ms +# vulcanize:check-config Checking postgraphile GQL endpoint http://127.0.0.1:5000/graphql +1ms +# vulcanize:check-config postgraphile GQL endpoint working +12ms +# vulcanize:check-config Checking RPC endpoint http://127.0.0.1:8081 +1ms +# vulcanize:check-config RPC endpoint working +25ms +``` + #### Note * In `vulcanize/ipld-eth-server`, add the following statement to `[ethereum]` section in `environments/config.toml`: diff --git a/packages/erc20-watcher/src/database.ts b/packages/erc20-watcher/src/database.ts index 87b5a0d0..ff503cda 100644 --- a/packages/erc20-watcher/src/database.ts +++ b/packages/erc20-watcher/src/database.ts @@ -10,6 +10,9 @@ import { Database as BaseDatabase, QueryOptions, Where } from '@vulcanize/util'; import { Allowance } from './entity/Allowance'; import { Balance } from './entity/Balance'; +import { Name } from './entity/Name'; +import { Symbol } from './entity/Symbol'; +import { Decimals } from './entity/Decimals'; import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; @@ -62,6 +65,31 @@ export class Database { .getOne(); } + async getName ({ blockHash, token }: { blockHash: string, token: string }): Promise { + return this._conn.getRepository(Name) + .findOne({ + blockHash, + token + }); + } + + // eslint-disable-next-line @typescript-eslint/ban-types + async getSymbol ({ blockHash, token }: { blockHash: string, token: string }): Promise { + return this._conn.getRepository(Symbol) + .findOne({ + blockHash, + token + }); + } + + async getDecimals ({ blockHash, token }: { blockHash: string, token: string }): Promise { + return this._conn.getRepository(Decimals) + .findOne({ + blockHash, + token + }); + } + async saveBalance ({ blockHash, blockNumber, token, owner, value, proof }: DeepPartial): Promise { const repo = this._conn.getRepository(Balance); const entity = repo.create({ blockHash, blockNumber, token, owner, value, proof }); @@ -74,6 +102,25 @@ export class Database { return repo.save(entity); } + async saveName ({ blockHash, blockNumber, token, value, proof }: DeepPartial): Promise { + const repo = this._conn.getRepository(Name); + const entity = repo.create({ blockHash, blockNumber, token, value, proof }); + return repo.save(entity); + } + + // eslint-disable-next-line @typescript-eslint/ban-types + async saveSymbol ({ blockHash, blockNumber, token, value, proof }: DeepPartial): Promise { + const repo = this._conn.getRepository(Symbol); + const entity = repo.create({ blockHash, blockNumber, token, value, proof }); + return repo.save(entity); + } + + async saveDecimals ({ blockHash, blockNumber, token, value, proof }: DeepPartial): Promise { + const repo = this._conn.getRepository(Decimals); + const entity = repo.create({ blockHash, blockNumber, token, value, proof }); + return repo.save(entity); + } + async getContracts (): Promise { const repo = this._conn.getRepository(Contract); diff --git a/packages/erc20-watcher/src/entity/Allowance.ts b/packages/erc20-watcher/src/entity/Allowance.ts index 76d9e92b..1f60b087 100644 --- a/packages/erc20-watcher/src/entity/Allowance.ts +++ b/packages/erc20-watcher/src/entity/Allowance.ts @@ -6,7 +6,7 @@ import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() -@Index(['blockHash', 'blockNumber', 'token', 'owner', 'spender'], { unique: true }) +@Index(['blockHash', 'token', 'owner', 'spender'], { unique: true }) export class Allowance { @PrimaryGeneratedColumn() id!: number; diff --git a/packages/erc20-watcher/src/entity/Balance.ts b/packages/erc20-watcher/src/entity/Balance.ts index 5d8d8588..c114fd40 100644 --- a/packages/erc20-watcher/src/entity/Balance.ts +++ b/packages/erc20-watcher/src/entity/Balance.ts @@ -6,7 +6,7 @@ import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() -@Index(['blockHash', 'blockNumber', 'token', 'owner'], { unique: true }) +@Index(['blockHash', 'token', 'owner'], { unique: true }) export class Balance { @PrimaryGeneratedColumn() id!: number; diff --git a/packages/erc20-watcher/src/entity/Decimals.ts b/packages/erc20-watcher/src/entity/Decimals.ts new file mode 100644 index 00000000..5016c362 --- /dev/null +++ b/packages/erc20-watcher/src/entity/Decimals.ts @@ -0,0 +1,27 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; + +@Entity() +@Index(['blockHash', 'token'], { unique: true }) +export class Decimals { + @PrimaryGeneratedColumn() + id!: number; + + @Column('varchar', { length: 66 }) + blockHash!: string; + + @Column('integer') + blockNumber!: number; + + @Column('varchar', { length: 42 }) + token!: string; + + @Column('integer') + value!: number; + + @Column('text', { nullable: true }) + proof!: string; +} diff --git a/packages/erc20-watcher/src/entity/Name.ts b/packages/erc20-watcher/src/entity/Name.ts new file mode 100644 index 00000000..4baefe79 --- /dev/null +++ b/packages/erc20-watcher/src/entity/Name.ts @@ -0,0 +1,27 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; + +@Entity() +@Index(['blockHash', 'token'], { unique: true }) +export class Name { + @PrimaryGeneratedColumn() + id!: number; + + @Column('varchar', { length: 66 }) + blockHash!: string; + + @Column('integer') + blockNumber!: number; + + @Column('varchar', { length: 42 }) + token!: string; + + @Column('varchar') + value!: string; + + @Column('text', { nullable: true }) + proof!: string; +} diff --git a/packages/erc20-watcher/src/entity/Symbol.ts b/packages/erc20-watcher/src/entity/Symbol.ts new file mode 100644 index 00000000..038ae2ce --- /dev/null +++ b/packages/erc20-watcher/src/entity/Symbol.ts @@ -0,0 +1,27 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; + +@Entity() +@Index(['blockHash', 'token'], { unique: true }) +export class Symbol { + @PrimaryGeneratedColumn() + id!: number; + + @Column('varchar', { length: 66 }) + blockHash!: string; + + @Column('integer') + blockNumber!: number; + + @Column('varchar', { length: 42 }) + token!: string; + + @Column('varchar') + value!: string; + + @Column('text', { nullable: true }) + proof!: string; +} diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts index 358ff1b0..b4e97639 100644 --- a/packages/erc20-watcher/src/indexer.ts +++ b/packages/erc20-watcher/src/indexer.ts @@ -180,8 +180,21 @@ export class Indexer { } async name (blockHash: string, token: string): Promise { + const entity = await this._db.getName({ blockHash, token }); + if (entity) { + log('name: db hit.'); + + return { + value: entity.value, + proof: JSON.parse(entity.proof) + }; + } + + log('name: db miss, fetching from upstream server'); let result: ValueResult; + const { block: { number: blockNumber } } = await this._ethClient.getBlockByHash(blockHash); + if (this._serverMode === ETH_CALL_MODE) { const value = await fetchTokenName(this._ethProvider, blockHash, token); @@ -190,14 +203,27 @@ export class Indexer { result = await this._baseIndexer.getStorageValue(this._storageLayout, blockHash, token, '_name'); } - // log(JSONbig.stringify(result, null, 2)); + await this._db.saveName({ blockHash, blockNumber, token, value: result.value, proof: JSONbig.stringify(result.proof) }); return result; } async symbol (blockHash: string, token: string): Promise { + const entity = await this._db.getSymbol({ blockHash, token }); + if (entity) { + log('symbol: db hit.'); + + return { + value: entity.value, + proof: JSON.parse(entity.proof) + }; + } + + log('symbol: db miss, fetching from upstream server'); let result: ValueResult; + const { block: { number: blockNumber } } = await this._ethClient.getBlockByHash(blockHash); + if (this._serverMode === ETH_CALL_MODE) { const value = await fetchTokenSymbol(this._ethProvider, blockHash, token); @@ -206,14 +232,27 @@ export class Indexer { result = await this._baseIndexer.getStorageValue(this._storageLayout, blockHash, token, '_symbol'); } - // log(JSONbig.stringify(result, null, 2)); + await this._db.saveSymbol({ blockHash, blockNumber, token, value: result.value, proof: JSONbig.stringify(result.proof) }); return result; } async decimals (blockHash: string, token: string): Promise { + const entity = await this._db.getDecimals({ blockHash, token }); + if (entity) { + log('decimals: db hit.'); + + return { + value: entity.value, + proof: JSON.parse(entity.proof) + }; + } + + log('decimals: db miss, fetching from upstream server'); let result: ValueResult; + const { block: { number: blockNumber } } = await this._ethClient.getBlockByHash(blockHash); + if (this._serverMode === ETH_CALL_MODE) { const value = await fetchTokenDecimals(this._ethProvider, blockHash, token); @@ -224,6 +263,8 @@ export class Indexer { throw new Error('Not implemented.'); } + await this._db.saveDecimals({ blockHash, blockNumber, token, value: result.value, proof: JSONbig.stringify(result.proof) }); + return result; } diff --git a/packages/uni-info-watcher/package.json b/packages/uni-info-watcher/package.json index 80d34a62..89c2a0a2 100644 --- a/packages/uni-info-watcher/package.json +++ b/packages/uni-info-watcher/package.json @@ -28,6 +28,7 @@ "test": "mocha src/**/*.test.ts", "test:gpev": "mocha src/get-prev-entity.test.ts", "test:server": "mocha src/server.test.ts", + "test:init": "ts-node test/init.ts", "build": "tsc", "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js", "server:prof": "DEBUG=vulcanize:* node --require pprof --enable-source-maps dist/server.js", @@ -36,7 +37,7 @@ "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js", "job-runner:prof": "DEBUG=vulcanize:* node --require pprof --enable-source-maps dist/job-runner.js", "job-runner:dev": "DEBUG=vulcanize:* nodemon --watch src src/job-runner.ts", - "smoke-test": "mocha src/smoke.test.ts", + "smoke-test": "yarn test:init && mocha src/smoke.test.ts", "fill": "DEBUG=vulcanize:* node --enable-source-maps dist/fill.js", "fill:prof": "DEBUG=vulcanize:* node --require pprof --enable-source-maps dist/fill.js", "fill:dev": "DEBUG=vulcanize:* ts-node src/fill.ts", diff --git a/packages/uni-info-watcher/src/database.ts b/packages/uni-info-watcher/src/database.ts index 1f9ef33d..bb7fb666 100644 --- a/packages/uni-info-watcher/src/database.ts +++ b/packages/uni-info-watcher/src/database.ts @@ -123,7 +123,7 @@ export class Database implements DatabaseInterface { return entity; } - async getToken (queryRunner: QueryRunner, { id, blockHash, blockNumber }: DeepPartial): Promise { + async getToken (queryRunner: QueryRunner, { id, blockHash, blockNumber }: DeepPartial, loadRelations = false): Promise { const repo = queryRunner.manager.getRepository(Token); const whereOptions: FindConditions = { id }; @@ -139,8 +139,7 @@ export class Database implements DatabaseInterface { where: whereOptions, order: { blockNumber: 'DESC' - }, - relations: ['whitelistPools'] + } }; let entity = await repo.findOne(findOptions as FindOneOptions); @@ -149,7 +148,7 @@ export class Database implements DatabaseInterface { entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); } - if (entity) { + if (loadRelations && entity) { [entity] = await this._baseDatabase.loadRelations( queryRunner, { hash: blockHash, number: blockNumber }, @@ -157,8 +156,7 @@ export class Database implements DatabaseInterface { { entity: Pool, type: 'many-to-many', - property: 'whitelistPools', - field: 'pool' + field: 'whitelistPools' } ], [entity] @@ -182,7 +180,7 @@ export class Database implements DatabaseInterface { return res; } - async getPool (queryRunner: QueryRunner, { id, blockHash, blockNumber }: DeepPartial): Promise { + async getPool (queryRunner: QueryRunner, { id, blockHash, blockNumber }: DeepPartial, loadRelations = false): Promise { const repo = queryRunner.manager.getRepository(Pool); const whereOptions: FindConditions = { id }; @@ -207,7 +205,7 @@ export class Database implements DatabaseInterface { entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); } - if (entity) { + if (loadRelations && entity) { [entity] = await this._baseDatabase.loadRelations( queryRunner, { hash: blockHash, number: blockNumber }, @@ -215,14 +213,12 @@ export class Database implements DatabaseInterface { { entity: Token, type: 'one-to-one', - property: 'token0', - field: 'token0Id' + field: 'token0' }, { entity: Token, type: 'one-to-one', - property: 'token1', - field: 'token1Id' + field: 'token1' } ], [entity] @@ -246,7 +242,7 @@ export class Database implements DatabaseInterface { return res; } - async getPosition ({ id, blockHash }: DeepPartial): Promise { + async getPosition ({ id, blockHash }: DeepPartial, loadRelations = false): Promise { const queryRunner = this._conn.createQueryRunner(); let entity; @@ -272,7 +268,7 @@ export class Database implements DatabaseInterface { entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); } - if (entity) { + if (loadRelations && entity) { [entity] = await this._baseDatabase.loadRelations( queryRunner, { hash: blockHash }, @@ -280,38 +276,32 @@ export class Database implements DatabaseInterface { { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId' + field: 'pool' }, { entity: Token, type: 'one-to-one', - property: 'token0', - field: 'token0Id' + field: 'token0' }, { entity: Token, type: 'one-to-one', - property: 'token1', - field: 'token1Id' + field: 'token1' }, { entity: Tick, type: 'one-to-one', - property: 'tickLower', - field: 'tickLowerId' + field: 'tickLower' }, { entity: Tick, type: 'one-to-one', - property: 'tickUpper', - field: 'tickUpperId' + field: 'tickUpper' }, { entity: Transaction, type: 'one-to-one', - property: 'transaction', - field: 'transactionId' + field: 'transaction' } ], [entity] @@ -324,7 +314,7 @@ export class Database implements DatabaseInterface { return entity; } - async getTick (queryRunner: QueryRunner, { id, blockHash }: DeepPartial): Promise { + async getTick (queryRunner: QueryRunner, { id, blockHash }: DeepPartial, loadRelations = false): Promise { const repo = queryRunner.manager.getRepository(Tick); const whereOptions: FindConditions = { id }; @@ -345,7 +335,7 @@ export class Database implements DatabaseInterface { entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); } - if (entity) { + if (loadRelations && entity) { [entity] = await this._baseDatabase.loadRelations( queryRunner, { hash: blockHash }, @@ -353,8 +343,7 @@ export class Database implements DatabaseInterface { { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId' + field: 'pool' } ], [entity] @@ -378,7 +367,7 @@ export class Database implements DatabaseInterface { return res; } - async getPoolDayData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial): Promise { + async getPoolDayData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial, loadRelations = false): Promise { const repo = queryRunner.manager.getRepository(PoolDayData); const whereOptions: FindConditions = { id }; @@ -399,7 +388,7 @@ export class Database implements DatabaseInterface { entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); } - if (entity) { + if (loadRelations && entity) { [entity] = await this._baseDatabase.loadRelations( queryRunner, { hash: blockHash }, @@ -407,8 +396,7 @@ export class Database implements DatabaseInterface { { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId' + field: 'pool' } ], [entity] @@ -418,7 +406,7 @@ export class Database implements DatabaseInterface { return entity; } - async getPoolHourData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial): Promise { + async getPoolHourData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial, loadRelations = false): Promise { const repo = queryRunner.manager.getRepository(PoolHourData); const whereOptions: FindConditions = { id }; @@ -439,7 +427,7 @@ export class Database implements DatabaseInterface { entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); } - if (entity) { + if (loadRelations && entity) { [entity] = await this._baseDatabase.loadRelations( queryRunner, { hash: blockHash }, @@ -447,8 +435,7 @@ export class Database implements DatabaseInterface { { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId' + field: 'pool' } ], [entity] @@ -482,7 +469,7 @@ export class Database implements DatabaseInterface { return entity; } - async getTokenDayData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial): Promise { + async getTokenDayData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial, loadRelations = false): Promise { const repo = queryRunner.manager.getRepository(TokenDayData); const whereOptions: FindConditions = { id }; @@ -503,7 +490,7 @@ export class Database implements DatabaseInterface { entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); } - if (entity) { + if (loadRelations && entity) { [entity] = await this._baseDatabase.loadRelations( queryRunner, { hash: blockHash }, @@ -511,8 +498,7 @@ export class Database implements DatabaseInterface { { entity: Token, type: 'one-to-one', - property: 'token', - field: 'tokenId' + field: 'token' } ], [entity] @@ -522,7 +508,7 @@ export class Database implements DatabaseInterface { return entity; } - async getTokenHourData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial): Promise { + async getTokenHourData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial, loadRelations = false): Promise { const repo = queryRunner.manager.getRepository(TokenHourData); const whereOptions: FindConditions = { id }; @@ -543,7 +529,7 @@ export class Database implements DatabaseInterface { entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); } - if (entity) { + if (loadRelations && entity) { [entity] = await this._baseDatabase.loadRelations( queryRunner, { hash: blockHash }, @@ -551,8 +537,7 @@ export class Database implements DatabaseInterface { { entity: Token, type: 'one-to-one', - property: 'token', - field: 'tokenId' + field: 'token' } ], [entity] @@ -562,7 +547,7 @@ export class Database implements DatabaseInterface { return entity; } - async getTickDayData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial): Promise { + async getTickDayData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial, loadRelations = false): Promise { const repo = queryRunner.manager.getRepository(TickDayData); const whereOptions: FindConditions = { id }; @@ -583,7 +568,7 @@ export class Database implements DatabaseInterface { entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); } - if (entity) { + if (loadRelations && entity) { [entity] = await this._baseDatabase.loadRelations( queryRunner, { hash: blockHash }, @@ -591,14 +576,12 @@ export class Database implements DatabaseInterface { { entity: Tick, type: 'one-to-one', - property: 'tick', - field: 'tickId' + field: 'tick' }, { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'PoolId' + field: 'pool' } ], [entity] diff --git a/packages/uni-info-watcher/src/entity/BlockProgress.ts b/packages/uni-info-watcher/src/entity/BlockProgress.ts index 90270bc2..ee13c6a1 100644 --- a/packages/uni-info-watcher/src/entity/BlockProgress.ts +++ b/packages/uni-info-watcher/src/entity/BlockProgress.ts @@ -10,6 +10,7 @@ import { BlockProgressInterface } from '@vulcanize/util'; @Index(['blockHash'], { unique: true }) @Index(['blockNumber']) @Index(['parentHash']) +@Index(['blockHash', 'isPruned']) export class BlockProgress implements BlockProgressInterface { @PrimaryGeneratedColumn() id!: number; diff --git a/packages/uni-info-watcher/src/entity/Burn.ts b/packages/uni-info-watcher/src/entity/Burn.ts index 95c10678..92375740 100644 --- a/packages/uni-info-watcher/src/entity/Burn.ts +++ b/packages/uni-info-watcher/src/entity/Burn.ts @@ -2,16 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer, graphDecimalTransformer, GraphDecimal } from '@vulcanize/util'; -import { Transaction } from './Transaction'; -import { Pool } from './Pool'; -import { Token } from './Token'; - @Entity() @Index(['id', 'blockNumber']) -@Index(['transactionId']) +@Index(['transaction']) export class Burn { @PrimaryColumn('varchar') id!: string; @@ -23,26 +19,20 @@ export class Burn { @Column('integer') blockNumber!: number; - @Column('varchar', { nullable: true }) - transactionId!: string; - - @ManyToOne(() => Transaction, transaction => transaction.burns, { onDelete: 'CASCADE' }) - transaction!: Transaction + @Column('varchar') + transaction!: string; @Column('numeric', { transformer: bigintTransformer }) timestamp!: bigint; - @Column('varchar', { length: 42, nullable: true }) - poolId!: string; - - @ManyToOne(() => Pool, { onDelete: 'CASCADE' }) - pool!: Pool + @Column('varchar', { length: 42 }) + pool!: string; - @ManyToOne(() => Token, { onDelete: 'CASCADE' }) - token0!: Token + @Column('varchar', { length: 42 }) + token0!: string - @ManyToOne(() => Token, { onDelete: 'CASCADE' }) - token1!: Token + @Column('varchar', { length: 42 }) + token1!: string @Column('varchar', { length: 42 }) owner!: string diff --git a/packages/uni-info-watcher/src/entity/Mint.ts b/packages/uni-info-watcher/src/entity/Mint.ts index b5532581..26201f3c 100644 --- a/packages/uni-info-watcher/src/entity/Mint.ts +++ b/packages/uni-info-watcher/src/entity/Mint.ts @@ -2,16 +2,14 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal, bigintTransformer } from '@vulcanize/util'; -import { Transaction } from './Transaction'; -import { Pool } from './Pool'; -import { Token } from './Token'; - @Entity() @Index(['id', 'blockNumber']) -@Index(['transactionId']) +@Index(['transaction']) +@Index(['blockHash', 'id', 'token0']) +@Index(['blockHash', 'id', 'token1']) export class Mint { @PrimaryColumn('varchar') id!: string; @@ -23,26 +21,20 @@ export class Mint { @Column('integer') blockNumber!: number; - @Column('varchar', { nullable: true }) - transactionId!: string; - - @ManyToOne(() => Transaction, transaction => transaction.mints, { onDelete: 'CASCADE' }) - transaction!: Transaction + @Column('varchar') + transaction!: string; @Column('numeric', { transformer: bigintTransformer }) timestamp!: bigint; - @Column('varchar', { length: 42, nullable: true }) - poolId!: string; - - @ManyToOne(() => Pool, { onDelete: 'CASCADE' }) - pool!: Pool + @Column('varchar', { length: 42 }) + pool!: string; - @ManyToOne(() => Token, { onDelete: 'CASCADE' }) - token0!: Token + @Column('varchar', { length: 42 }) + token0!: string - @ManyToOne(() => Token, { onDelete: 'CASCADE' }) - token1!: Token + @Column('varchar', { length: 42 }) + token1!: string @Column('varchar', { length: 42 }) owner!: string diff --git a/packages/uni-info-watcher/src/entity/Pool.ts b/packages/uni-info-watcher/src/entity/Pool.ts index 37c5cb8e..404e6dfa 100644 --- a/packages/uni-info-watcher/src/entity/Pool.ts +++ b/packages/uni-info-watcher/src/entity/Pool.ts @@ -2,11 +2,9 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal, bigintTransformer } from '@vulcanize/util'; -import { Token } from './Token'; - @Entity() @Index(['id', 'blockNumber']) export class Pool { @@ -20,17 +18,11 @@ export class Pool { @Column('integer') blockNumber!: number; - @Column('varchar', { length: 42, nullable: true, name: 'token0_id' }) - token0Id!: string; - - @ManyToOne(() => Token, { onDelete: 'CASCADE' }) - token0!: Token; - - @Column('varchar', { length: 42, nullable: true, name: 'token1_id' }) - token1Id!: string; + @Column('varchar', { length: 42 }) + token0!: string; - @ManyToOne(() => Token, { onDelete: 'CASCADE' }) - token1!: Token; + @Column('varchar', { length: 42 }) + token1!: string; @Column('numeric', { default: 0, transformer: graphDecimalTransformer }) token0Price!: GraphDecimal diff --git a/packages/uni-info-watcher/src/entity/PoolDayData.ts b/packages/uni-info-watcher/src/entity/PoolDayData.ts index 2d47e385..08492ac3 100644 --- a/packages/uni-info-watcher/src/entity/PoolDayData.ts +++ b/packages/uni-info-watcher/src/entity/PoolDayData.ts @@ -2,14 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal, bigintTransformer } from '@vulcanize/util'; -import { Pool } from './Pool'; - @Entity() @Index(['id', 'blockNumber']) -@Index(['date', 'poolId']) +@Index(['date', 'pool']) export class PoolDayData { @PrimaryColumn('varchar') id!: string; @@ -24,11 +22,8 @@ export class PoolDayData { @Column('integer') date!: number; - @Column('varchar', { length: 42, nullable: true }) - poolId!: string; - - @ManyToOne(() => Pool, { onDelete: 'CASCADE' }) - pool!: Pool; + @Column('varchar', { length: 42 }) + pool!: string; @Column('numeric', { transformer: graphDecimalTransformer }) high!: GraphDecimal; diff --git a/packages/uni-info-watcher/src/entity/PoolHourData.ts b/packages/uni-info-watcher/src/entity/PoolHourData.ts index d2065f49..e0988ab2 100644 --- a/packages/uni-info-watcher/src/entity/PoolHourData.ts +++ b/packages/uni-info-watcher/src/entity/PoolHourData.ts @@ -2,11 +2,9 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal, bigintTransformer } from '@vulcanize/util'; -import { Pool } from './Pool'; - @Entity() @Index(['id', 'blockNumber']) export class PoolHourData { @@ -23,11 +21,8 @@ export class PoolHourData { @Column('integer') periodStartUnix!: number; - @Column('varchar', { length: 42, nullable: true }) - poolId!: string; - - @ManyToOne(() => Pool, { onDelete: 'CASCADE' }) - pool!: Pool; + @Column('varchar', { length: 42 }) + pool!: string; @Column('numeric', { transformer: graphDecimalTransformer }) high!: GraphDecimal; diff --git a/packages/uni-info-watcher/src/entity/Position.ts b/packages/uni-info-watcher/src/entity/Position.ts index 8559f3a7..f75511ca 100644 --- a/packages/uni-info-watcher/src/entity/Position.ts +++ b/packages/uni-info-watcher/src/entity/Position.ts @@ -2,13 +2,9 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal, bigintTransformer } from '@vulcanize/util'; -import { Pool } from './Pool'; -import { Token } from './Token'; -import { Tick } from './Tick'; -import { Transaction } from './Transaction'; import { ADDRESS_ZERO } from '../utils/constants'; @Entity() @@ -54,39 +50,21 @@ export class Position { @Column('numeric', { default: 0, transformer: graphDecimalTransformer }) collectedFeesToken1!: GraphDecimal - @Column('varchar', { length: 42, nullable: true }) - poolId!: string; + @Column('varchar', { length: 42 }) + pool!: string; - @ManyToOne(() => Pool, { onDelete: 'CASCADE' }) - pool!: Pool + @Column('varchar', { length: 42 }) + token0!: string; - @Column('varchar', { length: 42, nullable: true, name: 'token0_id' }) - token0Id!: string; + @Column('varchar', { length: 42 }) + token1!: string; - @ManyToOne(() => Token) - token0!: Token + @Column('varchar') + tickLower!: string; - @Column('varchar', { length: 42, nullable: true, name: 'token1_id' }) - token1Id!: string; + @Column('varchar') + tickUpper!: string; - @ManyToOne(() => Token) - token1!: Token - - @Column('varchar', { nullable: true }) - tickLowerId!: string; - - @ManyToOne(() => Tick) - tickLower!: Tick - - @Column('varchar', { nullable: true }) - tickUpperId!: string; - - @ManyToOne(() => Tick) - tickUpper!: Tick - - @Column('varchar', { nullable: true }) - transactionId!: string; - - @ManyToOne(() => Transaction) - transaction!: Transaction + @Column('varchar') + transaction!: string; } diff --git a/packages/uni-info-watcher/src/entity/PositionSnapshot.ts b/packages/uni-info-watcher/src/entity/PositionSnapshot.ts index f9f76494..beabe8e9 100644 --- a/packages/uni-info-watcher/src/entity/PositionSnapshot.ts +++ b/packages/uni-info-watcher/src/entity/PositionSnapshot.ts @@ -2,13 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal, bigintTransformer } from '@vulcanize/util'; -import { Pool } from './Pool'; -import { Transaction } from './Transaction'; import { ADDRESS_ZERO } from '../utils/constants'; -import { Position } from './Position'; @Entity() @Index(['id', 'blockNumber']) @@ -56,12 +53,12 @@ export class PositionSnapshot { @Column('numeric', { default: 0, transformer: graphDecimalTransformer }) collectedFeesToken1!: GraphDecimal - @ManyToOne(() => Pool, { onDelete: 'CASCADE' }) - pool!: Pool + @Column('varchar', { length: 42 }) + pool!: string - @ManyToOne(() => Position, { onDelete: 'CASCADE' }) - position!: Position + @Column('varchar') + position!: string - @ManyToOne(() => Transaction, { onDelete: 'CASCADE' }) - transaction!: Transaction + @Column('varchar') + transaction!: string } diff --git a/packages/uni-info-watcher/src/entity/Swap.ts b/packages/uni-info-watcher/src/entity/Swap.ts index f64a1b69..af3779b4 100644 --- a/packages/uni-info-watcher/src/entity/Swap.ts +++ b/packages/uni-info-watcher/src/entity/Swap.ts @@ -2,16 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal, bigintTransformer } from '@vulcanize/util'; -import { Transaction } from './Transaction'; -import { Pool } from './Pool'; -import { Token } from './Token'; - @Entity() @Index(['id', 'blockNumber']) -@Index(['transactionId']) +@Index(['transaction']) export class Swap { @PrimaryColumn('varchar') id!: string; @@ -23,26 +19,20 @@ export class Swap { @Column('integer') blockNumber!: number; - @Column('varchar', { nullable: true }) - transactionId!: string; - - @ManyToOne(() => Transaction, transaction => transaction.swaps, { onDelete: 'CASCADE' }) - transaction!: Transaction + @Column('varchar') + transaction!: string; @Column('numeric', { transformer: bigintTransformer }) timestamp!: bigint; - @Column('varchar', { length: 42, nullable: true }) - poolId!: string; - - @ManyToOne(() => Pool, { onDelete: 'CASCADE' }) - pool!: Pool + @Column('varchar', { length: 42 }) + pool!: string; - @ManyToOne(() => Token, { onDelete: 'CASCADE' }) - token0!: Token + @Column('varchar', { length: 42 }) + token0!: string - @ManyToOne(() => Token, { onDelete: 'CASCADE' }) - token1!: Token + @Column('varchar', { length: 42 }) + token1!: string @Column('varchar', { length: 42 }) sender!: string diff --git a/packages/uni-info-watcher/src/entity/Tick.ts b/packages/uni-info-watcher/src/entity/Tick.ts index 089224f5..d15100a9 100644 --- a/packages/uni-info-watcher/src/entity/Tick.ts +++ b/packages/uni-info-watcher/src/entity/Tick.ts @@ -2,11 +2,9 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal, bigintTransformer } from '@vulcanize/util'; -import { Pool } from './Pool'; - @Entity() @Index(['id', 'blockNumber']) export class Tick { @@ -23,11 +21,8 @@ export class Tick { @Column('numeric', { transformer: bigintTransformer }) tickIdx!: bigint; - @Column('varchar', { length: 42, nullable: true }) - poolId!: string; - - @ManyToOne(() => Pool, { onDelete: 'CASCADE' }) - pool!: Pool + @Column('varchar', { length: 42 }) + pool!: string; @Column('varchar', { length: 42 }) poolAddress!: string diff --git a/packages/uni-info-watcher/src/entity/TickDayData.ts b/packages/uni-info-watcher/src/entity/TickDayData.ts index 61284230..9e912b94 100644 --- a/packages/uni-info-watcher/src/entity/TickDayData.ts +++ b/packages/uni-info-watcher/src/entity/TickDayData.ts @@ -2,12 +2,9 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; -import { Pool } from './Pool'; -import { Tick } from './Tick'; - @Entity() @Index(['id', 'blockNumber']) export class TickDayData { @@ -24,17 +21,11 @@ export class TickDayData { @Column('integer') date!: number - @Column('varchar', { length: 42, nullable: true }) - poolId!: string; - - @ManyToOne(() => Pool, { onDelete: 'CASCADE' }) - pool!: Pool; - - @Column('varchar', { nullable: true }) - tickId!: string; + @Column('varchar', { length: 42 }) + pool!: string; - @ManyToOne(() => Tick, { onDelete: 'CASCADE' }) - tick!: Tick + @Column('varchar') + tick!: string; @Column('numeric', { transformer: bigintTransformer }) liquidityGross!: bigint; diff --git a/packages/uni-info-watcher/src/entity/Token.ts b/packages/uni-info-watcher/src/entity/Token.ts index f7f03125..99ba7b1d 100644 --- a/packages/uni-info-watcher/src/entity/Token.ts +++ b/packages/uni-info-watcher/src/entity/Token.ts @@ -2,11 +2,9 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToMany, JoinTable, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal, bigintTransformer } from '@vulcanize/util'; -import { Pool } from './Pool'; - @Entity() @Index(['id', 'blockNumber']) export class Token { @@ -56,9 +54,8 @@ export class Token { @Column('numeric', { default: 0, transformer: graphDecimalTransformer }) feesUSD!: GraphDecimal; - @ManyToMany(() => Pool) - @JoinTable() - whitelistPools!: Pool[]; + @Column('varchar', { length: 42, array: true, default: [] }) + whitelistPools!: string[]; // TODO: Add remaining fields when they are used. } diff --git a/packages/uni-info-watcher/src/entity/TokenDayData.ts b/packages/uni-info-watcher/src/entity/TokenDayData.ts index a3865efb..bb155fb4 100644 --- a/packages/uni-info-watcher/src/entity/TokenDayData.ts +++ b/packages/uni-info-watcher/src/entity/TokenDayData.ts @@ -2,14 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal } from '@vulcanize/util'; -import { Token } from './Token'; - @Entity() @Index(['id', 'blockNumber']) -@Index(['date', 'tokenId']) +@Index(['date', 'token']) export class TokenDayData { @PrimaryColumn('varchar') id!: string; @@ -24,11 +22,8 @@ export class TokenDayData { @Column('integer') date!: number - @Column('varchar', { length: 42, nullable: true }) - tokenId!: string; - - @ManyToOne(() => Token, { onDelete: 'CASCADE' }) - token!: Token + @Column('varchar', { length: 42 }) + token!: string; @Column('numeric', { transformer: graphDecimalTransformer }) high!: GraphDecimal; diff --git a/packages/uni-info-watcher/src/entity/TokenHourData.ts b/packages/uni-info-watcher/src/entity/TokenHourData.ts index 3401cb1f..d26a4ee0 100644 --- a/packages/uni-info-watcher/src/entity/TokenHourData.ts +++ b/packages/uni-info-watcher/src/entity/TokenHourData.ts @@ -2,14 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, ManyToOne, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal } from '@vulcanize/util'; -import { Token } from './Token'; - @Entity() @Index(['id', 'blockNumber']) -@Index(['periodStartUnix', 'tokenId']) +@Index(['periodStartUnix', 'token']) export class TokenHourData { @PrimaryColumn('varchar') id!: string; @@ -24,11 +22,8 @@ export class TokenHourData { @Column('integer') periodStartUnix!: number - @Column('varchar', { length: 42, nullable: true }) - tokenId!: string; - - @ManyToOne(() => Token, { onDelete: 'CASCADE' }) - token!: Token + @Column('varchar', { length: 42 }) + token!: string; @Column('numeric', { transformer: graphDecimalTransformer }) high!: GraphDecimal; diff --git a/packages/uni-info-watcher/src/entity/Transaction.ts b/packages/uni-info-watcher/src/entity/Transaction.ts index 7a69a3f8..d4a82c81 100644 --- a/packages/uni-info-watcher/src/entity/Transaction.ts +++ b/packages/uni-info-watcher/src/entity/Transaction.ts @@ -2,13 +2,9 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column, OneToMany, Index } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { graphDecimalTransformer, GraphDecimal, bigintTransformer } from '@vulcanize/util'; -import { Mint } from './Mint'; -import { Burn } from './Burn'; -import { Swap } from './Swap'; - @Entity() @Index(['id', 'blockNumber']) export class Transaction { @@ -27,13 +23,4 @@ export class Transaction { @Column('numeric', { transformer: bigintTransformer }) timestamp!: bigint; - - @OneToMany(() => Mint, mint => mint.transaction) - mints!: Mint[]; - - @OneToMany(() => Burn, burn => burn.transaction) - burns!: Burn[]; - - @OneToMany(() => Swap, swap => swap.transaction) - swaps!: Swap[]; } diff --git a/packages/uni-info-watcher/src/indexer.ts b/packages/uni-info-watcher/src/indexer.ts index 5e4d317d..d2f4eab9 100644 --- a/packages/uni-info-watcher/src/indexer.ts +++ b/packages/uni-info-watcher/src/indexer.ts @@ -18,7 +18,7 @@ import { updatePoolDayData, updatePoolHourData, updateTickDayData, updateTokenDa import { Token } from './entity/Token'; import { convertTokenToDecimal, loadFactory, loadTransaction, safeDiv } from './utils'; import { createTick, feeTierToTickSpacing } from './utils/tick'; -import { FACTORY_ADDRESS, WATCHED_CONTRACTS } from './utils/constants'; +import { WATCHED_CONTRACTS } from './utils/constants'; import { Position } from './entity/Position'; import { Database } from './database'; import { Event } from './entity/Event'; @@ -236,7 +236,7 @@ export class Indexer implements IndexerInterface { let res; try { - res = await this._db.getPool(dbTx, { id, blockHash: block.hash, blockNumber: block.number }); + res = await this._db.getPool(dbTx, { id, blockHash: block.hash, blockNumber: block.number }, true); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -253,7 +253,7 @@ export class Indexer implements IndexerInterface { let res; try { - res = await this._db.getToken(dbTx, { id, blockHash: block.hash, blockNumber: block.number }); + res = await this._db.getToken(dbTx, { id, blockHash: block.hash, blockNumber: block.number }, true); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -511,8 +511,8 @@ export class Indexer implements IndexerInterface { assert(token0); assert(token1); - pool.token0 = token0; - pool.token1 = token1; + pool.token0 = token0.id; + pool.token1 = token1.id; pool.feeTier = BigInt(fee); // Skipping adding createdAtTimestamp field as it is not queried in frontend subgraph. @@ -521,17 +521,17 @@ export class Indexer implements IndexerInterface { // Update white listed pools. if (WHITELIST_TOKENS.includes(token0.id) || this._isDemo) { - token1.whitelistPools.push(pool); + token1.whitelistPools.push(pool.id); } if (WHITELIST_TOKENS.includes(token1.id) || this._isDemo) { - token0.whitelistPools.push(pool); + token0.whitelistPools.push(pool.id); } token0 = await this._db.saveToken(dbTx, token0, block); token1 = await this._db.saveToken(dbTx, token1, block); - pool.token0 = token0; - pool.token1 = token1; + pool.token0 = token0.id; + pool.token1 = token1.id; await this._db.savePool(dbTx, pool, block); await this._db.saveFactory(dbTx, factory, block); await dbTx.commitTransaction(); @@ -592,8 +592,8 @@ export class Indexer implements IndexerInterface { // Update token prices. const [token0, token1] = await Promise.all([ - this._db.getToken(dbTx, { id: pool.token0.id, blockHash: block.hash }), - this._db.getToken(dbTx, { id: pool.token1.id, blockHash: block.hash }) + this._db.getToken(dbTx, { id: pool.token0, blockHash: block.hash }), + this._db.getToken(dbTx, { id: pool.token1, blockHash: block.hash }) ]); assert(token0 && token1, 'Pool tokens not found.'); @@ -639,8 +639,11 @@ export class Indexer implements IndexerInterface { const factory = await loadFactory(this._db, dbTx, block, this._isDemo); - let token0 = await this._db.getToken(dbTx, { id: pool.token0.id, blockHash: block.hash }); - let token1 = await this._db.getToken(dbTx, { id: pool.token1.id, blockHash: block.hash }); + let [token0, token1] = await Promise.all([ + this._db.getToken(dbTx, { id: pool.token0, blockHash: block.hash }), + this._db.getToken(dbTx, { id: pool.token1, blockHash: block.hash }) + ]); + assert(token0); assert(token1); const amount0 = convertTokenToDecimal(BigInt(mintEvent.amount0), BigInt(token0.decimals)); @@ -696,9 +699,9 @@ export class Indexer implements IndexerInterface { const mint = new Mint(); mint.id = transaction.id + '#' + pool.txCount.toString(); - mint.transaction = transaction; + mint.transaction = transaction.id; mint.timestamp = transaction.timestamp; - mint.pool = pool; + mint.pool = pool.id; mint.token0 = pool.token0; mint.token1 = pool.token1; mint.owner = utils.hexlify(mintEvent.owner); @@ -752,19 +755,19 @@ export class Indexer implements IndexerInterface { this._db.saveToken(dbTx, token1, block) ]); - pool.token0 = token0; - pool.token1 = token1; + pool.token0 = token0.id; + pool.token1 = token1.id; pool = await this._db.savePool(dbTx, pool, block); await this._db.saveFactory(dbTx, factory, block); - mint.pool = pool; - mint.token0 = token0; - mint.token1 = token1; + mint.pool = pool.id; + mint.token0 = token0.id; + mint.token1 = token1.id; await this._db.saveMint(dbTx, mint, block); - lowerTick.pool = pool; - upperTick.pool = pool; + lowerTick.pool = pool.id; + upperTick.pool = pool.id; await Promise.all([ await this._db.saveTick(dbTx, lowerTick, block), await this._db.saveTick(dbTx, upperTick, block) @@ -797,8 +800,11 @@ export class Indexer implements IndexerInterface { const factory = await loadFactory(this._db, dbTx, block, this._isDemo); - let token0 = await this._db.getToken(dbTx, { id: pool.token0.id, blockHash: block.hash }); - let token1 = await this._db.getToken(dbTx, { id: pool.token1.id, blockHash: block.hash }); + let [token0, token1] = await Promise.all([ + this._db.getToken(dbTx, { id: pool.token0, blockHash: block.hash }), + this._db.getToken(dbTx, { id: pool.token1, blockHash: block.hash }) + ]); + assert(token0); assert(token1); const amount0 = convertTokenToDecimal(BigInt(burnEvent.amount0), BigInt(token0.decimals)); @@ -856,9 +862,9 @@ export class Indexer implements IndexerInterface { const burn = new Burn(); burn.id = transaction.id + '#' + pool.txCount.toString(); - burn.transaction = transaction; + burn.transaction = transaction.id; burn.timestamp = transaction.timestamp; - burn.pool = pool; + burn.pool = pool.id; burn.token0 = pool.token0; burn.token1 = pool.token1; burn.owner = utils.hexlify(burnEvent.owner); @@ -873,8 +879,8 @@ export class Indexer implements IndexerInterface { // Tick entities. const lowerTickId = poolAddress + '#' + (burnEvent.tickLower).toString(); const upperTickId = poolAddress + '#' + (burnEvent.tickUpper).toString(); - const lowerTick = await this._db.getTick(dbTx, { id: lowerTickId, blockHash: block.hash }); - const upperTick = await this._db.getTick(dbTx, { id: upperTickId, blockHash: block.hash }); + const lowerTick = await this._db.getTick(dbTx, { id: lowerTickId, blockHash: block.hash }, true); + const upperTick = await this._db.getTick(dbTx, { id: upperTickId, blockHash: block.hash }, true); assert(lowerTick && upperTick); const amount = BigInt(burnEvent.amount); lowerTick.liquidityGross = BigInt(lowerTick.liquidityGross) - amount; @@ -897,23 +903,23 @@ export class Indexer implements IndexerInterface { this._db.saveToken(dbTx, token1, block) ]); - pool.token0 = token0; - pool.token1 = token1; + pool.token0 = token0.id; + pool.token1 = token1.id; pool = await this._db.savePool(dbTx, pool, block); await this._db.saveFactory(dbTx, factory, block); // Skipping update Tick fee and Tick day data as they are not queried. - lowerTick.pool = pool; - upperTick.pool = pool; + lowerTick.pool = pool.id; + upperTick.pool = pool.id; await Promise.all([ await this._db.saveTick(dbTx, lowerTick, block), await this._db.saveTick(dbTx, upperTick, block) ]); - burn.pool = pool; - burn.token0 = token0; - burn.token1 = token1; + burn.pool = pool.id; + burn.token0 = token0.id; + burn.token1 = token1.id; await this._db.saveBurn(dbTx, burn, block); await dbTx.commitTransaction(); } catch (error) { @@ -944,8 +950,8 @@ export class Indexer implements IndexerInterface { } let [token0, token1] = await Promise.all([ - this._db.getToken(dbTx, { id: pool.token0.id, blockHash: block.hash }), - this._db.getToken(dbTx, { id: pool.token1.id, blockHash: block.hash }) + this._db.getToken(dbTx, { id: pool.token0, blockHash: block.hash }), + this._db.getToken(dbTx, { id: pool.token1, blockHash: block.hash }) ]); assert(token0 && token1, 'Pool tokens not found.'); @@ -1057,9 +1063,9 @@ export class Indexer implements IndexerInterface { const swap = new Swap(); swap.id = transaction.id + '#' + pool.txCount.toString(); - swap.transaction = transaction; + swap.transaction = transaction.id; swap.timestamp = transaction.timestamp; - swap.pool = pool; + swap.pool = pool.id; swap.token0 = pool.token0; swap.token1 = pool.token1; swap.sender = utils.hexlify(swapEvent.sender); @@ -1124,23 +1130,23 @@ export class Indexer implements IndexerInterface { this._db.saveToken(dbTx, token1, block) ]); - pool.token0 = token0; - pool.token1 = token1; + pool.token0 = token0.id; + pool.token1 = token1.id; pool = await this._db.savePool(dbTx, pool, block); - swap.token0 = token0; - swap.token1 = token1; - swap.pool = pool; + swap.token0 = token0.id; + swap.token1 = token1.id; + swap.pool = pool.id; await this._db.saveSwap(dbTx, swap, block); - token0DayData.token = token0; - token1DayData.token = token1; + token0DayData.token = token0.id; + token1DayData.token = token1.id; await this._db.saveTokenDayData(dbTx, token0DayData, block); await this._db.saveTokenDayData(dbTx, token1DayData, block); await this._db.saveUniswapDayData(dbTx, uniswapDayData, block); - poolDayData.pool = pool; + poolDayData.pool = pool.id; await this._db.savePoolDayData(dbTx, poolDayData, block); // Update inner vars of current or crossed ticks. @@ -1198,7 +1204,7 @@ export class Indexer implements IndexerInterface { } async _handleIncreaseLiquidity (block: Block, contractAddress: string, tx: Transaction, event: IncreaseLiquidityEvent): Promise { - let position = await this._getPosition(block, contractAddress, tx, BigInt(event.tokenId)); + const position = await this._getPosition(block, contractAddress, tx, BigInt(event.tokenId)); // position was not able to be fetched. if (position === null) { @@ -1206,11 +1212,7 @@ export class Indexer implements IndexerInterface { } // Temp fix from Subgraph mapping code. - // if (utils.getAddress(position.pool.id) === utils.getAddress('0x8fe8d9bb8eeba3ed688069c3d6b556c9ca258248')) { - // return; - // } - // For the above scenario position.pool is null which is computed in this._getPosition. - if (!position.pool) { + if (utils.getAddress(position.pool) === utils.getAddress('0x8fe8d9bb8eeba3ed688069c3d6b556c9ca258248')) { return; } @@ -1218,14 +1220,12 @@ export class Indexer implements IndexerInterface { const dbTx = await this._db.createTransactionRunner(); try { - if (!position.transaction) { - const transaction = await loadTransaction(this._db, dbTx, { block, tx }); - position.transaction = transaction; - position = await this._db.savePosition(dbTx, position, block); - } + const [token0, token1] = await Promise.all([ + this._db.getToken(dbTx, { id: position.token0, blockHash: block.hash }), + this._db.getToken(dbTx, { id: position.token1, blockHash: block.hash }) + ]); - const token0 = position.token0; - const token1 = position.token1; + assert(token0 && token1); const amount0 = convertTokenToDecimal(BigInt(event.amount0), BigInt(token0.decimals)); const amount1 = convertTokenToDecimal(BigInt(event.amount1), BigInt(token1.decimals)); @@ -1255,11 +1255,7 @@ export class Indexer implements IndexerInterface { } // Temp fix from Subgraph mapping code. - // if (utils.getAddress(position.pool.id) === utils.getAddress('0x8fe8d9bb8eeba3ed688069c3d6b556c9ca258248')) { - // return; - // } - // For the above scenario position.pool is null which is computed in this._getPosition. - if (!position.pool) { + if (utils.getAddress(position.pool) === utils.getAddress('0x8fe8d9bb8eeba3ed688069c3d6b556c9ca258248')) { return; } @@ -1267,14 +1263,13 @@ export class Indexer implements IndexerInterface { const dbTx = await this._db.createTransactionRunner(); try { - if (!position.transaction) { - const transaction = await loadTransaction(this._db, dbTx, { block, tx }); - position.transaction = transaction; - position = await this._db.savePosition(dbTx, position, block); - } + const [token0, token1] = await Promise.all([ + this._db.getToken(dbTx, { id: position.token0, blockHash: block.hash }), + this._db.getToken(dbTx, { id: position.token1, blockHash: block.hash }) + ]); + + assert(token0 && token1); - const token0 = position.token0; - const token1 = position.token1; const amount0 = convertTokenToDecimal(BigInt(event.amount0), BigInt(token0.decimals)); const amount1 = convertTokenToDecimal(BigInt(event.amount1), BigInt(token1.decimals)); @@ -1303,11 +1298,7 @@ export class Indexer implements IndexerInterface { } // Temp fix from Subgraph mapping code. - // if (utils.getAddress(position.pool.id) === utils.getAddress('0x8fe8d9bb8eeba3ed688069c3d6b556c9ca258248')) { - // return; - // } - // For the above scenario position.pool is null which is computed in this._getPosition. - if (!position.pool) { + if (utils.getAddress(position.pool) === utils.getAddress('0x8fe8d9bb8eeba3ed688069c3d6b556c9ca258248')) { return; } @@ -1315,14 +1306,13 @@ export class Indexer implements IndexerInterface { const dbTx = await this._db.createTransactionRunner(); try { - if (!position.transaction) { - const transaction = await loadTransaction(this._db, dbTx, { block, tx }); - position.transaction = transaction; - position = await this._db.savePosition(dbTx, position, block); - } + const [token0, token1] = await Promise.all([ + this._db.getToken(dbTx, { id: position.token0, blockHash: block.hash }), + this._db.getToken(dbTx, { id: position.token1, blockHash: block.hash }) + ]); + + assert(token0 && token1); - const token0 = position.token0; - const token1 = position.token1; const amount0 = convertTokenToDecimal(BigInt(event.amount0), BigInt(token0.decimals)); const amount1 = convertTokenToDecimal(BigInt(event.amount1), BigInt(token1.decimals)); @@ -1342,7 +1332,7 @@ export class Indexer implements IndexerInterface { } async _handleTransfer (block: Block, contractAddress: string, tx: Transaction, event: TransferEvent): Promise { - let position = await this._getPosition(block, contractAddress, tx, BigInt(event.tokenId)); + const position = await this._getPosition(block, contractAddress, tx, BigInt(event.tokenId)); // Position was not able to be fetched. if (position === null) { return; @@ -1351,12 +1341,6 @@ export class Indexer implements IndexerInterface { const dbTx = await this._db.createTransactionRunner(); try { - if (!position.transaction) { - const transaction = await loadTransaction(this._db, dbTx, { block, tx }); - position.transaction = transaction; - position = await this._db.savePosition(dbTx, position, block); - } - position.owner = utils.hexlify(event.to); await this._db.savePosition(dbTx, position, block); @@ -1385,7 +1369,8 @@ export class Indexer implements IndexerInterface { { id: poolAddress.concat('#').concat(tickId.toString()), blockHash: block.hash - } + }, + true ); if (tick) { @@ -1398,95 +1383,50 @@ export class Indexer implements IndexerInterface { let position = await this._db.getPosition({ id: tokenId.toString(), blockHash }); if (!position) { - let positionResult; - - try { - console.time('time:indexer#_getPosition-eth_call_for_positions'); - ({ value: positionResult } = await this._uniClient.positions(blockHash, contractAddress, tokenId)); - console.timeEnd('time:indexer#_getPosition-eth_call_for_positions'); - } catch (error: any) { - // The contract call reverts in situations where the position is minted and deleted in the same block. - // From my investigation this happens in calls from BancorSwap. - // (e.g. 0xf7867fa19aa65298fadb8d4f72d0daed5e836f3ba01f0b9b9631cdc6c36bed40) - - if (error.message !== utils.Logger.errors.CALL_EXCEPTION) { - log('nfpm positions eth_call failed'); - throw error; - } - } + console.time('time:indexer#_getPosition-storage_call_for_getPosition'); + const nfpmPosition = await this._uniClient.getPosition(blockHash, tokenId); + console.timeEnd('time:indexer#_getPosition-storage_call_for_getPosition'); - if (positionResult) { - let factoryAddress = FACTORY_ADDRESS; + // The contract call reverts in situations where the position is minted and deleted in the same block. + // From my investigation this happens in calls from BancorSwap. + // (e.g. 0xf7867fa19aa65298fadb8d4f72d0daed5e836f3ba01f0b9b9631cdc6c36bed40) - if (this._isDemo) { - // Currently fetching address from Factory entity in database as only one exists. - const [factory] = await this._db.getModelEntitiesNoTx(Factory, { hash: blockHash }, {}, { limit: 1 }); - factoryAddress = factory.id; - } + if (nfpmPosition) { + console.time('time:indexer#_getPosition-storage_call_for_poolIdToPoolKey'); + const { token0: token0Address, token1: token1Address, fee } = await this._uniClient.poolIdToPoolKey(blockHash, nfpmPosition.poolId); + console.timeEnd('time:indexer#_getPosition-storage_call_for_poolIdToPoolKey'); - console.time('time:indexer#_getPosition-eth_call_for_getPool'); - let { value: poolAddress } = await this._uniClient.callGetPool(blockHash, factoryAddress, positionResult.token0, positionResult.token1, positionResult.fee); - console.timeEnd('time:indexer#_getPosition-eth_call_for_getPool'); + console.time('time:indexer#_getPosition-storage_call_for_getPool'); + let { pool: poolAddress } = await this._uniClient.getPool(blockHash, token0Address, token1Address, fee); + console.timeEnd('time:indexer#_getPosition-storage_call_for_getPool'); // Get the pool address in lowercase. poolAddress = utils.hexlify(poolAddress); position = new Position(); position.id = tokenId.toString(); + position.pool = poolAddress; + position.token0 = utils.hexlify(token0Address); + position.token1 = utils.hexlify(token1Address); + position.tickLower = poolAddress.concat('#').concat(nfpmPosition.tickLower.toString()); + position.tickUpper = poolAddress.concat('#').concat(nfpmPosition.tickUpper.toString()); - const pool = await this._db.getPoolNoTx({ id: poolAddress, blockHash }); - - // No pool present for nfpm token related to pool 0x8fe8d9bb8eeba3ed688069c3d6b556c9ca258248 skipped in mapping code. - // Skipping assigning relation fields to follow subgraph behaviour for Position entity id 2074. - if (pool) { - position.pool = pool; - - const [token0, token1] = await Promise.all([ - this._db.getTokenNoTx({ id: utils.hexlify(positionResult.token0), blockHash }), - this._db.getTokenNoTx({ id: utils.hexlify(positionResult.token1), blockHash }) - ]); - assert(token0 && token1); - position.token0 = token0; - position.token1 = token1; - - const lowerTickIdx = positionResult.tickLower; - const upperTickIdx = positionResult.tickUpper; - - const lowerTickId = poolAddress.concat('#').concat(positionResult.tickLower.toString()); - const upperTickId = poolAddress.concat('#').concat(positionResult.tickUpper.toString()); - - let [tickLower, tickUpper] = await Promise.all([ - this._db.getTickNoTx({ id: lowerTickId, blockHash }), - this._db.getTickNoTx({ id: upperTickId, blockHash }) - ]); - - const dbTx = await this._db.createTransactionRunner(); - - try { - // Tick entities not present when Transfer event is processed before Pool Mint event. - // TODO: Save entity ids similar to subgraph mapping code. - if (!tickLower) { - tickLower = await createTick(this._db, dbTx, lowerTickId, BigInt(lowerTickIdx), pool, block); - } - - if (!tickUpper) { - tickUpper = await createTick(this._db, dbTx, upperTickId, BigInt(upperTickIdx), pool, block); - } - - await dbTx.commitTransaction(); - } catch (error) { - await dbTx.rollbackTransaction(); - throw error; - } finally { - await dbTx.release(); - } + const dbTx = await this._db.createTransactionRunner(); + + try { + const transaction = await loadTransaction(this._db, dbTx, { block, tx }); + position.transaction = transaction.id; - position.tickLower = tickLower; - position.tickUpper = tickUpper; + await dbTx.commitTransaction(); + } catch (error) { + await dbTx.rollbackTransaction(); + throw error; + } finally { + await dbTx.release(); } - position.feeGrowthInside0LastX128 = BigInt(positionResult.feeGrowthInside0LastX128.toString()); - position.feeGrowthInside1LastX128 = BigInt(positionResult.feeGrowthInside1LastX128.toString()); + position.feeGrowthInside0LastX128 = BigInt(nfpmPosition.feeGrowthInside0LastX128.toString()); + position.feeGrowthInside1LastX128 = BigInt(nfpmPosition.feeGrowthInside1LastX128.toString()); } } @@ -1494,18 +1434,13 @@ export class Indexer implements IndexerInterface { } async _updateFeeVars (position: Position, block: Block, contractAddress: string, tokenId: bigint): Promise { - try { - console.time('time:indexer#_updateFeeVars-eth_call_for_positions'); - const { value: positionResult } = await this._uniClient.positions(block.hash, contractAddress, tokenId); - console.timeEnd('time:indexer#_updateFeeVars-eth_call_for_positions'); + console.time('time:indexer#_updateFeeVars-storage_call_for_getPosition'); + const nfpmPosition = await this._uniClient.getPosition(block.hash, tokenId); + console.timeEnd('time:indexer#_updateFeeVars-storage_call_for_getPosition'); - if (positionResult) { - position.feeGrowthInside0LastX128 = BigInt(positionResult.feeGrowthInside0LastX128.toString()); - position.feeGrowthInside1LastX128 = BigInt(positionResult.feeGrowthInside1LastX128.toString()); - } - } catch (error) { - log('nfpm positions eth_call failed'); - log(error); + if (nfpmPosition) { + position.feeGrowthInside0LastX128 = BigInt(nfpmPosition.feeGrowthInside0LastX128.toString()); + position.feeGrowthInside1LastX128 = BigInt(nfpmPosition.feeGrowthInside1LastX128.toString()); } return position; @@ -1517,7 +1452,7 @@ export class Indexer implements IndexerInterface { positionSnapshot.blockNumber = block.number; positionSnapshot.owner = position.owner; positionSnapshot.pool = position.pool; - positionSnapshot.position = position; + positionSnapshot.position = position.id; positionSnapshot.timestamp = BigInt(block.timestamp); positionSnapshot.liquidity = position.liquidity; positionSnapshot.depositedToken0 = position.depositedToken0; @@ -1526,7 +1461,8 @@ export class Indexer implements IndexerInterface { positionSnapshot.withdrawnToken1 = position.withdrawnToken1; positionSnapshot.collectedFeesToken0 = position.collectedFeesToken0; positionSnapshot.collectedFeesToken1 = position.collectedFeesToken1; - positionSnapshot.transaction = await loadTransaction(this._db, dbTx, { block, tx }); + const transaction = await loadTransaction(this._db, dbTx, { block, tx }); + positionSnapshot.transaction = transaction.id; positionSnapshot.feeGrowthInside0LastX128 = position.feeGrowthInside0LastX128; positionSnapshot.feeGrowthInside1LastX128 = position.feeGrowthInside1LastX128; diff --git a/packages/uni-info-watcher/src/job-runner.ts b/packages/uni-info-watcher/src/job-runner.ts index d3162b2f..08dbaf45 100644 --- a/packages/uni-info-watcher/src/job-runner.ts +++ b/packages/uni-info-watcher/src/job-runner.ts @@ -134,7 +134,9 @@ export const main = async (): Promise => { const indexer = new Indexer(db, uniClient, erc20Client, ethClient, postgraphileClient, ethProvider, jobQueue, mode); await indexer.init(); - await indexer.addContracts(); + if (mode !== 'demo') { + await indexer.addContracts(); + } const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue); await jobRunner.start(); diff --git a/packages/uni-info-watcher/src/resolvers.ts b/packages/uni-info-watcher/src/resolvers.ts index 18d5bda6..872834af 100644 --- a/packages/uni-info-watcher/src/resolvers.ts +++ b/packages/uni-info-watcher/src/resolvers.ts @@ -86,28 +86,24 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId', + field: 'pool', childRelations: [ { entity: Token, type: 'one-to-one', - property: 'token0', - field: 'token0Id' + field: 'token0' }, { entity: Token, type: 'one-to-one', - property: 'token1', - field: 'token1Id' + field: 'token1' } ] }, { entity: Transaction, type: 'one-to-one', - property: 'transaction', - field: 'transactionId' + field: 'transaction' } ] ); @@ -131,28 +127,24 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId', + field: 'pool', childRelations: [ { entity: Token, type: 'one-to-one', - property: 'token0', - field: 'token0Id' + field: 'token0' }, { entity: Token, type: 'one-to-one', - property: 'token1', - field: 'token1Id' + field: 'token1' } ] }, { entity: Transaction, type: 'one-to-one', - property: 'transaction', - field: 'transactionId' + field: 'transaction' } ] ); @@ -182,14 +174,12 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch { entity: Token, type: 'one-to-one', - property: 'token0', - field: 'token0Id' + field: 'token0' }, { entity: Token, type: 'one-to-one', - property: 'token1', - field: 'token1Id' + field: 'token1' } ] ); @@ -207,28 +197,24 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId', + field: 'pool', childRelations: [ { entity: Token, type: 'one-to-one', - property: 'token0', - field: 'token0Id' + field: 'token0' }, { entity: Token, type: 'one-to-one', - property: 'token1', - field: 'token1Id' + field: 'token1' } ] }, { entity: Transaction, type: 'one-to-one', - property: 'transaction', - field: 'transactionId' + field: 'transaction' } ] ); @@ -258,8 +244,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch { entity: Pool, type: 'many-to-many', - property: 'whitelistPools', - field: 'pool' + field: 'whitelistPools' } ] ); @@ -289,32 +274,28 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch { entity: Mint, type: 'one-to-many', - property: 'mints', - field: 'transactionId', + field: 'mints', + foreignKey: 'transaction', childRelations: [ { entity: Transaction, type: 'one-to-one', - property: 'transaction', - field: 'transactionId' + field: 'transaction' }, { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId', + field: 'pool', childRelations: [ { entity: Token, type: 'one-to-one', - property: 'token0', - field: 'token0Id' + field: 'token0' }, { entity: Token, type: 'one-to-one', - property: 'token1', - field: 'token1Id' + field: 'token1' } ] } @@ -323,32 +304,28 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch { entity: Burn, type: 'one-to-many', - property: 'burns', - field: 'transactionId', + field: 'burns', + foreignKey: 'transaction', childRelations: [ { entity: Transaction, type: 'one-to-one', - property: 'transaction', - field: 'transactionId' + field: 'transaction' }, { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId', + field: 'pool', childRelations: [ { entity: Token, type: 'one-to-one', - property: 'token0', - field: 'token0Id' + field: 'token0' }, { entity: Token, type: 'one-to-one', - property: 'token1', - field: 'token1Id' + field: 'token1' } ] } @@ -357,32 +334,28 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch { entity: Swap, type: 'one-to-many', - property: 'swaps', - field: 'transactionId', + field: 'swaps', + foreignKey: 'transaction', childRelations: [ { entity: Transaction, type: 'one-to-one', - property: 'transaction', - field: 'transactionId' + field: 'transaction' }, { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId', + field: 'pool', childRelations: [ { entity: Token, type: 'one-to-one', - property: 'token0', - field: 'token0Id' + field: 'token0' }, { entity: Token, type: 'one-to-one', - property: 'token1', - field: 'token1Id' + field: 'token1' } ] } @@ -410,38 +383,32 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch { entity: Pool, type: 'one-to-one', - property: 'pool', - field: 'poolId' + field: 'pool' }, { entity: Token, type: 'one-to-one', - property: 'token0', - field: 'token0Id' + field: 'token0' }, { entity: Token, type: 'one-to-one', - property: 'token1', - field: 'token1Id' + field: 'token1' }, { entity: Tick, type: 'one-to-one', - property: 'tickLower', - field: 'tickLowerId' + field: 'tickLower' }, { entity: Tick, type: 'one-to-one', - property: 'tickUpper', - field: 'tickUpperId' + field: 'tickUpper' }, { entity: Transaction, type: 'one-to-one', - property: 'transaction', - field: 'transactionId' + field: 'transaction' } ] ); diff --git a/packages/uni-info-watcher/src/smoke.test.ts b/packages/uni-info-watcher/src/smoke.test.ts index 2ab0d3bc..cf27089c 100644 --- a/packages/uni-info-watcher/src/smoke.test.ts +++ b/packages/uni-info-watcher/src/smoke.test.ts @@ -53,6 +53,7 @@ describe('uni-info-watcher', () => { let token1Address: string; let nfpm: Contract; + let nfpmTokenId: number; let tickLower: number; let tickUpper: number; let signer: Signer; @@ -85,35 +86,29 @@ describe('uni-info-watcher', () => { signer = provider.getSigner(); recipient = await signer.getAddress(); - // Deadline set to 2 days from current date. - const deadlineDate = new Date(); - deadlineDate.setDate(deadlineDate.getDate() + 2); - deadline = Math.floor(deadlineDate.getTime() / 1000); - }); + // Get the factory contract address. + const factoryContract = await uniClient.getContract('factory'); + expect(factoryContract).to.not.be.empty; - it('should have a Factory entity', async () => { - // Getting the Factory from uni-info-watcher graphQL endpoint. - const factories = await client.getFactories(1); - expect(factories).to.not.be.empty; + // Initialize the factory contract. + factory = new Contract(factoryContract.address, FACTORY_ABI, signer); - // Initializing the factory variable. - const factoryAddress = factories[0].id; - factory = new ethers.Contract(factoryAddress, FACTORY_ABI, signer); - expect(factory.address).to.not.be.empty; - }); + // Get the NFPM contract address. + const nfpmContract = await uniClient.getContract('nfpm'); + expect(nfpmContract).to.not.be.empty; - it('should have a Bundle entity', async () => { - // Getting the Bundle from uni-info-watcher graphQL endpoint. - const bundles = await client.getBundles(1); - expect(bundles).to.not.be.empty; + // Initialize the NFPM contract. + nfpm = new Contract(nfpmContract.address, NFPM_ABI, signer); - const bundleId = '1'; - expect(bundles[0].id).to.equal(bundleId); + // Deadline set to 2 days from current date. + const deadlineDate = new Date(); + deadlineDate.setDate(deadlineDate.getDate() + 2); + deadline = Math.floor(deadlineDate.getTime() / 1000); }); describe('PoolCreatedEvent', () => { // NOTE: Skipping checking entity updates that cannot be gotten/derived using queries. - // Checked entities: Token, Pool. + // Checked entities: Token, Factory, Bundle, Pool. const fee = 500; @@ -158,6 +153,21 @@ describe('uni-info-watcher', () => { expect(token1).to.not.be.null; }); + it('should create a Factory entity', async () => { + // Check that a Factory entity is present. + const factories = await client.getFactories(1); + expect(factories).to.not.be.empty; + }); + + it('should create a Bundle entity', async () => { + // Check that a Bundle entity is present. + const bundles = await client.getBundles(1); + expect(bundles).to.not.be.empty; + + const bundleId = '1'; + expect(bundles[0].id).to.equal(bundleId); + }); + it('should create a Pool entity', async () => { // Checked values: feeTier @@ -783,6 +793,9 @@ describe('uni-info-watcher', () => { transaction ]); + // Store tokenId for further usage. + nfpmTokenId = Number(eventValue.event.tokenId); + // Sleeping for 15 sec for the events to be processed. await wait(15000); }); @@ -833,7 +846,6 @@ describe('uni-info-watcher', () => { let oldPosition: any; let eventValue: any; - const tokenId = 1; const amount0Desired = 15; const amount1Desired = 15; const amount0Min = 0; @@ -841,14 +853,15 @@ describe('uni-info-watcher', () => { before(async () => { // Get initial entity values. - const positions = await client.getPositions({ id: Number(tokenId) }, 1); + const positions = await client.getPositions({ id: nfpmTokenId }, 1); + expect(positions).to.not.be.empty; oldPosition = positions[0]; }); it('should trigger IncreaseLiquidityEvent', async () => { // Position manger increase liquidity and wait for MintEvent. const transaction = nfpm.increaseLiquidity({ - tokenId, + tokenId: nfpmTokenId, amount0Desired, amount1Desired, amount0Min, @@ -905,21 +918,20 @@ describe('uni-info-watcher', () => { let oldPosition: any; let eventValue: any; - const tokenId = 1; const liquidity = 5; const amount0Min = 0; const amount1Min = 0; before(async () => { // Get initial entity values. - const positions = await client.getPositions({ id: Number(tokenId) }, 1); + const positions = await client.getPositions({ id: nfpmTokenId }, 1); oldPosition = positions[0]; }); it('should trigger DecreaseLiquidityEvent', async () => { // Position manger decrease liquidity and wait for BurnEvent. const transaction = nfpm.decreaseLiquidity({ - tokenId, + tokenId: nfpmTokenId, liquidity, amount0Min, amount1Min, @@ -973,14 +985,13 @@ describe('uni-info-watcher', () => { // Checked entities: Transaction. // Unchecked entities: Position. - const tokenId = 1; const amount0Max = 15; const amount1Max = 15; it('should trigger CollectEvent', async () => { // Position manger collect and wait for BurnEvent. const transaction = nfpm.collect({ - tokenId, + tokenId: nfpmTokenId, recipient, amount0Max, amount1Max diff --git a/packages/uni-info-watcher/src/utils/interval-updates.ts b/packages/uni-info-watcher/src/utils/interval-updates.ts index eb3d77dd..b94bc00c 100644 --- a/packages/uni-info-watcher/src/utils/interval-updates.ts +++ b/packages/uni-info-watcher/src/utils/interval-updates.ts @@ -72,7 +72,7 @@ export const updatePoolDayData = async (db: Database, dbTx: QueryRunner, event: poolDayData = new PoolDayData(); poolDayData.id = dayPoolID; poolDayData.date = dayStartTimestamp; - poolDayData.pool = pool; + poolDayData.pool = pool.id; poolDayData.open = pool.token0Price; poolDayData.high = pool.token0Price; poolDayData.low = pool.token0Price; @@ -120,7 +120,7 @@ export const updatePoolHourData = async (db: Database, dbTx: QueryRunner, event: poolHourData = new PoolHourData(); poolHourData.id = hourPoolID; poolHourData.periodStartUnix = hourStartUnix; - poolHourData.pool = pool; + poolHourData.pool = pool.id; poolHourData.open = pool.token0Price; poolHourData.high = pool.token0Price; poolHourData.low = pool.token0Price; @@ -169,7 +169,7 @@ export const updateTokenDayData = async (db: Database, dbTx: QueryRunner, token: tokenDayData = new TokenDayData(); tokenDayData.id = tokenDayID; tokenDayData.date = dayStartTimestamp; - tokenDayData.token = token; + tokenDayData.token = token.id; tokenDayData.open = tokenPrice; tokenDayData.high = tokenPrice; tokenDayData.low = tokenPrice; @@ -213,7 +213,7 @@ export const updateTokenHourData = async (db: Database, dbTx: QueryRunner, token tokenHourData = new TokenHourData(); tokenHourData.id = tokenHourID; tokenHourData.periodStartUnix = hourStartUnix; - tokenHourData.token = token; + tokenHourData.token = token.id; tokenHourData.open = tokenPrice; tokenHourData.high = tokenPrice; tokenHourData.low = tokenPrice; @@ -255,7 +255,7 @@ export const updateTickDayData = async (db: Database, dbTx: QueryRunner, tick: T tickDayData.id = tickDayDataID; tickDayData.date = dayStartTimestamp; tickDayData.pool = tick.pool; - tickDayData.tick = tick; + tickDayData.tick = tick.id; } tickDayData.liquidityGross = tick.liquidityGross; diff --git a/packages/uni-info-watcher/src/utils/pricing.ts b/packages/uni-info-watcher/src/utils/pricing.ts index f8bef249..33fe91cd 100644 --- a/packages/uni-info-watcher/src/utils/pricing.ts +++ b/packages/uni-info-watcher/src/utils/pricing.ts @@ -97,14 +97,14 @@ export const findEthPerToken = async (db: Database, dbTx: QueryRunner, token: To let priceSoFar = new GraphDecimal(0); for (let i = 0; i < whiteList.length; ++i) { - const poolAddress = whiteList[i].id; + const poolAddress = whiteList[i]; const pool = await db.getPool(dbTx, { id: poolAddress, blockHash: block.hash }); assert(pool); if (BigNumber.from(pool.liquidity).gt(0)) { - if (pool.token0.id === token.id) { + if (pool.token0 === token.id) { // whitelist token is token1 - const token1 = await db.getToken(dbTx, { id: pool.token1.id, blockHash: block.hash }); + const token1 = await db.getToken(dbTx, { id: pool.token1, blockHash: block.hash }); assert(token1); // get the derived ETH in pool @@ -116,8 +116,8 @@ export const findEthPerToken = async (db: Database, dbTx: QueryRunner, token: To priceSoFar = pool.token1Price.times(token1.derivedETH); } } - if (pool.token1.id === token.id) { - const token0 = await db.getToken(dbTx, { id: pool.token0.id, blockHash: block.hash }); + if (pool.token1 === token.id) { + const token0 = await db.getToken(dbTx, { id: pool.token0, blockHash: block.hash }); assert(token0); // get the derived ETH in pool diff --git a/packages/uni-info-watcher/src/utils/tick.ts b/packages/uni-info-watcher/src/utils/tick.ts index 93288250..d5fc319e 100644 --- a/packages/uni-info-watcher/src/utils/tick.ts +++ b/packages/uni-info-watcher/src/utils/tick.ts @@ -16,7 +16,7 @@ export const createTick = async (db: Database, dbTx: QueryRunner, tickId: string const tick = new Tick(); tick.id = tickId; tick.tickIdx = tickIdx; - tick.pool = pool; + tick.pool = pool.id; tick.poolAddress = pool.id; // 1.0001^tick is token1/token0. diff --git a/packages/uni-info-watcher/test/init.ts b/packages/uni-info-watcher/test/init.ts new file mode 100644 index 00000000..72d77ce6 --- /dev/null +++ b/packages/uni-info-watcher/test/init.ts @@ -0,0 +1,83 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import assert from 'assert'; + +import { + getConfig, + getResetConfig, + JobQueue +} from '@vulcanize/util'; +import { Client as ERC20Client } from '@vulcanize/erc20-watcher'; +import { Client as UniClient } from '@vulcanize/uni-watcher'; + +import { Database } from '../src/database'; +import { Indexer } from '../src/indexer'; + +const CONFIG_FILE = './environments/test.toml'; + +const watchContract = async (indexer: Indexer, address: string, kind: string): Promise => { + const watchedContract = indexer.isWatchedContract(address); + + if (!watchedContract) { + await indexer.watchContract(address, kind, 100); + } +}; + +const main = async () => { + // Get config. + const config = await getConfig(CONFIG_FILE); + + const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: { mode } } = config; + + assert(dbConfig, 'Missing dbConfig.'); + // Initialize database. + const db = new Database(dbConfig); + await db.init(); + + // Initialize uniClient. + assert(upstream, 'Missing upstream config'); + const { uniWatcher: { gqlEndpoint, gqlSubscriptionEndpoint }, tokenWatcher } = upstream; + const uniClient = new UniClient({ + gqlEndpoint, + gqlSubscriptionEndpoint + }); + + const erc20Client = new ERC20Client(tokenWatcher); + + const { ethClient, postgraphileClient, ethProvider } = await getResetConfig(config); + + assert(jobQueueConfig, 'Missing job queue config'); + const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; + assert(dbConnectionString, 'Missing job queue db connection string'); + + const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); + await jobQueue.start(); + + const indexer = new Indexer(db, uniClient, erc20Client, ethClient, postgraphileClient, ethProvider, jobQueue, mode); + await indexer.init(); + + // Get the factory contract address. + const factoryContract = await uniClient.getContract('factory'); + assert(factoryContract !== null, 'Factory contract not available'); + + // Get the NFPM contract address. + const nfpmContract = await uniClient.getContract('nfpm'); + assert(nfpmContract !== null, 'NFPM contract not available'); + + // Watch factory contract. + await watchContract(indexer, factoryContract.address, 'factory'); + // Watch NFPM contract. + await watchContract(indexer, nfpmContract.address, 'nfpm'); + + // Closing the database. + await db.close(); +}; + +main() + .then(() => process.exit(0)) + .catch(error => { + console.error(error); + process.exit(1); + }); diff --git a/packages/util/package.json b/packages/util/package.json index 9701795f..0588ed62 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -40,6 +40,8 @@ "lint": "eslint .", "build": "tsc", "build:contracts": "hardhat compile", - "estimate-event-counts": "ts-node src/estimate-event-counts.ts" + "estimate-event-counts": "ts-node src/estimate-event-counts.ts", + "check-config": "DEBUG=vulcanize:* node --enable-source-maps dist/src/cli/check-config.js", + "check-config:dev": "DEBUG=vulcanize:* ts-node src/cli/check-config.ts" } } diff --git a/packages/util/src/cli/check-config.ts b/packages/util/src/cli/check-config.ts new file mode 100644 index 00000000..c32fccb3 --- /dev/null +++ b/packages/util/src/cli/check-config.ts @@ -0,0 +1,56 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import debug from 'debug'; +import yargs from 'yargs'; +import assert from 'assert'; +import { EthClient } from '@vulcanize/ipld-eth-client'; + +import { getConfig } from '../config'; +import { getCustomProvider } from '../misc'; + +const log = debug('vulcanize:check-config'); + +const main = async () => { + const argv = await yargs.options({ + configFile: { + alias: 'f', + type: 'string', + require: true, + demandOption: true, + describe: 'configuration file path (toml)' + } + }).argv; + + const { upstream: { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint } } } = await getConfig(argv.configFile); + + // Get latest block in chain using ipld-eth-server GQL. + log(`Checking ipld-eth-server GQL endpoint ${gqlApiEndpoint}`); + const ethClient = new EthClient({ gqlEndpoint: gqlApiEndpoint, cache: undefined }); + const { block: currentBlock } = await ethClient.getBlockByHash(); + assert(currentBlock && currentBlock.number); + log('ipld-eth-server GQL endpoint working'); + + // Get block by number using postgraphile. + log(`Checking postgraphile GQL endpoint ${gqlPostgraphileEndpoint}`); + const postgraphileClient = new EthClient({ gqlEndpoint: gqlPostgraphileEndpoint, cache: undefined }); + const { allEthHeaderCids: { nodes } } = await postgraphileClient.getBlocks({ blockNumber: currentBlock.number }); + assert(nodes.length); + const [{ blockHash }] = nodes; + assert(blockHash === currentBlock.hash); + log('postgraphile GQL endpoint working'); + + // Get block by hash using RPC endpoint. + log(`Checking RPC endpoint ${rpcProviderEndpoint}`); + const ethProvider = getCustomProvider(rpcProviderEndpoint); + const ethBlock = await ethProvider.getBlock(blockHash); + assert(ethBlock.number === currentBlock.number); + log('RPC endpoint working'); +}; + +main().then(() => { + process.exit(); +}).catch(error => { + log(error); +}); diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index bebdcba8..a32128ce 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -62,7 +62,7 @@ export interface Where { }] } -export type Relation = { entity: any, type: RelationType, property: string, field: string, childRelations?: Relation[] } +export type Relation = { entity: any, type: RelationType, field: string, foreignKey?: string, childRelations?: Relation[] } export class Database { _config: ConnectionOptions @@ -263,16 +263,10 @@ export class Database { } } - async removeEntities (queryRunner: QueryRunner, entity: new () => Entity, findConditions?: FindConditions): Promise { + async removeEntities (queryRunner: QueryRunner, entity: new () => Entity, findConditions: FindConditions = {}): Promise { const repo = queryRunner.manager.getRepository(entity); - if (findConditions) { - await repo.delete(findConditions); - - return; - } - - await repo.clear(); + await repo.delete(findConditions); } async getAncestorAtDepth (blockHash: string, depth: number): Promise { @@ -402,36 +396,6 @@ export class Database { return []; } - const manyToManyRelations = relations.filter(relation => relation.type === 'many-to-many'); - - if (manyToManyRelations.length) { - // Load entities separately for many to many relations as limit does not work with relation joins. - let relationQueryBuilder = repo.createQueryBuilder(tableName) - .whereInIds( - entities.map( - (entity: any) => ({ id: entity.id, blockHash: entity.blockHash }) - ) - ); - - if (queryOptions.orderBy) { - relationQueryBuilder = this._orderQuery(repo, relationQueryBuilder, queryOptions); - } - - relationQueryBuilder = this._orderQuery(repo, relationQueryBuilder, { ...queryOptions, orderBy: 'id' }); - - // Load entity ids for many to many relations. - manyToManyRelations.forEach(relation => { - const { property } = relation; - - relationQueryBuilder = relationQueryBuilder.leftJoinAndSelect(`${relationQueryBuilder.alias}.${property}`, property) - .addOrderBy(`${property}.id`); - - // TODO: Get only ids from many to many join table instead of joining with related entity table. - }); - - entities = await relationQueryBuilder.getMany(); - } - entities = await this.loadRelations(queryRunner, block, relations, entities); return entities; @@ -439,12 +403,14 @@ export class Database { async loadRelations (queryRunner: QueryRunner, block: BlockHeight, relations: Relation[], entities: Entity[]): Promise { const relationPromises = relations.map(async relation => { - const { entity: relationEntity, type, property, field, childRelations = [] } = relation; + const { entity: relationEntity, type, field, foreignKey, childRelations = [] } = relation; switch (type) { case 'one-to-many': { + assert(foreignKey); + const where: Where = { - [field]: [{ + [foreignKey]: [{ value: entities.map((entity: any) => entity.id), not: false, operator: 'in' @@ -461,20 +427,23 @@ export class Database { ); const relatedEntitiesMap = relatedEntities.reduce((acc: {[key:string]: any[]}, entity: any) => { - if (!acc[entity[field]]) { - acc[entity[field]] = []; + // Related entity might be loaded with data. + const parentEntityId = entity[foreignKey].id ?? entity[foreignKey]; + + if (!acc[parentEntityId]) { + acc[parentEntityId] = []; } - acc[entity[field]].push(entity); + acc[parentEntityId].push(entity); return acc; }, {}); entities.forEach((entity: any) => { if (relatedEntitiesMap[entity.id]) { - entity[property] = relatedEntitiesMap[entity.id]; + entity[field] = relatedEntitiesMap[entity.id]; } else { - entity[property] = []; + entity[field] = []; } }); @@ -483,7 +452,7 @@ export class Database { case 'many-to-many': { const relatedIds = entities.reduce((acc, entity: any) => { - entity[property].forEach((relatedEntity: any) => acc.add(relatedEntity.id)); + entity[field].forEach((relatedEntityId: any) => acc.add(relatedEntityId)); return acc; }, new Set()); @@ -512,10 +481,10 @@ export class Database { }, {}); entities.forEach((entity: any) => { - const relatedField = entity[property] as any[]; + const relatedField = entity[field] as any[]; - relatedField.forEach((relatedEntity, index) => { - relatedField[index] = relatedEntitiesMap[relatedEntity.id]; + relatedField.forEach((relatedEntityId, index) => { + relatedField[index] = relatedEntitiesMap[relatedEntityId]; }); }); @@ -549,7 +518,7 @@ export class Database { entities.forEach((entity: any) => { if (relatedEntitiesMap[entity[field]]) { - entity[property] = relatedEntitiesMap[entity[field]]; + entity[field] = relatedEntitiesMap[entity[field]]; } }); diff --git a/packages/util/src/job-queue.ts b/packages/util/src/job-queue.ts index 947b3d27..1b44237c 100644 --- a/packages/util/src/job-queue.ts +++ b/packages/util/src/job-queue.ts @@ -35,9 +35,11 @@ export class JobQueue { retryBackoff: true, // Time before active job fails by expiration. - expireInHours: 24 * 7, // 7 days + expireInHours: 24 * 1, // 1 day - retentionDays: 30, // 30 days + retentionHours: 4, // 4 hours + + deleteAfterHours: 1, // 1 hour newJobCheckInterval: 100 });