From 51e876066ec84c6cd45b4109576de754a338eb7e Mon Sep 17 00:00:00 2001 From: prathamesh Date: Mon, 4 Oct 2021 11:53:13 +0530 Subject: [PATCH 01/21] Add ipld-blocks entity generation --- .../codegen/src/data/entities/Blocks.yaml | 38 +++++++++++++++++++ packages/codegen/src/entity.ts | 7 +++- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 packages/codegen/src/data/entities/Blocks.yaml diff --git a/packages/codegen/src/data/entities/Blocks.yaml b/packages/codegen/src/data/entities/Blocks.yaml new file mode 100644 index 00000000..0e218ad4 --- /dev/null +++ b/packages/codegen/src/data/entities/Blocks.yaml @@ -0,0 +1,38 @@ +className: Blocks +indexOn: + - columns: + - blockHash + - contractAddress + unique: true +columns: + - name: block + tsType: BlockProgress + columnType: ManyToOne + lhs: () + rhs: BlockProgress + - name: contractAddress + pgType: varchar + tsType: string + columnType: Column + columnOptions: + - option: length + value: 42 + - name: cid + pgType: varchar + tsType: string + columnType: Column + - name: data + pgType: text + tsType: string + columnType: Column +imports: + - toImport: + - Entity + - PrimaryGeneratedColumn + - Column + - Index + - ManyToOne + from: typeorm + - toImport: + - BlockProgress + from: ./BlockProgress diff --git a/packages/codegen/src/entity.ts b/packages/codegen/src/entity.ts index 4b91a764..84467a3b 100644 --- a/packages/codegen/src/entity.ts +++ b/packages/codegen/src/entity.ts @@ -37,7 +37,6 @@ export class Entity { } const entityObject: any = { - // Capitalize the first letter of name. className: '', indexOn: [], columns: [], @@ -188,6 +187,7 @@ export class Entity { this._addSyncStatusEntity(); this._addContractEntity(); this._addBlockProgressEntity(); + this._addBlocksEntity(); const template = Handlebars.compile(this._templateString); this._entities.forEach(entityObj => { @@ -218,4 +218,9 @@ export class Entity { const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'BlockProgress.yaml'), 'utf8')); this._entities.push(entity); } + + _addBlocksEntity (): void { + const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'Blocks.yaml'), 'utf8')); + this._entities.push(entity); + } } From 0877ed29225a4e04d2025c5ea9ede612f6697758 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Mon, 4 Oct 2021 18:01:27 +0530 Subject: [PATCH 02/21] Populate ipld-blocks table --- packages/codegen/package.json | 1 + packages/codegen/src/blocks-hook.ts | 21 +++++ .../codegen/src/data/entities/Blocks.yaml | 3 +- packages/codegen/src/generate-code.ts | 6 ++ .../templates/blocks-hook-template.handlebars | 91 +++++++++++++++++++ .../templates/database-template.handlebars | 7 ++ .../src/templates/events-template.handlebars | 12 ++- .../src/templates/indexer-template.handlebars | 4 + .../src/templates/package-template.handlebars | 3 + 9 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 packages/codegen/src/blocks-hook.ts create mode 100644 packages/codegen/src/templates/blocks-hook-template.handlebars diff --git a/packages/codegen/package.json b/packages/codegen/package.json index 19e757c5..283c9ee9 100644 --- a/packages/codegen/package.json +++ b/packages/codegen/package.json @@ -21,6 +21,7 @@ "dependencies": { "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git", "@solidity-parser/parser": "^0.13.2", + "@vulcanize/util": "^0.1.0", "gql-generator": "https://github.com/vulcanize/gql-generator.git", "graphql": "^15.5.0", "graphql-compose": "^9.0.3", diff --git a/packages/codegen/src/blocks-hook.ts b/packages/codegen/src/blocks-hook.ts new file mode 100644 index 00000000..25b9e726 --- /dev/null +++ b/packages/codegen/src/blocks-hook.ts @@ -0,0 +1,21 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import fs from 'fs'; +import path from 'path'; +import Handlebars from 'handlebars'; +import { Writable } from 'stream'; + +const TEMPLATE_FILE = './templates/blocks-hook-template.handlebars'; + +/** + * Writes the blocks-hook.ts file generated from a template to a stream. + * @param outStream A writable output stream to write the blocks-hook.ts file to. + */ +export function exportBlocksHook (outStream: Writable): void { + const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); + const template = Handlebars.compile(templateString); + const blocksHook = template({}); + outStream.write(blocksHook); +} diff --git a/packages/codegen/src/data/entities/Blocks.yaml b/packages/codegen/src/data/entities/Blocks.yaml index 0e218ad4..8330bdc7 100644 --- a/packages/codegen/src/data/entities/Blocks.yaml +++ b/packages/codegen/src/data/entities/Blocks.yaml @@ -1,9 +1,8 @@ className: Blocks indexOn: - columns: - - blockHash + - block - contractAddress - unique: true columns: - name: block tsType: BlockProgress diff --git a/packages/codegen/src/generate-code.ts b/packages/codegen/src/generate-code.ts index b3fed713..6eff250a 100644 --- a/packages/codegen/src/generate-code.ts +++ b/packages/codegen/src/generate-code.ts @@ -27,6 +27,7 @@ import { exportLint } from './lint'; import { registerHandlebarHelpers } from './utils/handlebar-helpers'; import { exportHooks } from './hooks'; import { exportFill } from './fill'; +import { exportBlocksHook } from './blocks-hook'; const main = async (): Promise => { const argv = await yargs(hideBin(process.argv)) @@ -239,6 +240,11 @@ function generateWatcher (data: string, visitor: Visitor, argv: any) { ? fs.createWriteStream(path.join(outputDir, 'src/client.ts')) : process.stdout; visitor.exportClient(outStream, schemaContent, path.join(outputDir, 'src/gql')); + + outStream = outputDir + ? fs.createWriteStream(path.join(outputDir, 'src/blocks-hook.ts')) + : process.stdout; + exportBlocksHook(outStream); } main().catch(err => { diff --git a/packages/codegen/src/templates/blocks-hook-template.handlebars b/packages/codegen/src/templates/blocks-hook-template.handlebars new file mode 100644 index 00000000..292f9720 --- /dev/null +++ b/packages/codegen/src/templates/blocks-hook-template.handlebars @@ -0,0 +1,91 @@ +import { encode, code as codecCode } from '@ipld/dag-json'; +import { sha256 } from 'multiformats/hashes/sha2'; +import { CID } from 'multiformats/cid'; + +import { BlockProgressInterface } from '@vulcanize/util'; + +import { Indexer } from './indexer'; + +export async function blockProcessingCompleteHandler (indexer: Indexer, jobData: any): Promise { + // Get events for current block and make an entry of updated values in an IPLD Block. + + const { blockHash } = jobData; + const events = await indexer.getBlockEvents(blockHash); + + if (events) { + events.forEach(async (event) => { + const contractAddress = event.contract; + + const ipldBlockData: any = {}; + + const eventData = indexer.getResultEvent(event); + + switch (event.eventName) { + case 'Transfer': { + const { from, to } = eventData.event; + + const fromBalance = await indexer.balanceOf(blockHash, contractAddress, from); + const toBalance = await indexer.balanceOf(blockHash, contractAddress, to); + + // { + // "_balances": { + // "0xCA6D29232D1435D8198E3E5302495417dD073d61": { + // "value": 100 + // }, + // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": { + // "value": 999999999999999999900 + // } + // } + // } + ipldBlockData._balances = {}; + ipldBlockData._balances[from] = fromBalance; + ipldBlockData._balances[to] = toBalance; + break; + } + + case 'Approval': { + const { owner, spender } = eventData.event; + const allowance = await indexer.allowance(blockHash, contractAddress, owner, spender); + + // { + // "_allowances": { + // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": { + // "0xCA6D29232D1435D8198E3E5302495417dD073d61": { + // "value": 10 + // } + // } + // } + // } + ipldBlockData._allowances = {}; + ipldBlockData._allowances[owner] = {}; + ipldBlockData._allowances[owner][spender] = allowance; + break; + } + } + + const blockProgress = event.block; + if (blockProgress) { + const ipldBlock = await prepareIPLDBlock(blockProgress, contractAddress, ipldBlockData); + await indexer.saveIPLDBlock(ipldBlock); + } + }); + } +} + +async function prepareIPLDBlock (blockProgress: BlockProgressInterface, contractAddress: string, data: any):Promise { + // Encoding the data using dag-json codec. + const bytes = encode(data); + + // Calculating sha256 (multi)hash of the encoded data. + const hash = await sha256.digest(bytes); + + // Calculating the CID: v1, code: dag-json, hash. + const cid = CID.create(1, codecCode, hash); + + return { + block: blockProgress, + contractAddress, + cid: cid.toString(), + data: bytes + }; +} diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 31e5f2da..4d69361c 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -12,6 +12,7 @@ import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; import { BlockProgress } from './entity/BlockProgress'; +import { Blocks } from './entity/Blocks'; {{#each queries as | query |}} import { {{query.entityName}} } from './entity/{{query.entityName}}'; @@ -78,6 +79,12 @@ export class Database { } {{/each}} + async saveIPLDBlock (ipldBlock: DeepPartial): Promise { + const repo = this._conn.getRepository(Blocks); + const entity = repo.create(ipldBlock); + return repo.save(entity); + } + async getContract (address: string): Promise { const repo = this._conn.getRepository(Contract); diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index c6b5d751..ea4e2457 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -12,11 +12,13 @@ import { EventWatcher as BaseEventWatcher, QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, - UNKNOWN_EVENT_NAME + UNKNOWN_EVENT_NAME, + JOB_KIND_INDEX } from '@vulcanize/util'; import { Indexer } from './indexer'; import { Event } from './entity/Event'; +import { blockProcessingCompleteHandler } from './blocks-hook'; const EVENT = 'event'; @@ -71,6 +73,14 @@ export class EventWatcher { async initBlockProcessingOnCompleteHandler (): Promise { this._jobQueue.onComplete(QUEUE_BLOCK_PROCESSING, async (job) => { await this._baseEventWatcher.blockProcessingCompleteHandler(job); + + // Call custom hook function to update Blocks table. + const { data: { request: { data } } } = job; + const { kind } = data; + + if (kind === JOB_KIND_INDEX) { + await blockProcessingCompleteHandler(this._indexer, data); + } }); } diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 6eeedb20..43b34007 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -169,6 +169,10 @@ export class Indexer { } {{/each}} + async saveIPLDBlock (ipldBlock: any):Promise { + await this._db.saveIPLDBlock(ipldBlock); + } + async triggerIndexingOnEvent (event: Event): Promise { const resultEvent = this.getResultEvent(event); diff --git a/packages/codegen/src/templates/package-template.handlebars b/packages/codegen/src/templates/package-template.handlebars index 1358b7de..600df981 100644 --- a/packages/codegen/src/templates/package-template.handlebars +++ b/packages/codegen/src/templates/package-template.handlebars @@ -23,7 +23,9 @@ }, "homepage": "https://github.com/vulcanize/watcher-ts#readme", "dependencies": { + "@apollo/client": "^3.3.19", "@ethersproject/providers": "5.3.0", + "@ipld/dag-json": "^8.0.1", "@vulcanize/cache": "^0.1.0", "@vulcanize/ipld-eth-client": "^0.1.0", "@vulcanize/solidity-mapper": "^0.1.0", @@ -36,6 +38,7 @@ "graphql": "^15.5.0", "graphql-import-node": "^0.0.4", "json-bigint": "^1.0.0", + "multiformats": "^9.4.8", "reflect-metadata": "^0.1.13", "typeorm": "^0.2.32", "yargs": "^17.0.1" From 1aa6d768a143b1d6fffd8bf2859e576c9f42dbc9 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Tue, 5 Oct 2021 09:42:07 +0530 Subject: [PATCH 03/21] Rename ipld-block entity and update after each event --- packages/codegen/src/blocks-hook.ts | 21 ---- .../entities/{Blocks.yaml => IPLDBlock.yaml} | 2 +- packages/codegen/src/entity.ts | 6 +- packages/codegen/src/generate-code.ts | 6 +- packages/codegen/src/ipld-hook.ts | 21 ++++ .../templates/blocks-hook-template.handlebars | 91 ---------------- .../templates/database-template.handlebars | 23 +++- .../src/templates/events-template.handlebars | 2 +- .../src/templates/indexer-template.handlebars | 9 +- .../templates/ipld-hook-template.handlebars | 103 ++++++++++++++++++ 10 files changed, 158 insertions(+), 126 deletions(-) delete mode 100644 packages/codegen/src/blocks-hook.ts rename packages/codegen/src/data/entities/{Blocks.yaml => IPLDBlock.yaml} (96%) create mode 100644 packages/codegen/src/ipld-hook.ts delete mode 100644 packages/codegen/src/templates/blocks-hook-template.handlebars create mode 100644 packages/codegen/src/templates/ipld-hook-template.handlebars diff --git a/packages/codegen/src/blocks-hook.ts b/packages/codegen/src/blocks-hook.ts deleted file mode 100644 index 25b9e726..00000000 --- a/packages/codegen/src/blocks-hook.ts +++ /dev/null @@ -1,21 +0,0 @@ -// -// Copyright 2021 Vulcanize, Inc. -// - -import fs from 'fs'; -import path from 'path'; -import Handlebars from 'handlebars'; -import { Writable } from 'stream'; - -const TEMPLATE_FILE = './templates/blocks-hook-template.handlebars'; - -/** - * Writes the blocks-hook.ts file generated from a template to a stream. - * @param outStream A writable output stream to write the blocks-hook.ts file to. - */ -export function exportBlocksHook (outStream: Writable): void { - const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); - const template = Handlebars.compile(templateString); - const blocksHook = template({}); - outStream.write(blocksHook); -} diff --git a/packages/codegen/src/data/entities/Blocks.yaml b/packages/codegen/src/data/entities/IPLDBlock.yaml similarity index 96% rename from packages/codegen/src/data/entities/Blocks.yaml rename to packages/codegen/src/data/entities/IPLDBlock.yaml index 8330bdc7..8879d9b4 100644 --- a/packages/codegen/src/data/entities/Blocks.yaml +++ b/packages/codegen/src/data/entities/IPLDBlock.yaml @@ -1,4 +1,4 @@ -className: Blocks +className: IPLDBlock indexOn: - columns: - block diff --git a/packages/codegen/src/entity.ts b/packages/codegen/src/entity.ts index 84467a3b..1d4bf753 100644 --- a/packages/codegen/src/entity.ts +++ b/packages/codegen/src/entity.ts @@ -187,7 +187,7 @@ export class Entity { this._addSyncStatusEntity(); this._addContractEntity(); this._addBlockProgressEntity(); - this._addBlocksEntity(); + this._addIPLDBlockEntity(); const template = Handlebars.compile(this._templateString); this._entities.forEach(entityObj => { @@ -219,8 +219,8 @@ export class Entity { this._entities.push(entity); } - _addBlocksEntity (): void { - const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'Blocks.yaml'), 'utf8')); + _addIPLDBlockEntity (): void { + const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'IPLDBlock.yaml'), 'utf8')); this._entities.push(entity); } } diff --git a/packages/codegen/src/generate-code.ts b/packages/codegen/src/generate-code.ts index 6eff250a..73d01807 100644 --- a/packages/codegen/src/generate-code.ts +++ b/packages/codegen/src/generate-code.ts @@ -27,7 +27,7 @@ import { exportLint } from './lint'; import { registerHandlebarHelpers } from './utils/handlebar-helpers'; import { exportHooks } from './hooks'; import { exportFill } from './fill'; -import { exportBlocksHook } from './blocks-hook'; +import { exportIpldHook } from './ipld-hook'; const main = async (): Promise => { const argv = await yargs(hideBin(process.argv)) @@ -242,9 +242,9 @@ function generateWatcher (data: string, visitor: Visitor, argv: any) { visitor.exportClient(outStream, schemaContent, path.join(outputDir, 'src/gql')); outStream = outputDir - ? fs.createWriteStream(path.join(outputDir, 'src/blocks-hook.ts')) + ? fs.createWriteStream(path.join(outputDir, 'src/ipld-hook.ts')) : process.stdout; - exportBlocksHook(outStream); + exportIpldHook(outStream); } main().catch(err => { diff --git a/packages/codegen/src/ipld-hook.ts b/packages/codegen/src/ipld-hook.ts new file mode 100644 index 00000000..b41aa071 --- /dev/null +++ b/packages/codegen/src/ipld-hook.ts @@ -0,0 +1,21 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import fs from 'fs'; +import path from 'path'; +import Handlebars from 'handlebars'; +import { Writable } from 'stream'; + +const TEMPLATE_FILE = './templates/ipld-hook-template.handlebars'; + +/** + * Writes the ipld-hook.ts file generated from a template to a stream. + * @param outStream A writable output stream to write the ipld-hook.ts file to. + */ +export function exportIpldHook (outStream: Writable): void { + const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); + const template = Handlebars.compile(templateString); + const ipldHook = template({}); + outStream.write(ipldHook); +} diff --git a/packages/codegen/src/templates/blocks-hook-template.handlebars b/packages/codegen/src/templates/blocks-hook-template.handlebars deleted file mode 100644 index 292f9720..00000000 --- a/packages/codegen/src/templates/blocks-hook-template.handlebars +++ /dev/null @@ -1,91 +0,0 @@ -import { encode, code as codecCode } from '@ipld/dag-json'; -import { sha256 } from 'multiformats/hashes/sha2'; -import { CID } from 'multiformats/cid'; - -import { BlockProgressInterface } from '@vulcanize/util'; - -import { Indexer } from './indexer'; - -export async function blockProcessingCompleteHandler (indexer: Indexer, jobData: any): Promise { - // Get events for current block and make an entry of updated values in an IPLD Block. - - const { blockHash } = jobData; - const events = await indexer.getBlockEvents(blockHash); - - if (events) { - events.forEach(async (event) => { - const contractAddress = event.contract; - - const ipldBlockData: any = {}; - - const eventData = indexer.getResultEvent(event); - - switch (event.eventName) { - case 'Transfer': { - const { from, to } = eventData.event; - - const fromBalance = await indexer.balanceOf(blockHash, contractAddress, from); - const toBalance = await indexer.balanceOf(blockHash, contractAddress, to); - - // { - // "_balances": { - // "0xCA6D29232D1435D8198E3E5302495417dD073d61": { - // "value": 100 - // }, - // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": { - // "value": 999999999999999999900 - // } - // } - // } - ipldBlockData._balances = {}; - ipldBlockData._balances[from] = fromBalance; - ipldBlockData._balances[to] = toBalance; - break; - } - - case 'Approval': { - const { owner, spender } = eventData.event; - const allowance = await indexer.allowance(blockHash, contractAddress, owner, spender); - - // { - // "_allowances": { - // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": { - // "0xCA6D29232D1435D8198E3E5302495417dD073d61": { - // "value": 10 - // } - // } - // } - // } - ipldBlockData._allowances = {}; - ipldBlockData._allowances[owner] = {}; - ipldBlockData._allowances[owner][spender] = allowance; - break; - } - } - - const blockProgress = event.block; - if (blockProgress) { - const ipldBlock = await prepareIPLDBlock(blockProgress, contractAddress, ipldBlockData); - await indexer.saveIPLDBlock(ipldBlock); - } - }); - } -} - -async function prepareIPLDBlock (blockProgress: BlockProgressInterface, contractAddress: string, data: any):Promise { - // Encoding the data using dag-json codec. - const bytes = encode(data); - - // Calculating sha256 (multi)hash of the encoded data. - const hash = await sha256.digest(bytes); - - // Calculating the CID: v1, code: dag-json, hash. - const cid = CID.create(1, codecCode, hash); - - return { - block: blockProgress, - contractAddress, - cid: cid.toString(), - data: bytes - }; -} diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 4d69361c..284796b2 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -12,7 +12,7 @@ import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { Blocks } from './entity/Blocks'; +import { IPLDBlock } from './entity/IPLDBlock'; {{#each queries as | query |}} import { {{query.entityName}} } from './entity/{{query.entityName}}'; @@ -79,9 +79,24 @@ export class Database { } {{/each}} - async saveIPLDBlock (ipldBlock: DeepPartial): Promise { - const repo = this._conn.getRepository(Blocks); - const entity = repo.create(ipldBlock); + async getIPLDBlock (block: BlockProgress, contractAddress: string): Promise { + const repo = this._conn.getRepository(IPLDBlock); + return repo.findOne({ block, contractAddress }); + } + + async saveOrUpdateIPLDBlock (ipldBlock: DeepPartial): Promise { + const repo = this._conn.getRepository(IPLDBlock); + const oldIPLDBlock = await repo.findOne({ block: ipldBlock.block, contractAddress: ipldBlock.contractAddress }); + + let entity: IPLDBlock; + if (oldIPLDBlock) { + entity = oldIPLDBlock; + entity.cid = ipldBlock.cid || ''; + entity.data = ipldBlock.data || ''; + } else { + entity = repo.create(ipldBlock); + } + return repo.save(entity); } diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index ea4e2457..721beb18 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -18,7 +18,7 @@ import { import { Indexer } from './indexer'; import { Event } from './entity/Event'; -import { blockProcessingCompleteHandler } from './blocks-hook'; +import { blockProcessingCompleteHandler } from './ipld-hook'; const EVENT = 'event'; diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 43b34007..352c3f9a 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -19,6 +19,7 @@ import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; import { BlockProgress } from './entity/BlockProgress'; +import { IPLDBlock } from './entity/IPLDBlock'; import artifacts from './artifacts/{{inputFileName}}.json'; import { handleEvent } from './hooks'; @@ -169,8 +170,12 @@ export class Indexer { } {{/each}} - async saveIPLDBlock (ipldBlock: any):Promise { - await this._db.saveIPLDBlock(ipldBlock); + async getIPLDBlock (block: BlockProgress, contractAddress: string): Promise { + return this._db.getIPLDBlock(block, contractAddress); + } + + async saveOrUpdateIPLDBlock (ipldBlock: DeepPartial):Promise { + await this._db.saveOrUpdateIPLDBlock(ipldBlock); } async triggerIndexingOnEvent (event: Event): Promise { diff --git a/packages/codegen/src/templates/ipld-hook-template.handlebars b/packages/codegen/src/templates/ipld-hook-template.handlebars new file mode 100644 index 00000000..4e060464 --- /dev/null +++ b/packages/codegen/src/templates/ipld-hook-template.handlebars @@ -0,0 +1,103 @@ +import * as codec from '@ipld/dag-json'; +import { sha256 } from 'multiformats/hashes/sha2'; +import { CID } from 'multiformats/cid'; + +import { BlockProgressInterface } from '@vulcanize/util'; + +import { Indexer } from './indexer'; +import { IPLDBlock } from './entity/IPLDBlock'; + +export async function blockProcessingCompleteHandler (indexer: Indexer, jobData: any): Promise { + // Get events for current block and make an entry of updated values in IPLDBlock. + + const { blockHash } = jobData; + const events = await indexer.getBlockEvents(blockHash); + + if (!events) return; + + const ipldUpdates = events.map(async (event): Promise => { + const block = event.block; + const contractAddress = event.contract; + + const oldIpldBlock = await indexer.getIPLDBlock(block, contractAddress); + + const ipldBlockData: any = {}; + const eventData = indexer.getResultEvent(event); + + switch (event.eventName) { + case 'Transfer': { + const { from, to } = eventData.event; + + const fromBalance = await indexer.balanceOf(blockHash, contractAddress, from); + const toBalance = await indexer.balanceOf(blockHash, contractAddress, to); + + // { + // "_balances": { + // "0xCA6D29232D1435D8198E3E5302495417dD073d61": { + // "value": 100 + // }, + // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": { + // "value": 999999999999999999900 + // } + // } + // } + ipldBlockData._balances = {}; + ipldBlockData._balances[from] = fromBalance; + ipldBlockData._balances[to] = toBalance; + + break; + } + + case 'Approval': { + const { owner, spender } = eventData.event; + const allowance = await indexer.allowance(blockHash, contractAddress, owner, spender); + + // { + // "_allowances": { + // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": { + // "0xCA6D29232D1435D8198E3E5302495417dD073d61": { + // "value": 10 + // } + // } + // } + // } + ipldBlockData._allowances = {}; + ipldBlockData._allowances[owner] = {}; + ipldBlockData._allowances[owner][spender] = allowance; + + break; + } + } + + if (block) { + const ipldBlock = await prepareIPLDBlock(block, contractAddress, ipldBlockData, oldIpldBlock); + await indexer.saveOrUpdateIPLDBlock(ipldBlock); + } + }); + + await Promise.all(ipldUpdates); +} + +async function prepareIPLDBlock (block: BlockProgressInterface, contractAddress: string, data: any, oldIpldBlock?: IPLDBlock):Promise { + // If an IPLDBlock for { block, contractAddress } already exists, update the data field. + if (oldIpldBlock) { + const oldData = codec.decode(Buffer.from(oldIpldBlock.data)); + data = Object.assign(oldData, data); + } + + // Encoding the data using dag-json codec. + const bytes = codec.encode(data); + + // Calculating sha256 (multi)hash of the encoded data. + const hash = await sha256.digest(bytes); + + // Calculating the CID: v1, code: dag-json, hash. + const cid = CID.create(1, codec.code, hash); + + return { + block, + contractAddress, + cid: cid.toString(), + data: bytes + }; +} From af4c6d4eaf02f191e776209796b192b15f756981 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Tue, 5 Oct 2021 15:17:29 +0530 Subject: [PATCH 04/21] Move ipld-hook to hooks.ts --- packages/codegen/src/generate-code.ts | 11 +- packages/codegen/src/hooks.ts | 13 +- packages/codegen/src/ipld-hook.ts | 21 --- .../templates/database-template.handlebars | 15 +- .../src/templates/events-template.handlebars | 4 +- .../hooks-example-template.handlebars | 51 ------- .../src/templates/hooks-template.handlebars | 131 ++++++++++++++++++ .../src/templates/indexer-template.handlebars | 6 +- .../templates/ipld-hook-template.handlebars | 103 -------------- .../src/templates/package-template.handlebars | 1 + .../templates/resolvers-template.handlebars | 4 +- packages/util/src/indexer.ts | 2 +- 12 files changed, 145 insertions(+), 217 deletions(-) delete mode 100644 packages/codegen/src/ipld-hook.ts delete mode 100644 packages/codegen/src/templates/hooks-example-template.handlebars delete mode 100644 packages/codegen/src/templates/ipld-hook-template.handlebars diff --git a/packages/codegen/src/generate-code.ts b/packages/codegen/src/generate-code.ts index 73d01807..42da717e 100644 --- a/packages/codegen/src/generate-code.ts +++ b/packages/codegen/src/generate-code.ts @@ -27,7 +27,6 @@ import { exportLint } from './lint'; import { registerHandlebarHelpers } from './utils/handlebar-helpers'; import { exportHooks } from './hooks'; import { exportFill } from './fill'; -import { exportIpldHook } from './ipld-hook'; const main = async (): Promise => { const argv = await yargs(hideBin(process.argv)) @@ -210,15 +209,12 @@ function generateWatcher (data: string, visitor: Visitor, argv: any) { exportWatchContract(outStream); let hooksOutStream; - let exampleOutStream; if (outputDir) { hooksOutStream = fs.createWriteStream(path.join(outputDir, 'src/hooks.ts')); - exampleOutStream = fs.createWriteStream(path.join(outputDir, 'src/hooks.example.ts')); } else { hooksOutStream = process.stdout; - exampleOutStream = process.stdout; } - exportHooks(hooksOutStream, exampleOutStream); + exportHooks(hooksOutStream); outStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/fill.ts')) @@ -240,11 +236,6 @@ function generateWatcher (data: string, visitor: Visitor, argv: any) { ? fs.createWriteStream(path.join(outputDir, 'src/client.ts')) : process.stdout; visitor.exportClient(outStream, schemaContent, path.join(outputDir, 'src/gql')); - - outStream = outputDir - ? fs.createWriteStream(path.join(outputDir, 'src/ipld-hook.ts')) - : process.stdout; - exportIpldHook(outStream); } main().catch(err => { diff --git a/packages/codegen/src/hooks.ts b/packages/codegen/src/hooks.ts index 6c2783be..7b0aba04 100644 --- a/packages/codegen/src/hooks.ts +++ b/packages/codegen/src/hooks.ts @@ -8,23 +8,14 @@ import Handlebars from 'handlebars'; import { Writable } from 'stream'; const HOOKS_TEMPLATE_FILE = './templates/hooks-template.handlebars'; -const EXAMPLE_TEMPLATE_FILE = './templates/hooks-example-template.handlebars'; /** - * Writes the hooks and hooks.example files generated from templates to a stream. + * Writes the hooks file generated from template to a stream. * @param outStream A writable output stream to write the hooks file to. - * @param exampleOutStream A writable output stream to write the hooks.example file to. */ -export function exportHooks (hooksOutStream: Writable, exampleOutStream: Writable): void { +export function exportHooks (hooksOutStream: Writable): void { const hooksTemplateString = fs.readFileSync(path.resolve(__dirname, HOOKS_TEMPLATE_FILE)).toString(); - const exampleTemplateString = fs.readFileSync(path.resolve(__dirname, EXAMPLE_TEMPLATE_FILE)).toString(); - const hooksTemplate = Handlebars.compile(hooksTemplateString); - const exampleTemplate = Handlebars.compile(exampleTemplateString); - const hooks = hooksTemplate({}); - const example = exampleTemplate({}); - hooksOutStream.write(hooks); - exampleOutStream.write(example); } diff --git a/packages/codegen/src/ipld-hook.ts b/packages/codegen/src/ipld-hook.ts deleted file mode 100644 index b41aa071..00000000 --- a/packages/codegen/src/ipld-hook.ts +++ /dev/null @@ -1,21 +0,0 @@ -// -// Copyright 2021 Vulcanize, Inc. -// - -import fs from 'fs'; -import path from 'path'; -import Handlebars from 'handlebars'; -import { Writable } from 'stream'; - -const TEMPLATE_FILE = './templates/ipld-hook-template.handlebars'; - -/** - * Writes the ipld-hook.ts file generated from a template to a stream. - * @param outStream A writable output stream to write the ipld-hook.ts file to. - */ -export function exportIpldHook (outStream: Writable): void { - const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); - const template = Handlebars.compile(templateString); - const ipldHook = template({}); - outStream.write(ipldHook); -} diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 284796b2..8b62a6b7 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -84,20 +84,9 @@ export class Database { return repo.findOne({ block, contractAddress }); } - async saveOrUpdateIPLDBlock (ipldBlock: DeepPartial): Promise { + async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { const repo = this._conn.getRepository(IPLDBlock); - const oldIPLDBlock = await repo.findOne({ block: ipldBlock.block, contractAddress: ipldBlock.contractAddress }); - - let entity: IPLDBlock; - if (oldIPLDBlock) { - entity = oldIPLDBlock; - entity.cid = ipldBlock.cid || ''; - entity.data = ipldBlock.data || ''; - } else { - entity = repo.create(ipldBlock); - } - - return repo.save(entity); + return repo.save(ipldBlock); } async getContract (address: string): Promise { diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index 721beb18..67de7b75 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -18,7 +18,7 @@ import { import { Indexer } from './indexer'; import { Event } from './entity/Event'; -import { blockProcessingCompleteHandler } from './ipld-hook'; +import { handleBlock } from './hooks'; const EVENT = 'event'; @@ -79,7 +79,7 @@ export class EventWatcher { const { kind } = data; if (kind === JOB_KIND_INDEX) { - await blockProcessingCompleteHandler(this._indexer, data); + await handleBlock(this._indexer, data); } }); } diff --git a/packages/codegen/src/templates/hooks-example-template.handlebars b/packages/codegen/src/templates/hooks-example-template.handlebars deleted file mode 100644 index 04b5ebf2..00000000 --- a/packages/codegen/src/templates/hooks-example-template.handlebars +++ /dev/null @@ -1,51 +0,0 @@ -// -// Copyright 2021 Vulcanize, Inc. -// - -import assert from 'assert'; - -import { Indexer, ResultEvent } from './indexer'; - -/** - * Event hook function. - * @param indexer Indexer instance that contains methods to fetch and update the contract values in the database. - * @param eventData ResultEvent object containing necessary information. - */ -export async function handleEvent (indexer: Indexer, eventData: ResultEvent): Promise { - assert(indexer); - assert(eventData); - - // The following code is for ERC20 contract implementation. - - // Perform indexing based on the type of event. - switch (eventData.event.__typename) { - // In case of ERC20 'Transfer' event. - case 'TransferEvent': { - // On a transfer, balances for both parties change. - // Therefore, trigger indexing for both sender and receiver. - - // Get event fields from eventData. - // const { from, to } = eventData.event; - - // Update balance entry for sender in the database. - // await indexer.balanceOf(eventData.block.hash, eventData.contract, from); - - // Update balance entry for receiver in the database. - // await indexer.balanceOf(eventData.block.hash, eventData.contract, to); - - break; - } - // In case of ERC20 'Approval' event. - case 'ApprovalEvent': { - // On an approval, allowance for (owner, spender) combination changes. - - // Get event fields from eventData. - // const { owner, spender } = eventData.event; - - // Update allowance entry for (owner, spender) combination in the database. - // await indexer.allowance(eventData.block.hash, eventData.contract, owner, spender); - - break; - } - } -} diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 27e856ae..2eb4af08 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -3,8 +3,107 @@ // import assert from 'assert'; +import * as codec from '@ipld/dag-json'; +import { sha256 } from 'multiformats/hashes/sha2'; +import { CID } from 'multiformats/cid'; +import _ from 'lodash'; + +import { BlockProgressInterface } from '@vulcanize/util'; import { Indexer, ResultEvent } from './indexer'; +import { IPLDBlock } from './entity/IPLDBlock'; + +export async function handleBlock (indexer: Indexer, jobData: any): Promise { + // Get events for current block and make an entry of updated values in IPLDBlock. + + const { blockHash } = jobData; + const events = await indexer.getEventsByFilter(blockHash); + + // No IPLDBlock entry if there are no events. + if (!events) return; + + for (const event of events) { + const block = event.block; + const contractAddress = event.contract; + + const oldIpldBlock = await indexer.getIPLDBlock(block, contractAddress); + + const ipldBlockData: any = {}; + const eventData = indexer.getResultEvent(event); + + switch (event.eventName) { + case 'Transfer': { + const { from, to } = eventData.event; + + const fromBalance = await indexer.balanceOf(blockHash, contractAddress, from); + const toBalance = await indexer.balanceOf(blockHash, contractAddress, to); + + // { + // "_balances": { + // "0xCA6D29232D1435D8198E3E5302495417dD073d61": { + // "value": 100 + // }, + // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": { + // "value": 999999999999999999900 + // } + // } + // } + _.set(ipldBlockData, `_balances[${from}]`, fromBalance); + _.set(ipldBlockData, `_balances[${to}]`, toBalance); + + break; + } + + case 'Approval': { + const { owner, spender } = eventData.event; + const allowance = await indexer.allowance(blockHash, contractAddress, owner, spender); + + // { + // "_allowances": { + // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": { + // "0xCA6D29232D1435D8198E3E5302495417dD073d61": { + // "value": 10 + // } + // } + // } + // } + _.set(ipldBlockData, `_allowances[${owner}][${spender}]`, allowance); + + break; + } + } + + const ipldBlock = await prepareIPLDBlock(block, contractAddress, ipldBlockData, oldIpldBlock); + await indexer.saveOrUpdateIPLDBlock(ipldBlock); + } +} + +async function prepareIPLDBlock (block: BlockProgressInterface, contractAddress: string, data: any, oldIpldBlock?: IPLDBlock):Promise { + // If an IPLDBlock for { block, contractAddress } already exists, update the data field. + if (oldIpldBlock) { + const oldData = codec.decode(Buffer.from(oldIpldBlock.data)); + data = Object.assign(oldData, data); + } + + // Encoding the data using dag-json codec. + const bytes = codec.encode(data); + + // Calculating sha256 (multi)hash of the encoded data. + const hash = await sha256.digest(bytes); + + // Calculating the CID: v1, code: dag-json, hash. + const cid = CID.create(1, codec.code, hash); + + let ipldBlock = oldIpldBlock || new IPLDBlock(); + ipldBlock = Object.assign(ipldBlock, { + block, + contractAddress, + cid: cid.toString(), + data: bytes + }); + + return ipldBlock; +} /** * Event hook function. @@ -15,5 +114,37 @@ export async function handleEvent (indexer: Indexer, eventData: ResultEvent): Pr assert(indexer); assert(eventData); + // The following code is for ERC20 contract implementation. + // Perform indexing based on the type of event. + switch (eventData.event.__typename) { + // In case of ERC20 'Transfer' event. + case 'TransferEvent': { + // On a transfer, balances for both parties change. + // Therefore, trigger indexing for both sender and receiver. + + // Get event fields from eventData. + // const { from, to } = eventData.event; + + // Update balance entry for sender in the database. + // await indexer.balanceOf(eventData.block.hash, eventData.contract, from); + + // Update balance entry for receiver in the database. + // await indexer.balanceOf(eventData.block.hash, eventData.contract, to); + + break; + } + // In case of ERC20 'Approval' event. + case 'ApprovalEvent': { + // On an approval, allowance for (owner, spender) combination changes. + + // Get event fields from eventData. + // const { owner, spender } = eventData.event; + + // Update allowance entry for (owner, spender) combination in the database. + // await indexer.allowance(eventData.block.hash, eventData.contract, owner, spender); + + break; + } + } } diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 352c3f9a..df561250 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -174,8 +174,8 @@ export class Indexer { return this._db.getIPLDBlock(block, contractAddress); } - async saveOrUpdateIPLDBlock (ipldBlock: DeepPartial):Promise { - await this._db.saveOrUpdateIPLDBlock(ipldBlock); + async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock):Promise { + return this._db.saveOrUpdateIPLDBlock(ipldBlock); } async triggerIndexingOnEvent (event: Event): Promise { @@ -228,7 +228,7 @@ export class Indexer { return true; } - async getEventsByFilter (blockHash: string, contract: string, name: string | null): Promise> { + async getEventsByFilter (blockHash: string, contract?: string, name?: string): Promise> { return this._baseIndexer.getEventsByFilter(blockHash, contract, name); } diff --git a/packages/codegen/src/templates/ipld-hook-template.handlebars b/packages/codegen/src/templates/ipld-hook-template.handlebars deleted file mode 100644 index 4e060464..00000000 --- a/packages/codegen/src/templates/ipld-hook-template.handlebars +++ /dev/null @@ -1,103 +0,0 @@ -import * as codec from '@ipld/dag-json'; -import { sha256 } from 'multiformats/hashes/sha2'; -import { CID } from 'multiformats/cid'; - -import { BlockProgressInterface } from '@vulcanize/util'; - -import { Indexer } from './indexer'; -import { IPLDBlock } from './entity/IPLDBlock'; - -export async function blockProcessingCompleteHandler (indexer: Indexer, jobData: any): Promise { - // Get events for current block and make an entry of updated values in IPLDBlock. - - const { blockHash } = jobData; - const events = await indexer.getBlockEvents(blockHash); - - if (!events) return; - - const ipldUpdates = events.map(async (event): Promise => { - const block = event.block; - const contractAddress = event.contract; - - const oldIpldBlock = await indexer.getIPLDBlock(block, contractAddress); - - const ipldBlockData: any = {}; - const eventData = indexer.getResultEvent(event); - - switch (event.eventName) { - case 'Transfer': { - const { from, to } = eventData.event; - - const fromBalance = await indexer.balanceOf(blockHash, contractAddress, from); - const toBalance = await indexer.balanceOf(blockHash, contractAddress, to); - - // { - // "_balances": { - // "0xCA6D29232D1435D8198E3E5302495417dD073d61": { - // "value": 100 - // }, - // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": { - // "value": 999999999999999999900 - // } - // } - // } - ipldBlockData._balances = {}; - ipldBlockData._balances[from] = fromBalance; - ipldBlockData._balances[to] = toBalance; - - break; - } - - case 'Approval': { - const { owner, spender } = eventData.event; - const allowance = await indexer.allowance(blockHash, contractAddress, owner, spender); - - // { - // "_allowances": { - // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": { - // "0xCA6D29232D1435D8198E3E5302495417dD073d61": { - // "value": 10 - // } - // } - // } - // } - ipldBlockData._allowances = {}; - ipldBlockData._allowances[owner] = {}; - ipldBlockData._allowances[owner][spender] = allowance; - - break; - } - } - - if (block) { - const ipldBlock = await prepareIPLDBlock(block, contractAddress, ipldBlockData, oldIpldBlock); - await indexer.saveOrUpdateIPLDBlock(ipldBlock); - } - }); - - await Promise.all(ipldUpdates); -} - -async function prepareIPLDBlock (block: BlockProgressInterface, contractAddress: string, data: any, oldIpldBlock?: IPLDBlock):Promise { - // If an IPLDBlock for { block, contractAddress } already exists, update the data field. - if (oldIpldBlock) { - const oldData = codec.decode(Buffer.from(oldIpldBlock.data)); - data = Object.assign(oldData, data); - } - - // Encoding the data using dag-json codec. - const bytes = codec.encode(data); - - // Calculating sha256 (multi)hash of the encoded data. - const hash = await sha256.digest(bytes); - - // Calculating the CID: v1, code: dag-json, hash. - const cid = CID.create(1, codec.code, hash); - - return { - block, - contractAddress, - cid: cid.toString(), - data: bytes - }; -} diff --git a/packages/codegen/src/templates/package-template.handlebars b/packages/codegen/src/templates/package-template.handlebars index 600df981..af502ff7 100644 --- a/packages/codegen/src/templates/package-template.handlebars +++ b/packages/codegen/src/templates/package-template.handlebars @@ -38,6 +38,7 @@ "graphql": "^15.5.0", "graphql-import-node": "^0.0.4", "json-bigint": "^1.0.0", + "lodash": "^4.17.21", "multiformats": "^9.4.8", "reflect-metadata": "^0.1.13", "typeorm": "^0.2.32", diff --git a/packages/codegen/src/templates/resolvers-template.handlebars b/packages/codegen/src/templates/resolvers-template.handlebars index 56e979d3..3bd3362b 100644 --- a/packages/codegen/src/templates/resolvers-template.handlebars +++ b/packages/codegen/src/templates/resolvers-template.handlebars @@ -52,8 +52,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch }, {{/each}} - events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name: string }) => { - log('events', blockHash, contractAddress, name || ''); + events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name?: string }) => { + log('events', blockHash, contractAddress, name); const block = await indexer.getBlockProgress(blockHash); if (!block || !block.isComplete) { diff --git a/packages/util/src/indexer.ts b/packages/util/src/indexer.ts index 6268ca5a..16035724 100644 --- a/packages/util/src/indexer.ts +++ b/packages/util/src/indexer.ts @@ -170,7 +170,7 @@ export class Indexer { return this._db.getBlockEvents(blockHash); } - async getEventsByFilter (blockHash: string, contract: string, name: string | null): Promise> { + async getEventsByFilter (blockHash: string, contract?: string, name?: string): Promise> { if (contract) { const watchedContract = await this.isWatchedContract(contract); if (!watchedContract) { From 13dfd9ebb4bef808716a265bf8ece4b7b9c8f89d Mon Sep 17 00:00:00 2001 From: prathamesh Date: Wed, 6 Oct 2021 12:58:06 +0530 Subject: [PATCH 05/21] Change IPLD block structure --- .../src/templates/hooks-template.handlebars | 24 +++++++++++++++---- .../src/templates/indexer-template.handlebars | 18 ++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 2eb4af08..82e51195 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -26,9 +26,25 @@ export async function handleBlock (indexer: Indexer, jobData: any): Promise { + assert(blockHash); + + const { + allEthHeaderCids: { + nodes: [ + { + cid + } + ] + } + } = await this._postgraphileClient.getBlockWithTransactions({ blockHash }); + + assert(cid); + + return cid; + } + async triggerIndexingOnEvent (event: Event): Promise { const resultEvent = this.getResultEvent(event); From 224f3ffe522c3ef9588fbc275d3f63404040710c Mon Sep 17 00:00:00 2001 From: prathamesh Date: Wed, 6 Oct 2021 15:34:11 +0530 Subject: [PATCH 06/21] Add cid field in blocks --- .../src/data/entities/BlockProgress.yaml | 4 +++ packages/codegen/src/schema.ts | 1 + .../src/templates/hooks-template.handlebars | 4 +-- .../src/templates/indexer-template.handlebars | 7 +++-- packages/erc20-watcher/src/fill.ts | 8 +++++- packages/erc20-watcher/src/indexer.ts | 8 +++--- packages/erc20-watcher/src/job-runner.ts | 8 +++++- packages/erc20-watcher/src/server.ts | 8 +++++- packages/ipld-eth-client/src/eth-queries.ts | 1 + packages/uni-info-watcher/src/fill.ts | 8 +++++- packages/uni-info-watcher/src/indexer.ts | 7 +++-- packages/uni-info-watcher/src/job-runner.ts | 7 ++++- packages/uni-info-watcher/src/server.ts | 8 +++++- packages/uni-watcher/src/indexer.ts | 4 +-- packages/util/src/database.ts | 2 ++ packages/util/src/events.ts | 7 ++--- packages/util/src/indexer.ts | 27 ++++++++++++++++--- packages/util/src/job-runner.ts | 7 ++--- packages/util/src/types.ts | 1 + 19 files changed, 101 insertions(+), 26 deletions(-) diff --git a/packages/codegen/src/data/entities/BlockProgress.yaml b/packages/codegen/src/data/entities/BlockProgress.yaml index ba0307d8..5dcb3a7f 100644 --- a/packages/codegen/src/data/entities/BlockProgress.yaml +++ b/packages/codegen/src/data/entities/BlockProgress.yaml @@ -9,6 +9,10 @@ indexOn: - columns: - parentHash columns: + - name: cid + pgType: varchar + tsType: string + columnType: Column - name: blockHash pgType: varchar tsType: string diff --git a/packages/codegen/src/schema.ts b/packages/codegen/src/schema.ts index 5e75cfb2..d91597e5 100644 --- a/packages/codegen/src/schema.ts +++ b/packages/codegen/src/schema.ts @@ -173,6 +173,7 @@ export class Schema { this._composer.createObjectTC({ name: blockName, fields: { + cid: 'String!', hash: 'String!', number: 'Int!', timestamp: 'Int!', diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 82e51195..5be15d8b 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -40,7 +40,7 @@ export async function handleBlock (indexer: Indexer, jobData: any): Promise): Promise { + async _fetchAndSaveEvents ({ cid: blockCid, blockHash }: DeepPartial): Promise { assert(blockHash); let { block, logs } = await this._ethClient.getLogs({ blockHash }); @@ -397,6 +399,7 @@ export class Indexer { try { block = { + cid: blockCid, blockHash, blockNumber: block.number, blockTimestamp: block.timestamp, diff --git a/packages/erc20-watcher/src/fill.ts b/packages/erc20-watcher/src/fill.ts index e8ed9bd6..c6375791 100644 --- a/packages/erc20-watcher/src/fill.ts +++ b/packages/erc20-watcher/src/fill.ts @@ -62,18 +62,24 @@ export const main = async (): Promise => { assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); const cache = await getCache(cacheConfig); + const ethClient = new EthClient({ gqlEndpoint: gqlPostgraphileEndpoint, gqlSubscriptionEndpoint: gqlPostgraphileEndpoint, cache }); + const postgraphileClient = new EthClient({ + gqlEndpoint: gqlPostgraphileEndpoint, + cache + }); + const ethProvider = getDefaultProvider(rpcProviderEndpoint); // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries const pubsub = new PubSub(); - const indexer = new Indexer(db, ethClient, ethProvider, mode); + const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, mode); const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; assert(dbConnectionString, 'Missing job queue db connection string'); diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts index dfbeb56e..badc714e 100644 --- a/packages/erc20-watcher/src/indexer.ts +++ b/packages/erc20-watcher/src/indexer.ts @@ -44,6 +44,7 @@ interface EventResult { export class Indexer { _db: Database _ethClient: EthClient + _postgraphileClient: EthClient; _ethProvider: BaseProvider _baseIndexer: BaseIndexer @@ -52,15 +53,16 @@ export class Indexer { _contract: ethers.utils.Interface _serverMode: string - constructor (db: Database, ethClient: EthClient, ethProvider: BaseProvider, serverMode: string) { + constructor (db: Database, ethClient: EthClient, postgraphileClient: EthClient, ethProvider: BaseProvider, serverMode: string) { assert(db); assert(ethClient); this._db = db; this._ethClient = ethClient; + this._postgraphileClient = postgraphileClient; this._ethProvider = ethProvider; this._serverMode = serverMode; - this._baseIndexer = new BaseIndexer(this._db, this._ethClient); + this._baseIndexer = new BaseIndexer(this._db, this._ethClient, this._postgraphileClient); const { abi, storageLayout } = artifacts; @@ -295,7 +297,7 @@ export class Indexer { return true; } - async getEventsByFilter (blockHash: string, contract: string, name: string | null): Promise> { + async getEventsByFilter (blockHash: string, contract: string, name?: string): Promise> { return this._baseIndexer.getEventsByFilter(blockHash, contract, name); } diff --git a/packages/erc20-watcher/src/job-runner.ts b/packages/erc20-watcher/src/job-runner.ts index aa68e8fd..9167d98b 100644 --- a/packages/erc20-watcher/src/job-runner.ts +++ b/packages/erc20-watcher/src/job-runner.ts @@ -94,14 +94,20 @@ export const main = async (): Promise => { assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); const cache = await getCache(cacheConfig); + const ethClient = new EthClient({ gqlEndpoint: gqlApiEndpoint, gqlSubscriptionEndpoint: gqlPostgraphileEndpoint, cache }); + const postgraphileClient = new EthClient({ + gqlEndpoint: gqlPostgraphileEndpoint, + cache + }); + const ethProvider = getDefaultProvider(rpcProviderEndpoint); - const indexer = new Indexer(db, ethClient, ethProvider, mode); + const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, mode); assert(jobQueueConfig, 'Missing job queue config'); diff --git a/packages/erc20-watcher/src/server.ts b/packages/erc20-watcher/src/server.ts index 45d81673..b270bdd7 100644 --- a/packages/erc20-watcher/src/server.ts +++ b/packages/erc20-watcher/src/server.ts @@ -57,18 +57,24 @@ export const main = async (): Promise => { assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); const cache = await getCache(cacheConfig); + const ethClient = new EthClient({ gqlEndpoint: gqlApiEndpoint, gqlSubscriptionEndpoint: gqlPostgraphileEndpoint, cache }); + const postgraphileClient = new EthClient({ + gqlEndpoint: gqlPostgraphileEndpoint, + cache + }); + const ethProvider = getDefaultProvider(rpcProviderEndpoint); // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries const pubsub = new PubSub(); - const indexer = new Indexer(db, ethClient, ethProvider, mode); + const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, mode); assert(jobQueueConfig, 'Missing job queue config'); diff --git a/packages/ipld-eth-client/src/eth-queries.ts b/packages/ipld-eth-client/src/eth-queries.ts index dcf45b2c..9129cc06 100644 --- a/packages/ipld-eth-client/src/eth-queries.ts +++ b/packages/ipld-eth-client/src/eth-queries.ts @@ -82,6 +82,7 @@ subscription { listen(topic: "header_cids") { relatedNode { ... on EthHeaderCid { + cid blockHash blockNumber parentHash diff --git a/packages/uni-info-watcher/src/fill.ts b/packages/uni-info-watcher/src/fill.ts index 141a095a..721b4136 100644 --- a/packages/uni-info-watcher/src/fill.ts +++ b/packages/uni-info-watcher/src/fill.ts @@ -63,19 +63,25 @@ export const main = async (): Promise => { assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); const cache = await getCache(cacheConfig); + const ethClient = new EthClient({ gqlEndpoint: gqlPostgraphileEndpoint, gqlSubscriptionEndpoint: gqlPostgraphileEndpoint, cache }); + const postgraphileClient = new EthClient({ + gqlEndpoint: gqlPostgraphileEndpoint, + cache + }); + const uniClient = new UniClient(uniWatcher); const erc20Client = new ERC20Client(tokenWatcher); // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries const pubsub = new PubSub(); - const indexer = new Indexer(db, uniClient, erc20Client, ethClient, mode); + const indexer = new Indexer(db, uniClient, erc20Client, ethClient, postgraphileClient, mode); assert(jobQueueConfig, 'Missing job queue config'); const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; diff --git a/packages/uni-info-watcher/src/indexer.ts b/packages/uni-info-watcher/src/indexer.ts index bff41eb3..93ebf000 100644 --- a/packages/uni-info-watcher/src/indexer.ts +++ b/packages/uni-info-watcher/src/indexer.ts @@ -44,20 +44,23 @@ export class Indexer implements IndexerInterface { _uniClient: UniClient _erc20Client: ERC20Client _ethClient: EthClient + _postgraphileClient: EthClient; _baseIndexer: BaseIndexer _isDemo: boolean - constructor (db: Database, uniClient: UniClient, erc20Client: ERC20Client, ethClient: EthClient, mode: string) { + constructor (db: Database, uniClient: UniClient, erc20Client: ERC20Client, ethClient: EthClient, postgraphileClient: EthClient, mode: string) { assert(db); assert(uniClient); assert(erc20Client); assert(ethClient); + assert(postgraphileClient); this._db = db; this._uniClient = uniClient; this._erc20Client = erc20Client; this._ethClient = ethClient; - this._baseIndexer = new BaseIndexer(this._db, this._ethClient); + this._postgraphileClient = postgraphileClient; + this._baseIndexer = new BaseIndexer(this._db, this._ethClient, this._postgraphileClient); this._isDemo = mode === 'demo'; } diff --git a/packages/uni-info-watcher/src/job-runner.ts b/packages/uni-info-watcher/src/job-runner.ts index 18e0e5e4..2f6886ff 100644 --- a/packages/uni-info-watcher/src/job-runner.ts +++ b/packages/uni-info-watcher/src/job-runner.ts @@ -106,9 +106,14 @@ export const main = async (): Promise => { gqlSubscriptionEndpoint }); + const postgraphileClient = new EthClient({ + gqlEndpoint: gqlPostgraphileEndpoint, + cache + }); + const erc20Client = new ERC20Client(tokenWatcher); - const indexer = new Indexer(db, uniClient, erc20Client, ethClient, mode); + const indexer = new Indexer(db, uniClient, erc20Client, ethClient, postgraphileClient, mode); assert(jobQueueConfig, 'Missing job queue config'); diff --git a/packages/uni-info-watcher/src/server.ts b/packages/uni-info-watcher/src/server.ts index a197e536..b716c31a 100644 --- a/packages/uni-info-watcher/src/server.ts +++ b/packages/uni-info-watcher/src/server.ts @@ -67,15 +67,21 @@ export const main = async (): Promise => { assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); const cache = await getCache(cacheConfig); + const ethClient = new EthClient({ gqlEndpoint: gqlApiEndpoint, gqlSubscriptionEndpoint: gqlPostgraphileEndpoint, cache }); + const postgraphileClient = new EthClient({ + gqlEndpoint: gqlPostgraphileEndpoint, + cache + }); + const uniClient = new UniClient(uniWatcher); const erc20Client = new ERC20Client(tokenWatcher); - const indexer = new Indexer(db, uniClient, erc20Client, ethClient, mode); + const indexer = new Indexer(db, uniClient, erc20Client, ethClient, postgraphileClient, mode); assert(jobQueueConfig, 'Missing job queue config'); diff --git a/packages/uni-watcher/src/indexer.ts b/packages/uni-watcher/src/indexer.ts index 624c0afe..637007d8 100644 --- a/packages/uni-watcher/src/indexer.ts +++ b/packages/uni-watcher/src/indexer.ts @@ -49,7 +49,7 @@ export class Indexer implements IndexerInterface { this._db = db; this._ethClient = ethClient; this._postgraphileClient = postgraphileClient; - this._baseIndexer = new BaseIndexer(this._db, this._ethClient); + this._baseIndexer = new BaseIndexer(this._db, this._ethClient, this._postgraphileClient); this._factoryContract = new ethers.utils.Interface(factoryABI); this._poolContract = new ethers.utils.Interface(poolABI); @@ -295,7 +295,7 @@ export class Indexer implements IndexerInterface { return contract; } - async getEventsByFilter (blockHash: string, contract: string, name: string | null): Promise> { + async getEventsByFilter (blockHash: string, contract: string, name?: string): Promise> { return this._baseIndexer.getEventsByFilter(blockHash, contract, name); } diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index 6623ad40..9b68111f 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -196,6 +196,7 @@ export class Database { async saveEvents (blockRepo: Repository, eventRepo: Repository, block: DeepPartial, events: DeepPartial[]): Promise { const { + cid, blockHash, blockNumber, blockTimestamp, @@ -216,6 +217,7 @@ export class Database { if (!blockProgress) { const entity = blockRepo.create({ + cid, blockHash, parentHash, blockNumber, diff --git a/packages/util/src/events.ts b/packages/util/src/events.ts index 41ec7a1c..2abb2ce8 100644 --- a/packages/util/src/events.ts +++ b/packages/util/src/events.ts @@ -37,13 +37,13 @@ export class EventWatcher { } async blocksHandler (value: any): Promise { - const { blockHash, blockNumber, parentHash, timestamp } = _.get(value, 'data.listen.relatedNode'); + const { cid, blockHash, blockNumber, parentHash, timestamp } = _.get(value, 'data.listen.relatedNode'); await this._indexer.updateSyncStatusChainHead(blockHash, blockNumber); log('watchBlock', blockHash, blockNumber); - await this._jobQueue.pushJob(QUEUE_BLOCK_PROCESSING, { kind: JOB_KIND_INDEX, blockHash, blockNumber, parentHash, timestamp }); + await this._jobQueue.pushJob(QUEUE_BLOCK_PROCESSING, { kind: JOB_KIND_INDEX, cid, blockHash, blockNumber, parentHash, timestamp }); } async blockProcessingCompleteHandler (job: any): Promise { @@ -80,11 +80,12 @@ export class EventWatcher { } async publishBlockProgressToSubscribers (blockProgress: BlockProgressInterface): Promise { - const { blockHash, blockNumber, numEvents, numProcessedEvents, isComplete } = blockProgress; + const { cid, blockHash, blockNumber, numEvents, numProcessedEvents, isComplete } = blockProgress; // Publishing the event here will result in pushing the payload to GQL subscribers for `onAddressEvent(address)`. await this._pubsub.publish(BlockProgressEvent, { onBlockProgressEvent: { + cid, blockHash, blockNumber, numEvents, diff --git a/packages/util/src/indexer.ts b/packages/util/src/indexer.ts index 16035724..51035644 100644 --- a/packages/util/src/indexer.ts +++ b/packages/util/src/indexer.ts @@ -27,11 +27,13 @@ export interface ValueResult { export class Indexer { _db: DatabaseInterface; _ethClient: EthClient; + _postgraphileClient: EthClient; _getStorageAt: GetStorageAt - constructor (db: DatabaseInterface, ethClient: EthClient) { + constructor (db: DatabaseInterface, ethClient: EthClient, postgraphileClient: EthClient) { this._db = db; this._ethClient = ethClient; + this._postgraphileClient = postgraphileClient; this._getStorageAt = this._ethClient.getStorageAt.bind(this._ethClient); } @@ -104,8 +106,27 @@ export class Indexer { } async getBlock (blockHash: string): Promise { - const { block } = await this._ethClient.getLogs({ blockHash }); - return block; + const { + allEthHeaderCids: { + nodes: [ + { + cid, + blockNumber, + timestamp, + parentHash + } + ] + } + } = await this._postgraphileClient.getBlockWithTransactions({ blockHash }); + + return { + cid, + number: blockNumber, + parent: { + hash: parentHash + }, + timestamp + }; } async getBlockProgress (blockHash: string): Promise { diff --git a/packages/util/src/job-runner.ts b/packages/util/src/job-runner.ts index 08bd773c..f8fb50d4 100644 --- a/packages/util/src/job-runner.ts +++ b/packages/util/src/job-runner.ts @@ -121,7 +121,7 @@ export class JobRunner { } async _indexBlock (job: any, syncStatus: SyncStatusInterface): Promise { - const { data: { blockHash, blockNumber, parentHash, priority, timestamp } } = job; + const { data: { cid, blockHash, blockNumber, parentHash, priority, timestamp } } = job; log(`Processing block number ${blockNumber} hash ${blockHash} `); // Check if chain pruning is caught up. @@ -139,13 +139,14 @@ export class JobRunner { const parent = await this._indexer.getBlockProgress(parentHash); if (!parent) { - const { number: parentBlockNumber, parent: grandparent, timestamp: parentTimestamp } = await this._indexer.getBlock(parentHash); + const { cid: parentCid, number: parentBlockNumber, parent: grandparent, timestamp: parentTimestamp } = await this._indexer.getBlock(parentHash); // Create a higher priority job to index parent block and then abort. // We don't have to worry about aborting as this job will get retried later. const newPriority = (priority || 0) + 1; await this._jobQueue.pushJob(QUEUE_BLOCK_PROCESSING, { kind: JOB_KIND_INDEX, + cid: parentCid, blockHash: parentHash, blockNumber: parentBlockNumber, parentHash: grandparent?.hash, @@ -176,7 +177,7 @@ export class JobRunner { // Delay required to process block. await wait(jobDelayInMilliSecs); - const events = await this._indexer.getOrFetchBlockEvents({ blockHash, blockNumber, parentHash, blockTimestamp: timestamp }); + const events = await this._indexer.getOrFetchBlockEvents({ cid, blockHash, blockNumber, parentHash, blockTimestamp: timestamp }); for (let ei = 0; ei < events.length; ei++) { await this._jobQueue.pushJob(QUEUE_EVENT_PROCESSING, { id: events[ei].id, publish: true }); diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index 0c19a263..bb626942 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -6,6 +6,7 @@ import { DeepPartial, FindConditions, QueryRunner } from 'typeorm'; export interface BlockProgressInterface { id: number; + cid?: string; blockHash: string; parentHash: string; blockNumber: number; From 7e6311727a8f7f95fce0e28cd0a538d59c452b59 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Thu, 7 Oct 2021 07:58:12 +0530 Subject: [PATCH 07/21] Fetch prev. IPLDBlock for a contract --- .../templates/database-template.handlebars | 70 ++++++++++++++++++- .../src/templates/hooks-template.handlebars | 18 ++--- .../src/templates/indexer-template.handlebars | 32 ++++----- .../erc20-watcher/src/entity/BlockProgress.ts | 3 + .../src/entity/BlockProgress.ts | 3 + .../uni-watcher/src/entity/BlockProgress.ts | 3 + packages/util/src/types.ts | 2 +- 7 files changed, 103 insertions(+), 28 deletions(-) diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 8b62a6b7..17e9952d 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -6,7 +6,7 @@ import assert from 'assert'; import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner } from 'typeorm'; import path from 'path'; -import { Database as BaseDatabase } from '@vulcanize/util'; +import { Database as BaseDatabase, MAX_REORG_DEPTH } from '@vulcanize/util'; import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; @@ -89,6 +89,74 @@ export class Database { return repo.save(ipldBlock); } + async getPrevIPLDBlock (queryRunner: QueryRunner, blockHash: string, contractAddress: string): Promise { + const repo = this._conn.getRepository(IPLDBlock); + const heirerchicalQuery = ` + WITH RECURSIVE cte_query AS + ( + SELECT + b.block_hash, + b.block_number, + b.parent_hash, + 1 as depth, + i.id + FROM + block_progress b + LEFT JOIN + ipld_block i ON i.block_id = b.id + WHERE + b.block_hash = $1 + UNION ALL + SELECT + b.block_hash, + b.block_number, + b.parent_hash, + c.depth + 1, + i.id + FROM + block_progress b + LEFT JOIN + ipld_block i + ON i.block_id = b.id + AND i.contract_address = $2 + INNER JOIN + cte_query c ON c.parent_hash = b.block_hash + WHERE + c.id IS NULL AND c.depth < $3 + ) + SELECT + block_number, id + FROM + cte_query + ORDER BY block_number ASC + LIMIT 1; + `; + + // Fetching block and id for previous IPLDBlock in frothy region. + const [{ block_number: blockNumber, id }] = await queryRunner.query(heirerchicalQuery, [blockHash, contractAddress, MAX_REORG_DEPTH]); + + let result: IPLDBlock | undefined; + if (id) { + result = await repo.findOne({ id }); + } else { + // If IPLDBlock not found in frothy region get latest IPLDBlock in the pruned region. + // Filter out IPLDBlocks from pruned blocks. + const canonicalBlockNumber = blockNumber + 1; + const ipldBlockInPrunedRegion:any = await repo.createQueryBuilder('ipld_block') + .innerJoinAndSelect('block_progress', 'block', 'block.id = ipld_block.block_id') + .where('block.is_pruned = false') + .andWhere('ipld_block.contract_address = :contractAddress', { contractAddress }) + .andWhere('block.block_number <= :canonicalBlockNumber', { canonicalBlockNumber }) + .orderBy('block.block_number', 'DESC') + .limit(1) + .getOne(); + + result = ipldBlockInPrunedRegion; + } + + return result; + } + async getContract (address: string): Promise { const repo = this._conn.getRepository(Contract); diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 5be15d8b..c98113bf 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -28,6 +28,7 @@ export async function handleBlock (indexer: Indexer, jobData: any): Promise { + async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { return this._db.saveOrUpdateIPLDBlock(ipldBlock); } - async getBlockCID (blockHash: string): Promise { - assert(blockHash); - - const { - allEthHeaderCids: { - nodes: [ - { - cid - } - ] - } - } = await this._postgraphileClient.getBlockWithTransactions({ blockHash }); - - assert(cid); + async getPrevIPLDBlock (blockHash: string, contractAddress: string): Promise { + const dbTx = await this._db.createTransactionRunner(); + let res; - return cid; + try { + res = await this._db.getPrevIPLDBlock(dbTx, blockHash, contractAddress); + await dbTx.commitTransaction(); + } catch (error) { + await dbTx.rollbackTransaction(); + throw error; + } finally { + await dbTx.release(); + } + return res; } async triggerIndexingOnEvent (event: Event): Promise { diff --git a/packages/erc20-watcher/src/entity/BlockProgress.ts b/packages/erc20-watcher/src/entity/BlockProgress.ts index e67cf2dd..9e42d306 100644 --- a/packages/erc20-watcher/src/entity/BlockProgress.ts +++ b/packages/erc20-watcher/src/entity/BlockProgress.ts @@ -14,6 +14,9 @@ export class BlockProgress implements BlockProgressInterface { @PrimaryGeneratedColumn() id!: number; + @Column('varchar', { nullable: true }) + cid!: string; + @Column('varchar', { length: 66 }) blockHash!: string; diff --git a/packages/uni-info-watcher/src/entity/BlockProgress.ts b/packages/uni-info-watcher/src/entity/BlockProgress.ts index b6adf6f4..96c70603 100644 --- a/packages/uni-info-watcher/src/entity/BlockProgress.ts +++ b/packages/uni-info-watcher/src/entity/BlockProgress.ts @@ -14,6 +14,9 @@ export class BlockProgress implements BlockProgressInterface { @PrimaryGeneratedColumn() id!: number; + @Column('varchar', { nullable: true }) + cid!: string; + @Column('varchar', { length: 66 }) blockHash!: string; diff --git a/packages/uni-watcher/src/entity/BlockProgress.ts b/packages/uni-watcher/src/entity/BlockProgress.ts index b6adf6f4..96c70603 100644 --- a/packages/uni-watcher/src/entity/BlockProgress.ts +++ b/packages/uni-watcher/src/entity/BlockProgress.ts @@ -14,6 +14,9 @@ export class BlockProgress implements BlockProgressInterface { @PrimaryGeneratedColumn() id!: number; + @Column('varchar', { nullable: true }) + cid!: string; + @Column('varchar', { length: 66 }) blockHash!: string; diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index bb626942..fbb1a216 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -6,7 +6,7 @@ import { DeepPartial, FindConditions, QueryRunner } from 'typeorm'; export interface BlockProgressInterface { id: number; - cid?: string; + cid: string; blockHash: string; parentHash: string; blockNumber: number; From 04ed30549b57adf924c0f77902db693f5cb17797 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Thu, 7 Oct 2021 11:55:49 +0530 Subject: [PATCH 08/21] GQL API to query IPLDBlock by CID --- packages/codegen/src/schema.ts | 26 +++++++ .../templates/database-template.handlebars | 72 +++++++++---------- .../src/templates/hooks-template.handlebars | 4 +- .../src/templates/indexer-template.handlebars | 48 ++++++++++++- .../templates/resolvers-template.handlebars | 8 +++ packages/util/src/database.ts | 1 + 6 files changed, 120 insertions(+), 39 deletions(-) diff --git a/packages/codegen/src/schema.ts b/packages/codegen/src/schema.ts index d91597e5..3f4c7cab 100644 --- a/packages/codegen/src/schema.ts +++ b/packages/codegen/src/schema.ts @@ -97,6 +97,9 @@ export class Schema { // Add a mutation for watching a contract. this._addWatchContractMutation(); + this._addIPLDType(); + this._addIPLDBlockQuery(); + return this._composer.buildSchema(); } @@ -235,6 +238,29 @@ export class Schema { }); } + _addIPLDType (): void { + this._composer.createObjectTC({ + name: 'ResultIPLDBlock', + fields: { + block: () => this._composer.getOTC('Block').NonNull, + contractAddress: 'String!', + cid: 'String!', + data: 'String!' + } + }); + } + + _addIPLDBlockQuery (): void { + this._composer.Query.addFields({ + ipldBlock: { + type: this._composer.getOTC('ResultIPLDBlock'), + args: { + cid: 'String!' + } + } + }); + } + /** * Adds an event subscription to the schema. */ diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 17e9952d..579d6f01 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -79,9 +79,9 @@ export class Database { } {{/each}} - async getIPLDBlock (block: BlockProgress, contractAddress: string): Promise { + async getIPLDBlocks (where: FindConditions): Promise { const repo = this._conn.getRepository(IPLDBlock); - return repo.findOne({ block, contractAddress }); + return repo.find({ where, relations: ['block'] }); } async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { @@ -91,46 +91,47 @@ export class Database { async getPrevIPLDBlock (queryRunner: QueryRunner, blockHash: string, contractAddress: string): Promise { const repo = this._conn.getRepository(IPLDBlock); + const heirerchicalQuery = ` - WITH RECURSIVE cte_query AS - ( - SELECT - b.block_hash, - b.block_number, - b.parent_hash, - 1 as depth, - i.id - FROM - block_progress b - LEFT JOIN - ipld_block i ON i.block_id = b.id - WHERE - b.block_hash = $1 - UNION ALL + WITH RECURSIVE cte_query AS + ( SELECT b.block_hash, b.block_number, b.parent_hash, - c.depth + 1, + 1 as depth, i.id FROM block_progress b LEFT JOIN - ipld_block i - ON i.block_id = b.id - AND i.contract_address = $2 - INNER JOIN - cte_query c ON c.parent_hash = b.block_hash - WHERE - c.id IS NULL AND c.depth < $3 - ) - SELECT - block_number, id - FROM - cte_query - ORDER BY block_number ASC - LIMIT 1; - `; + ipld_block i ON i.block_id = b.id + WHERE + b.block_hash = $1 + UNION ALL + SELECT + b.block_hash, + b.block_number, + b.parent_hash, + c.depth + 1, + i.id + FROM + block_progress b + LEFT JOIN + ipld_block i + ON i.block_id = b.id + AND i.contract_address = $2 + INNER JOIN + cte_query c ON c.parent_hash = b.block_hash + WHERE + c.id IS NULL AND c.depth < $3 + ) + SELECT + block_number, id + FROM + cte_query + ORDER BY block_number ASC + LIMIT 1; + `; // Fetching block and id for previous IPLDBlock in frothy region. const [{ block_number: blockNumber, id }] = await queryRunner.query(heirerchicalQuery, [blockHash, contractAddress, MAX_REORG_DEPTH]); @@ -142,7 +143,8 @@ export class Database { // If IPLDBlock not found in frothy region get latest IPLDBlock in the pruned region. // Filter out IPLDBlocks from pruned blocks. const canonicalBlockNumber = blockNumber + 1; - const ipldBlockInPrunedRegion:any = await repo.createQueryBuilder('ipld_block') + + result = await repo.createQueryBuilder('ipld_block') .innerJoinAndSelect('block_progress', 'block', 'block.id = ipld_block.block_id') .where('block.is_pruned = false') .andWhere('ipld_block.contract_address = :contractAddress', { contractAddress }) @@ -150,8 +152,6 @@ export class Database { .orderBy('block.block_number', 'DESC') .limit(1) .getOne(); - - result = ipldBlockInPrunedRegion; } return result; diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index c98113bf..89ea22be 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -39,7 +39,9 @@ export async function handleBlock (indexer: Indexer, jobData: any): Promise { @@ -173,7 +203,21 @@ export class Indexer { {{/each}} async getIPLDBlock (block: BlockProgress, contractAddress: string): Promise { - return this._db.getIPLDBlock(block, contractAddress); + const ipldBlocks = await this._db.getIPLDBlocks({ block, contractAddress }); + + // There can be only one IPLDBlock for a { block, contractAddress } combination. + assert(ipldBlocks.length <= 1); + + return ipldBlocks[0]; + } + + async getIPLDBlockByCid (cid: string): Promise { + const ipldBlocks = await this._db.getIPLDBlocks({ cid }); + + // There can be only one IPLDBlock with a particular cid. + assert(ipldBlocks.length <= 1); + + return ipldBlocks[0]; } async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { diff --git a/packages/codegen/src/templates/resolvers-template.handlebars b/packages/codegen/src/templates/resolvers-template.handlebars index 3bd3362b..6ef3b621 100644 --- a/packages/codegen/src/templates/resolvers-template.handlebars +++ b/packages/codegen/src/templates/resolvers-template.handlebars @@ -74,6 +74,14 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch const events = await indexer.getEventsInRange(fromBlockNumber, toBlockNumber); return events.map(event => indexer.getResultEvent(event)); + }, + + ipldBlock: async (_: any, { cid }: { cid: string }) => { + log('ipldBlock', cid); + + const ipldBlock = await indexer.getIPLDBlockByCid(cid); + + return ipldBlock ? indexer.getResultIPLDBlock(ipldBlock) : undefined; } } }; diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index 9b68111f..591a9c6c 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -491,6 +491,7 @@ export class Database { // If entity not found in frothy region get latest entity in the pruned region. // Filter out entities from pruned blocks. const canonicalBlockNumber = blockNumber + 1; + const entityInPrunedRegion:any = await repo.createQueryBuilder('entity') .innerJoinAndSelect('block_progress', 'block', 'block.block_hash = entity.block_hash') .where('block.is_pruned = false') From 0d8322f61d0919681e643170c0533a1746c2b61d Mon Sep 17 00:00:00 2001 From: prathamesh Date: Thu, 7 Oct 2021 13:42:26 +0530 Subject: [PATCH 09/21] Save cid in blocks in existing watchers --- packages/erc20-watcher/src/entity/BlockProgress.ts | 2 +- packages/erc20-watcher/src/indexer.ts | 3 ++- packages/uni-info-watcher/src/entity/BlockProgress.ts | 2 +- packages/uni-info-watcher/src/events.ts | 1 + packages/uni-info-watcher/src/indexer.ts | 1 + packages/uni-watcher/src/entity/BlockProgress.ts | 2 +- packages/uni-watcher/src/indexer.ts | 4 +++- packages/uni-watcher/src/schema.ts | 1 + 8 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/erc20-watcher/src/entity/BlockProgress.ts b/packages/erc20-watcher/src/entity/BlockProgress.ts index 9e42d306..50140904 100644 --- a/packages/erc20-watcher/src/entity/BlockProgress.ts +++ b/packages/erc20-watcher/src/entity/BlockProgress.ts @@ -14,7 +14,7 @@ export class BlockProgress implements BlockProgressInterface { @PrimaryGeneratedColumn() id!: number; - @Column('varchar', { nullable: true }) + @Column('varchar') cid!: string; @Column('varchar', { length: 66 }) diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts index badc714e..20d1b435 100644 --- a/packages/erc20-watcher/src/indexer.ts +++ b/packages/erc20-watcher/src/indexer.ts @@ -369,7 +369,7 @@ export class Indexer { return this._baseIndexer.getAncestorAtDepth(blockHash, depth); } - async _fetchAndSaveEvents ({ blockHash }: DeepPartial): Promise { + async _fetchAndSaveEvents ({ cid: blockCid, blockHash }: DeepPartial): Promise { assert(blockHash); let { block, logs } = await this._ethClient.getLogs({ blockHash }); @@ -434,6 +434,7 @@ export class Indexer { try { block = { + cid: blockCid, blockHash, blockNumber: block.number, blockTimestamp: block.timestamp, diff --git a/packages/uni-info-watcher/src/entity/BlockProgress.ts b/packages/uni-info-watcher/src/entity/BlockProgress.ts index 96c70603..1809cfe7 100644 --- a/packages/uni-info-watcher/src/entity/BlockProgress.ts +++ b/packages/uni-info-watcher/src/entity/BlockProgress.ts @@ -14,7 +14,7 @@ export class BlockProgress implements BlockProgressInterface { @PrimaryGeneratedColumn() id!: number; - @Column('varchar', { nullable: true }) + @Column('varchar') cid!: string; @Column('varchar', { length: 66 }) diff --git a/packages/uni-info-watcher/src/events.ts b/packages/uni-info-watcher/src/events.ts index 3202ca53..ebe25b8b 100644 --- a/packages/uni-info-watcher/src/events.ts +++ b/packages/uni-info-watcher/src/events.ts @@ -92,6 +92,7 @@ export interface TransferEvent { } export interface Block { + cid: string; number: number; hash: string; timestamp: number; diff --git a/packages/uni-info-watcher/src/indexer.ts b/packages/uni-info-watcher/src/indexer.ts index 93ebf000..047e8448 100644 --- a/packages/uni-info-watcher/src/indexer.ts +++ b/packages/uni-info-watcher/src/indexer.ts @@ -71,6 +71,7 @@ export class Indexer implements IndexerInterface { return { block: { + cid: block.cid, hash: block.blockHash, number: block.blockNumber, timestamp: block.blockTimestamp, diff --git a/packages/uni-watcher/src/entity/BlockProgress.ts b/packages/uni-watcher/src/entity/BlockProgress.ts index 96c70603..1809cfe7 100644 --- a/packages/uni-watcher/src/entity/BlockProgress.ts +++ b/packages/uni-watcher/src/entity/BlockProgress.ts @@ -14,7 +14,7 @@ export class BlockProgress implements BlockProgressInterface { @PrimaryGeneratedColumn() id!: number; - @Column('varchar', { nullable: true }) + @Column('varchar') cid!: string; @Column('varchar', { length: 66 }) diff --git a/packages/uni-watcher/src/indexer.ts b/packages/uni-watcher/src/indexer.ts index 637007d8..95b8551f 100644 --- a/packages/uni-watcher/src/indexer.ts +++ b/packages/uni-watcher/src/indexer.ts @@ -63,6 +63,7 @@ export class Indexer implements IndexerInterface { return { block: { + cid: block.cid, hash: block.blockHash, number: block.blockNumber, timestamp: block.blockTimestamp, @@ -368,7 +369,7 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.getAncestorAtDepth(blockHash, depth); } - async _fetchAndSaveEvents ({ blockHash }: DeepPartial): Promise { + async _fetchAndSaveEvents ({ cid: blockCid, blockHash }: DeepPartial): Promise { assert(blockHash); let { block, logs } = await this._ethClient.getLogs({ blockHash }); @@ -451,6 +452,7 @@ export class Indexer implements IndexerInterface { try { block = { + cid: blockCid, blockHash, blockNumber: block.number, blockTimestamp: block.timestamp, diff --git a/packages/uni-watcher/src/schema.ts b/packages/uni-watcher/src/schema.ts index eda70ef7..099e4dfb 100644 --- a/packages/uni-watcher/src/schema.ts +++ b/packages/uni-watcher/src/schema.ts @@ -158,6 +158,7 @@ union Event = TransferEvent | PoolCreatedEvent | IncreaseLiquidityEvent | Decrea # Ethereum types type Block { + cid: String! hash: String! number: Int! timestamp: Int! From 5ad734ce061c13aeaeb9f14cba4cd04e3118e69e Mon Sep 17 00:00:00 2001 From: prathamesh Date: Thu, 7 Oct 2021 14:57:18 +0530 Subject: [PATCH 10/21] Update codegen docs --- packages/codegen/README.md | 12 +++++++----- .../codegen/src/templates/hooks-template.handlebars | 6 +++--- .../codegen/src/templates/readme-template.handlebars | 6 ++++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/codegen/README.md b/packages/codegen/README.md index fe6ac4fb..563bcf6c 100644 --- a/packages/codegen/README.md +++ b/packages/codegen/README.md @@ -53,13 +53,13 @@ yarn codegen --input-file ./test/examples/contracts/ERC721.sol --contract-name ERC721 --output-folder ../my-erc721-watcher --mode storage --kind lazy ``` - Generate code for `ERC721` contract in both `eth_call` and `storage` mode, `active` kind: + Generate code for `ERC20` contract in both `eth_call` and `storage` mode, `active` kind: ```bash - yarn codegen --input-file ../../node_modules/@openzeppelin/contracts/token/ERC721/ERC721.sol --contract-name ERC721 --output-folder ../demo-erc721-watcher --mode all --kind active + yarn codegen --input-file ../../node_modules/@openzeppelin/contracts/token/ERC20/ERC20.sol --contract-name ERC20 --output-folder ../demo-erc20-watcher --mode all --kind active ``` - This will create a folder called `demo-erc721-watcher` containing the generated code at the specified path. Follow the steps in [Run Generated Watcher](#run-generated-watcher) to setup and run the generated watcher. + This will create a folder called `demo-erc20-watcher` containing the generated code at the specified path. Follow the steps in [Run Generated Watcher](#run-generated-watcher) to setup and run the generated watcher. ## Run Generated Watcher @@ -79,7 +79,9 @@ * Edit the custom hook function `handleEvent` (triggered on an event) in `src/hooks.ts` to perform corresponding indexing using the `Indexer` object. - * Refer to `src/hooks.example.ts` for an example hook function for events in an ERC20 contract. + * Edit the custom hook function `handleBlock` (triggered on a block) in `src/hooks.ts` to save `IPLDBlock`s using the `Indexer` object. + + * The existing example hooks in `src/hooks.ts` are for an `ERC20` contract. ### Run @@ -106,7 +108,7 @@ * To watch a contract: ```bash - yarn watch:contract --address --kind ERC721 --starting-block [block-number] + yarn watch:contract --address --kind --starting-block [block-number] ``` * To fill a block range: diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 89ea22be..3d10014f 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -53,8 +53,8 @@ export async function handleBlock (indexer: Indexer, jobData: any): Promise Date: Thu, 7 Oct 2021 16:47:50 +0530 Subject: [PATCH 11/21] GQL API for getting last derived state (#3) * GQL API for getting last derived state * Rename query to getState * Change query names to getState and getStateByCid * Save BigInt as string --- packages/codegen/src/schema.ts | 16 +++++++++++++--- .../src/templates/database-template.handlebars | 4 ++-- .../src/templates/hooks-template.handlebars | 6 +++--- .../src/templates/resolvers-template.handlebars | 14 +++++++++++--- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/packages/codegen/src/schema.ts b/packages/codegen/src/schema.ts index 3f4c7cab..d8772ebd 100644 --- a/packages/codegen/src/schema.ts +++ b/packages/codegen/src/schema.ts @@ -98,7 +98,7 @@ export class Schema { this._addWatchContractMutation(); this._addIPLDType(); - this._addIPLDBlockQuery(); + this._addIPLDQuery(); return this._composer.buildSchema(); } @@ -250,15 +250,25 @@ export class Schema { }); } - _addIPLDBlockQuery (): void { + _addIPLDQuery (): void { this._composer.Query.addFields({ - ipldBlock: { + getStateByCID: { type: this._composer.getOTC('ResultIPLDBlock'), args: { cid: 'String!' } } }); + + this._composer.Query.addFields({ + getState: { + type: this._composer.getOTC('ResultIPLDBlock'), + args: { + blockHash: 'String!', + contractAddress: 'String!' + } + } + }); } /** diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 579d6f01..ea75b9a5 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -138,14 +138,14 @@ export class Database { let result: IPLDBlock | undefined; if (id) { - result = await repo.findOne({ id }); + result = await repo.findOne({ id }, { relations: ['block'] }); } else { // If IPLDBlock not found in frothy region get latest IPLDBlock in the pruned region. // Filter out IPLDBlocks from pruned blocks. const canonicalBlockNumber = blockNumber + 1; result = await repo.createQueryBuilder('ipld_block') - .innerJoinAndSelect('block_progress', 'block', 'block.id = ipld_block.block_id') + .leftJoinAndSelect('ipld_block.block', 'block') .where('block.is_pruned = false') .andWhere('ipld_block.contract_address = :contractAddress', { contractAddress }) .andWhere('block.block_number <= :canonicalBlockNumber', { canonicalBlockNumber }) diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 3d10014f..f063273b 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -66,8 +66,8 @@ export async function handleBlock (indexer: Indexer, jobData: any): Promise indexer.getResultEvent(event)); }, - ipldBlock: async (_: any, { cid }: { cid: string }) => { - log('ipldBlock', cid); + getStateByCID: async (_: any, { cid }: { cid: string }) => { + log('getStateByCID', cid); const ipldBlock = await indexer.getIPLDBlockByCid(cid); - return ipldBlock ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + }, + + getState: async (_: any, { blockHash, contractAddress }: { blockHash: string, contractAddress: string }) => { + log('getState', blockHash, contractAddress); + + const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress); + + return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; } } }; From 40f72aaee8ee0920a705bcdd392d7fd493c53d3e Mon Sep 17 00:00:00 2001 From: prathamesh Date: Thu, 7 Oct 2021 17:43:37 +0530 Subject: [PATCH 12/21] Move function to prepare IPLDBlock to indexer --- .../src/templates/hooks-template.handlebars | 57 ++++--------------- .../src/templates/indexer-template.handlebars | 31 ++++++++++ 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index f063273b..6fff2516 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -3,15 +3,9 @@ // import assert from 'assert'; -import * as codec from '@ipld/dag-json'; -import { sha256 } from 'multiformats/hashes/sha2'; -import { CID } from 'multiformats/cid'; import _ from 'lodash'; -import { BlockProgressInterface } from '@vulcanize/util'; - import { Indexer, ResultEvent } from './indexer'; -import { IPLDBlock } from './entity/IPLDBlock'; export async function handleBlock (indexer: Indexer, jobData: any): Promise { // Get events for current block and make an entry of updated values in IPLDBlock. @@ -29,14 +23,18 @@ export async function handleBlock (indexer: Indexer, jobData: any): Promise { - // If an IPLDBlock for { block, contractAddress } already exists, update the data field. - if (oldIpldBlock) { - const oldData = codec.decode(Buffer.from(oldIpldBlock.data)); - data = _.merge(oldData, data); - } - - // Encoding the data using dag-json codec. - const bytes = codec.encode(data); - - // Calculating sha256 (multi)hash of the encoded data. - const hash = await sha256.digest(bytes); - - // Calculating the CID: v1, code: dag-json, hash. - const cid = CID.create(1, codec.code, hash); - - let ipldBlock = oldIpldBlock || new IPLDBlock(); - ipldBlock = Object.assign(ipldBlock, { - block, - contractAddress, - cid: cid.toString(), - data: bytes - }); - - return ipldBlock; -} - /** * Event hook function. * @param indexer Indexer instance that contains methods to fetch and update the contract values in the database. diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 221eb748..c7d07519 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -9,6 +9,10 @@ import { DeepPartial } from 'typeorm'; import JSONbig from 'json-bigint'; import { ethers } from 'ethers'; import { BaseProvider } from '@ethersproject/providers'; +import * as codec from '@ipld/dag-json'; +import { sha256 } from 'multiformats/hashes/sha2'; +import { CID } from 'multiformats/cid'; +import _ from 'lodash'; import { EthClient } from '@vulcanize/ipld-eth-client'; import { StorageLayout } from '@vulcanize/solidity-mapper'; @@ -202,6 +206,33 @@ export class Indexer { } {{/each}} + async prepareIPLDBlock (block: BlockProgress, contractAddress: string, data: any, currentIPLDBlock?: IPLDBlock):Promise { + // If an IPLDBlock for { block, contractAddress } already exists, update the data field. + if (currentIPLDBlock) { + const oldData = codec.decode(Buffer.from(currentIPLDBlock.data)); + data = _.merge(oldData, data); + } + + // Encoding the data using dag-json codec. + const bytes = codec.encode(data); + + // Calculating sha256 (multi)hash of the encoded data. + const hash = await sha256.digest(bytes); + + // Calculating the CID: v1, code: dag-json, hash. + const cid = CID.create(1, codec.code, hash); + + let ipldBlock = currentIPLDBlock || new IPLDBlock(); + ipldBlock = Object.assign(ipldBlock, { + block, + contractAddress, + cid: cid.toString(), + data: bytes + }); + + return ipldBlock; + } + async getIPLDBlock (block: BlockProgress, contractAddress: string): Promise { const ipldBlocks = await this._db.getIPLDBlocks({ block, contractAddress }); From 1831e18a845323c804f40dbee54e1c4123f1ed76 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Fri, 8 Oct 2021 11:40:06 +0530 Subject: [PATCH 13/21] Refactor IPLDBlock hook --- .../src/templates/events-template.handlebars | 9 +- .../src/templates/hooks-template.handlebars | 38 ++------- .../src/templates/indexer-template.handlebars | 83 ++++++++++++------- .../templates/job-runner-template.handlebars | 2 + 4 files changed, 69 insertions(+), 63 deletions(-) diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index 67de7b75..aa4ae94b 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -74,12 +74,13 @@ export class EventWatcher { this._jobQueue.onComplete(QUEUE_BLOCK_PROCESSING, async (job) => { await this._baseEventWatcher.blockProcessingCompleteHandler(job); + // TODO Call post-block hook here. + // Call custom hook function to update Blocks table. - const { data: { request: { data } } } = job; - const { kind } = data; + const { data: { request: { data: { kind, blockHash } } } } = job; if (kind === JOB_KIND_INDEX) { - await handleBlock(this._indexer, data); + await handleBlock(this._indexer, blockHash); } }); } @@ -88,6 +89,8 @@ export class EventWatcher { await this._jobQueue.onComplete(QUEUE_EVENT_PROCESSING, async (job) => { const dbEvent = await this._baseEventWatcher.eventProcessingCompleteHandler(job); + // TODO Can call IPLDBlock hook here if blockProgress.isComplete() is true (get blockProgress from eventProcessingCompleteHandler). + const { data: { request, failed, state, createdOn } } = job; const timeElapsedInSeconds = (Date.now() - Date.parse(createdOn)) / 1000; diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 6fff2516..01508419 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -7,10 +7,13 @@ import _ from 'lodash'; import { Indexer, ResultEvent } from './indexer'; -export async function handleBlock (indexer: Indexer, jobData: any): Promise { +/** + * IPLDBlock hook function. + * @param indexer Indexer instance that contains methods to fetch the contract varaiable values. + * @param blockHash Block hash of the concerned block. + */ +export async function handleBlock (indexer: Indexer, blockHash: string): Promise { // Get events for current block and make an entry of updated values in IPLDBlock. - - const { blockHash } = jobData; const events = await indexer.getEventsByFilter(blockHash); // No IPLDBlock entry if there are no events. @@ -20,33 +23,10 @@ export async function handleBlock (indexer: Indexer, jobData: any): Promise { - // If an IPLDBlock for { block, contractAddress } already exists, update the data field. - if (currentIPLDBlock) { - const oldData = codec.decode(Buffer.from(currentIPLDBlock.data)); - data = _.merge(oldData, data); - } - - // Encoding the data using dag-json codec. - const bytes = codec.encode(data); - - // Calculating sha256 (multi)hash of the encoded data. - const hash = await sha256.digest(bytes); - - // Calculating the CID: v1, code: dag-json, hash. - const cid = CID.create(1, codec.code, hash); - - let ipldBlock = currentIPLDBlock || new IPLDBlock(); - ipldBlock = Object.assign(ipldBlock, { - block, - contractAddress, - cid: cid.toString(), - data: bytes - }); - - return ipldBlock; - } - async getIPLDBlock (block: BlockProgress, contractAddress: string): Promise { const ipldBlocks = await this._db.getIPLDBlocks({ block, contractAddress }); @@ -251,10 +224,6 @@ export class Indexer { return ipldBlocks[0]; } - async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { - return this._db.saveOrUpdateIPLDBlock(ipldBlock); - } - async getPrevIPLDBlock (blockHash: string, contractAddress: string): Promise { const dbTx = await this._db.createTransactionRunner(); let res; @@ -271,6 +240,58 @@ export class Indexer { return res; } + async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { + return this._db.saveOrUpdateIPLDBlock(ipldBlock); + } + + async prepareIPLDBlock (block: BlockProgress, contractAddress: string, data: any):Promise { + // Get an existing IPLDBlock for current block and contractAddress. + const currentIPLDBlock = await this.getIPLDBlock(block, contractAddress); + + // If an IPLDBlock for { block, contractAddress } already exists, update the data field. + if (currentIPLDBlock) { + const oldData = codec.decode(Buffer.from(currentIPLDBlock.data)); + data = _.merge(oldData, data); + } else { + // Fetch the parent IPLDBlock. + const parentIPLDBlock = await this.getPrevIPLDBlock(block.blockHash, contractAddress); + + // Setting the meta-data for an IPLDBlock (done only once per block). + data.meta = { + id: contractAddress, + kind: 'diff', + parent: { + '/': parentIPLDBlock ? parentIPLDBlock.cid : null + }, + ethBlock: { + cid: { + '/': block.cid + }, + num: block.blockNumber + } + }; + } + + // Encoding the data using dag-json codec. + const bytes = codec.encode(data); + + // Calculating sha256 (multi)hash of the encoded data. + const hash = await sha256.digest(bytes); + + // Calculating the CID: v1, code: dag-json, hash. + const cid = CID.create(1, codec.code, hash); + + let ipldBlock = currentIPLDBlock || new IPLDBlock(); + ipldBlock = Object.assign(ipldBlock, { + block, + contractAddress, + cid: cid.toString(), + data: bytes + }); + + return ipldBlock; + } + async triggerIndexingOnEvent (event: Event): Promise { const resultEvent = this.getResultEvent(event); diff --git a/packages/codegen/src/templates/job-runner-template.handlebars b/packages/codegen/src/templates/job-runner-template.handlebars index c1f6c804..4ace1537 100644 --- a/packages/codegen/src/templates/job-runner-template.handlebars +++ b/packages/codegen/src/templates/job-runner-template.handlebars @@ -46,6 +46,8 @@ export class JobRunner { async subscribeBlockProcessingQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => { + // TODO Call pre-block hook here (Directly or indirectly (Like done through indexer.processEvent for events)). + await this._baseJobRunner.processBlock(job); await this._jobQueue.markComplete(job); From 6d5cafc084dde3eef92276e0158d9ddd9cbe96bc Mon Sep 17 00:00:00 2001 From: prathamesh Date: Fri, 8 Oct 2021 15:43:58 +0530 Subject: [PATCH 14/21] Add genesis hook --- .../templates/database-template.handlebars | 7 +++- .../src/templates/events-template.handlebars | 8 ++--- .../src/templates/hooks-template.handlebars | 21 ++++++++++-- .../src/templates/indexer-template.handlebars | 24 +++++++++++--- .../watch-contract-template.handlebars | 32 +++++++++++++++++-- 5 files changed, 76 insertions(+), 16 deletions(-) diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index ea75b9a5..73f3e56b 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -197,7 +197,7 @@ export class Database { return this._baseDatabase.saveEvents(blockRepo, eventRepo, block, events); } - async saveContract (address: string, kind: string, startingBlock: number): Promise { + async saveContract (address: string, startingBlock: number, kind: string): Promise { await this._conn.transaction(async (tx) => { const repo = tx.getRepository(Contract); @@ -252,6 +252,11 @@ export class Database { return this._baseDatabase.getBlockProgress(repo, blockHash); } + async getLatestBlockProgress (): Promise { + const repo = this._conn.getRepository(BlockProgress); + return repo.findOne({ order: { blockNumber: 'DESC' } }); + } + async updateBlockProgress (queryRunner: QueryRunner, blockHash: string, lastProcessedEventIndex: number): Promise { const repo = queryRunner.manager.getRepository(BlockProgress); diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index aa4ae94b..cd66398f 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -18,7 +18,6 @@ import { import { Indexer } from './indexer'; import { Event } from './entity/Event'; -import { handleBlock } from './hooks'; const EVENT = 'event'; @@ -74,13 +73,10 @@ export class EventWatcher { this._jobQueue.onComplete(QUEUE_BLOCK_PROCESSING, async (job) => { await this._baseEventWatcher.blockProcessingCompleteHandler(job); - // TODO Call post-block hook here. - - // Call custom hook function to update Blocks table. const { data: { request: { data: { kind, blockHash } } } } = job; if (kind === JOB_KIND_INDEX) { - await handleBlock(this._indexer, blockHash); + await this._indexer.processBlock(blockHash); } }); } @@ -89,7 +85,7 @@ export class EventWatcher { await this._jobQueue.onComplete(QUEUE_EVENT_PROCESSING, async (job) => { const dbEvent = await this._baseEventWatcher.eventProcessingCompleteHandler(job); - // TODO Can call IPLDBlock hook here if blockProgress.isComplete() is true (get blockProgress from eventProcessingCompleteHandler). + // TODO Can call IPLDBlock hook here if dbEvent.blockProgress.isComplete() is true. const { data: { request, failed, state, createdOn } } = job; diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 01508419..8f588a47 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -6,13 +6,30 @@ import assert from 'assert'; import _ from 'lodash'; import { Indexer, ResultEvent } from './indexer'; +import { BlockProgress } from './entity/BlockProgress'; /** - * IPLDBlock hook function. + * Genesis hook function. + * @param indexer Indexer instance. + * @param block Concerned block. + * @param contractAddress Address of the concerned contract. + */ +export async function genesisHook (indexer: Indexer, block: BlockProgress, contractAddress: string): Promise { + // Store the genesis state values in an IPLDBlock. + const ipldBlockData: any = {}; + _.set(ipldBlockData, 'state._balances[0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc]', BigInt(1000000000000000000000).toString()); + _.set(ipldBlockData, 'meta.kind', 'checkpoint'); + + const ipldBlock = await indexer.prepareIPLDBlock(block, contractAddress, ipldBlockData); + await indexer.saveOrUpdateIPLDBlock(ipldBlock); +} + +/** + * Post-block hook function. * @param indexer Indexer instance that contains methods to fetch the contract varaiable values. * @param blockHash Block hash of the concerned block. */ -export async function handleBlock (indexer: Indexer, blockHash: string): Promise { +export async function postBlockHook (indexer: Indexer, blockHash: string): Promise { // Get events for current block and make an entry of updated values in IPLDBlock. const events = await indexer.getEventsByFilter(blockHash); diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 83475ad6..5efc4e30 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -25,7 +25,7 @@ import { SyncStatus } from './entity/SyncStatus'; import { BlockProgress } from './entity/BlockProgress'; import { IPLDBlock } from './entity/IPLDBlock'; import artifacts from './artifacts/{{inputFileName}}.json'; -import { handleEvent } from './hooks'; +import { genesisHook, handleEvent, postBlockHook } from './hooks'; const log = debug('vulcanize:indexer'); @@ -206,6 +206,11 @@ export class Indexer { } {{/each}} + async processBlock (blockHash: string): Promise { + // Call custom post-block hook. + await postBlockHook(this, blockHash); + } + async getIPLDBlock (block: BlockProgress, contractAddress: string): Promise { const ipldBlocks = await this._db.getIPLDBlocks({ block, contractAddress }); @@ -257,7 +262,7 @@ export class Indexer { const parentIPLDBlock = await this.getPrevIPLDBlock(block.blockHash, contractAddress); // Setting the meta-data for an IPLDBlock (done only once per block). - data.meta = { + const metaData = { id: contractAddress, kind: 'diff', parent: { @@ -270,6 +275,8 @@ export class Indexer { num: block.blockNumber } }; + + data.meta = _.merge(metaData, data.meta); } // Encoding the data using dag-json codec. @@ -335,9 +342,18 @@ export class Indexer { return { eventName, eventInfo }; } - async watchContract (address: string, startingBlock: number): Promise { + async watchContract (address: string, startingBlock: number, kind?: string): Promise { + kind = kind || 'ERC20'; + // Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress). - await this._db.saveContract(ethers.utils.getAddress(address), '{{contractName}}', startingBlock); + await this._db.saveContract(ethers.utils.getAddress(address), startingBlock, kind); + + // Getting the current block. + const currentBlock = await this._db.getLatestBlockProgress(); + assert(currentBlock); + + // Call genesis hook here. + await genesisHook(this, currentBlock, address); return true; } diff --git a/packages/codegen/src/templates/watch-contract-template.handlebars b/packages/codegen/src/templates/watch-contract-template.handlebars index a9b9640e..056700a3 100644 --- a/packages/codegen/src/templates/watch-contract-template.handlebars +++ b/packages/codegen/src/templates/watch-contract-template.handlebars @@ -5,10 +5,14 @@ import assert from 'assert'; import yargs from 'yargs'; import 'reflect-metadata'; +import { getDefaultProvider } from 'ethers'; import { Config, DEFAULT_CONFIG_PATH, getConfig } from '@vulcanize/util'; +import { getCache } from '@vulcanize/cache'; +import { EthClient } from '@vulcanize/ipld-eth-client'; import { Database } from '../database'; +import { Indexer } from '../indexer'; (async () => { const argv = await yargs.parserConfiguration({ @@ -42,14 +46,36 @@ import { Database } from '../database'; }).argv; const config: Config = await getConfig(argv.configFile); - const { database: dbConfig } = config; + assert(config.server, 'Missing server config'); - assert(dbConfig); + const { upstream, database: dbConfig } = config; + assert(upstream, 'Missing upstream config'); + assert(dbConfig, 'Missing database config'); const db = new Database(dbConfig); await db.init(); - await db.saveContract(argv.address, argv.kind, argv.startingBlock); + const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream; + assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint'); + assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); + + const cache = await getCache(cacheConfig); + + const ethClient = new EthClient({ + gqlEndpoint: gqlApiEndpoint, + gqlSubscriptionEndpoint: gqlPostgraphileEndpoint, + cache + }); + + const postgraphileClient = new EthClient({ + gqlEndpoint: gqlPostgraphileEndpoint, + cache + }); + + const ethProvider = getDefaultProvider(rpcProviderEndpoint); + + const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider); + await indexer.watchContract(argv.address, argv.startingBlock, argv.kind); await db.close(); })(); From 0592a4de4c264374f6a598c56daa38e512307a86 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Fri, 8 Oct 2021 17:26:33 +0530 Subject: [PATCH 15/21] Call post-block hook after a block is marked as complete --- .../src/templates/events-template.handlebars | 9 ++------- .../codegen/src/templates/hooks-template.handlebars | 13 ++++++++++++- .../src/templates/indexer-template.handlebars | 2 +- packages/util/src/events.ts | 1 + 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index cd66398f..9d653c6a 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -72,12 +72,6 @@ export class EventWatcher { async initBlockProcessingOnCompleteHandler (): Promise { this._jobQueue.onComplete(QUEUE_BLOCK_PROCESSING, async (job) => { await this._baseEventWatcher.blockProcessingCompleteHandler(job); - - const { data: { request: { data: { kind, blockHash } } } } = job; - - if (kind === JOB_KIND_INDEX) { - await this._indexer.processBlock(blockHash); - } }); } @@ -85,7 +79,8 @@ export class EventWatcher { await this._jobQueue.onComplete(QUEUE_EVENT_PROCESSING, async (job) => { const dbEvent = await this._baseEventWatcher.eventProcessingCompleteHandler(job); - // TODO Can call IPLDBlock hook here if dbEvent.blockProgress.isComplete() is true. + // Use an indexer method to call a post-block hook if the block is marked as complete. + if (dbEvent.block.isComplete && dbEvent.eventName !== UNKNOWN_EVENT_NAME) await this._indexer.processBlock(dbEvent.block.blockHash); const { data: { request, failed, state, createdOn } } = job; diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 8f588a47..477d65f3 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -17,7 +17,18 @@ import { BlockProgress } from './entity/BlockProgress'; export async function genesisHook (indexer: Indexer, block: BlockProgress, contractAddress: string): Promise { // Store the genesis state values in an IPLDBlock. const ipldBlockData: any = {}; - _.set(ipldBlockData, 'state._balances[0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc]', BigInt(1000000000000000000000).toString()); + + const accounts = [ + '0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc' + ]; + + // Setting the initial balances of accounts. + for (const account of accounts) { + const balance = await indexer._balances(block.blockHash, contractAddress, account); + _.set(ipldBlockData, `state._balances[${account}]`, balance.value.toString()); + } + + // Setting the IPLDBlock kind as checkpoint. _.set(ipldBlockData, 'meta.kind', 'checkpoint'); const ipldBlock = await indexer.prepareIPLDBlock(block, contractAddress, ipldBlockData); diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 5efc4e30..cc11bc0f 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -352,7 +352,7 @@ export class Indexer { const currentBlock = await this._db.getLatestBlockProgress(); assert(currentBlock); - // Call genesis hook here. + // Call custom genesis hook. await genesisHook(this, currentBlock, address); return true; diff --git a/packages/util/src/events.ts b/packages/util/src/events.ts index 2abb2ce8..db1612c5 100644 --- a/packages/util/src/events.ts +++ b/packages/util/src/events.ts @@ -74,6 +74,7 @@ export class EventWatcher { const blockProgress = await this._indexer.getBlockProgress(dbEvent.block.blockHash); if (blockProgress) { await this.publishBlockProgressToSubscribers(blockProgress); + dbEvent.block = blockProgress; } return dbEvent; From f0f79c7d45a9598ad0b82e1708cbafc13ad8bef7 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Mon, 11 Oct 2021 14:08:46 +0530 Subject: [PATCH 16/21] Add IPLDBlock checkpointing --- .../codegen/src/data/entities/IPLDBlock.yaml | 4 ++ packages/codegen/src/schema.ts | 3 +- .../src/templates/config-template.handlebars | 2 + .../templates/database-template.handlebars | 70 +++++++++++++++++-- .../src/templates/events-template.handlebars | 13 ++-- .../src/templates/hooks-template.handlebars | 19 ++--- .../src/templates/indexer-template.handlebars | 68 +++++++++++++++--- .../templates/job-runner-template.handlebars | 18 +++-- .../templates/resolvers-template.handlebars | 6 +- packages/uni-info-watcher/src/indexer.ts | 5 ++ packages/uni-watcher/src/indexer.ts | 5 ++ packages/util/src/config.ts | 2 + packages/util/src/constants.ts | 1 + packages/util/src/job-runner.ts | 14 +++- packages/util/src/types.ts | 1 + 15 files changed, 194 insertions(+), 37 deletions(-) diff --git a/packages/codegen/src/data/entities/IPLDBlock.yaml b/packages/codegen/src/data/entities/IPLDBlock.yaml index 8879d9b4..145d46fd 100644 --- a/packages/codegen/src/data/entities/IPLDBlock.yaml +++ b/packages/codegen/src/data/entities/IPLDBlock.yaml @@ -20,6 +20,10 @@ columns: pgType: varchar tsType: string columnType: Column + - name: kind + pgType: varchar + tsType: string + columnType: Column - name: data pgType: text tsType: string diff --git a/packages/codegen/src/schema.ts b/packages/codegen/src/schema.ts index d8772ebd..2fc0f885 100644 --- a/packages/codegen/src/schema.ts +++ b/packages/codegen/src/schema.ts @@ -291,7 +291,8 @@ export class Schema { type: 'Boolean!', args: { contractAddress: 'String!', - startingBlock: 'Int' + startingBlock: 'Int', + kind: 'String!' } } }); diff --git a/packages/codegen/src/templates/config-template.handlebars b/packages/codegen/src/templates/config-template.handlebars index fe2be89c..8f25a1b8 100644 --- a/packages/codegen/src/templates/config-template.handlebars +++ b/packages/codegen/src/templates/config-template.handlebars @@ -2,6 +2,8 @@ host = "127.0.0.1" port = 3008 kind = "{{watcherKind}}" + checkpointing = true + checkpointInterval = 30 [database] type = "postgres" diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 73f3e56b..ba95ce1c 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -3,7 +3,7 @@ // import assert from 'assert'; -import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner } from 'typeorm'; +import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner, In, Between } from 'typeorm'; import path from 'path'; import { Database as BaseDatabase, MAX_REORG_DEPTH } from '@vulcanize/util'; @@ -84,11 +84,6 @@ export class Database { return repo.find({ where, relations: ['block'] }); } - async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { - const repo = this._conn.getRepository(IPLDBlock); - return repo.save(ipldBlock); - } - async getPrevIPLDBlock (queryRunner: QueryRunner, blockHash: string, contractAddress: string): Promise { const repo = this._conn.getRepository(IPLDBlock); @@ -157,6 +152,69 @@ export class Database { return result; } + async getPrevIPLDBlocks (queryRunner: QueryRunner, blockHash: string, blockNumber: number, contractAddress: string, checkpointBlockNumber: number): Promise { + const repo = this._conn.getRepository(IPLDBlock); + + const heirerchicalQuery = ` + WITH RECURSIVE cte_query AS + ( + SELECT + b.block_hash, + b.parent_hash, + 1 as depth, + i.id + FROM + block_progress b + LEFT JOIN + ipld_block i ON i.block_id = b.id + WHERE + b.block_hash = $1 + UNION ALL + SELECT + b.block_hash, + b.parent_hash, + c.depth + 1, + i.id + FROM + block_progress b + LEFT JOIN + ipld_block i + ON i.block_id = b.id + AND i.contract_address = $2 + INNER JOIN + cte_query c ON c.parent_hash = b.block_hash + WHERE + c.depth < $3 + ) + SELECT + id + FROM + cte_query + WHERE id IS NOT NULL + `; + + // Fetching ids for previous IPLDBlocks in the frothy region. + const queryResult = await queryRunner.query(heirerchicalQuery, [blockHash, contractAddress, MAX_REORG_DEPTH]); + const frothyIds = queryResult.map((obj: any) => obj.id); + + // Fetching all diff blocks along with the previous checkpoint. + const ipldBlocks = await repo.find({ + relations: ['block'], + where: [ + { contractAddress, block: { isPruned: false, blockNumber: Between(checkpointBlockNumber, blockNumber - MAX_REORG_DEPTH) } }, + { id: In(frothyIds) } + ], + order: { block: 'ASC' } + }); + + return ipldBlocks; + } + + async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { + const repo = this._conn.getRepository(IPLDBlock); + return repo.save(ipldBlock); + } + async getContract (address: string): Promise { const repo = this._conn.getRepository(Contract); diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index 9d653c6a..57de8a7b 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -12,8 +12,8 @@ import { EventWatcher as BaseEventWatcher, QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, - UNKNOWN_EVENT_NAME, - JOB_KIND_INDEX + QUEUE_BLOCK_CHECKPOINT, + UNKNOWN_EVENT_NAME } from '@vulcanize/util'; import { Indexer } from './indexer'; @@ -79,8 +79,13 @@ export class EventWatcher { await this._jobQueue.onComplete(QUEUE_EVENT_PROCESSING, async (job) => { const dbEvent = await this._baseEventWatcher.eventProcessingCompleteHandler(job); - // Use an indexer method to call a post-block hook if the block is marked as complete. - if (dbEvent.block.isComplete && dbEvent.eventName !== UNKNOWN_EVENT_NAME) await this._indexer.processBlock(dbEvent.block.blockHash); + // If the block is marked as complete: + // a. Use an indexer method to call a post-block hook. + // b. Push a block checkpointing job. + if (dbEvent.block.isComplete) { + await this._indexer.processBlock(dbEvent.block.blockHash); + await this._jobQueue.pushJob(QUEUE_BLOCK_CHECKPOINT, { blockHash: dbEvent.block.blockHash, blockNumber: dbEvent.block.blockNumber }); + } const { data: { request, failed, state, createdOn } } = job; diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 477d65f3..0ff100a1 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -5,9 +5,15 @@ import assert from 'assert'; import _ from 'lodash'; +import { UNKNOWN_EVENT_NAME } from '@vulcanize/util'; + import { Indexer, ResultEvent } from './indexer'; import { BlockProgress } from './entity/BlockProgress'; +const ACCOUNTS = [ + '0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc' +]; + /** * Genesis hook function. * @param indexer Indexer instance. @@ -18,20 +24,13 @@ export async function genesisHook (indexer: Indexer, block: BlockProgress, contr // Store the genesis state values in an IPLDBlock. const ipldBlockData: any = {}; - const accounts = [ - '0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc' - ]; - // Setting the initial balances of accounts. - for (const account of accounts) { + for (const account of ACCOUNTS) { const balance = await indexer._balances(block.blockHash, contractAddress, account); _.set(ipldBlockData, `state._balances[${account}]`, balance.value.toString()); } - // Setting the IPLDBlock kind as checkpoint. - _.set(ipldBlockData, 'meta.kind', 'checkpoint'); - - const ipldBlock = await indexer.prepareIPLDBlock(block, contractAddress, ipldBlockData); + const ipldBlock = await indexer.prepareIPLDBlock(block, contractAddress, ipldBlockData, 'checkpoint'); await indexer.saveOrUpdateIPLDBlock(ipldBlock); } @@ -48,6 +47,8 @@ export async function postBlockHook (indexer: Indexer, blockHash: string): Promi if (!events) return; for (const event of events) { + if (event.eventName === UNKNOWN_EVENT_NAME) continue; + const block = event.block; const contractAddress = event.contract; diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index cc11bc0f..a8641910 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -75,12 +75,13 @@ export class Indexer { _ethProvider: BaseProvider _postgraphileClient: EthClient; _baseIndexer: BaseIndexer + _checkpointInterval: number _abi: JsonFragment[] _storageLayout: StorageLayout _contract: ethers.utils.Interface - constructor (db: Database, ethClient: EthClient, postgraphileClient: EthClient, ethProvider: BaseProvider) { + constructor (db: Database, ethClient: EthClient, postgraphileClient: EthClient, ethProvider: BaseProvider, checkpointInterval?: number) { assert(db); assert(ethClient); @@ -89,6 +90,7 @@ export class Indexer { this._ethProvider = ethProvider; this._postgraphileClient = postgraphileClient; this._baseIndexer = new BaseIndexer(this._db, this._ethClient, this._postgraphileClient); + this._checkpointInterval = checkpointInterval || 0; const { abi, storageLayout } = artifacts; @@ -211,6 +213,40 @@ export class Indexer { await postBlockHook(this, blockHash); } + async processCheckpoint (job: any): Promise { + // Create a checkpoint IPLDBlock for contracts that were checkpointed checkPointInterval blocks before. + + // Return if checkpointInterval is <= 0. + if (this._checkpointInterval <= 0) return; + + const { data: { blockNumber: currentBlockNumber, blockHash: currentBlockHash } } = job; + + // Get checkpoint IPLDBlocks with blockNumber: current-checkPointInterval. + // Assuming checkPointInterval < MAX_REORG_DEPTH. + const prevCheckpointBlocks = await this._db.getIPLDBlocks({ block: { blockNumber: currentBlockNumber - this._checkpointInterval }, kind: 'checkpoint' }); + + // For each contractAddress, merge the diff till now. + for (const checkpointBlock of prevCheckpointBlocks) { + const { contractAddress } = checkpointBlock; + + // Fetching all diff blocks along with the previous checkpoint. + const diffBlocks = await this.getPrevIPLDBlocks(currentBlockHash, currentBlockNumber, contractAddress, currentBlockNumber - this._checkpointInterval); + + let checkPoint: any = {}; + for (const diffBlock of diffBlocks) { + const diff = codec.decode(Buffer.from(diffBlock.data)); + checkPoint = _.merge(checkPoint, diff); + } + + // Getting the current block. + const block = await this.getBlockProgress(currentBlockHash); + assert(block); + + const ipldBlock = await this.prepareIPLDBlock(block, contractAddress, checkPoint, 'checkpoint'); + await this.saveOrUpdateIPLDBlock(ipldBlock); + } + } + async getIPLDBlock (block: BlockProgress, contractAddress: string): Promise { const ipldBlocks = await this._db.getIPLDBlocks({ block, contractAddress }); @@ -245,11 +281,28 @@ export class Indexer { return res; } + async getPrevIPLDBlocks (blockHash: string, blockNumber: number, contractAddress: string, checkpointBlockNumber: number): Promise { + const dbTx = await this._db.createTransactionRunner(); + let res; + + try { + res = await this._db.getPrevIPLDBlocks(dbTx, blockHash, blockNumber, contractAddress, checkpointBlockNumber); + await dbTx.commitTransaction(); + } catch (error) { + await dbTx.rollbackTransaction(); + throw error; + } finally { + await dbTx.release(); + } + + return res; + } + async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { return this._db.saveOrUpdateIPLDBlock(ipldBlock); } - async prepareIPLDBlock (block: BlockProgress, contractAddress: string, data: any):Promise { + async prepareIPLDBlock (block: BlockProgress, contractAddress: string, data: any, kind?: string):Promise { // Get an existing IPLDBlock for current block and contractAddress. const currentIPLDBlock = await this.getIPLDBlock(block, contractAddress); @@ -262,9 +315,9 @@ export class Indexer { const parentIPLDBlock = await this.getPrevIPLDBlock(block.blockHash, contractAddress); // Setting the meta-data for an IPLDBlock (done only once per block). - const metaData = { + data.meta = { id: contractAddress, - kind: 'diff', + kind: kind || 'diff', parent: { '/': parentIPLDBlock ? parentIPLDBlock.cid : null }, @@ -275,8 +328,6 @@ export class Indexer { num: block.blockNumber } }; - - data.meta = _.merge(metaData, data.meta); } // Encoding the data using dag-json codec. @@ -293,6 +344,7 @@ export class Indexer { block, contractAddress, cid: cid.toString(), + kind: data.meta.kind, data: bytes }); @@ -342,9 +394,7 @@ export class Indexer { return { eventName, eventInfo }; } - async watchContract (address: string, startingBlock: number, kind?: string): Promise { - kind = kind || 'ERC20'; - + async watchContract (address: string, startingBlock: number, kind: string): Promise { // Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress). await this._db.saveContract(ethers.utils.getAddress(address), startingBlock, kind); diff --git a/packages/codegen/src/templates/job-runner-template.handlebars b/packages/codegen/src/templates/job-runner-template.handlebars index 4ace1537..a8c2baf0 100644 --- a/packages/codegen/src/templates/job-runner-template.handlebars +++ b/packages/codegen/src/templates/job-runner-template.handlebars @@ -17,6 +17,7 @@ import { JobRunner as BaseJobRunner, QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, + QUEUE_BLOCK_CHECKPOINT, JobQueueConfig, DEFAULT_CONFIG_PATH } from '@vulcanize/util'; @@ -39,9 +40,10 @@ export class JobRunner { this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._indexer, this._jobQueue); } - async start (): Promise { + async start (checkpointing: boolean): Promise { await this.subscribeBlockProcessingQueue(); await this.subscribeEventProcessingQueue(); + if (checkpointing) await this.subscribeBlockCheckpointQueue(); } async subscribeBlockProcessingQueue (): Promise { @@ -66,6 +68,14 @@ export class JobRunner { await this._jobQueue.markComplete(job); }); } + + async subscribeBlockCheckpointQueue (): Promise { + await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => { + await this._indexer.processCheckpoint(job); + + await this._jobQueue.markComplete(job); + }); + } } export const main = async (): Promise => { @@ -83,7 +93,7 @@ export const main = async (): Promise => { assert(config.server, 'Missing server config'); - const { upstream, database: dbConfig, jobQueue: jobQueueConfig } = config; + const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: { checkpointing, checkpointInterval } } = config; assert(dbConfig, 'Missing database config'); @@ -109,7 +119,7 @@ export const main = async (): Promise => { }); const ethProvider = getDefaultProvider(rpcProviderEndpoint); - const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider); + const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, checkpointInterval); assert(jobQueueConfig, 'Missing job queue config'); @@ -120,7 +130,7 @@ export const main = async (): Promise => { await jobQueue.start(); const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue); - await jobRunner.start(); + await jobRunner.start(checkpointing); }; main().then(() => { diff --git a/packages/codegen/src/templates/resolvers-template.handlebars b/packages/codegen/src/templates/resolvers-template.handlebars index 6466d1a0..b408a682 100644 --- a/packages/codegen/src/templates/resolvers-template.handlebars +++ b/packages/codegen/src/templates/resolvers-template.handlebars @@ -34,9 +34,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch }, Mutation: { - watchContract: (_: any, { contractAddress, startingBlock = 1 }: { contractAddress: string, startingBlock: number }): Promise => { - log('watchContract', contractAddress, startingBlock); - return indexer.watchContract(contractAddress, startingBlock); + watchContract: (_: any, { contractAddress, startingBlock = 1, kind }: { contractAddress: string, startingBlock: number, kind: string }): Promise => { + log('watchContract', contractAddress, startingBlock, kind); + return indexer.watchContract(contractAddress, startingBlock, kind); } }, diff --git a/packages/uni-info-watcher/src/indexer.ts b/packages/uni-info-watcher/src/indexer.ts index 047e8448..2802f2ce 100644 --- a/packages/uni-info-watcher/src/indexer.ts +++ b/packages/uni-info-watcher/src/indexer.ts @@ -152,6 +152,11 @@ export class Indexer implements IndexerInterface { log('Event processing completed for', eventName); } + async processBlock (blockHash: string): Promise { + // Empty post-block method. + assert(blockHash); + } + async getBlocks (where: { [key: string]: any } = {}, queryOptions: QueryOptions): Promise { if (where.timestamp_gt) { where.blockTimestamp_gt = where.timestamp_gt; diff --git a/packages/uni-watcher/src/indexer.ts b/packages/uni-watcher/src/indexer.ts index 95b8551f..080c65c3 100644 --- a/packages/uni-watcher/src/indexer.ts +++ b/packages/uni-watcher/src/indexer.ts @@ -116,6 +116,11 @@ export class Indexer implements IndexerInterface { } } + async processBlock (blockHash: string): Promise { + // Empty post-block method. + assert(blockHash); + } + parseEventNameAndArgs (kind: string, logObj: any): any { let eventName = UNKNOWN_EVENT_NAME; let eventInfo = {}; diff --git a/packages/util/src/config.ts b/packages/util/src/config.ts index c15f8d14..972e89c8 100644 --- a/packages/util/src/config.ts +++ b/packages/util/src/config.ts @@ -24,6 +24,8 @@ export interface Config { port: number; mode: string; kind: string; + checkpointing: boolean; + checkpointInterval: number; }; database: ConnectionOptions; upstream: { diff --git a/packages/util/src/constants.ts b/packages/util/src/constants.ts index 8f54bf4d..ec28aad4 100644 --- a/packages/util/src/constants.ts +++ b/packages/util/src/constants.ts @@ -7,6 +7,7 @@ export const MAX_REORG_DEPTH = 16; export const QUEUE_BLOCK_PROCESSING = 'block-processing'; export const QUEUE_EVENT_PROCESSING = 'event-processing'; export const QUEUE_CHAIN_PRUNING = 'chain-pruning'; +export const QUEUE_BLOCK_CHECKPOINT = 'block-checkpoint'; export const JOB_KIND_INDEX = 'index'; export const JOB_KIND_PRUNE = 'prune'; diff --git a/packages/util/src/job-runner.ts b/packages/util/src/job-runner.ts index f8fb50d4..e5c2fecb 100644 --- a/packages/util/src/job-runner.ts +++ b/packages/util/src/job-runner.ts @@ -8,7 +8,14 @@ import { wait } from '.'; import { createPruningJob } from './common'; import { JobQueueConfig } from './config'; -import { JOB_KIND_INDEX, JOB_KIND_PRUNE, MAX_REORG_DEPTH, QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING } from './constants'; +import { + JOB_KIND_INDEX, + JOB_KIND_PRUNE, + MAX_REORG_DEPTH, + QUEUE_BLOCK_PROCESSING, + QUEUE_EVENT_PROCESSING, + QUEUE_BLOCK_CHECKPOINT +} from './constants'; import { JobQueue } from './job-queue'; import { EventInterface, IndexerInterface, SyncStatusInterface } from './types'; @@ -179,6 +186,11 @@ export class JobRunner { await wait(jobDelayInMilliSecs); const events = await this._indexer.getOrFetchBlockEvents({ cid, blockHash, blockNumber, parentHash, blockTimestamp: timestamp }); + if (!events.length) { + await this._indexer.processBlock(blockHash); + await this._jobQueue.pushJob(QUEUE_BLOCK_CHECKPOINT, { blockHash, blockNumber }); + } + for (let ei = 0; ei < events.length; ei++) { await this._jobQueue.pushJob(QUEUE_EVENT_PROCESSING, { id: events[ei].id, publish: true }); } diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index fbb1a216..e994bf58 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -61,6 +61,7 @@ export interface IndexerInterface { updateSyncStatusIndexedBlock (blockHash: string, blockNumber: number): Promise updateSyncStatusCanonicalBlock (blockHash: string, blockNumber: number): Promise markBlocksAsPruned (blocks: BlockProgressInterface[]): Promise; + processBlock(blockHash: string): Promise; } export interface EventWatcherInterface { From 31d84422fe682f3bdc182c84f152d76b7534ca6d Mon Sep 17 00:00:00 2001 From: prathamesh Date: Mon, 11 Oct 2021 15:54:07 +0530 Subject: [PATCH 17/21] Use queryRunner instead of a new repo for queries --- .../templates/database-template.handlebars | 28 ++++++++++--------- .../src/templates/indexer-template.handlebars | 13 +++++---- packages/erc20-watcher/src/indexer.ts | 9 ++++-- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index ba95ce1c..4f023801 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -85,8 +85,6 @@ export class Database { } async getPrevIPLDBlock (queryRunner: QueryRunner, blockHash: string, contractAddress: string): Promise { - const repo = this._conn.getRepository(IPLDBlock); - const heirerchicalQuery = ` WITH RECURSIVE cte_query AS ( @@ -133,13 +131,13 @@ export class Database { let result: IPLDBlock | undefined; if (id) { - result = await repo.findOne({ id }, { relations: ['block'] }); + result = await queryRunner.manager.findOne(IPLDBlock, { id }, { relations: ['block'] }); } else { // If IPLDBlock not found in frothy region get latest IPLDBlock in the pruned region. // Filter out IPLDBlocks from pruned blocks. const canonicalBlockNumber = blockNumber + 1; - result = await repo.createQueryBuilder('ipld_block') + result = await queryRunner.manager.createQueryBuilder(IPLDBlock, 'ipld_block') .leftJoinAndSelect('ipld_block.block', 'block') .where('block.is_pruned = false') .andWhere('ipld_block.contract_address = :contractAddress', { contractAddress }) @@ -152,14 +150,13 @@ export class Database { return result; } - async getPrevIPLDBlocks (queryRunner: QueryRunner, blockHash: string, blockNumber: number, contractAddress: string, checkpointBlockNumber: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); - + async getPrevIPLDBlocksAfterCheckpoint (queryRunner: QueryRunner, blockHash: string, checkpointBlockNumber: number, contractAddress: string): Promise { const heirerchicalQuery = ` WITH RECURSIVE cte_query AS ( SELECT b.block_hash, + b.block_number, b.parent_hash, 1 as depth, i.id @@ -172,6 +169,7 @@ export class Database { UNION ALL SELECT b.block_hash, + b.block_number, b.parent_hash, c.depth + 1, i.id @@ -187,21 +185,25 @@ export class Database { c.depth < $3 ) SELECT - id + block_number, id FROM cte_query - WHERE id IS NOT NULL + ORDER BY block_number ASC `; // Fetching ids for previous IPLDBlocks in the frothy region. const queryResult = await queryRunner.query(heirerchicalQuery, [blockHash, contractAddress, MAX_REORG_DEPTH]); - const frothyIds = queryResult.map((obj: any) => obj.id); - // Fetching all diff blocks along with the previous checkpoint. - const ipldBlocks = await repo.find({ + let frothyIds = queryResult.map((obj: any) => obj.id); + frothyIds = frothyIds.filter((id: any) => id !== null); + + const frothyBlockNumber = queryResult[0].block_number; + + // Fetching all diff blocks after checkpoint till current blockNumber. + const ipldBlocks = await queryRunner.manager.find(IPLDBlock, { relations: ['block'], where: [ - { contractAddress, block: { isPruned: false, blockNumber: Between(checkpointBlockNumber, blockNumber - MAX_REORG_DEPTH) } }, + { contractAddress, block: { isPruned: false, blockNumber: Between(checkpointBlockNumber + 1, frothyBlockNumber - 1) } }, { id: In(frothyIds) } ], order: { block: 'ASC' } diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index a8641910..585854aa 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -227,12 +227,13 @@ export class Indexer { // For each contractAddress, merge the diff till now. for (const checkpointBlock of prevCheckpointBlocks) { - const { contractAddress } = checkpointBlock; + const { contractAddress, block: { blockNumber: checkpointBlockNumber } } = checkpointBlock; - // Fetching all diff blocks along with the previous checkpoint. - const diffBlocks = await this.getPrevIPLDBlocks(currentBlockHash, currentBlockNumber, contractAddress, currentBlockNumber - this._checkpointInterval); + // Fetching all diff blocks after checkpoint. + const diffBlocks = await this.getPrevIPLDBlocksAfterCheckpoint(currentBlockHash, checkpointBlockNumber, contractAddress); + + let checkPoint = codec.decode(Buffer.from(checkpointBlock.data)) as any; - let checkPoint: any = {}; for (const diffBlock of diffBlocks) { const diff = codec.decode(Buffer.from(diffBlock.data)); checkPoint = _.merge(checkPoint, diff); @@ -281,12 +282,12 @@ export class Indexer { return res; } - async getPrevIPLDBlocks (blockHash: string, blockNumber: number, contractAddress: string, checkpointBlockNumber: number): Promise { + async getPrevIPLDBlocksAfterCheckpoint (blockHash: string, checkpointBlockNumber: number, contractAddress: string): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.getPrevIPLDBlocks(dbTx, blockHash, blockNumber, contractAddress, checkpointBlockNumber); + res = await this._db.getPrevIPLDBlocksAfterCheckpoint(dbTx, blockHash, checkpointBlockNumber, contractAddress); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts index 20d1b435..76b945dd 100644 --- a/packages/erc20-watcher/src/indexer.ts +++ b/packages/erc20-watcher/src/indexer.ts @@ -12,7 +12,7 @@ import { BaseProvider } from '@ethersproject/providers'; import { EthClient } from '@vulcanize/ipld-eth-client'; import { StorageLayout } from '@vulcanize/solidity-mapper'; -import { EventInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME } from '@vulcanize/util'; +import { EventInterface, IndexerInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME } from '@vulcanize/util'; import { Database } from './database'; import { Event } from './entity/Event'; @@ -41,7 +41,7 @@ interface EventResult { proof?: string; } -export class Indexer { +export class Indexer implements IndexerInterface { _db: Database _ethClient: EthClient _postgraphileClient: EthClient; @@ -255,6 +255,11 @@ export class Indexer { await this.triggerIndexingOnEvent(event); } + async processBlock (blockHash: string): Promise { + // Empty post-block method. + assert(blockHash); + } + parseEventNameAndArgs (kind: string, logObj: any): any { let eventName = UNKNOWN_EVENT_NAME; let eventInfo = {}; From afba86a53106cb127057317afb9ffbba552f9438 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Mon, 11 Oct 2021 18:47:12 +0530 Subject: [PATCH 18/21] Add a query to get block in ipld-eth-client --- packages/codegen/src/generate-code.ts | 2 +- packages/codegen/src/schema.ts | 5 ++-- .../src/templates/client-template.handlebars | 1 - .../src/templates/config-template.handlebars | 6 ++++- .../templates/database-template.handlebars | 2 +- .../src/templates/fill-template.handlebars | 8 +++--- .../src/templates/indexer-template.handlebars | 25 +++++++++++-------- .../templates/job-runner-template.handlebars | 22 ++++++++-------- .../templates/resolvers-template.handlebars | 6 ++--- .../src/templates/server-template.handlebars | 13 +++++----- packages/ipld-eth-client/src/eth-client.ts | 10 ++++++++ packages/ipld-eth-client/src/eth-queries.ts | 15 +++++++++++ packages/util/src/config.ts | 18 +++++++------ packages/util/src/indexer.ts | 6 ++--- 14 files changed, 87 insertions(+), 52 deletions(-) diff --git a/packages/codegen/src/generate-code.ts b/packages/codegen/src/generate-code.ts index 42da717e..6b613272 100644 --- a/packages/codegen/src/generate-code.ts +++ b/packages/codegen/src/generate-code.ts @@ -7,8 +7,8 @@ import fetch from 'node-fetch'; import path from 'path'; import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; -import { flatten } from '@poanet/solidity-flattener'; +import { flatten } from '@poanet/solidity-flattener'; import { parse, visit } from '@solidity-parser/parser'; import { KIND_ACTIVE, KIND_LAZY } from '@vulcanize/util'; diff --git a/packages/codegen/src/schema.ts b/packages/codegen/src/schema.ts index 2fc0f885..738c71f6 100644 --- a/packages/codegen/src/schema.ts +++ b/packages/codegen/src/schema.ts @@ -245,6 +245,7 @@ export class Schema { block: () => this._composer.getOTC('Block').NonNull, contractAddress: 'String!', cid: 'String!', + kind: 'String!', data: 'String!' } }); @@ -291,8 +292,8 @@ export class Schema { type: 'Boolean!', args: { contractAddress: 'String!', - startingBlock: 'Int', - kind: 'String!' + kind: 'String!', + startingBlock: 'Int' } } }); diff --git a/packages/codegen/src/templates/client-template.handlebars b/packages/codegen/src/templates/client-template.handlebars index 172e75e0..a8548c1b 100644 --- a/packages/codegen/src/templates/client-template.handlebars +++ b/packages/codegen/src/templates/client-template.handlebars @@ -3,7 +3,6 @@ // import { gql } from '@apollo/client/core'; - import { GraphQLClient, GraphQLConfig } from '@vulcanize/ipld-eth-client'; import { queries, mutations, subscriptions } from './gql'; diff --git a/packages/codegen/src/templates/config-template.handlebars b/packages/codegen/src/templates/config-template.handlebars index 8f25a1b8..664705d0 100644 --- a/packages/codegen/src/templates/config-template.handlebars +++ b/packages/codegen/src/templates/config-template.handlebars @@ -2,8 +2,12 @@ host = "127.0.0.1" port = 3008 kind = "{{watcherKind}}" + + # Checkpointing derived state. checkpointing = true - checkpointInterval = 30 + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 [database] type = "postgres" diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 4f023801..c3f9c9e7 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -257,7 +257,7 @@ export class Database { return this._baseDatabase.saveEvents(blockRepo, eventRepo, block, events); } - async saveContract (address: string, startingBlock: number, kind: string): Promise { + async saveContract (address: string, kind: string, startingBlock: number): Promise { await this._conn.transaction(async (tx) => { const repo = tx.getRepository(Contract); diff --git a/packages/codegen/src/templates/fill-template.handlebars b/packages/codegen/src/templates/fill-template.handlebars index b5855137..ccca0d7e 100644 --- a/packages/codegen/src/templates/fill-template.handlebars +++ b/packages/codegen/src/templates/fill-template.handlebars @@ -45,11 +45,11 @@ export const main = async (): Promise => { const config = await getConfig(argv.configFile); - assert(config.server, 'Missing server config'); - - const { upstream, database: dbConfig, jobQueue: jobQueueConfig } = config; + const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: serverConfig } = config; + assert(upstream, 'Missing upstream config'); assert(dbConfig, 'Missing database config'); + assert(serverConfig, 'Missing server config'); const db = new Database(dbConfig); await db.init(); @@ -76,7 +76,7 @@ export const main = async (): Promise => { // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries const pubsub = new PubSub(); - const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider); + const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider); const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; assert(dbConnectionString, 'Missing job queue db connection string'); diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 585854aa..25fe1dee 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -4,19 +4,19 @@ import assert from 'assert'; import debug from 'debug'; -import { JsonFragment } from '@ethersproject/abi'; import { DeepPartial } from 'typeorm'; import JSONbig from 'json-bigint'; import { ethers } from 'ethers'; -import { BaseProvider } from '@ethersproject/providers'; -import * as codec from '@ipld/dag-json'; import { sha256 } from 'multiformats/hashes/sha2'; import { CID } from 'multiformats/cid'; import _ from 'lodash'; +import { JsonFragment } from '@ethersproject/abi'; +import { BaseProvider } from '@ethersproject/providers'; +import * as codec from '@ipld/dag-json'; import { EthClient } from '@vulcanize/ipld-eth-client'; import { StorageLayout } from '@vulcanize/solidity-mapper'; -import { EventInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME } from '@vulcanize/util'; +import { EventInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, ServerConfig } from '@vulcanize/util'; import { Database } from './database'; import { Contract } from './entity/Contract'; @@ -66,6 +66,7 @@ export type ResultIPLDBlock = { }; contractAddress: string; cid: string; + kind: string; data: string; }; @@ -75,13 +76,13 @@ export class Indexer { _ethProvider: BaseProvider _postgraphileClient: EthClient; _baseIndexer: BaseIndexer - _checkpointInterval: number + _serverConfig: ServerConfig _abi: JsonFragment[] _storageLayout: StorageLayout _contract: ethers.utils.Interface - constructor (db: Database, ethClient: EthClient, postgraphileClient: EthClient, ethProvider: BaseProvider, checkpointInterval?: number) { + constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, postgraphileClient: EthClient, ethProvider: BaseProvider) { assert(db); assert(ethClient); @@ -89,8 +90,8 @@ export class Indexer { this._ethClient = ethClient; this._ethProvider = ethProvider; this._postgraphileClient = postgraphileClient; + this._serverConfig = serverConfig; this._baseIndexer = new BaseIndexer(this._db, this._ethClient, this._postgraphileClient); - this._checkpointInterval = checkpointInterval || 0; const { abi, storageLayout } = artifacts; @@ -150,6 +151,7 @@ export class Indexer { }, contractAddress: ipldBlock.contractAddress, cid: ipldBlock.cid, + kind: ipldBlock.kind, data: ipldBlock.data }; } @@ -217,13 +219,14 @@ export class Indexer { // Create a checkpoint IPLDBlock for contracts that were checkpointed checkPointInterval blocks before. // Return if checkpointInterval is <= 0. - if (this._checkpointInterval <= 0) return; + const checkpointInterval = this._serverConfig.checkpointInterval; + if (checkpointInterval <= 0) return; const { data: { blockNumber: currentBlockNumber, blockHash: currentBlockHash } } = job; // Get checkpoint IPLDBlocks with blockNumber: current-checkPointInterval. // Assuming checkPointInterval < MAX_REORG_DEPTH. - const prevCheckpointBlocks = await this._db.getIPLDBlocks({ block: { blockNumber: currentBlockNumber - this._checkpointInterval }, kind: 'checkpoint' }); + const prevCheckpointBlocks = await this._db.getIPLDBlocks({ block: { blockNumber: currentBlockNumber - checkpointInterval }, kind: 'checkpoint' }); // For each contractAddress, merge the diff till now. for (const checkpointBlock of prevCheckpointBlocks) { @@ -395,9 +398,9 @@ export class Indexer { return { eventName, eventInfo }; } - async watchContract (address: string, startingBlock: number, kind: string): Promise { + async watchContract (address: string, kind: string, startingBlock: number): Promise { // Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress). - await this._db.saveContract(ethers.utils.getAddress(address), startingBlock, kind); + await this._db.saveContract(ethers.utils.getAddress(address), kind, startingBlock); // Getting the current block. const currentBlock = await this._db.getLatestBlockProgress(); diff --git a/packages/codegen/src/templates/job-runner-template.handlebars b/packages/codegen/src/templates/job-runner-template.handlebars index a8c2baf0..eaa61705 100644 --- a/packages/codegen/src/templates/job-runner-template.handlebars +++ b/packages/codegen/src/templates/job-runner-template.handlebars @@ -19,6 +19,7 @@ import { QUEUE_EVENT_PROCESSING, QUEUE_BLOCK_CHECKPOINT, JobQueueConfig, + ServerConfig, DEFAULT_CONFIG_PATH } from '@vulcanize/util'; @@ -32,18 +33,20 @@ export class JobRunner { _jobQueue: JobQueue _baseJobRunner: BaseJobRunner _jobQueueConfig: JobQueueConfig + _serverConfig: ServerConfig - constructor (jobQueueConfig: JobQueueConfig, indexer: Indexer, jobQueue: JobQueue) { + constructor (jobQueueConfig: JobQueueConfig, serverConfig: ServerConfig, indexer: Indexer, jobQueue: JobQueue) { this._indexer = indexer; this._jobQueue = jobQueue; this._jobQueueConfig = jobQueueConfig; + this._serverConfig = serverConfig; this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._indexer, this._jobQueue); } - async start (checkpointing: boolean): Promise { + async start (): Promise { await this.subscribeBlockProcessingQueue(); await this.subscribeEventProcessingQueue(); - if (checkpointing) await this.subscribeBlockCheckpointQueue(); + if (this._serverConfig.checkpointing) await this.subscribeBlockCheckpointQueue(); } async subscribeBlockProcessingQueue (): Promise { @@ -91,16 +94,15 @@ export const main = async (): Promise => { const config = await getConfig(argv.f); - assert(config.server, 'Missing server config'); - - const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: { checkpointing, checkpointInterval } } = config; + const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: serverConfig } = config; + assert(upstream, 'Missing upstream config'); assert(dbConfig, 'Missing database config'); + assert(serverConfig, 'Missing server config'); const db = new Database(dbConfig); await db.init(); - assert(upstream, 'Missing upstream config'); const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream; assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint'); assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); @@ -119,7 +121,7 @@ export const main = async (): Promise => { }); const ethProvider = getDefaultProvider(rpcProviderEndpoint); - const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, checkpointInterval); + const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider); assert(jobQueueConfig, 'Missing job queue config'); @@ -129,8 +131,8 @@ export const main = async (): Promise => { const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); await jobQueue.start(); - const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue); - await jobRunner.start(checkpointing); + const jobRunner = new JobRunner(jobQueueConfig, serverConfig, indexer, jobQueue); + await jobRunner.start(); }; main().then(() => { diff --git a/packages/codegen/src/templates/resolvers-template.handlebars b/packages/codegen/src/templates/resolvers-template.handlebars index b408a682..96a8b7d3 100644 --- a/packages/codegen/src/templates/resolvers-template.handlebars +++ b/packages/codegen/src/templates/resolvers-template.handlebars @@ -34,9 +34,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch }, Mutation: { - watchContract: (_: any, { contractAddress, startingBlock = 1, kind }: { contractAddress: string, startingBlock: number, kind: string }): Promise => { - log('watchContract', contractAddress, startingBlock, kind); - return indexer.watchContract(contractAddress, startingBlock, kind); + watchContract: (_: any, { contractAddress, kind, startingBlock = 1 }: { contractAddress: string, kind: string, startingBlock: number }): Promise => { + log('watchContract', contractAddress, kind, startingBlock); + return indexer.watchContract(contractAddress, kind, startingBlock); } }, diff --git a/packages/codegen/src/templates/server-template.handlebars b/packages/codegen/src/templates/server-template.handlebars index ed588f74..637b8a91 100644 --- a/packages/codegen/src/templates/server-template.handlebars +++ b/packages/codegen/src/templates/server-template.handlebars @@ -39,18 +39,17 @@ export const main = async (): Promise => { const config = await getConfig(argv.f); - assert(config.server, 'Missing server config'); - - const { host, port, kind: watcherKind } = config.server; - - const { upstream, database: dbConfig, jobQueue: jobQueueConfig } = config; + const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: serverConfig } = config; + assert(upstream, 'Missing upstream config'); assert(dbConfig, 'Missing database config'); + assert(serverConfig, 'Missing server config'); + + const { host, port, kind: watcherKind } = serverConfig; const db = new Database(dbConfig); await db.init(); - assert(upstream, 'Missing upstream config'); const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream; assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint'); assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); @@ -70,7 +69,7 @@ export const main = async (): Promise => { const ethProvider = getDefaultProvider(rpcProviderEndpoint); - const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider); + const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider); // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries diff --git a/packages/ipld-eth-client/src/eth-client.ts b/packages/ipld-eth-client/src/eth-client.ts index a1ba84fc..2ebb9386 100644 --- a/packages/ipld-eth-client/src/eth-client.ts +++ b/packages/ipld-eth-client/src/eth-client.ts @@ -72,6 +72,16 @@ export class EthClient { ); } + async getBlock ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise { + return this._graphqlClient.query( + ethQueries.getBlock, + { + blockNumber: blockNumber?.toString(), + blockHash + } + ); + } + async getBlockByHash (blockHash: string): Promise { return this._graphqlClient.query(ethQueries.getBlockByHash, { blockHash }); } diff --git a/packages/ipld-eth-client/src/eth-queries.ts b/packages/ipld-eth-client/src/eth-queries.ts index 9129cc06..143338f9 100644 --- a/packages/ipld-eth-client/src/eth-queries.ts +++ b/packages/ipld-eth-client/src/eth-queries.ts @@ -64,6 +64,20 @@ query allEthHeaderCids($blockNumber: BigInt, $blockHash: String) { } `; +export const getBlock = gql` +query allEthHeaderCids($blockNumber: BigInt, $blockHash: String) { + allEthHeaderCids(condition: { blockNumber: $blockNumber, blockHash: $blockHash }) { + nodes { + cid + blockNumber + blockHash + parentHash + timestamp + } + } +} +`; + export const getBlockByHash = gql` query block($blockHash: Bytes32) { block(hash: $blockHash) { @@ -114,6 +128,7 @@ export default { getStorageAt, getLogs, getBlockWithTransactions, + getBlock, getBlockByHash, subscribeBlocks, subscribeTransactions diff --git a/packages/util/src/config.ts b/packages/util/src/config.ts index 972e89c8..ed05ece2 100644 --- a/packages/util/src/config.ts +++ b/packages/util/src/config.ts @@ -18,15 +18,17 @@ export interface JobQueueConfig { jobDelayInMilliSecs?: number; } +export interface ServerConfig { + host: string; + port: number; + mode: string; + kind: string; + checkpointing: boolean; + checkpointInterval: number; +} + export interface Config { - server: { - host: string; - port: number; - mode: string; - kind: string; - checkpointing: boolean; - checkpointInterval: number; - }; + server: ServerConfig; database: ConnectionOptions; upstream: { cache: CacheConfig, diff --git a/packages/util/src/indexer.ts b/packages/util/src/indexer.ts index 51035644..016d81e5 100644 --- a/packages/util/src/indexer.ts +++ b/packages/util/src/indexer.ts @@ -112,12 +112,12 @@ export class Indexer { { cid, blockNumber, - timestamp, - parentHash + parentHash, + timestamp } ] } - } = await this._postgraphileClient.getBlockWithTransactions({ blockHash }); + } = await this._postgraphileClient.getBlock({ blockHash }); return { cid, From 74b154356167951247d850b366e500b43524600a Mon Sep 17 00:00:00 2001 From: prathamesh Date: Tue, 12 Oct 2021 14:11:06 +0530 Subject: [PATCH 19/21] Get latest checkpoints for all contracts for checkpointing. --- .../templates/database-template.handlebars | 20 ++++++-- .../src/templates/events-template.handlebars | 6 ++- .../src/templates/indexer-template.handlebars | 47 ++++++++++++++++--- .../templates/job-runner-template.handlebars | 4 +- packages/erc20-watcher/src/job-runner.ts | 18 +++---- packages/uni-info-watcher/src/job-runner.ts | 18 +++---- packages/uni-watcher/src/job-runner.ts | 16 ++++--- packages/util/src/job-runner.ts | 19 +++++--- 8 files changed, 107 insertions(+), 41 deletions(-) diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index c3f9c9e7..87d14433 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -3,7 +3,7 @@ // import assert from 'assert'; -import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner, In, Between } from 'typeorm'; +import { Connection, ConnectionOptions, DeepPartial, FindConditions, FindManyOptions, QueryRunner, In, Between } from 'typeorm'; import path from 'path'; import { Database as BaseDatabase, MAX_REORG_DEPTH } from '@vulcanize/util'; @@ -79,9 +79,23 @@ export class Database { } {{/each}} - async getIPLDBlocks (where: FindConditions): Promise { + async getIPLDBlocks (findOptions: FindManyOptions): Promise { const repo = this._conn.getRepository(IPLDBlock); - return repo.find({ where, relations: ['block'] }); + return repo.find(findOptions); + } + + async getLatestCheckpoints (queryRunner: QueryRunner): Promise { + // Get the latest checkpoints for all the contracts. + const result = await queryRunner.manager.createQueryBuilder(IPLDBlock, 'ipld_block') + .distinctOn(['contract_address']) + .innerJoinAndSelect(Contract, 'contract', 'contract_address = contract.address') + .leftJoinAndSelect('ipld_block.block', 'block') + .where('ipld_block.kind = :kind', { kind: 'checkpoint' }) + .orderBy('contract_address') + .addOrderBy('ipld_block.block_id', 'DESC') + .getMany(); + + return result; } async getPrevIPLDBlock (queryRunner: QueryRunner, blockHash: string, contractAddress: string): Promise { diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index 57de8a7b..6482294c 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -84,7 +84,11 @@ export class EventWatcher { // b. Push a block checkpointing job. if (dbEvent.block.isComplete) { await this._indexer.processBlock(dbEvent.block.blockHash); - await this._jobQueue.pushJob(QUEUE_BLOCK_CHECKPOINT, { blockHash: dbEvent.block.blockHash, blockNumber: dbEvent.block.blockNumber }); + + // Push checkpointing job if checkpointing is on. + if (this._indexer._serverConfig.checkpointing) { + await this._jobQueue.pushJob(QUEUE_BLOCK_CHECKPOINT, { blockHash: dbEvent.block.blockHash, blockNumber: dbEvent.block.blockNumber }); + } } const { data: { request, failed, state, createdOn } } = job; diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 25fe1dee..7379d35d 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -224,12 +224,17 @@ export class Indexer { const { data: { blockNumber: currentBlockNumber, blockHash: currentBlockHash } } = job; - // Get checkpoint IPLDBlocks with blockNumber: current-checkPointInterval. - // Assuming checkPointInterval < MAX_REORG_DEPTH. - const prevCheckpointBlocks = await this._db.getIPLDBlocks({ block: { blockNumber: currentBlockNumber - checkpointInterval }, kind: 'checkpoint' }); + // Get latest checkpoints for all the contracts. + // Assuming checkPointInterval > MAX_REORG_DEPTH. + const latestCheckpointBlocks = await this.getLatestCheckpoints(); // For each contractAddress, merge the diff till now. - for (const checkpointBlock of prevCheckpointBlocks) { + for (const checkpointBlock of latestCheckpointBlocks) { + // Check if it is time for a new checkpoint. + if (checkpointBlock.block.blockNumber > currentBlockNumber - checkpointInterval) { + continue; + } + const { contractAddress, block: { blockNumber: checkpointBlockNumber } } = checkpointBlock; // Fetching all diff blocks after checkpoint. @@ -251,8 +256,32 @@ export class Indexer { } } + async getLatestCheckpoints (): Promise { + // Get the latest checkpoints for all the contracts. + const dbTx = await this._db.createTransactionRunner(); + let res; + + try { + res = await this._db.getLatestCheckpoints(dbTx); + await dbTx.commitTransaction(); + } catch (error) { + await dbTx.rollbackTransaction(); + throw error; + } finally { + await dbTx.release(); + } + return res; + } + async getIPLDBlock (block: BlockProgress, contractAddress: string): Promise { - const ipldBlocks = await this._db.getIPLDBlocks({ block, contractAddress }); + // Setting up find options for the query. + const where = { + block, + contractAddress + }; + const relations = ['block']; + + const ipldBlocks = await this._db.getIPLDBlocks({ where, relations }); // There can be only one IPLDBlock for a { block, contractAddress } combination. assert(ipldBlocks.length <= 1); @@ -261,7 +290,13 @@ export class Indexer { } async getIPLDBlockByCid (cid: string): Promise { - const ipldBlocks = await this._db.getIPLDBlocks({ cid }); + // Setting up find options for the query. + const where = { + cid + }; + const relations = ['block']; + + const ipldBlocks = await this._db.getIPLDBlocks({ where, relations }); // There can be only one IPLDBlock with a particular cid. assert(ipldBlocks.length <= 1); diff --git a/packages/codegen/src/templates/job-runner-template.handlebars b/packages/codegen/src/templates/job-runner-template.handlebars index eaa61705..bdcc4761 100644 --- a/packages/codegen/src/templates/job-runner-template.handlebars +++ b/packages/codegen/src/templates/job-runner-template.handlebars @@ -40,13 +40,13 @@ export class JobRunner { this._jobQueue = jobQueue; this._jobQueueConfig = jobQueueConfig; this._serverConfig = serverConfig; - this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._indexer, this._jobQueue); + this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._serverConfig, this._indexer, this._jobQueue); } async start (): Promise { await this.subscribeBlockProcessingQueue(); await this.subscribeEventProcessingQueue(); - if (this._serverConfig.checkpointing) await this.subscribeBlockCheckpointQueue(); + await this.subscribeBlockCheckpointQueue(); } async subscribeBlockProcessingQueue (): Promise { diff --git a/packages/erc20-watcher/src/job-runner.ts b/packages/erc20-watcher/src/job-runner.ts index 9167d98b..89a966aa 100644 --- a/packages/erc20-watcher/src/job-runner.ts +++ b/packages/erc20-watcher/src/job-runner.ts @@ -18,6 +18,7 @@ import { QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, JobQueueConfig, + ServerConfig, DEFAULT_CONFIG_PATH } from '@vulcanize/util'; @@ -31,12 +32,14 @@ export class JobRunner { _jobQueue: JobQueue _baseJobRunner: BaseJobRunner _jobQueueConfig: JobQueueConfig + _serverConfig: ServerConfig - constructor (jobQueueConfig: JobQueueConfig, indexer: Indexer, jobQueue: JobQueue) { + constructor (jobQueueConfig: JobQueueConfig, serverConfig: ServerConfig, indexer: Indexer, jobQueue: JobQueue) { this._indexer = indexer; this._jobQueue = jobQueue; this._jobQueueConfig = jobQueueConfig; - this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._indexer, this._jobQueue); + this._serverConfig = serverConfig; + this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._serverConfig, this._indexer, this._jobQueue); } async start (): Promise { @@ -79,16 +82,15 @@ export const main = async (): Promise => { const config = await getConfig(argv.f); - assert(config.server, 'Missing server config'); - - const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: { mode } } = config; + const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: serverConfig } = config; + assert(upstream, 'Missing upstream config'); assert(dbConfig, 'Missing database config'); + assert(serverConfig, 'Missing server config'); const db = new Database(dbConfig); await db.init(); - assert(upstream, 'Missing upstream config'); const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream; assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint'); assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); @@ -107,7 +109,7 @@ export const main = async (): Promise => { }); const ethProvider = getDefaultProvider(rpcProviderEndpoint); - const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, mode); + const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, serverConfig.mode); assert(jobQueueConfig, 'Missing job queue config'); @@ -117,7 +119,7 @@ export const main = async (): Promise => { const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); await jobQueue.start(); - const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue); + const jobRunner = new JobRunner(jobQueueConfig, serverConfig, indexer, jobQueue); await jobRunner.start(); }; diff --git a/packages/uni-info-watcher/src/job-runner.ts b/packages/uni-info-watcher/src/job-runner.ts index 2f6886ff..5a3ec7e3 100644 --- a/packages/uni-info-watcher/src/job-runner.ts +++ b/packages/uni-info-watcher/src/job-runner.ts @@ -19,6 +19,7 @@ import { QUEUE_EVENT_PROCESSING, JobRunner as BaseJobRunner, JobQueueConfig, + ServerConfig, DEFAULT_CONFIG_PATH } from '@vulcanize/util'; @@ -32,12 +33,14 @@ export class JobRunner { _jobQueue: JobQueue _baseJobRunner: BaseJobRunner _jobQueueConfig: JobQueueConfig + _serverConfig: ServerConfig - constructor (jobQueueConfig: JobQueueConfig, indexer: Indexer, jobQueue: JobQueue) { + constructor (jobQueueConfig: JobQueueConfig, serverConfig: ServerConfig, indexer: Indexer, jobQueue: JobQueue) { this._indexer = indexer; this._jobQueue = jobQueue; this._jobQueueConfig = jobQueueConfig; - this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._indexer, this._jobQueue); + this._serverConfig = serverConfig; + this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._serverConfig, this._indexer, this._jobQueue); } async start (): Promise { @@ -80,16 +83,15 @@ export const main = async (): Promise => { const config = await getConfig(argv.f); - assert(config.server, 'Missing server config'); - - const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: { mode } } = config; + const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: serverConfig } = config; + assert(upstream, 'Missing upstream config'); assert(dbConfig, 'Missing database config'); + assert(serverConfig, 'Missing server config'); const db = new Database(dbConfig); await db.init(); - assert(upstream, 'Missing upstream config'); const { uniWatcher: { gqlEndpoint, gqlSubscriptionEndpoint }, tokenWatcher, cache: cacheConfig, ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint } } = upstream; assert(gqlEndpoint, 'Missing upstream uniWatcher.gqlEndpoint'); assert(gqlSubscriptionEndpoint, 'Missing upstream uniWatcher.gqlSubscriptionEndpoint'); @@ -113,7 +115,7 @@ export const main = async (): Promise => { const erc20Client = new ERC20Client(tokenWatcher); - const indexer = new Indexer(db, uniClient, erc20Client, ethClient, postgraphileClient, mode); + const indexer = new Indexer(db, uniClient, erc20Client, ethClient, postgraphileClient, serverConfig.mode); assert(jobQueueConfig, 'Missing job queue config'); @@ -123,7 +125,7 @@ export const main = async (): Promise => { const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); await jobQueue.start(); - const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue); + const jobRunner = new JobRunner(jobQueueConfig, serverConfig, indexer, jobQueue); await jobRunner.start(); }; diff --git a/packages/uni-watcher/src/job-runner.ts b/packages/uni-watcher/src/job-runner.ts index d4f4a945..cc0c50e4 100644 --- a/packages/uni-watcher/src/job-runner.ts +++ b/packages/uni-watcher/src/job-runner.ts @@ -17,6 +17,7 @@ import { QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, JobQueueConfig, + ServerConfig, DEFAULT_CONFIG_PATH } from '@vulcanize/util'; @@ -31,12 +32,14 @@ export class JobRunner { _jobQueue: JobQueue _baseJobRunner: BaseJobRunner _jobQueueConfig: JobQueueConfig + _serverConfig: ServerConfig - constructor (jobQueueConfig: JobQueueConfig, indexer: Indexer, jobQueue: JobQueue) { + constructor (jobQueueConfig: JobQueueConfig, serverConfig: ServerConfig, indexer: Indexer, jobQueue: JobQueue) { this._indexer = indexer; this._jobQueue = jobQueue; this._jobQueueConfig = jobQueueConfig; - this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._indexer, this._jobQueue); + this._serverConfig = serverConfig; + this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._serverConfig, this._indexer, this._jobQueue); } async start (): Promise { @@ -95,16 +98,15 @@ export const main = async (): Promise => { const config = await getConfig(argv.f); - assert(config.server, 'Missing server config'); - - const { upstream, database: dbConfig, jobQueue: jobQueueConfig } = config; + const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: serverConfig } = config; + assert(upstream, 'Missing upstream config'); assert(dbConfig, 'Missing database config'); + assert(serverConfig, 'Missing server config'); const db = new Database(dbConfig); await db.init(); - assert(upstream, 'Missing upstream config'); const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint }, cache: cacheConfig } = upstream; assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint'); assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); @@ -131,7 +133,7 @@ export const main = async (): Promise => { const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); await jobQueue.start(); - const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue); + const jobRunner = new JobRunner(jobQueueConfig, serverConfig, indexer, jobQueue); await jobRunner.start(); }; diff --git a/packages/util/src/job-runner.ts b/packages/util/src/job-runner.ts index e5c2fecb..7201ea4d 100644 --- a/packages/util/src/job-runner.ts +++ b/packages/util/src/job-runner.ts @@ -7,7 +7,7 @@ import debug from 'debug'; import { wait } from '.'; import { createPruningJob } from './common'; -import { JobQueueConfig } from './config'; +import { JobQueueConfig, ServerConfig } from './config'; import { JOB_KIND_INDEX, JOB_KIND_PRUNE, @@ -25,11 +25,13 @@ export class JobRunner { _indexer: IndexerInterface _jobQueue: JobQueue _jobQueueConfig: JobQueueConfig + _serverConfig: ServerConfig - constructor (jobQueueConfig: JobQueueConfig, indexer: IndexerInterface, jobQueue: JobQueue) { + constructor (jobQueueConfig: JobQueueConfig, serverConfig: ServerConfig, indexer: IndexerInterface, jobQueue: JobQueue) { this._indexer = indexer; this._jobQueue = jobQueue; this._jobQueueConfig = jobQueueConfig; + this._serverConfig = serverConfig; } async processBlock (job: any): Promise { @@ -186,13 +188,18 @@ export class JobRunner { await wait(jobDelayInMilliSecs); const events = await this._indexer.getOrFetchBlockEvents({ cid, blockHash, blockNumber, parentHash, blockTimestamp: timestamp }); + for (let ei = 0; ei < events.length; ei++) { + await this._jobQueue.pushJob(QUEUE_EVENT_PROCESSING, { id: events[ei].id, publish: true }); + } + + // Call post-block hook and checkpointing if there are no events as the block is already marked as complete. if (!events.length) { await this._indexer.processBlock(blockHash); - await this._jobQueue.pushJob(QUEUE_BLOCK_CHECKPOINT, { blockHash, blockNumber }); - } - for (let ei = 0; ei < events.length; ei++) { - await this._jobQueue.pushJob(QUEUE_EVENT_PROCESSING, { id: events[ei].id, publish: true }); + // Push checkpointing job if checkpointing is on. + if (this._serverConfig.checkpointing) { + await this._jobQueue.pushJob(QUEUE_BLOCK_CHECKPOINT, { blockHash, blockNumber }); + } } } } From 556c2111fe497ef28951de24f058ea8b9760e775 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Tue, 12 Oct 2021 15:30:39 +0530 Subject: [PATCH 20/21] Call post-block hook in a queue --- .../templates/database-template.handlebars | 11 +++++----- .../src/templates/events-template.handlebars | 5 +++-- .../src/templates/hooks-template.handlebars | 8 +++++-- .../src/templates/indexer-template.handlebars | 21 +++++-------------- .../templates/job-runner-template.handlebars | 11 ++++++++++ packages/erc20-watcher/src/job-runner.ts | 9 ++++++++ packages/uni-info-watcher/src/job-runner.ts | 9 ++++++++ packages/uni-watcher/src/job-runner.ts | 9 ++++++++ packages/util/src/constants.ts | 1 + packages/util/src/job-runner.ts | 9 ++++---- 10 files changed, 64 insertions(+), 29 deletions(-) diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 87d14433..32f01950 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -3,7 +3,7 @@ // import assert from 'assert'; -import { Connection, ConnectionOptions, DeepPartial, FindConditions, FindManyOptions, QueryRunner, In, Between } from 'typeorm'; +import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner, In, Between } from 'typeorm'; import path from 'path'; import { Database as BaseDatabase, MAX_REORG_DEPTH } from '@vulcanize/util'; @@ -79,19 +79,20 @@ export class Database { } {{/each}} - async getIPLDBlocks (findOptions: FindManyOptions): Promise { + async getIPLDBlocks (where: FindConditions): Promise { const repo = this._conn.getRepository(IPLDBlock); - return repo.find(findOptions); + return repo.find({ where, relations: ['block'] }); } async getLatestCheckpoints (queryRunner: QueryRunner): Promise { // Get the latest checkpoints for all the contracts. const result = await queryRunner.manager.createQueryBuilder(IPLDBlock, 'ipld_block') .distinctOn(['contract_address']) + .orderBy('contract_address') .innerJoinAndSelect(Contract, 'contract', 'contract_address = contract.address') .leftJoinAndSelect('ipld_block.block', 'block') - .where('ipld_block.kind = :kind', { kind: 'checkpoint' }) - .orderBy('contract_address') + .where('block.is_pruned = false') + .andWhere('ipld_block.kind = :kind', { kind: 'checkpoint' }) .addOrderBy('ipld_block.block_id', 'DESC') .getMany(); diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index 6482294c..1b0c580b 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -13,6 +13,7 @@ import { QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, QUEUE_BLOCK_CHECKPOINT, + QUEUE_HOOKS, UNKNOWN_EVENT_NAME } from '@vulcanize/util'; @@ -80,10 +81,10 @@ export class EventWatcher { const dbEvent = await this._baseEventWatcher.eventProcessingCompleteHandler(job); // If the block is marked as complete: - // a. Use an indexer method to call a post-block hook. + // a. Push a post-block hook job. // b. Push a block checkpointing job. if (dbEvent.block.isComplete) { - await this._indexer.processBlock(dbEvent.block.blockHash); + await this._jobQueue.pushJob(QUEUE_HOOKS, { blockHash: dbEvent.block.blockHash }); // Push checkpointing job if checkpointing is on. if (this._indexer._serverConfig.checkpointing) { diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 0ff100a1..b7ad4ca0 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -44,10 +44,14 @@ export async function postBlockHook (indexer: Indexer, blockHash: string): Promi const events = await indexer.getEventsByFilter(blockHash); // No IPLDBlock entry if there are no events. - if (!events) return; + if (!events) { + return; + } for (const event of events) { - if (event.eventName === UNKNOWN_EVENT_NAME) continue; + if (event.eventName === UNKNOWN_EVENT_NAME) { + continue; + } const block = event.block; const contractAddress = event.contract; diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 7379d35d..94fb0be7 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -210,7 +210,9 @@ export class Indexer { } {{/each}} - async processBlock (blockHash: string): Promise { + async processBlock (job: any): Promise { + const { data: { blockHash } } = job; + // Call custom post-block hook. await postBlockHook(this, blockHash); } @@ -274,14 +276,7 @@ export class Indexer { } async getIPLDBlock (block: BlockProgress, contractAddress: string): Promise { - // Setting up find options for the query. - const where = { - block, - contractAddress - }; - const relations = ['block']; - - const ipldBlocks = await this._db.getIPLDBlocks({ where, relations }); + const ipldBlocks = await this._db.getIPLDBlocks({ block, contractAddress }); // There can be only one IPLDBlock for a { block, contractAddress } combination. assert(ipldBlocks.length <= 1); @@ -290,13 +285,7 @@ export class Indexer { } async getIPLDBlockByCid (cid: string): Promise { - // Setting up find options for the query. - const where = { - cid - }; - const relations = ['block']; - - const ipldBlocks = await this._db.getIPLDBlocks({ where, relations }); + const ipldBlocks = await this._db.getIPLDBlocks({ cid }); // There can be only one IPLDBlock with a particular cid. assert(ipldBlocks.length <= 1); diff --git a/packages/codegen/src/templates/job-runner-template.handlebars b/packages/codegen/src/templates/job-runner-template.handlebars index bdcc4761..1c61c032 100644 --- a/packages/codegen/src/templates/job-runner-template.handlebars +++ b/packages/codegen/src/templates/job-runner-template.handlebars @@ -18,6 +18,7 @@ import { QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, QUEUE_BLOCK_CHECKPOINT, + QUEUE_HOOKS, JobQueueConfig, ServerConfig, DEFAULT_CONFIG_PATH @@ -47,6 +48,7 @@ export class JobRunner { await this.subscribeBlockProcessingQueue(); await this.subscribeEventProcessingQueue(); await this.subscribeBlockCheckpointQueue(); + await this.subscribeHooksQueue(); } async subscribeBlockProcessingQueue (): Promise { @@ -79,6 +81,15 @@ export class JobRunner { await this._jobQueue.markComplete(job); }); } + + // TODO: Make sure the hooks run in order. + async subscribeHooksQueue (): Promise { + await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => { + await this._indexer.processBlock(job); + + await this._jobQueue.markComplete(job); + }); + } } export const main = async (): Promise => { diff --git a/packages/erc20-watcher/src/job-runner.ts b/packages/erc20-watcher/src/job-runner.ts index 89a966aa..1a979963 100644 --- a/packages/erc20-watcher/src/job-runner.ts +++ b/packages/erc20-watcher/src/job-runner.ts @@ -17,6 +17,7 @@ import { JobRunner as BaseJobRunner, QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, + QUEUE_HOOKS, JobQueueConfig, ServerConfig, DEFAULT_CONFIG_PATH @@ -67,6 +68,14 @@ export class JobRunner { await this._jobQueue.markComplete(job); }); } + + async subscribeHooksQueue (): Promise { + await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => { + await this._indexer.processBlock(job); + + await this._jobQueue.markComplete(job); + }); + } } export const main = async (): Promise => { diff --git a/packages/uni-info-watcher/src/job-runner.ts b/packages/uni-info-watcher/src/job-runner.ts index 5a3ec7e3..4ed91d1b 100644 --- a/packages/uni-info-watcher/src/job-runner.ts +++ b/packages/uni-info-watcher/src/job-runner.ts @@ -17,6 +17,7 @@ import { JobQueue, QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, + QUEUE_HOOKS, JobRunner as BaseJobRunner, JobQueueConfig, ServerConfig, @@ -68,6 +69,14 @@ export class JobRunner { await this._jobQueue.markComplete(job); }); } + + async subscribeHooksQueue (): Promise { + await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => { + await this._indexer.processBlock(job); + + await this._jobQueue.markComplete(job); + }); + } } export const main = async (): Promise => { diff --git a/packages/uni-watcher/src/job-runner.ts b/packages/uni-watcher/src/job-runner.ts index cc0c50e4..adc3c02c 100644 --- a/packages/uni-watcher/src/job-runner.ts +++ b/packages/uni-watcher/src/job-runner.ts @@ -16,6 +16,7 @@ import { JobRunner as BaseJobRunner, QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, + QUEUE_HOOKS, JobQueueConfig, ServerConfig, DEFAULT_CONFIG_PATH @@ -83,6 +84,14 @@ export class JobRunner { await this._jobQueue.markComplete(job); }); } + + async subscribeHooksQueue (): Promise { + await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => { + await this._indexer.processBlock(job); + + await this._jobQueue.markComplete(job); + }); + } } export const main = async (): Promise => { diff --git a/packages/util/src/constants.ts b/packages/util/src/constants.ts index ec28aad4..8f82b931 100644 --- a/packages/util/src/constants.ts +++ b/packages/util/src/constants.ts @@ -8,6 +8,7 @@ export const QUEUE_BLOCK_PROCESSING = 'block-processing'; export const QUEUE_EVENT_PROCESSING = 'event-processing'; export const QUEUE_CHAIN_PRUNING = 'chain-pruning'; export const QUEUE_BLOCK_CHECKPOINT = 'block-checkpoint'; +export const QUEUE_HOOKS = 'hooks'; export const JOB_KIND_INDEX = 'index'; export const JOB_KIND_PRUNE = 'prune'; diff --git a/packages/util/src/job-runner.ts b/packages/util/src/job-runner.ts index 7201ea4d..b559d95d 100644 --- a/packages/util/src/job-runner.ts +++ b/packages/util/src/job-runner.ts @@ -14,7 +14,8 @@ import { MAX_REORG_DEPTH, QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING, - QUEUE_BLOCK_CHECKPOINT + QUEUE_BLOCK_CHECKPOINT, + QUEUE_HOOKS } from './constants'; import { JobQueue } from './job-queue'; import { EventInterface, IndexerInterface, SyncStatusInterface } from './types'; @@ -192,11 +193,11 @@ export class JobRunner { await this._jobQueue.pushJob(QUEUE_EVENT_PROCESSING, { id: events[ei].id, publish: true }); } - // Call post-block hook and checkpointing if there are no events as the block is already marked as complete. + // Push post-block hook and checkpointing jobs if there are no events as the block is already marked as complete. if (!events.length) { - await this._indexer.processBlock(blockHash); + await this._jobQueue.pushJob(QUEUE_HOOKS, { blockHash }); - // Push checkpointing job if checkpointing is on. + // Push checkpointing job only if checkpointing is on. if (this._serverConfig.checkpointing) { await this._jobQueue.pushJob(QUEUE_BLOCK_CHECKPOINT, { blockHash, blockNumber }); } From b91052162ea993cd49aa513cf223218b77ddf2ac Mon Sep 17 00:00:00 2001 From: prathamesh Date: Tue, 12 Oct 2021 15:57:14 +0530 Subject: [PATCH 21/21] Pass server config to Indexer in watch-contract cli --- .../src/templates/watch-contract-template.handlebars | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/codegen/src/templates/watch-contract-template.handlebars b/packages/codegen/src/templates/watch-contract-template.handlebars index 056700a3..6e3dd45d 100644 --- a/packages/codegen/src/templates/watch-contract-template.handlebars +++ b/packages/codegen/src/templates/watch-contract-template.handlebars @@ -46,11 +46,12 @@ import { Indexer } from '../indexer'; }).argv; const config: Config = await getConfig(argv.configFile); - assert(config.server, 'Missing server config'); - const { upstream, database: dbConfig } = config; + const { upstream, database: dbConfig, server: serverConfig } = config; + assert(upstream, 'Missing upstream config'); assert(dbConfig, 'Missing database config'); + assert(serverConfig, 'Missing server config'); const db = new Database(dbConfig); await db.init(); @@ -74,8 +75,8 @@ import { Indexer } from '../indexer'; const ethProvider = getDefaultProvider(rpcProviderEndpoint); - const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider); - await indexer.watchContract(argv.address, argv.startingBlock, argv.kind); + const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider); + await indexer.watchContract(argv.address, argv.kind, argv.startingBlock); await db.close(); })();