From 1d6222297fc5b48c3b58e48e17a5116423fd1241 Mon Sep 17 00:00:00 2001 From: Shreerang Kale Date: Tue, 10 Oct 2023 17:44:45 +0530 Subject: [PATCH 01/10] Add validation for config file --- packages/util/src/constants.ts | 2 + packages/util/src/index.ts | 1 + packages/util/src/validate-config.ts | 110 +++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 packages/util/src/validate-config.ts diff --git a/packages/util/src/constants.ts b/packages/util/src/constants.ts index 5cf1d0103..d376721b3 100644 --- a/packages/util/src/constants.ts +++ b/packages/util/src/constants.ts @@ -27,3 +27,5 @@ export const KIND_LAZY = 'lazy'; export const DEFAULT_PREFETCH_BATCH_SIZE = 10; export const DEFAULT_MAX_GQL_CACHE_SIZE = Math.pow(2, 20) * 8; // 8 MB + +export const VALID_ETH_RPC_METHODS = ['eth_getBlockByHash', 'eth_getStorageAt', 'eth_getBlockByNumber']; diff --git a/packages/util/src/index.ts b/packages/util/src/index.ts index 5a4399c54..679c1537d 100644 --- a/packages/util/src/index.ts +++ b/packages/util/src/index.ts @@ -26,3 +26,4 @@ export * from './graph/types'; export * from './payments'; export * from './eth'; export * from './consensus'; +export * from './validate-config'; diff --git a/packages/util/src/validate-config.ts b/packages/util/src/validate-config.ts new file mode 100644 index 000000000..23a9a3640 --- /dev/null +++ b/packages/util/src/validate-config.ts @@ -0,0 +1,110 @@ +import { ethers } from 'ethers'; +import { Client } from 'pg'; +import debug from 'debug'; +import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions'; +import WebSocket from 'ws'; + +import { VALID_ETH_RPC_METHODS } from './constants'; + +const log = debug('vulcanize:server'); + +async function validateContractDeployment (rpcEndpoint: string, contractAddress: string): Promise { + try { + const provider = await new ethers.providers.JsonRpcProvider(rpcEndpoint); + const code = await provider.getCode(contractAddress); + if (code === '0x') { + log(`WARNING: Contract is not deployed at address ${contractAddress}`); + } else { + log(`SUCCESS: Contract is deployed at address ${contractAddress}`); + } + } catch (error) { + log(error); + } +} + +function validateContractAddressFormat (contractAddress: string): void { + if (ethers.utils.isAddress(contractAddress)) { + log(`SUCCESS: Given contract address ${contractAddress} is in a valid format`); + } else { + log(`WARNING: Given contract address ${contractAddress} is not in a valid format`); + } +} + +export async function validateContracts (contractsArr: string[], rpcProviderMutationEndpoint: string): Promise { + contractsArr.forEach((contractAddr: string) => { + validateContractAddressFormat(contractAddr); + validateContractDeployment(rpcProviderMutationEndpoint, contractAddr); + }); +} + +export async function validateEndpoint (endPoint: string, kind: string): Promise { + try { + const response = await fetch(endPoint); + if (!response.ok) { + log(`WARNING: HTTP error! Status: ${response.status}`); + } else { + log(`SUCCESS: The ${endPoint} is up`); + } + } catch (error:any) { + log(`WARNING: could not connect to ${endPoint}. Please check if the ${kind} is correct and up.`); + log(error); + } +} + +async function checkDBEndpoint (connectionString: string, dbKind: string): Promise { + const client = new Client({ + connectionString + }); + + try { + await client.connect(); + log(`SUCCESS: ${dbKind} endpoint is up!`); + } catch (error) { + log('WARNING: Error connecting to job queue database. Please check if job queue config is setup and database is running \n', error); + } finally { + await client.end(); + } +} + +export async function validateDatabaseEndpoint (database: PostgresConnectionOptions): Promise { + const connectionString = `${database.type}://${database.username}:${database.password}@${database.host}:${database.port}/${database.database}`; + await checkDBEndpoint(connectionString, 'postgresQL'); +} + +export async function validateJobQueueEndpoint (connString: string): Promise { + await checkDBEndpoint(connString, 'Job queue database'); +} + +async function checkWebSocket (wsEndpoint: string) { + const socket = new WebSocket(wsEndpoint); + + return new Promise((resolve, reject) => { + socket.on('open', () => { + socket.close(); + resolve(true); + }); + + socket.on('error', (error) => { + reject(error); + }); + }); +} + +export async function validateNitroChainUrl (wsEndpoint: string): Promise { + try { + await checkWebSocket(wsEndpoint); + log(`The WebSocket endpoint ${wsEndpoint} is running.`); + } catch (error) { + log(`WARNING: Error connecting to websocket endpoint ${wsEndpoint}. Please check if server.p2p.nitro.chainUrl is correct.`, error); + } +} + +export async function validateEthRPCMethods (paidRPCMethods: string[]): Promise { + paidRPCMethods.forEach((method) => { + if (VALID_ETH_RPC_METHODS.includes(method)) { + log(`SUCESS: ${method} is a valid JsonRpcMethod`); + } else { + log(`WARNING: ${method} is not a valid JsonRpcMethod`); + } + }); +} From 681eddea7d90e33f6d18e3394f8b9a734a1d88e1 Mon Sep 17 00:00:00 2001 From: Shreerang Kale Date: Fri, 13 Oct 2023 15:52:58 +0530 Subject: [PATCH 02/10] Add support for websocket endpoints to be checked for contract deployment --- packages/util/src/constants.ts | 2 +- packages/util/src/validate-config.ts | 55 ++++++++++++++++++---------- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/packages/util/src/constants.ts b/packages/util/src/constants.ts index d376721b3..ba1b464a5 100644 --- a/packages/util/src/constants.ts +++ b/packages/util/src/constants.ts @@ -28,4 +28,4 @@ export const DEFAULT_PREFETCH_BATCH_SIZE = 10; export const DEFAULT_MAX_GQL_CACHE_SIZE = Math.pow(2, 20) * 8; // 8 MB -export const VALID_ETH_RPC_METHODS = ['eth_getBlockByHash', 'eth_getStorageAt', 'eth_getBlockByNumber']; +export const SUPPORTED_PAID_RPC_METHODS = ['eth_getBlockByHash', 'eth_getStorageAt', 'eth_getBlockByNumber']; diff --git a/packages/util/src/validate-config.ts b/packages/util/src/validate-config.ts index 23a9a3640..b032400dd 100644 --- a/packages/util/src/validate-config.ts +++ b/packages/util/src/validate-config.ts @@ -3,45 +3,52 @@ import { Client } from 'pg'; import debug from 'debug'; import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions'; import WebSocket from 'ws'; +import path from 'path'; +import fs from 'fs-extra'; -import { VALID_ETH_RPC_METHODS } from './constants'; +import { SUPPORTED_PAID_RPC_METHODS } from './constants'; const log = debug('vulcanize:server'); -async function validateContractDeployment (rpcEndpoint: string, contractAddress: string): Promise { +async function validateContractDeployment (rpcEndpoint: string, contractInfo: {address:string, name?:string}, isWs: boolean): Promise { try { - const provider = await new ethers.providers.JsonRpcProvider(rpcEndpoint); - const code = await provider.getCode(contractAddress); + let provider; + if (isWs) { + provider = new ethers.providers.WebSocketProvider(rpcEndpoint); + } else { + provider = new ethers.providers.JsonRpcProvider(rpcEndpoint); + } + const code = await provider.getCode(contractInfo.address); if (code === '0x') { - log(`WARNING: Contract is not deployed at address ${contractAddress}`); + log(`WARNING: Contract ${contractInfo.name ? contractInfo.name : ''} is not deployed at ${contractInfo.address}`); } else { - log(`SUCCESS: Contract is deployed at address ${contractAddress}`); + log(`SUCCESS: Contract ${contractInfo.name ? contractInfo.name : ''} is deployed at ${contractInfo.address}`); } } catch (error) { log(error); } } -function validateContractAddressFormat (contractAddress: string): void { - if (ethers.utils.isAddress(contractAddress)) { - log(`SUCCESS: Given contract address ${contractAddress} is in a valid format`); +function validateContractAddressFormat (contractInfo: {address:string, name?:string}): void { + if (ethers.utils.isAddress(contractInfo.address)) { + log(`SUCCESS: The ${contractInfo.name ? contractInfo.name : 'contract'} ${contractInfo.address} is in a valid format`); } else { - log(`WARNING: Given contract address ${contractAddress} is not in a valid format`); + log(`WARNING: The ${contractInfo.name ? contractInfo.name : 'contract'} ${contractInfo.address} is not in a valid format`); } } -export async function validateContracts (contractsArr: string[], rpcProviderMutationEndpoint: string): Promise { - contractsArr.forEach((contractAddr: string) => { - validateContractAddressFormat(contractAddr); - validateContractDeployment(rpcProviderMutationEndpoint, contractAddr); +export async function validateContracts (contractsArr: {address:string, name?:string}[], rpcProviderMutationEndpoint: string, isWs: boolean): Promise { + contractsArr.forEach((contract) => { + validateContractAddressFormat(contract); + validateContractDeployment(rpcProviderMutationEndpoint, contract, isWs); }); } -export async function validateEndpoint (endPoint: string, kind: string): Promise { +export async function validateHttpEndpoint (endPoint: string, kind: string): Promise { try { const response = await fetch(endPoint); if (!response.ok) { - log(`WARNING: HTTP error! Status: ${response.status}`); + log(`WARNING: HTTP error! for endpoint ${endPoint} Status: ${response.status}`); } else { log(`SUCCESS: The ${endPoint} is up`); } @@ -90,7 +97,7 @@ async function checkWebSocket (wsEndpoint: string) { }); } -export async function validateNitroChainUrl (wsEndpoint: string): Promise { +export async function validateWebSocketEndpoint (wsEndpoint: string): Promise { try { await checkWebSocket(wsEndpoint); log(`The WebSocket endpoint ${wsEndpoint} is running.`); @@ -99,12 +106,22 @@ export async function validateNitroChainUrl (wsEndpoint: string): Promise } } -export async function validateEthRPCMethods (paidRPCMethods: string[]): Promise { +export async function validatePaidRPCMethods (paidRPCMethods: string[]): Promise { paidRPCMethods.forEach((method) => { - if (VALID_ETH_RPC_METHODS.includes(method)) { + if (SUPPORTED_PAID_RPC_METHODS.includes(method)) { log(`SUCESS: ${method} is a valid JsonRpcMethod`); } else { log(`WARNING: ${method} is not a valid JsonRpcMethod`); } }); } + +export async function validateFilePath (configFile: string): Promise { + const configFilePath = path.resolve(configFile); + const fileExists = await fs.pathExists(configFilePath); + if (!fileExists) { + log(`WARNING: Config file not found: ${configFilePath}`); + } else { + log(`SUCCESS: Config file found: ${configFilePath}`); + } +} From fd1e23648d8442d67bfa52b021e56842843be73a Mon Sep 17 00:00:00 2001 From: Shreerang Kale Date: Mon, 16 Oct 2023 09:48:58 +0530 Subject: [PATCH 03/10] Add flag to toggle validation --- packages/util/src/config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/util/src/config.ts b/packages/util/src/config.ts index 318c21324..15063c5de 100644 --- a/packages/util/src/config.ts +++ b/packages/util/src/config.ts @@ -194,6 +194,7 @@ export interface ServerConfig { port: number; mode: string; kind: string; + enableValidation: boolean; checkpointing: boolean; checkpointInterval: number; subgraphPath: string; From 1284843a3a26f2be1ae671c3f093369a83db0bc7 Mon Sep 17 00:00:00 2001 From: Shreerang Kale Date: Mon, 16 Oct 2023 10:37:41 +0530 Subject: [PATCH 04/10] Update log message for paid rpc method validation --- packages/util/src/validate-config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/util/src/validate-config.ts b/packages/util/src/validate-config.ts index b032400dd..c66594b65 100644 --- a/packages/util/src/validate-config.ts +++ b/packages/util/src/validate-config.ts @@ -109,9 +109,9 @@ export async function validateWebSocketEndpoint (wsEndpoint: string): Promise { paidRPCMethods.forEach((method) => { if (SUPPORTED_PAID_RPC_METHODS.includes(method)) { - log(`SUCESS: ${method} is a valid JsonRpcMethod`); + log(`SUCESS: ${method} is a supported paid RPC method`); } else { - log(`WARNING: ${method} is not a valid JsonRpcMethod`); + log(`WARNING: ${method} is not a supported paid RPC method`); } }); } From 4a9fcdce0ba5b5d9d4d39fe85453db741b0f7d24 Mon Sep 17 00:00:00 2001 From: Shreerang Kale Date: Mon, 16 Oct 2023 11:09:11 +0530 Subject: [PATCH 05/10] Update log message for validation --- packages/util/src/validate-config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/util/src/validate-config.ts b/packages/util/src/validate-config.ts index c66594b65..0c44e58ab 100644 --- a/packages/util/src/validate-config.ts +++ b/packages/util/src/validate-config.ts @@ -31,9 +31,9 @@ async function validateContractDeployment (rpcEndpoint: string, contractInfo: {a function validateContractAddressFormat (contractInfo: {address:string, name?:string}): void { if (ethers.utils.isAddress(contractInfo.address)) { - log(`SUCCESS: The ${contractInfo.name ? contractInfo.name : 'contract'} ${contractInfo.address} is in a valid format`); + log(`SUCCESS: Address ${contractInfo.address} ${contractInfo.name ? `for ${contractInfo.name}` : ''} is in a valid format`); } else { - log(`WARNING: The ${contractInfo.name ? contractInfo.name : 'contract'} ${contractInfo.address} is not in a valid format`); + log(`WARNING: Address ${contractInfo.address} ${contractInfo.name ? `for ${contractInfo.name}` : ''} is not in a valid format`); } } From adaf666b463aec45b4e944bfc9fc3b01e35a6ff4 Mon Sep 17 00:00:00 2001 From: Shreerang Kale Date: Mon, 16 Oct 2023 11:51:40 +0530 Subject: [PATCH 06/10] Rename enableValidation to enableConfigValidation --- packages/util/src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/util/src/config.ts b/packages/util/src/config.ts index 15063c5de..211adf6fe 100644 --- a/packages/util/src/config.ts +++ b/packages/util/src/config.ts @@ -194,7 +194,7 @@ export interface ServerConfig { port: number; mode: string; kind: string; - enableValidation: boolean; + enableConfigValidation: boolean; checkpointing: boolean; checkpointInterval: number; subgraphPath: string; From f12aec7826dc656e73513399ae51a3b7d270b836 Mon Sep 17 00:00:00 2001 From: Shreerang Kale Date: Mon, 16 Oct 2023 13:47:25 +0530 Subject: [PATCH 07/10] Check deployment only if contract address is in a valid format --- packages/util/src/validate-config.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/util/src/validate-config.ts b/packages/util/src/validate-config.ts index 0c44e58ab..1a148f2bc 100644 --- a/packages/util/src/validate-config.ts +++ b/packages/util/src/validate-config.ts @@ -29,18 +29,23 @@ async function validateContractDeployment (rpcEndpoint: string, contractInfo: {a } } -function validateContractAddressFormat (contractInfo: {address:string, name?:string}): void { +function validateContractAddressFormat (contractInfo: {address:string, name?:string}): boolean { if (ethers.utils.isAddress(contractInfo.address)) { log(`SUCCESS: Address ${contractInfo.address} ${contractInfo.name ? `for ${contractInfo.name}` : ''} is in a valid format`); + return true; } else { log(`WARNING: Address ${contractInfo.address} ${contractInfo.name ? `for ${contractInfo.name}` : ''} is not in a valid format`); + return false; } } export async function validateContracts (contractsArr: {address:string, name?:string}[], rpcProviderMutationEndpoint: string, isWs: boolean): Promise { contractsArr.forEach((contract) => { - validateContractAddressFormat(contract); - validateContractDeployment(rpcProviderMutationEndpoint, contract, isWs); + const isValidFormat = validateContractAddressFormat(contract); + + if (isValidFormat) { + validateContractDeployment(rpcProviderMutationEndpoint, contract, isWs); + } }); } From f3a6bd8d2ffb8dd646acfeaf5b9473c804dcba53 Mon Sep 17 00:00:00 2001 From: Shreerang Kale Date: Mon, 16 Oct 2023 15:23:50 +0530 Subject: [PATCH 08/10] Update httpValidateEndpoint method to check if endpoint is up instead of response as ok --- packages/util/src/validate-config.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/util/src/validate-config.ts b/packages/util/src/validate-config.ts index 1a148f2bc..123d60945 100644 --- a/packages/util/src/validate-config.ts +++ b/packages/util/src/validate-config.ts @@ -52,11 +52,7 @@ export async function validateContracts (contractsArr: {address:string, name?:st export async function validateHttpEndpoint (endPoint: string, kind: string): Promise { try { const response = await fetch(endPoint); - if (!response.ok) { - log(`WARNING: HTTP error! for endpoint ${endPoint} Status: ${response.status}`); - } else { - log(`SUCCESS: The ${endPoint} is up`); - } + log(`SUCCESS: The ${endPoint} is up. Status ${response.status}`); } catch (error:any) { log(`WARNING: could not connect to ${endPoint}. Please check if the ${kind} is correct and up.`); log(error); From 9fd06abb4ed1b52077a7bde68861ed5180eee9bc Mon Sep 17 00:00:00 2001 From: Shreerang Kale Date: Mon, 16 Oct 2023 16:04:26 +0530 Subject: [PATCH 09/10] Update log messages --- packages/util/src/validate-config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/util/src/validate-config.ts b/packages/util/src/validate-config.ts index 123d60945..933de90e3 100644 --- a/packages/util/src/validate-config.ts +++ b/packages/util/src/validate-config.ts @@ -68,7 +68,7 @@ async function checkDBEndpoint (connectionString: string, dbKind: string): Promi await client.connect(); log(`SUCCESS: ${dbKind} endpoint is up!`); } catch (error) { - log('WARNING: Error connecting to job queue database. Please check if job queue config is setup and database is running \n', error); + log(`WARNING: Error connecting to ${dbKind} database. Please check if job queue config is setup and database is running \n`, error); } finally { await client.end(); } @@ -101,7 +101,7 @@ async function checkWebSocket (wsEndpoint: string) { export async function validateWebSocketEndpoint (wsEndpoint: string): Promise { try { await checkWebSocket(wsEndpoint); - log(`The WebSocket endpoint ${wsEndpoint} is running.`); + log(`SUCCESS: The WebSocket endpoint ${wsEndpoint} is running.`); } catch (error) { log(`WARNING: Error connecting to websocket endpoint ${wsEndpoint}. Please check if server.p2p.nitro.chainUrl is correct.`, error); } @@ -110,7 +110,7 @@ export async function validateWebSocketEndpoint (wsEndpoint: string): Promise { paidRPCMethods.forEach((method) => { if (SUPPORTED_PAID_RPC_METHODS.includes(method)) { - log(`SUCESS: ${method} is a supported paid RPC method`); + log(`SUCCESS: ${method} is a supported paid RPC method`); } else { log(`WARNING: ${method} is not a supported paid RPC method`); } From 2081225c2768299bf1a00128a3f65c780c8a0ac7 Mon Sep 17 00:00:00 2001 From: Shreerang Kale Date: Mon, 16 Oct 2023 16:35:05 +0530 Subject: [PATCH 10/10] Update log messages --- packages/util/src/validate-config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/util/src/validate-config.ts b/packages/util/src/validate-config.ts index 933de90e3..9603bd31d 100644 --- a/packages/util/src/validate-config.ts +++ b/packages/util/src/validate-config.ts @@ -66,7 +66,7 @@ async function checkDBEndpoint (connectionString: string, dbKind: string): Promi try { await client.connect(); - log(`SUCCESS: ${dbKind} endpoint is up!`); + log(`SUCCESS: ${dbKind} endpoint is up.`); } catch (error) { log(`WARNING: Error connecting to ${dbKind} database. Please check if job queue config is setup and database is running \n`, error); } finally { @@ -101,7 +101,7 @@ async function checkWebSocket (wsEndpoint: string) { export async function validateWebSocketEndpoint (wsEndpoint: string): Promise { try { await checkWebSocket(wsEndpoint); - log(`SUCCESS: The WebSocket endpoint ${wsEndpoint} is running.`); + log(`SUCCESS: The WebSocket endpoint ${wsEndpoint} is up.`); } catch (error) { log(`WARNING: Error connecting to websocket endpoint ${wsEndpoint}. Please check if server.p2p.nitro.chainUrl is correct.`, error); }