From 9b345ae44e8b89636cd9fbbb86644b220c6528f5 Mon Sep 17 00:00:00 2001 From: Robert Raynor <35671663+mooselumph@users.noreply.github.com> Date: Thu, 11 Apr 2024 18:52:15 +0000 Subject: [PATCH] Update flags --- core/thegraph/config.go | 53 ++++++++++++++++++++++++++++ core/thegraph/state.go | 11 ++++++ disperser/cmd/batcher/config.go | 5 +-- disperser/cmd/batcher/flags/flags.go | 9 ++--- disperser/cmd/batcher/main.go | 10 ++---- operators/churner/cmd/main.go | 9 ++--- operators/churner/config.go | 11 +++--- operators/churner/flags/flags.go | 9 ++--- retriever/cmd/main.go | 10 ++---- retriever/config.go | 15 ++++---- retriever/flags/flags.go | 9 ++--- 11 files changed, 94 insertions(+), 57 deletions(-) create mode 100644 core/thegraph/config.go diff --git a/core/thegraph/config.go b/core/thegraph/config.go new file mode 100644 index 0000000000..f38bb84ff5 --- /dev/null +++ b/core/thegraph/config.go @@ -0,0 +1,53 @@ +package thegraph + +import ( + "time" + + "github.com/Layr-Labs/eigenda/common" + "github.com/urfave/cli" +) + +const ( + EndpointFlagName = "thegraph.endpoint" + PullIntervalFlagName = "thegraph.pull_interval" + MaxRetriesFlagName = "thegraph.max_retries" +) + +type Config struct { + Endpoint string // The Graph endpoint + PullInterval time.Duration // The interval to pull data from The Graph + MaxRetries int // The maximum number of retries to pull data from The Graph +} + +func CLIFlags(envPrefix string) []cli.Flag { + return []cli.Flag{ + cli.StringFlag{ + Name: EndpointFlagName, + Usage: "The Graph endpoint", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "GRAPH_URL"), + }, + cli.DurationFlag{ + Name: PullIntervalFlagName, + Usage: "Pull interval for retries", + Value: 100 * time.Millisecond, + EnvVar: common.PrefixEnvVar(envPrefix, "GRAPH_PULL_INTERVAL"), + }, + cli.UintFlag{ + Name: MaxRetriesFlagName, + Usage: "The maximum number of retries", + Value: 5, + EnvVar: common.PrefixEnvVar(envPrefix, "GRAPH_MAX_RETRIES"), + }, + } +} + +func ReadCLIConfig(ctx *cli.Context) Config { + + return Config{ + Endpoint: ctx.String(EndpointFlagName), + PullInterval: ctx.Duration(PullIntervalFlagName), + MaxRetries: ctx.Int(MaxRetriesFlagName), + } + +} diff --git a/core/thegraph/state.go b/core/thegraph/state.go index c797e51960..084a1af185 100644 --- a/core/thegraph/state.go +++ b/core/thegraph/state.go @@ -87,6 +87,17 @@ type ( var _ IndexedChainState = (*indexedChainState)(nil) +func MakeIndexedChainState(config Config, cs core.ChainState, logger logging.Logger) *indexedChainState { + + logger.Info("Using graph node") + querier := graphql.NewClient(config.Endpoint, nil) + + // RetryQuerier is a wrapper around the GraphQLQuerier that retries queries on failure + retryQuerier := NewRetryQuerier(querier, config.PullInterval, config.MaxRetries) + + return NewIndexedChainState(cs, retryQuerier, logger) +} + func NewIndexedChainState(cs core.ChainState, querier GraphQLQuerier, logger logging.Logger) *indexedChainState { return &indexedChainState{ ChainState: cs, diff --git a/disperser/cmd/batcher/config.go b/disperser/cmd/batcher/config.go index 3b1028ffd0..89febaf5c6 100644 --- a/disperser/cmd/batcher/config.go +++ b/disperser/cmd/batcher/config.go @@ -4,6 +4,7 @@ import ( "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/common/aws" "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/thegraph" "github.com/Layr-Labs/eigenda/disperser/batcher" "github.com/Layr-Labs/eigenda/disperser/cmd/batcher/flags" "github.com/Layr-Labs/eigenda/disperser/common/blobstore" @@ -23,7 +24,7 @@ type Config struct { MetricsConfig batcher.MetricsConfig IndexerConfig indexer.Config FireblocksConfig common.FireblocksConfig - GraphUrl string + ChainStateConfig thegraph.Config UseGraph bool IndexerDataDir string @@ -75,8 +76,8 @@ func NewConfig(ctx *cli.Context) (Config, error) { HTTPPort: ctx.GlobalString(flags.MetricsHTTPPort.Name), EnableMetrics: ctx.GlobalBool(flags.EnableMetrics.Name), }, + ChainStateConfig: thegraph.ReadCLIConfig(ctx), UseGraph: ctx.Bool(flags.UseGraphFlag.Name), - GraphUrl: ctx.GlobalString(flags.GraphUrlFlag.Name), BLSOperatorStateRetrieverAddr: ctx.GlobalString(flags.BlsOperatorStateRetrieverFlag.Name), EigenDAServiceManagerAddr: ctx.GlobalString(flags.EigenDAServiceManagerFlag.Name), IndexerDataDir: ctx.GlobalString(flags.IndexerDataDirFlag.Name), diff --git a/disperser/cmd/batcher/flags/flags.go b/disperser/cmd/batcher/flags/flags.go index 62f29b516c..204f3c4090 100644 --- a/disperser/cmd/batcher/flags/flags.go +++ b/disperser/cmd/batcher/flags/flags.go @@ -6,6 +6,7 @@ import ( "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/common/aws" "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/thegraph" "github.com/Layr-Labs/eigenda/indexer" "github.com/urfave/cli" ) @@ -59,12 +60,6 @@ var ( Required: true, EnvVar: common.PrefixEnvVar(envVarPrefix, "ENABLE_METRICS"), } - GraphUrlFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "graph-url"), - Usage: "The url of the graph node", - Required: true, - EnvVar: common.PrefixEnvVar(envVarPrefix, "GRAPH_URL"), - } UseGraphFlag = cli.BoolFlag{ Name: common.PrefixFlag(FlagPrefix, "use-graph"), Usage: "Whether to use the graph node", @@ -194,7 +189,6 @@ var requiredFlags = []cli.Flag{ EigenDAServiceManagerFlag, EncoderSocket, EnableMetrics, - GraphUrlFlag, BatchSizeLimitFlag, UseGraphFlag, SRSOrderFlag, @@ -227,4 +221,5 @@ func init() { Flags = append(Flags, indexer.CLIFlags(envVarPrefix)...) Flags = append(Flags, aws.ClientFlags(envVarPrefix, FlagPrefix)...) Flags = append(Flags, common.FireblocksCLIFlags(envVarPrefix, FlagPrefix)...) + Flags = append(Flags, thegraph.CLIFlags(envVarPrefix)...) } diff --git a/disperser/cmd/batcher/main.go b/disperser/cmd/batcher/main.go index 3ffc1b5917..5bcda004ff 100644 --- a/disperser/cmd/batcher/main.go +++ b/disperser/cmd/batcher/main.go @@ -8,8 +8,6 @@ import ( "os" "time" - "github.com/shurcooL/graphql" - "github.com/Layr-Labs/eigenda/common" coreindexer "github.com/Layr-Labs/eigenda/core/indexer" "github.com/Layr-Labs/eigenda/core/thegraph" @@ -140,13 +138,9 @@ func RunBatcher(ctx *cli.Context) error { var ics core.IndexedChainState if config.UseGraph { logger.Info("Using graph node") - querier := graphql.NewClient(config.GraphUrl, nil) - - // RetryQuerier is a wrapper around the GraphQLQuerier that retries queries on failure - retryQuerier := thegraph.NewRetryQuerier(querier, 100*time.Millisecond, config.EthClientConfig.NumRetries) - logger.Info("Connecting to subgraph", "url", config.GraphUrl) - ics = thegraph.NewIndexedChainState(cs, retryQuerier, logger) + logger.Info("Connecting to subgraph", "url", config.ChainStateConfig.Endpoint) + ics = thegraph.MakeIndexedChainState(config.ChainStateConfig, cs, logger) } else { logger.Info("Using built-in indexer") diff --git a/operators/churner/cmd/main.go b/operators/churner/cmd/main.go index b298c37dcd..8f4aaaf74d 100644 --- a/operators/churner/cmd/main.go +++ b/operators/churner/cmd/main.go @@ -5,7 +5,6 @@ import ( "log" "net" "os" - "time" pb "github.com/Layr-Labs/eigenda/api/grpc/churner" "github.com/Layr-Labs/eigenda/common" @@ -17,7 +16,6 @@ import ( "github.com/Layr-Labs/eigenda/operators/churner" "github.com/Layr-Labs/eigenda/operators/churner/flags" gethcommon "github.com/ethereum/go-ethereum/common" - "github.com/shurcooL/graphql" "github.com/urfave/cli" "google.golang.org/grpc" "google.golang.org/grpc/reflection" @@ -83,12 +81,11 @@ func run(ctx *cli.Context) error { cs := coreeth.NewChainState(tx, gethClient) - querier := graphql.NewClient(config.GraphUrl, nil) + logger.Info("Using graph node") - // RetryQuerier is a wrapper around the GraphQLQuerier that retries queries on failure - retryQuerier := thegraph.NewRetryQuerier(querier, 100*time.Millisecond, config.EthClientConfig.NumRetries) + logger.Info("Connecting to subgraph", "url", config.ChainStateConfig.Endpoint) + indexer := thegraph.MakeIndexedChainState(config.ChainStateConfig, cs, logger) - indexer := thegraph.NewIndexedChainState(cs, retryQuerier, logger) metrics := churner.NewMetrics(config.MetricsConfig.HTTPPort, logger) cn, err := churner.NewChurner(config, indexer, tx, logger, metrics) diff --git a/operators/churner/config.go b/operators/churner/config.go index 6ddf380b1d..ae7832c6d7 100644 --- a/operators/churner/config.go +++ b/operators/churner/config.go @@ -5,15 +5,16 @@ import ( "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/thegraph" "github.com/Layr-Labs/eigenda/operators/churner/flags" "github.com/urfave/cli" ) type Config struct { - EthClientConfig geth.EthClientConfig - LoggerConfig common.LoggerConfig - GraphUrl string - MetricsConfig MetricsConfig + EthClientConfig geth.EthClientConfig + LoggerConfig common.LoggerConfig + MetricsConfig MetricsConfig + ChainStateConfig thegraph.Config BLSOperatorStateRetrieverAddr string EigenDAServiceManagerAddr string @@ -29,7 +30,7 @@ func NewConfig(ctx *cli.Context) (*Config, error) { return &Config{ EthClientConfig: geth.ReadEthClientConfig(ctx), LoggerConfig: *loggerConfig, - GraphUrl: ctx.GlobalString(flags.GraphUrlFlag.Name), + ChainStateConfig: thegraph.ReadCLIConfig(ctx), BLSOperatorStateRetrieverAddr: ctx.GlobalString(flags.BlsOperatorStateRetrieverFlag.Name), EigenDAServiceManagerAddr: ctx.GlobalString(flags.EigenDAServiceManagerFlag.Name), PerPublicKeyRateLimit: ctx.GlobalDuration(flags.PerPublicKeyRateLimit.Name), diff --git a/operators/churner/flags/flags.go b/operators/churner/flags/flags.go index 963156d02d..6a5dbf795d 100644 --- a/operators/churner/flags/flags.go +++ b/operators/churner/flags/flags.go @@ -5,6 +5,7 @@ import ( "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/thegraph" "github.com/Layr-Labs/eigenda/indexer" "github.com/urfave/cli" ) @@ -31,12 +32,6 @@ var ( Required: true, EnvVar: common.PrefixEnvVar(envPrefix, "GRPC_PORT"), } - GraphUrlFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "indexer-graph-url"), - Usage: "The url of the subgraph to query", - Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "GRAPH_URL"), - } BlsOperatorStateRetrieverFlag = cli.StringFlag{ Name: common.PrefixFlag(FlagPrefix, "bls-operator-state-retriever"), Usage: "Address of the BLS Operator State Retriever", @@ -75,7 +70,6 @@ var ( var requiredFlags = []cli.Flag{ HostnameFlag, GrpcPortFlag, - GraphUrlFlag, BlsOperatorStateRetrieverFlag, EigenDAServiceManagerFlag, EnableMetrics, @@ -94,4 +88,5 @@ func init() { Flags = append(Flags, geth.EthClientFlags(envPrefix)...) Flags = append(Flags, common.LoggerCLIFlags(envPrefix, FlagPrefix)...) Flags = append(Flags, indexer.CLIFlags(envPrefix)...) + Flags = append(Flags, thegraph.CLIFlags(envPrefix)...) } diff --git a/retriever/cmd/main.go b/retriever/cmd/main.go index 9b0106cf3c..6d291fa1eb 100644 --- a/retriever/cmd/main.go +++ b/retriever/cmd/main.go @@ -6,7 +6,6 @@ import ( "log" "net" "os" - "time" pb "github.com/Layr-Labs/eigenda/api/grpc/retriever" "github.com/Layr-Labs/eigenda/clients" @@ -23,7 +22,6 @@ import ( "github.com/Layr-Labs/eigenda/retriever/flags" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rpc" - "github.com/shurcooL/graphql" "github.com/urfave/cli" "google.golang.org/grpc" "google.golang.org/grpc/reflection" @@ -108,13 +106,9 @@ func RetrieverMain(ctx *cli.Context) error { var ics core.IndexedChainState if config.UseGraph { logger.Info("Using graph node") - querier := graphql.NewClient(config.GraphUrl, nil) - // RetryQuerier is a wrapper around the GraphQLQuerier that retries queries on failure - retryQuerier := thegraph.NewRetryQuerier(querier, 100*time.Millisecond, config.EthClientConfig.NumRetries) - - logger.Info("Connecting to subgraph", "url", config.GraphUrl) - ics = thegraph.NewIndexedChainState(cs, retryQuerier, logger) + logger.Info("Connecting to subgraph", "url", config.ChainStateConfig.Endpoint) + ics = thegraph.MakeIndexedChainState(config.ChainStateConfig, cs, logger) } else { logger.Info("Using built-in indexer") diff --git a/retriever/config.go b/retriever/config.go index 11e3e9fd41..3f9a2b0d1c 100644 --- a/retriever/config.go +++ b/retriever/config.go @@ -5,6 +5,7 @@ import ( "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/thegraph" "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/indexer" "github.com/Layr-Labs/eigenda/retriever/flags" @@ -12,18 +13,18 @@ import ( ) type Config struct { - EncoderConfig kzg.KzgConfig - EthClientConfig geth.EthClientConfig - LoggerConfig common.LoggerConfig - IndexerConfig indexer.Config - MetricsConfig MetricsConfig + EncoderConfig kzg.KzgConfig + EthClientConfig geth.EthClientConfig + LoggerConfig common.LoggerConfig + IndexerConfig indexer.Config + MetricsConfig MetricsConfig + ChainStateConfig thegraph.Config IndexerDataDir string Timeout time.Duration NumConnections int BLSOperatorStateRetrieverAddr string EigenDAServiceManagerAddr string - GraphUrl string UseGraph bool } @@ -40,12 +41,12 @@ func NewConfig(ctx *cli.Context) (*Config, error) { MetricsConfig: MetricsConfig{ HTTPPort: ctx.GlobalString(flags.MetricsHTTPPortFlag.Name), }, + ChainStateConfig: thegraph.ReadCLIConfig(ctx), IndexerDataDir: ctx.GlobalString(flags.IndexerDataDirFlag.Name), Timeout: ctx.Duration(flags.TimeoutFlag.Name), NumConnections: ctx.Int(flags.NumConnectionsFlag.Name), BLSOperatorStateRetrieverAddr: ctx.GlobalString(flags.BlsOperatorStateRetrieverFlag.Name), EigenDAServiceManagerAddr: ctx.GlobalString(flags.EigenDAServiceManagerFlag.Name), - GraphUrl: ctx.GlobalString(flags.GraphUrlFlag.Name), UseGraph: ctx.GlobalBool(flags.UseGraphFlag.Name), }, nil } diff --git a/retriever/flags/flags.go b/retriever/flags/flags.go index 4cd5499f00..61ac03a368 100644 --- a/retriever/flags/flags.go +++ b/retriever/flags/flags.go @@ -3,6 +3,7 @@ package flags import ( "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/thegraph" "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/indexer" "github.com/urfave/cli" @@ -67,12 +68,6 @@ var ( Value: "9100", EnvVar: common.PrefixEnvVar(envPrefix, "METRICS_HTTP_PORT"), } - GraphUrlFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "graph-url"), - Usage: "The url of the graph node", - Required: false, - EnvVar: common.PrefixEnvVar(envPrefix, "GRAPH_URL"), - } UseGraphFlag = cli.BoolFlag{ Name: common.PrefixFlag(FlagPrefix, "use-graph"), Usage: "Whether to use the graph node", @@ -93,7 +88,6 @@ var optionalFlags = []cli.Flag{ NumConnectionsFlag, IndexerDataDirFlag, MetricsHTTPPortFlag, - GraphUrlFlag, UseGraphFlag, } @@ -106,4 +100,5 @@ func init() { Flags = append(Flags, geth.EthClientFlags(envPrefix)...) Flags = append(Flags, common.LoggerCLIFlags(envPrefix, FlagPrefix)...) Flags = append(Flags, indexer.CLIFlags(envPrefix)...) + Flags = append(Flags, thegraph.CLIFlags(envPrefix)...) }