From af3ac5930097597a43932c7a9188d02f2ca53276 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 17 Jul 2024 10:14:10 -0500 Subject: [PATCH 01/74] Added comments to the generator config. Signed-off-by: Cody Littley --- tools/traffic/config.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/traffic/config.go b/tools/traffic/config.go index 590c607590..ca84125706 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -10,14 +10,22 @@ import ( "github.com/urfave/cli" ) +// Config configures a traffic generator. type Config struct { clients.Config - NumInstances uint - RequestInterval time.Duration - DataSize uint64 - LoggingConfig common.LoggerConfig - RandomizeBlobs bool + // The number of worker threads that generate write traffic. + NumInstances uint + // The period of the submission rate of new blobs for each worker thread. + RequestInterval time.Duration + // The size of each blob dispersed, in bytes. + DataSize uint64 + // Configures logging for the traffic generator. + LoggingConfig common.LoggerConfig + // If true, then each blob will contain unique random data. If false, the same random data + // will be dispersed for each blob by a particular worker thread. + RandomizeBlobs bool + // The amount of time to sleep after launching each worker thread. InstanceLaunchInterval time.Duration SignerPrivateKey string From 8191b7fd4b49a6e138a75eaaca3202007ab17bdf Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 17 Jul 2024 13:09:37 -0500 Subject: [PATCH 02/74] Incremental progress. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 8 +++--- tools/traffic/config.go | 27 ++++++++++++++----- tools/traffic/flags/flags.go | 20 +++++++------- tools/traffic/generator.go | 47 ++++++++++++++++++++++++--------- tools/traffic/generator_test.go | 14 +++++----- 5 files changed, 77 insertions(+), 39 deletions(-) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index c9cea11d4b..3fa0618d83 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -8,10 +8,12 @@ build: clean run: build TRAFFIC_GENERATOR_HOSTNAME=localhost \ - TRAFFIC_GENERATOR_GRPC_PORT=32001 \ + TRAFFIC_GENERATOR_GRPC_PORT=32003 \ TRAFFIC_GENERATOR_TIMEOUT=10s \ - TRAFFIC_GENERATOR_NUM_INSTANCES=1 \ - TRAFFIC_GENERATOR_REQUEST_INTERVAL=1s \ + TRAFFIC_GENERATOR_NUM_WRITE_INSTANCES=1 \ + TRAFFIC_GENERATOR_WRITE_REQUEST_INTERVAL=1s \ TRAFFIC_GENERATOR_DATA_SIZE=1000 \ TRAFFIC_GENERATOR_RANDOMIZE_BLOBS=true \ ./bin/server + +# TODO do not merge change TRAFFIC_GENERATOR_GRPC_PORT=32003 \ No newline at end of file diff --git a/tools/traffic/config.go b/tools/traffic/config.go index ca84125706..215951a10b 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -14,17 +14,30 @@ import ( type Config struct { clients.Config + // TODO add to flags.go + // The number of worker threads that generate read traffic. + NumReadInstances uint + // The period of the submission rate of read requests for each read worker thread. + ReadRequestInterval time.Duration + // For each blob, how many times should it be downloaded? If between 0.0 and 1.0, blob will be downloaded + // 0 or 1 times with the specified probability (e.g. 0.2 means each blob has a 20% chance of being downloaded). + // If greater than 1.0, then each blob will be downloaded the specified number of times. + DownloadRate float64 + // The minimum amount of time that must pass after a blob is written prior to the first read attempt being made. + ReadDelay time.Duration + // The number of worker threads that generate write traffic. - NumInstances uint - // The period of the submission rate of new blobs for each worker thread. - RequestInterval time.Duration + NumWriteInstances uint + // The period of the submission rate of new blobs for each write worker thread. + WriteRequestInterval time.Duration // The size of each blob dispersed, in bytes. DataSize uint64 - // Configures logging for the traffic generator. - LoggingConfig common.LoggerConfig // If true, then each blob will contain unique random data. If false, the same random data // will be dispersed for each blob by a particular worker thread. RandomizeBlobs bool + + // Configures logging for the traffic generator. + LoggingConfig common.LoggerConfig // The amount of time to sleep after launching each worker thread. InstanceLaunchInterval time.Duration @@ -52,8 +65,8 @@ func NewConfig(ctx *cli.Context) (*Config, error) { ctx.Duration(flags.TimeoutFlag.Name), ctx.GlobalBool(flags.UseSecureGrpcFlag.Name), ), - NumInstances: ctx.GlobalUint(flags.NumInstancesFlag.Name), - RequestInterval: ctx.Duration(flags.RequestIntervalFlag.Name), + NumWriteInstances: ctx.GlobalUint(flags.NumWriteInstancesFlag.Name), + WriteRequestInterval: ctx.Duration(flags.WriteRequestIntervalFlag.Name), DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), LoggingConfig: *loggerConfig, RandomizeBlobs: ctx.GlobalBool(flags.RandomizeBlobsFlag.Name), diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index aca2eb11a3..57ee87cba3 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -34,17 +34,17 @@ var ( EnvVar: common.PrefixEnvVar(envPrefix, "TIMEOUT"), Value: 10 * time.Second, } - NumInstancesFlag = cli.UintFlag{ - Name: common.PrefixFlag(FlagPrefix, "num-instances"), - Usage: "Number of generator instances to run in parallel", + NumWriteInstancesFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "num-write-instances"), + Usage: "Number of generator instances producing write traffic to run in parallel", Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "NUM_INSTANCES"), + EnvVar: common.PrefixEnvVar(envPrefix, "NUM_WRITE_INSTANCES"), } - RequestIntervalFlag = cli.DurationFlag{ - Name: common.PrefixFlag(FlagPrefix, "request-interval"), - Usage: "Duration between requests", + WriteRequestIntervalFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "write-request-interval"), + Usage: "Duration between write requests", Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "REQUEST_INTERVAL"), + EnvVar: common.PrefixEnvVar(envPrefix, "WRITE_REQUEST_INTERVAL"), Value: 30 * time.Second, } DataSizeFlag = cli.Uint64Flag{ @@ -89,8 +89,8 @@ var ( var requiredFlags = []cli.Flag{ HostnameFlag, GrpcPortFlag, - NumInstancesFlag, - RequestIntervalFlag, + NumWriteInstancesFlag, + WriteRequestIntervalFlag, DataSizeFlag, } diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 2f8731ffd5..67a621cc18 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -37,14 +37,15 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi }, nil } +// Run instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. func (g *TrafficGenerator) Run() error { ctx, cancel := context.WithCancel(context.Background()) var wg sync.WaitGroup - for i := 0; i < int(g.Config.NumInstances); i++ { + for i := 0; i < int(g.Config.NumWriteInstances); i++ { wg.Add(1) go func() { defer wg.Done() - _ = g.StartTraffic(ctx) + _ = g.StartWriteWorker(ctx) }() time.Sleep(g.Config.InstanceLaunchInterval) } @@ -57,7 +58,29 @@ func (g *TrafficGenerator) Run() error { return nil } -func (g *TrafficGenerator) StartTraffic(ctx context.Context) error { +// TODO maybe split reader/writer into separate files + +// StartReadWorker periodically requests to download random blobs at a configured rate. +func (g *TrafficGenerator) StartReadWorker(ctx context.Context) error { + ticker := time.NewTicker(g.Config.WriteRequestInterval) + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + // TODO determine which blob to download + g.readRequest() // TODO add parameters + } + } +} + +// readRequest reads a blob. +func (g *TrafficGenerator) readRequest() { + // TODO +} + +// StartWriteWorker periodically sends (possibly) random blobs to a disperser at a configured rate. +func (g *TrafficGenerator) StartWriteWorker(ctx context.Context) error { data := make([]byte, g.Config.DataSize) _, err := rand.Read(data) if err != nil { @@ -66,7 +89,7 @@ func (g *TrafficGenerator) StartTraffic(ctx context.Context) error { paddedData := codec.ConvertByPaddingEmptyByte(data) - ticker := time.NewTicker(g.Config.RequestInterval) + ticker := time.NewTicker(g.Config.WriteRequestInterval) for { select { case <-ctx.Done(): @@ -79,13 +102,13 @@ func (g *TrafficGenerator) StartTraffic(ctx context.Context) error { } paddedData = codec.ConvertByPaddingEmptyByte(data) - err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) + _, err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) if err != nil { g.Logger.Error("failed to send blob request", "err:", err) } paddedData = nil } else { - err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) + _, err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) if err != nil { g.Logger.Error("failed to send blob request", "err:", err) } @@ -95,26 +118,26 @@ func (g *TrafficGenerator) StartTraffic(ctx context.Context) error { } } -func (g *TrafficGenerator) sendRequest(ctx context.Context, data []byte) error { +// sendRequest sends a blob to a disperser. +func (g *TrafficGenerator) sendRequest(ctx context.Context, data []byte) ([]byte /* key */, error) { ctxTimeout, cancel := context.WithTimeout(ctx, g.Config.Timeout) defer cancel() if g.Config.SignerPrivateKey != "" { blobStatus, key, err := g.DisperserClient.DisperseBlobAuthenticated(ctxTimeout, data, g.Config.CustomQuorums) if err != nil { - return err + return nil, err } g.Logger.Info("successfully dispersed new blob", "authenticated", true, "key", hex.EncodeToString(key), "status", blobStatus.String()) - return nil + return key, nil } else { blobStatus, key, err := g.DisperserClient.DisperseBlob(ctxTimeout, data, g.Config.CustomQuorums) if err != nil { - return err + return nil, err } g.Logger.Info("successfully dispersed new blob", "authenticated", false, "key", hex.EncodeToString(key), "status", blobStatus.String()) - return nil + return key, nil } - } diff --git a/tools/traffic/generator_test.go b/tools/traffic/generator_test.go index b530ef5bd2..b2ef6abb1a 100644 --- a/tools/traffic/generator_test.go +++ b/tools/traffic/generator_test.go @@ -23,8 +23,8 @@ func TestTrafficGenerator(t *testing.T) { Config: clients.Config{ Timeout: 1 * time.Second, }, - DataSize: 1000_000, - RequestInterval: 2 * time.Second, + DataSize: 1000_000, + WriteRequestInterval: 2 * time.Second, }, DisperserClient: disperserClient, } @@ -34,7 +34,7 @@ func TestTrafficGenerator(t *testing.T) { Return(&processing, []byte{1}, nil) ctx, cancel := context.WithCancel(context.Background()) go func() { - _ = trafficGenerator.StartTraffic(ctx) + _ = trafficGenerator.StartWriteWorker(ctx) }() time.Sleep(5 * time.Second) cancel() @@ -51,9 +51,9 @@ func TestTrafficGeneratorAuthenticated(t *testing.T) { Config: clients.Config{ Timeout: 1 * time.Second, }, - DataSize: 1000_000, - RequestInterval: 2 * time.Second, - SignerPrivateKey: "Hi", + DataSize: 1000_000, + WriteRequestInterval: 2 * time.Second, + SignerPrivateKey: "Hi", }, DisperserClient: disperserClient, } @@ -63,7 +63,7 @@ func TestTrafficGeneratorAuthenticated(t *testing.T) { Return(&processing, []byte{1}, nil) ctx, cancel := context.WithCancel(context.Background()) go func() { - _ = trafficGenerator.StartTraffic(ctx) + _ = trafficGenerator.StartWriteWorker(ctx) }() time.Sleep(5 * time.Second) cancel() From fff7c76a0bd30f042c4fcf7044903eca99cd248d Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 18 Jul 2024 14:13:55 -0500 Subject: [PATCH 03/74] Incremental progress. Signed-off-by: Cody Littley --- tools/traffic/blob_metadata.go | 54 ++++++++++++++ tools/traffic/blob_table.go | 91 +++++++++++++++++++++++ tools/traffic/config.go | 2 +- tools/traffic/generator.go | 15 +++- tools/traffic/status_verifier.go | 119 +++++++++++++++++++++++++++++++ 5 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 tools/traffic/blob_metadata.go create mode 100644 tools/traffic/blob_table.go create mode 100644 tools/traffic/status_verifier.go diff --git a/tools/traffic/blob_metadata.go b/tools/traffic/blob_metadata.go new file mode 100644 index 0000000000..10153b8395 --- /dev/null +++ b/tools/traffic/blob_metadata.go @@ -0,0 +1,54 @@ +package traffic + +// BlobMetadata encapsulates various information about a blob written by the traffic generator. +type BlobMetadata struct { + // key of the blob, set when the blob is initially uploaded. + key *[]byte + + // batchHeaderHash of the blob. + batchHeaderHash *[]byte + + // blobIndex of the blob. + blobIndex uint32 + + // remainingReadPermits describes the maximum number of remaining reads permitted against this blob. + // If -1 then an unlimited number of reads are permitted. + remainingReadPermits int32 + + // index describes the position of this blob within the blobTable. + index uint32 +} + +// NewBlobMetadata creates a new BlobMetadata instance. The readPermits parameter describes the maximum number of +// remaining reads permitted against this blob. If -1 then an unlimited number of reads are permitted. +func NewBlobMetadata( + key *[]byte, + batchHeaderHash *[]byte, + blobIndex uint32, + readPermits int32) *BlobMetadata { + + return &BlobMetadata{ + key: key, + batchHeaderHash: batchHeaderHash, + blobIndex: blobIndex, + remainingReadPermits: readPermits, + index: 0, + } +} + +// Key returns the key of the blob. +func (blob *BlobMetadata) Key() *[]byte { + return blob.key +} + +// BatchHeaderHash returns the batchHeaderHash of the blob. +func (blob *BlobMetadata) BatchHeaderHash() *[]byte { + return blob.batchHeaderHash +} + +// BlobIndex returns the blobIndex of the blob. +func (blob *BlobMetadata) BlobIndex() uint32 { + return blob.blobIndex +} + +// TODO method for decrementing read permits diff --git a/tools/traffic/blob_table.go b/tools/traffic/blob_table.go new file mode 100644 index 0000000000..d5ef8e37a4 --- /dev/null +++ b/tools/traffic/blob_table.go @@ -0,0 +1,91 @@ +package traffic + +import ( + "fmt" + "math/rand" + "sync" +) + +// BlobTable tracks blobs written by the traffic generator. This is a thread safe data structure. +type BlobTable struct { + + // blobs contains all blobs currently tracked by the table. + blobs []*BlobMetadata + + // size describes the total number of blobs currently tracked by the table. + // size may be smaller than the capacity of the blobs slice. + size uint32 + + // lock is used to synchronize access to the table. + lock sync.Mutex +} + +// NewBlobTable creates a new BlobTable instance. +func NewBlobTable() BlobTable { + return BlobTable{ + blobs: make([]*BlobMetadata, 1024), + size: 0, + } +} + +// Size returns the total number of blobs currently tracked by the table. +func (table *BlobTable) Size() uint32 { + table.lock.Lock() + defer table.lock.Unlock() + + return table.size +} + +// Add a blob to the table. +func (table *BlobTable) Add(blob *BlobMetadata) { + table.lock.Lock() + defer table.lock.Unlock() + + if table.size == uint32(len(table.blobs)) { + panic(fmt.Sprintf("blob table is full, cannot add blob %x", blob.Key)) + } + + blob.index = table.size + table.blobs[table.size] = blob + table.size++ +} + +// GetRandom returns a random blob currently tracked by the table. Returns nil if the table is empty. +// Optionally decrements the read permits of the blob if decrement is true. If the number of read permits +// reaches 0, the blob is removed from the table. +func (table *BlobTable) GetRandom(decrement bool) *BlobMetadata { + table.lock.Lock() + defer table.lock.Unlock() + + if table.size == 0 { + return nil + } + + blob := table.blobs[rand.Int31n(int32(table.size))] // TODO make sure we can get items if we overflow an int32 + + if decrement && blob.remainingReadPermits != -1 { + blob.remainingReadPermits-- + if blob.remainingReadPermits == 0 { + table.remove(blob) + } + } + + return blob +} + +// remove a blob from the table. +func (table *BlobTable) remove(blob *BlobMetadata) { + if table.blobs[blob.index] != blob { + panic(fmt.Sprintf("blob %x is not not present in the table at index %d", blob.Key, blob.index)) + } + + if table.size == 1 { + table.blobs[0] = nil + } else { + // Move the last blob to the position of the blob being removed. + table.blobs[blob.index] = table.blobs[table.size-1] + table.blobs[blob.index].index = blob.index + table.blobs[table.size-1] = nil + } + table.size-- +} diff --git a/tools/traffic/config.go b/tools/traffic/config.go index 215951a10b..66bb754249 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -30,7 +30,7 @@ type Config struct { NumWriteInstances uint // The period of the submission rate of new blobs for each write worker thread. WriteRequestInterval time.Duration - // The size of each blob dispersed, in bytes. + // The Size of each blob dispersed, in bytes. DataSize uint64 // If true, then each blob will contain unique random data. If false, the same random data // will be dispersed for each blob by a particular worker thread. diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 67a621cc18..780899f5ac 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/hex" + "fmt" "os" "os/signal" "sync" @@ -87,6 +88,11 @@ func (g *TrafficGenerator) StartWriteWorker(ctx context.Context) error { return err } + // TODO configuration for this stuff + var table BlobTable = NewBlobTable() + var verifier StatusVerifier = NewStatusVerifier(&table, &g.DisperserClient, -1) + verifier.Start(ctx, time.Second) + paddedData := codec.ConvertByPaddingEmptyByte(data) ticker := time.NewTicker(g.Config.WriteRequestInterval) @@ -95,6 +101,7 @@ func (g *TrafficGenerator) StartWriteWorker(ctx context.Context) error { case <-ctx.Done(): return nil case <-ticker.C: + var key []byte if g.Config.RandomizeBlobs { _, err := rand.Read(data) if err != nil { @@ -102,18 +109,22 @@ func (g *TrafficGenerator) StartWriteWorker(ctx context.Context) error { } paddedData = codec.ConvertByPaddingEmptyByte(data) - _, err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) + key, err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) + if err != nil { g.Logger.Error("failed to send blob request", "err:", err) } paddedData = nil } else { - _, err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) + key, err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) if err != nil { g.Logger.Error("failed to send blob request", "err:", err) } } + fmt.Println("passing key to verifier") // TODO remove + verifier.AddUnconfirmedKey(&key) + fmt.Println("done passing key") // TODO remove } } } diff --git a/tools/traffic/status_verifier.go b/tools/traffic/status_verifier.go new file mode 100644 index 0000000000..9ce9ba424f --- /dev/null +++ b/tools/traffic/status_verifier.go @@ -0,0 +1,119 @@ +package traffic + +import ( + "context" + "fmt" + "github.com/Layr-Labs/eigenda/api/clients" + "github.com/Layr-Labs/eigenda/api/grpc/disperser" + "time" +) + +// StatusVerifier periodically polls the disperser service to verify the status of blobs that were recently written. +// When blobs become confirmed, the status verifier updates the blob table accordingly. +// This is a thread safe data structure. +type StatusVerifier struct { + + // A table of confirmed blobs. Blobs are added here when they are confirmed by the disperser service. + table *BlobTable + + // The maximum number of reads permitted against an individual blob, or -1 if unlimited. + blobReadLimit int32 + + // The disperser client used to monitor the disperser service. + dispenser *clients.DisperserClient + + // The keys of blobs that have not yet been confirmed by the disperser service. + unconfirmedKeys []*[]byte + + // Newly added keys that require verification. + keyChannel chan *[]byte +} + +// NewStatusVerifier creates a new StatusVerifier instance. +func NewStatusVerifier( + table *BlobTable, + disperser *clients.DisperserClient, + blobReadLimit int32) StatusVerifier { + + return StatusVerifier{ + table: table, + blobReadLimit: blobReadLimit, + dispenser: disperser, + unconfirmedKeys: make([]*[]byte, 0), + keyChannel: make(chan *[]byte), + } +} + +// AddUnconfirmedKey adds a key to the list of unconfirmed keys. +func (verifier *StatusVerifier) AddUnconfirmedKey(key *[]byte) { + fmt.Println("Adding unconfirmed key") // TODO remove + verifier.keyChannel <- key + fmt.Println("Finished adding unconfirmed key") // TODO remove +} + +// Start begins the status goroutine, which periodically polls +// the disperser service to verify the status of blobs. +func (verifier *StatusVerifier) Start(ctx context.Context, period time.Duration) { + go verifier.monitor(ctx, period) +} + +// monitor periodically polls the disperser service to verify the status of blobs. +func (verifier *StatusVerifier) monitor(ctx context.Context, period time.Duration) { + fmt.Println("::: Starting status verifier :::") // TODO remove + + ticker := time.NewTicker(period) + for { + select { + case <-ctx.Done(): + return + case key := <-verifier.keyChannel: + fmt.Println("Got unconfirmed key") // TODO remove + verifier.unconfirmedKeys = append(verifier.unconfirmedKeys, key) + case <-ticker.C: + fmt.Println("polling") // TODO remove + verifier.poll(ctx) + fmt.Println("done polling") // TODO remove + } + } +} + +// poll checks all unconfirmed keys to see if they have been confirmed by the disperser service. +// If a key is confirmed, it is added to the blob table and removed from the list of unconfirmed keys. +func (verifier *StatusVerifier) poll(ctx context.Context) { + unconfirmedKeys := make([]*[]byte, 0) + for _, key := range verifier.unconfirmedKeys { + confirmed := verifier.checkStatusForBlob(ctx, key) + if !confirmed { + unconfirmedKeys = append(unconfirmedKeys, key) + } + } + verifier.unconfirmedKeys = unconfirmedKeys +} + +// checkStatusForBlob checks the status of a blob. Returns true if the blob is confirmed, false otherwise. +func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]byte) bool { + status, err := (*verifier.dispenser).GetBlobStatus(ctx, *key) + + if err != nil { + fmt.Println("Error getting blob status:", err) // TODO is this proper? + return false + } + + // TODO other statuses? + if status.GetStatus() == disperser.BlobStatus_CONFIRMED { + + fmt.Println(">>>>>>>>>>>>>>>>>>>>>> Confirmed key", key) // TODO remove + + batchHeaderHash := status.GetInfo().BlobVerificationProof.BatchMetadata.BatchHeaderHash + blobIndex := status.GetInfo().BlobVerificationProof.GetBlobIndex() + + blobMetadata := NewBlobMetadata(key, &batchHeaderHash, blobIndex, -1) // TODO permits + verifier.table.Add(blobMetadata) + + return true + } else { + fmt.Println("-------------- key not yet confirmed") // TODO remove + } + + return false +} From 3089bb0ac72ef919fe911eb22d9f2c9ac8b57faf Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 18 Jul 2024 15:41:06 -0500 Subject: [PATCH 04/74] Refactor blob writer. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 1 + tools/traffic/blob_writer.go | 124 ++++++++++++++++++++++++++++++++ tools/traffic/generator.go | 90 +++-------------------- tools/traffic/generator_test.go | 4 +- 4 files changed, 135 insertions(+), 84 deletions(-) create mode 100644 tools/traffic/blob_reader.go create mode 100644 tools/traffic/blob_writer.go diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go new file mode 100644 index 0000000000..afb546e827 --- /dev/null +++ b/tools/traffic/blob_reader.go @@ -0,0 +1 @@ +package traffic diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go new file mode 100644 index 0000000000..a35a177e04 --- /dev/null +++ b/tools/traffic/blob_writer.go @@ -0,0 +1,124 @@ +package traffic + +import ( + "context" + "crypto/rand" + "encoding/hex" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "sync" + "time" +) + +// TODO document + +// BlobWriter sends blobs to a disperser at a configured rate. +type BlobWriter struct { + ctx *context.Context + waitGroup *sync.WaitGroup + generator *TrafficGenerator + verifier *StatusVerifier + + // fixedRandomData contains random data for blobs if RandomizeBlobs is false, and nil otherwise. + fixedRandomData *[]byte +} + +// NewBlobWriter creates a new BlobWriter instance. +func NewBlobWriter( + ctx *context.Context, + waitGroup *sync.WaitGroup, + generator *TrafficGenerator, + verifier *StatusVerifier) BlobWriter { + + var fixedRandomData []byte + if generator.Config.RandomizeBlobs { + // New random data will be generated for each blob. + fixedRandomData = nil + } else { + // Use this random data for each blob. + fixedRandomData := make([]byte, generator.Config.DataSize) + _, err := rand.Read(fixedRandomData) + if err != nil { + panic(err) + } + fixedRandomData = codec.ConvertByPaddingEmptyByte(fixedRandomData) + } + + return BlobWriter{ + ctx: ctx, + waitGroup: waitGroup, + generator: generator, + verifier: verifier, + fixedRandomData: &fixedRandomData, + } +} + +// Start begins the blob writer goroutine. +func (writer *BlobWriter) Start() { + writer.waitGroup.Add(1) + go func() { + writer.run() + writer.waitGroup.Done() + }() +} + +// run sends blobs to a disperser at a configured rate. +// Continues and dues not return until the context is cancelled. +func (writer *BlobWriter) run() { + ticker := time.NewTicker(writer.generator.Config.WriteRequestInterval) + for { + select { + case <-(*writer.ctx).Done(): + return + case <-ticker.C: + key, err := writer.sendRequest(*writer.getRandomData()) + + if err != nil { + writer.generator.Logger.Error("failed to send blob request", "err:", err) + continue + } + + writer.verifier.AddUnconfirmedKey(&key) + } + } +} + +// getRandomData returns a slice of random data to be used for a blob. +func (writer *BlobWriter) getRandomData() *[]byte { + if *writer.fixedRandomData != nil { + return writer.fixedRandomData + } + + data := make([]byte, writer.generator.Config.DataSize) + _, err := rand.Read(data) + if err != nil { + panic(err) + } + data = codec.ConvertByPaddingEmptyByte(data) + + return &data +} + +// sendRequest sends a blob to a disperser. +func (writer *BlobWriter) sendRequest(data []byte) ([]byte /* key */, error) { + ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.generator.Config.Timeout) + defer cancel() + + if writer.generator.Config.SignerPrivateKey != "" { + blobStatus, key, err := + writer.generator.DisperserClient.DisperseBlobAuthenticated(ctxTimeout, data, writer.generator.Config.CustomQuorums) + if err != nil { + return nil, err + } + + writer.generator.Logger.Info("successfully dispersed new blob", "authenticated", true, "key", hex.EncodeToString(key), "status", blobStatus.String()) + return key, nil + } else { + blobStatus, key, err := writer.generator.DisperserClient.DisperseBlob(ctxTimeout, data, writer.generator.Config.CustomQuorums) + if err != nil { + return nil, err + } + + writer.generator.Logger.Info("successfully dispersed new blob", "authenticated", false, "key", hex.EncodeToString(key), "status", blobStatus.String()) + return key, nil + } +} diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 780899f5ac..88779ed63b 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -2,9 +2,6 @@ package traffic import ( "context" - "crypto/rand" - "encoding/hex" - "fmt" "os" "os/signal" "sync" @@ -14,7 +11,6 @@ import ( "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/core" - "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/Layr-Labs/eigensdk-go/logging" ) @@ -41,13 +37,16 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi // Run instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. func (g *TrafficGenerator) Run() error { ctx, cancel := context.WithCancel(context.Background()) + + // TODO add configuration + table := NewBlobTable() + verifier := NewStatusVerifier(&table, &g.DisperserClient, -1) + verifier.Start(ctx, time.Second) + var wg sync.WaitGroup for i := 0; i < int(g.Config.NumWriteInstances); i++ { - wg.Add(1) - go func() { - defer wg.Done() - _ = g.StartWriteWorker(ctx) - }() + writer := NewBlobWriter(&ctx, &wg, g, &verifier) + writer.Start() time.Sleep(g.Config.InstanceLaunchInterval) } signals := make(chan os.Signal, 1) @@ -79,76 +78,3 @@ func (g *TrafficGenerator) StartReadWorker(ctx context.Context) error { func (g *TrafficGenerator) readRequest() { // TODO } - -// StartWriteWorker periodically sends (possibly) random blobs to a disperser at a configured rate. -func (g *TrafficGenerator) StartWriteWorker(ctx context.Context) error { - data := make([]byte, g.Config.DataSize) - _, err := rand.Read(data) - if err != nil { - return err - } - - // TODO configuration for this stuff - var table BlobTable = NewBlobTable() - var verifier StatusVerifier = NewStatusVerifier(&table, &g.DisperserClient, -1) - verifier.Start(ctx, time.Second) - - paddedData := codec.ConvertByPaddingEmptyByte(data) - - ticker := time.NewTicker(g.Config.WriteRequestInterval) - for { - select { - case <-ctx.Done(): - return nil - case <-ticker.C: - var key []byte - if g.Config.RandomizeBlobs { - _, err := rand.Read(data) - if err != nil { - return err - } - paddedData = codec.ConvertByPaddingEmptyByte(data) - - key, err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) - - if err != nil { - g.Logger.Error("failed to send blob request", "err:", err) - } - paddedData = nil - } else { - key, err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) - if err != nil { - g.Logger.Error("failed to send blob request", "err:", err) - } - } - - fmt.Println("passing key to verifier") // TODO remove - verifier.AddUnconfirmedKey(&key) - fmt.Println("done passing key") // TODO remove - } - } -} - -// sendRequest sends a blob to a disperser. -func (g *TrafficGenerator) sendRequest(ctx context.Context, data []byte) ([]byte /* key */, error) { - ctxTimeout, cancel := context.WithTimeout(ctx, g.Config.Timeout) - defer cancel() - - if g.Config.SignerPrivateKey != "" { - blobStatus, key, err := g.DisperserClient.DisperseBlobAuthenticated(ctxTimeout, data, g.Config.CustomQuorums) - if err != nil { - return nil, err - } - - g.Logger.Info("successfully dispersed new blob", "authenticated", true, "key", hex.EncodeToString(key), "status", blobStatus.String()) - return key, nil - } else { - blobStatus, key, err := g.DisperserClient.DisperseBlob(ctxTimeout, data, g.Config.CustomQuorums) - if err != nil { - return nil, err - } - - g.Logger.Info("successfully dispersed new blob", "authenticated", false, "key", hex.EncodeToString(key), "status", blobStatus.String()) - return key, nil - } -} diff --git a/tools/traffic/generator_test.go b/tools/traffic/generator_test.go index b2ef6abb1a..05983e9b30 100644 --- a/tools/traffic/generator_test.go +++ b/tools/traffic/generator_test.go @@ -34,7 +34,7 @@ func TestTrafficGenerator(t *testing.T) { Return(&processing, []byte{1}, nil) ctx, cancel := context.WithCancel(context.Background()) go func() { - _ = trafficGenerator.StartWriteWorker(ctx) + _ = trafficGenerator.StartBlobWriter(ctx) }() time.Sleep(5 * time.Second) cancel() @@ -63,7 +63,7 @@ func TestTrafficGeneratorAuthenticated(t *testing.T) { Return(&processing, []byte{1}, nil) ctx, cancel := context.WithCancel(context.Background()) go func() { - _ = trafficGenerator.StartWriteWorker(ctx) + _ = trafficGenerator.StartBlobWriter(ctx) }() time.Sleep(5 * time.Second) cancel() From 463f88f4704f914f7b8400ccf7aa36711853c10c Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 19 Jul 2024 08:11:09 -0500 Subject: [PATCH 05/74] Add ability to read from disperser. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 80 ++++++++++++++++++++++++++++++++ tools/traffic/blob_writer.go | 11 ++++- tools/traffic/generator.go | 43 ++++++++--------- tools/traffic/status_verifier.go | 9 +--- 4 files changed, 112 insertions(+), 31 deletions(-) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index afb546e827..0be931a293 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -1 +1,81 @@ package traffic + +import ( + "context" + "fmt" + "sync" + "time" +) + +// BlobReader reads blobs from a disperser at a configured rate. +type BlobReader struct { + // The context for the generator. All work should cease when this context is cancelled. + ctx *context.Context + + // Tracks the number of active goroutines within the generator. + waitGroup *sync.WaitGroup + + // TODO this type should be refactored maybe + generator *TrafficGenerator + + // table of blobs to read from. + table *BlobTable +} + +// NewBlobReader creates a new BlobReader instance. +func NewBlobReader( + ctx *context.Context, + waitGroup *sync.WaitGroup, + generator *TrafficGenerator, + table *BlobTable) BlobReader { + + return BlobReader{ + ctx: ctx, + waitGroup: waitGroup, + generator: generator, + table: table, + } +} + +// Start begins a blob reader goroutine. +func (reader *BlobReader) Start() { + reader.waitGroup.Add(1) + go func() { + defer reader.waitGroup.Done() + reader.run() + }() +} + +// run periodically performs reads on blobs. +func (reader *BlobReader) run() { + ticker := time.NewTicker(time.Second) // TODO setting + for { + select { + case <-(*reader.ctx).Done(): + return + case <-ticker.C: + reader.randomRead() + } + } +} + +// randomRead reads a random blob. +func (reader *BlobReader) randomRead() { + + metadata := reader.table.GetRandom(true) + if metadata == nil { + // There are no blobs to read, do nothing. + return + } + + // TODO convert this to a proper read + data, err := reader.generator.DisperserClient.RetrieveBlob(*reader.ctx, *metadata.batchHeaderHash, metadata.blobIndex) + if err != nil { + fmt.Println("Error reading blob:", err) // TODO + return + } + + // TODO it would be nice to do some verification, perhaps just of the hash of the blob + + fmt.Println("Read blob:", data) // TODO +} diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index a35a177e04..ab2e0d8429 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -13,10 +13,17 @@ import ( // BlobWriter sends blobs to a disperser at a configured rate. type BlobWriter struct { - ctx *context.Context + // The context for the generator. All work should cease when this context is cancelled. + ctx *context.Context + + // Tracks the number of active goroutines within the generator. waitGroup *sync.WaitGroup + + // TODO this type should be refactored maybe generator *TrafficGenerator - verifier *StatusVerifier + + // Responsible for polling on the status of a recently written blob until it becomes confirmed. + verifier *StatusVerifier // fixedRandomData contains random data for blobs if RandomizeBlobs is false, and nil otherwise. fixedRandomData *[]byte diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 88779ed63b..85b073d330 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -14,6 +14,22 @@ import ( "github.com/Layr-Labs/eigensdk-go/logging" ) +// TrafficGenerator simulates read/write traffic to the DA service. +// +// ┌------------┐ ┌------------┐ +// | writer |-┐ ┌------------┐ | reader |-┐ +// └------------┘ |-┐ -------> | verifier | -------> └------------┘ |-┐ +// └------------┘ | └------------┘ └------------┘ | +// └------------┘ └------------┘ +// +// The traffic generator is built from three principal components: one or more writers +// that write blobs, a verifier that polls the dispenser service until blobs are confirmed, +// and one or more readers that read blobs. +// +// When a writer finishes writing a blob, it +// sends information about that blob to the verifier. When the verifier observes that a blob +// has been confirmed, it sends information about the blob to the readers. The readers +// only attempt to read blobs that have been confirmed by the verifier. type TrafficGenerator struct { Logger logging.Logger DisperserClient clients.DisperserClient @@ -44,11 +60,17 @@ func (g *TrafficGenerator) Run() error { verifier.Start(ctx, time.Second) var wg sync.WaitGroup + for i := 0; i < int(g.Config.NumWriteInstances); i++ { writer := NewBlobWriter(&ctx, &wg, g, &verifier) writer.Start() time.Sleep(g.Config.InstanceLaunchInterval) } + + // TODO start multiple readers + reader := NewBlobReader(&ctx, &wg, g, &table) + reader.Start() + signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt, syscall.SIGTERM) <-signals @@ -57,24 +79,3 @@ func (g *TrafficGenerator) Run() error { wg.Wait() return nil } - -// TODO maybe split reader/writer into separate files - -// StartReadWorker periodically requests to download random blobs at a configured rate. -func (g *TrafficGenerator) StartReadWorker(ctx context.Context) error { - ticker := time.NewTicker(g.Config.WriteRequestInterval) - for { - select { - case <-ctx.Done(): - return nil - case <-ticker.C: - // TODO determine which blob to download - g.readRequest() // TODO add parameters - } - } -} - -// readRequest reads a blob. -func (g *TrafficGenerator) readRequest() { - // TODO -} diff --git a/tools/traffic/status_verifier.go b/tools/traffic/status_verifier.go index 9ce9ba424f..aa47095498 100644 --- a/tools/traffic/status_verifier.go +++ b/tools/traffic/status_verifier.go @@ -46,9 +46,7 @@ func NewStatusVerifier( // AddUnconfirmedKey adds a key to the list of unconfirmed keys. func (verifier *StatusVerifier) AddUnconfirmedKey(key *[]byte) { - fmt.Println("Adding unconfirmed key") // TODO remove verifier.keyChannel <- key - fmt.Println("Finished adding unconfirmed key") // TODO remove } // Start begins the status goroutine, which periodically polls @@ -59,20 +57,15 @@ func (verifier *StatusVerifier) Start(ctx context.Context, period time.Duration) // monitor periodically polls the disperser service to verify the status of blobs. func (verifier *StatusVerifier) monitor(ctx context.Context, period time.Duration) { - fmt.Println("::: Starting status verifier :::") // TODO remove - ticker := time.NewTicker(period) for { select { case <-ctx.Done(): return case key := <-verifier.keyChannel: - fmt.Println("Got unconfirmed key") // TODO remove verifier.unconfirmedKeys = append(verifier.unconfirmedKeys, key) case <-ticker.C: - fmt.Println("polling") // TODO remove verifier.poll(ctx) - fmt.Println("done polling") // TODO remove } } } @@ -99,7 +92,7 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]b return false } - // TODO other statuses? + // TODO other statuses if status.GetStatus() == disperser.BlobStatus_CONFIRMED { fmt.Println(">>>>>>>>>>>>>>>>>>>>>> Confirmed key", key) // TODO remove From 8a1b37be5d5a9a2cd3733dce6889cfa912c5a866 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 19 Jul 2024 08:16:40 -0500 Subject: [PATCH 06/74] Fix formatting. Signed-off-by: Cody Littley --- tools/traffic/generator.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 85b073d330..aee55d128a 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -16,11 +16,11 @@ import ( // TrafficGenerator simulates read/write traffic to the DA service. // -// ┌------------┐ ┌------------┐ -// | writer |-┐ ┌------------┐ | reader |-┐ -// └------------┘ |-┐ -------> | verifier | -------> └------------┘ |-┐ -// └------------┘ | └------------┘ └------------┘ | -// └------------┘ └------------┘ +// ┌------------┐ ┌------------┐ +// | writer |-┐ ┌------------┐ | reader |-┐ +// └------------┘ |-┐ -------> | verifier | -------> └------------┘ |-┐ +// └------------┘ | └------------┘ └------------┘ | +// └------------┘ └------------┘ // // The traffic generator is built from three principal components: one or more writers // that write blobs, a verifier that polls the dispenser service until blobs are confirmed, From 86eb80aab9cfbf8c00d292eb72874603ac153a52 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 19 Jul 2024 10:05:36 -0500 Subject: [PATCH 07/74] Incremental progress. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 1 + tools/traffic/blob_reader.go | 16 +++++++++++++--- tools/traffic/blob_writer.go | 17 +++++++++++++++++ tools/traffic/cmd/main.go | 2 +- tools/traffic/generator.go | 31 ++++++++++++++++++++++++++----- 5 files changed, 58 insertions(+), 9 deletions(-) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 3fa0618d83..35b13bc6af 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -14,6 +14,7 @@ run: build TRAFFIC_GENERATOR_WRITE_REQUEST_INTERVAL=1s \ TRAFFIC_GENERATOR_DATA_SIZE=1000 \ TRAFFIC_GENERATOR_RANDOMIZE_BLOBS=true \ + TRAFFIC_GENERATOR_SIGNER_PRIVATE_KEY_HEX=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \ ./bin/server # TODO do not merge change TRAFFIC_GENERATOR_GRPC_PORT=32003 \ No newline at end of file diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 0be931a293..ad5051bd72 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -68,14 +68,24 @@ func (reader *BlobReader) randomRead() { return } - // TODO convert this to a proper read - data, err := reader.generator.DisperserClient.RetrieveBlob(*reader.ctx, *metadata.batchHeaderHash, metadata.blobIndex) + // TODO + //data, err := reader.generator.DisperserClient.RetrieveBlob(*reader.ctx, *metadata.batchHeaderHash, metadata.blobIndex) + //if err != nil { + // fmt.Println("Error reading blob:", err) // TODO + // return + //} + + fmt.Println("attempting to read blob") + fmt.Println("batch header hash:", *metadata.batchHeaderHash) + fmt.Println("blob index:", metadata.blobIndex) + fmt.Println("client: ", reader.generator.EigenDAClient) + data, err := reader.generator.EigenDAClient.GetBlob(*reader.ctx, *metadata.batchHeaderHash, metadata.blobIndex) if err != nil { fmt.Println("Error reading blob:", err) // TODO return } - // TODO it would be nice to do some verification, perhaps just of the hash of the blob + // TODO verify blob data fmt.Println("Read blob:", data) // TODO } diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index ab2e0d8429..29e991da36 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -100,13 +100,30 @@ func (writer *BlobWriter) getRandomData() *[]byte { if err != nil { panic(err) } + // TODO: get explanation why this is necessary data = codec.ConvertByPaddingEmptyByte(data) + // TODO remove + //if !fft.IsPowerOfTwo(uint64(len(data))) { + // p := uint32(math.Log2(float64(len(data)))) + 1 + // newSize := 1 << p + // bytesToAdd := newSize - len(data) + // for i := 0; i < bytesToAdd; i++ { + // data = append(data, 0) + // } + //} + return &data } // sendRequest sends a blob to a disperser. func (writer *BlobWriter) sendRequest(data []byte) ([]byte /* key */, error) { + // TODO remove + //if !fft.IsPowerOfTwo(uint64(len(data))) { + // return nil, fmt.Errorf("data length must be a power of two, data size = %d", len(data)) + //} + + // TODO add timeout to other types of requests ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.generator.Config.Timeout) defer cancel() diff --git a/tools/traffic/cmd/main.go b/tools/traffic/cmd/main.go index 62a508629c..160b04e931 100644 --- a/tools/traffic/cmd/main.go +++ b/tools/traffic/cmd/main.go @@ -45,7 +45,7 @@ func trafficGeneratorMain(ctx *cli.Context) error { generator, err := traffic.NewTrafficGenerator(config, signer) if err != nil { - panic("failed to create new traffic generator") + panic(fmt.Sprintf("failed to create new traffic generator\n%s", err)) } return generator.Run() diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index aee55d128a..432f02f4d7 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -2,6 +2,8 @@ package traffic import ( "context" + "fmt" + "github.com/ethereum/go-ethereum/log" "os" "os/signal" "sync" @@ -17,7 +19,7 @@ import ( // TrafficGenerator simulates read/write traffic to the DA service. // // ┌------------┐ ┌------------┐ -// | writer |-┐ ┌------------┐ | reader |-┐ +// | writer |-┐ ┌------------┐ | reader |-┐ // └------------┘ |-┐ -------> | verifier | -------> └------------┘ |-┐ // └------------┘ | └------------┘ └------------┘ | // └------------┘ └------------┘ @@ -26,13 +28,13 @@ import ( // that write blobs, a verifier that polls the dispenser service until blobs are confirmed, // and one or more readers that read blobs. // -// When a writer finishes writing a blob, it -// sends information about that blob to the verifier. When the verifier observes that a blob -// has been confirmed, it sends information about the blob to the readers. The readers -// only attempt to read blobs that have been confirmed by the verifier. +// When a writer finishes writing a blob, it sends information about that blob to the verifier. +// When the verifier observes that a blob has been confirmed, it sends information about the blob +// to the readers. The readers only attempt to read blobs that have been confirmed by the verifier. type TrafficGenerator struct { Logger logging.Logger DisperserClient clients.DisperserClient + EigenDAClient *clients.EigenDAClient Config *Config } @@ -43,9 +45,28 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi return nil, err } + fmt.Printf("Signer private key: '%s', length: %d\n", config.SignerPrivateKey, len(config.SignerPrivateKey)) + + clientConfig := clients.EigenDAClientConfig{ + RPC: "localhost:32003", // TODO make this configurable + DisableTLS: true, // TODO config + SignerPrivateKeyHex: config.SignerPrivateKey, + } + err = clientConfig.CheckAndSetDefaults() + if err != nil { + return nil, err + } + + logger2 := log.NewLogger(log.NewTerminalHandler(os.Stderr, true)) // TODO + client, err := clients.NewEigenDAClient(logger2, clientConfig) + if err != nil { + return nil, err + } + return &TrafficGenerator{ Logger: logger, DisperserClient: clients.NewDisperserClient(&config.Config, signer), + EigenDAClient: client, Config: config, }, nil } From f2591d6bafbf538ed4647311cf83f311322e9b25 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Mon, 22 Jul 2024 09:20:32 -0500 Subject: [PATCH 08/74] Incremental progress. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 5 ++--- tools/traffic/blob_writer.go | 16 +++++++++------ tools/traffic/status_verifier.go | 34 ++++++++++++++++++++++++-------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index ad5051bd72..7b3372a358 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -75,10 +75,9 @@ func (reader *BlobReader) randomRead() { // return //} + // TODO use NodeClient + fmt.Println("attempting to read blob") - fmt.Println("batch header hash:", *metadata.batchHeaderHash) - fmt.Println("blob index:", metadata.blobIndex) - fmt.Println("client: ", reader.generator.EigenDAClient) data, err := reader.generator.EigenDAClient.GetBlob(*reader.ctx, *metadata.batchHeaderHash, metadata.blobIndex) if err != nil { fmt.Println("Error reading blob:", err) // TODO diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 29e991da36..8fdba18345 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -3,7 +3,7 @@ package traffic import ( "context" "crypto/rand" - "encoding/hex" + "fmt" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "sync" "time" @@ -77,12 +77,14 @@ func (writer *BlobWriter) run() { case <-(*writer.ctx).Done(): return case <-ticker.C: - key, err := writer.sendRequest(*writer.getRandomData()) + data := writer.getRandomData() + key, err := writer.sendRequest(*data) if err != nil { writer.generator.Logger.Error("failed to send blob request", "err:", err) continue } + fmt.Println("Sent blob with length", len(*data)) // TODO remove writer.verifier.AddUnconfirmedKey(&key) } @@ -128,21 +130,23 @@ func (writer *BlobWriter) sendRequest(data []byte) ([]byte /* key */, error) { defer cancel() if writer.generator.Config.SignerPrivateKey != "" { - blobStatus, key, err := + _, key, err := writer.generator.DisperserClient.DisperseBlobAuthenticated(ctxTimeout, data, writer.generator.Config.CustomQuorums) if err != nil { return nil, err } - writer.generator.Logger.Info("successfully dispersed new blob", "authenticated", true, "key", hex.EncodeToString(key), "status", blobStatus.String()) + // TODO + //writer.generator.Logger.Info("successfully dispersed new blob", "authenticated", true, "key", hex.EncodeToString(key), "status", blobStatus.String()) return key, nil } else { - blobStatus, key, err := writer.generator.DisperserClient.DisperseBlob(ctxTimeout, data, writer.generator.Config.CustomQuorums) + _, key, err := writer.generator.DisperserClient.DisperseBlob(ctxTimeout, data, writer.generator.Config.CustomQuorums) if err != nil { return nil, err } - writer.generator.Logger.Info("successfully dispersed new blob", "authenticated", false, "key", hex.EncodeToString(key), "status", blobStatus.String()) + // TODO + //writer.generator.Logger.Info("successfully dispersed new blob", "authenticated", false, "key", hex.EncodeToString(key), "status", blobStatus.String()) return key, nil } } diff --git a/tools/traffic/status_verifier.go b/tools/traffic/status_verifier.go index aa47095498..d99dfbafe5 100644 --- a/tools/traffic/status_verifier.go +++ b/tools/traffic/status_verifier.go @@ -73,6 +73,9 @@ func (verifier *StatusVerifier) monitor(ctx context.Context, period time.Duratio // poll checks all unconfirmed keys to see if they have been confirmed by the disperser service. // If a key is confirmed, it is added to the blob table and removed from the list of unconfirmed keys. func (verifier *StatusVerifier) poll(ctx context.Context) { + + // TODO If the number of unconfirmed blobs is high and the time to confirm his high, this is not efficient. + unconfirmedKeys := make([]*[]byte, 0) for _, key := range verifier.unconfirmedKeys { confirmed := verifier.checkStatusForBlob(ctx, key) @@ -83,7 +86,7 @@ func (verifier *StatusVerifier) poll(ctx context.Context) { verifier.unconfirmedKeys = unconfirmedKeys } -// checkStatusForBlob checks the status of a blob. Returns true if the blob is confirmed, false otherwise. +// checkStatusForBlob checks the status of a blob. Returns true if the final blob status is known, false otherwise. func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]byte) bool { status, err := (*verifier.dispenser).GetBlobStatus(ctx, *key) @@ -92,21 +95,36 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]b return false } - // TODO other statuses - if status.GetStatus() == disperser.BlobStatus_CONFIRMED { + switch status.GetStatus() { + + case disperser.BlobStatus_UNKNOWN: + fallthrough + case disperser.BlobStatus_PROCESSING: + fallthrough + case disperser.BlobStatus_DISPERSING: + // Final status is not yet known. Check it again later. + return false - fmt.Println(">>>>>>>>>>>>>>>>>>>>>> Confirmed key", key) // TODO remove + case disperser.BlobStatus_FAILED: + fallthrough + case disperser.BlobStatus_INSUFFICIENT_SIGNATURES: + fmt.Println("Blob dispersal failed:", status.GetStatus()) // TODO use logger + return true + case disperser.BlobStatus_CONFIRMED: + fallthrough + case disperser.BlobStatus_FINALIZED: batchHeaderHash := status.GetInfo().BlobVerificationProof.BatchMetadata.BatchHeaderHash blobIndex := status.GetInfo().BlobVerificationProof.GetBlobIndex() blobMetadata := NewBlobMetadata(key, &batchHeaderHash, blobIndex, -1) // TODO permits verifier.table.Add(blobMetadata) + fmt.Println("Confirmed blob") return true - } else { - fmt.Println("-------------- key not yet confirmed") // TODO remove - } - return false + default: + fmt.Println("Unknown blob status:", status.GetStatus()) // TODO use logger + return true + } } From a60dfc5d4646678ac42cdb76d7ebb8a60d597d4e Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 23 Jul 2024 08:17:45 -0500 Subject: [PATCH 09/74] Able to instiantiate retrieval client. Signed-off-by: Cody Littley --- api/clients/retrieval_client.go | 2 +- retriever/Makefile | 10 ++-- retriever/cmd/main.go | 3 ++ tools/traffic/blob_reader.go | 21 ++++++-- tools/traffic/generator.go | 91 +++++++++++++++++++++++++++++++- tools/traffic/status_verifier.go | 4 +- 6 files changed, 119 insertions(+), 12 deletions(-) diff --git a/api/clients/retrieval_client.go b/api/clients/retrieval_client.go index 70693787b2..f1225231c9 100644 --- a/api/clients/retrieval_client.go +++ b/api/clients/retrieval_client.go @@ -42,7 +42,7 @@ func NewRetrievalClient( nodeClient NodeClient, verifier encoding.Verifier, numConnections int, -) (*retrievalClient, error) { +) (RetrievalClient, error) { return &retrievalClient{ logger: logger.With("component", "RetrievalClient"), diff --git a/retriever/Makefile b/retriever/Makefile index f71e1e49d9..7219fe79b3 100644 --- a/retriever/Makefile +++ b/retriever/Makefile @@ -12,13 +12,15 @@ run: build DA_RETRIEVER_TIMEOUT=10s \ ./bin/server \ --retriever.hostname localhost \ - --retriever.grpc-port 32011 \ + --retriever.grpc-port 32099 \ --retriever.timeout 10s \ - --retriever.bls-operator-state-retriever 0x9d4454B023096f34B160D6B654540c56A1F81688 \ - --retriever.eigenda-service-manager 0x67d269191c92Caf3cD7723F116c85e6E9bf55933 \ + --retriever.bls-operator-state-retriever 0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154 \ + --retriever.eigenda-service-manager 0x851356ae760d987E095750cCeb3bC6014560891C \ --kzg.g1-path ../inabox/resources/kzg/g1.point \ --kzg.g2-path ../inabox/resources/kzg/g2.point \ --kzg.cache-path ../inabox/resources/kzg/SRSTables \ + --kzg.srs-load 3000 \ --kzg.srs-order 3000 \ --chain.rpc http://localhost:8545 \ - --chain.private-key="" \ No newline at end of file + --chain.private-key="" \ + --thegraph.endpoint localhost:8000 \ No newline at end of file diff --git a/retriever/cmd/main.go b/retriever/cmd/main.go index ab03ec6eff..aa58c3dc5f 100644 --- a/retriever/cmd/main.go +++ b/retriever/cmd/main.go @@ -72,6 +72,9 @@ func RetrieverMain(ctx *cli.Context) error { if err != nil { log.Fatalf("failed to parse the command line flags: %v", err) } + + fmt.Printf(">>>>>>>>>>> Configuration\n%+v\n", config) + logger, err := common.NewLogger(config.LoggerConfig) if err != nil { log.Fatalf("failed to create logger: %v", err) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 7b3372a358..6329875332 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -3,10 +3,13 @@ package traffic import ( "context" "fmt" + "github.com/Layr-Labs/eigenda/api/clients" "sync" "time" ) +// TODO for all of these new types, decide if variables need to be pointers or not + // BlobReader reads blobs from a disperser at a configured rate. type BlobReader struct { // The context for the generator. All work should cease when this context is cancelled. @@ -15,8 +18,8 @@ type BlobReader struct { // Tracks the number of active goroutines within the generator. waitGroup *sync.WaitGroup - // TODO this type should be refactored maybe - generator *TrafficGenerator + // TODO use code from this class + retriever clients.RetrievalClient // table of blobs to read from. table *BlobTable @@ -26,13 +29,13 @@ type BlobReader struct { func NewBlobReader( ctx *context.Context, waitGroup *sync.WaitGroup, - generator *TrafficGenerator, + retriever clients.RetrievalClient, table *BlobTable) BlobReader { return BlobReader{ ctx: ctx, waitGroup: waitGroup, - generator: generator, + retriever: retriever, table: table, } } @@ -78,7 +81,15 @@ func (reader *BlobReader) randomRead() { // TODO use NodeClient fmt.Println("attempting to read blob") - data, err := reader.generator.EigenDAClient.GetBlob(*reader.ctx, *metadata.batchHeaderHash, metadata.blobIndex) + // TODO arguments probably not right + data, err := reader.retriever.RetrieveBlob( + *reader.ctx, + [32]byte(*metadata.batchHeaderHash), + metadata.blobIndex, + 0, + [32]byte{}, + 0) + if err != nil { fmt.Println("Error reading blob:", err) // TODO return diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 432f02f4d7..dae8b644c0 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -3,7 +3,15 @@ package traffic import ( "context" "fmt" + "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/eth" + coreindexer "github.com/Layr-Labs/eigenda/core/indexer" + "github.com/Layr-Labs/eigenda/encoding/kzg" + "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" + "github.com/Layr-Labs/eigenda/indexer" + gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" "os" "os/signal" "sync" @@ -71,6 +79,87 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi }, nil } +// buildRetriever creates a retriever client for the traffic generator. +func (g *TrafficGenerator) buildRetriever() clients.RetrievalClient { + + loggerConfig := common.LoggerConfig{ + Format: "text", + } + logger, err := common.NewLogger(loggerConfig) + if err != nil { + panic(err) // TODO + } + + ethClientConfig := geth.EthClientConfig{ + RPCURLs: []string{"http://localhost:8545"}, + } + gethClient, err := geth.NewMultiHomingClient(ethClientConfig, gethcommon.Address{}, logger) + + tx, err := eth.NewTransactor( + logger, + gethClient, + "0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154", + "0x851356ae760d987E095750cCeb3bC6014560891C") + + cs := eth.NewChainState(tx, gethClient) + + rpcClient, err := rpc.Dial("http://localhost:8545") + + indexerConfig := indexer.Config{ + PullInterval: time.Second, + } + indexer, err := coreindexer.CreateNewIndexer( + &indexerConfig, + gethClient, + rpcClient, + "0x851356ae760d987E095750cCeb3bC6014560891C", + logger, + ) + if err != nil { + panic(err) // TODO + } + chainState, err := coreindexer.NewIndexedChainState(cs, indexer) + if err != nil { + panic(err) // TODO + } + + //chainState := thegraph.MakeIndexedChainState(chainStateConfig, cs, logger) + + var assignmentCoordinator core.AssignmentCoordinator = &core.StdAssignmentCoordinator{} + + nodeClient := clients.NewNodeClient(10 * time.Second) + + encoderConfig := kzg.KzgConfig{ + G1Path: "../../inabox/resources/kzg/g1.point", + G2Path: "../../inabox/resources/kzg/g2.point", + CacheDir: "../../inabox/resources/kzg/SRSTables", + SRSOrder: 3000, + SRSNumberToLoad: 3000, + NumWorker: 12, + } + v, err := verifier.NewVerifier(&encoderConfig, true) + if err != nil { + panic(err) // TODO + } + + numConnections := 20 + + //var retriever *clients.RetrievalClient + retriever, err := clients.NewRetrievalClient( + logger, + chainState, + assignmentCoordinator, + nodeClient, + v, + numConnections) + + if err != nil { + panic(err) // TODO + } + + return retriever +} + // Run instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. func (g *TrafficGenerator) Run() error { ctx, cancel := context.WithCancel(context.Background()) @@ -89,7 +178,7 @@ func (g *TrafficGenerator) Run() error { } // TODO start multiple readers - reader := NewBlobReader(&ctx, &wg, g, &table) + reader := NewBlobReader(&ctx, &wg, g.buildRetriever(), &table) reader.Start() signals := make(chan os.Signal, 1) diff --git a/tools/traffic/status_verifier.go b/tools/traffic/status_verifier.go index d99dfbafe5..88182fb97e 100644 --- a/tools/traffic/status_verifier.go +++ b/tools/traffic/status_verifier.go @@ -2,6 +2,7 @@ package traffic import ( "context" + "encoding/base64" "fmt" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/api/grpc/disperser" @@ -119,7 +120,8 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]b blobMetadata := NewBlobMetadata(key, &batchHeaderHash, blobIndex, -1) // TODO permits verifier.table.Add(blobMetadata) - fmt.Println("Confirmed blob") + fmt.Printf("Confirmed blob, batch header hash: %s, blobIndex %d\n", base64.StdEncoding.EncodeToString(batchHeaderHash), blobIndex) + //fmt.Println("Confirmed blob") return true From 9bf4333ce438135232e9a5b79b88a73e030564bd Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 23 Jul 2024 10:31:04 -0500 Subject: [PATCH 10/74] Checkin. Signed-off-by: Cody Littley --- api/clients/retrieval_client.go | 5 +++- core/indexer/state.go | 9 +++--- tools/traffic/blob_reader.go | 50 +++++++++++++++++++++------------ tools/traffic/blob_writer.go | 7 +++-- tools/traffic/generator.go | 30 +++++++++++++++----- 5 files changed, 68 insertions(+), 33 deletions(-) diff --git a/api/clients/retrieval_client.go b/api/clients/retrieval_client.go index f1225231c9..c164b58f5b 100644 --- a/api/clients/retrieval_client.go +++ b/api/clients/retrieval_client.go @@ -61,13 +61,16 @@ func (r *retrievalClient) RetrieveBlob( referenceBlockNumber uint, batchRoot [32]byte, quorumID core.QuorumID) ([]byte, error) { + + fmt.Printf(" getting indexed operator state, referenceBlockNumber: %d, quorumId: %d\n", referenceBlockNumber, quorumID) // TODO indexedOperatorState, err := r.indexedChainState.GetIndexedOperatorState(ctx, referenceBlockNumber, []core.QuorumID{quorumID}) if err != nil { return nil, err } + fmt.Println("getting operators") // TODO operators, ok := indexedOperatorState.Operators[quorumID] if !ok { - return nil, fmt.Errorf("no quorum with ID: %d", quorumID) + return nil, fmt.Errorf(" no quorum with ID: %d", quorumID) } // Get blob header from any operator diff --git a/core/indexer/state.go b/core/indexer/state.go index 28c1f823f6..2078167ae1 100644 --- a/core/indexer/state.go +++ b/core/indexer/state.go @@ -3,6 +3,7 @@ package indexer import ( "context" "errors" + "fmt" "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/indexer" @@ -35,12 +36,12 @@ func (ics *IndexedChainState) GetIndexedOperatorState(ctx context.Context, block pubkeys, sockets, err := ics.getObjects(blockNumber) if err != nil { - return nil, err + return nil, fmt.Errorf("unable to complete ics.getObjects(%d): %s)", blockNumber, err) } operatorState, err := ics.ChainState.GetOperatorState(ctx, blockNumber, quorums) if err != nil { - return nil, err + return nil, fmt.Errorf("unable to complete ics.ChainState.GetOperatorState(%d, %v): %s", blockNumber, quorums, err) } ops := make(map[core.OperatorID]*core.IndexedOperatorInfo, len(pubkeys.Operators)) @@ -92,7 +93,7 @@ func (ics *IndexedChainState) getObjects(blockNumber uint) (*OperatorPubKeys, Op obj, err := ics.Indexer.GetObject(queryHeader, 0) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("(1) unable to call Indexer.GetObject({Number = %d}, 0): %s", blockNumber, err) } pubkeys, ok := obj.(*OperatorPubKeys) @@ -102,7 +103,7 @@ func (ics *IndexedChainState) getObjects(blockNumber uint) (*OperatorPubKeys, Op obj, err = ics.Indexer.GetObject(queryHeader, 1) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("(2) unable to call Indexer.GetObject(%v, 1): %s", queryHeader, err) } sockets, ok := obj.(OperatorSockets) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 6329875332..c7740517ba 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -4,6 +4,10 @@ import ( "context" "fmt" "github.com/Layr-Labs/eigenda/api/clients" + "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/retriever/eth" + gcommon "github.com/ethereum/go-ethereum/common" + "math/big" "sync" "time" ) @@ -19,7 +23,8 @@ type BlobReader struct { waitGroup *sync.WaitGroup // TODO use code from this class - retriever clients.RetrievalClient + retriever clients.RetrievalClient + chainClient eth.ChainClient // table of blobs to read from. table *BlobTable @@ -30,13 +35,15 @@ func NewBlobReader( ctx *context.Context, waitGroup *sync.WaitGroup, retriever clients.RetrievalClient, + chainClient eth.ChainClient, table *BlobTable) BlobReader { return BlobReader{ - ctx: ctx, - waitGroup: waitGroup, - retriever: retriever, - table: table, + ctx: ctx, + waitGroup: waitGroup, + retriever: retriever, + chainClient: chainClient, + table: table, } } @@ -71,24 +78,31 @@ func (reader *BlobReader) randomRead() { return } - // TODO - //data, err := reader.generator.DisperserClient.RetrieveBlob(*reader.ctx, *metadata.batchHeaderHash, metadata.blobIndex) - //if err != nil { - // fmt.Println("Error reading blob:", err) // TODO - // return - //} - // TODO use NodeClient - fmt.Println("attempting to read blob") - // TODO arguments probably not right + batchHeader, err := reader.chainClient.FetchBatchHeader( + *reader.ctx, + gcommon.HexToAddress("0x851356ae760d987E095750cCeb3bC6014560891C"), + *metadata.batchHeaderHash, + big.NewInt(int64(0)), + nil) + if err != nil { + fmt.Println("Error fetching batch header:", err) // TODO + return + } + + // TODO is this correct? + var batchHeaderHash [32]byte + copy(batchHeaderHash[:], *metadata.batchHeaderHash) + + fmt.Printf("attempting to read blob, header hash: %x, index: %d\n, batch header: %+v\n", *metadata.batchHeaderHash, metadata.blobIndex, batchHeader) data, err := reader.retriever.RetrieveBlob( *reader.ctx, - [32]byte(*metadata.batchHeaderHash), + batchHeaderHash, metadata.blobIndex, - 0, - [32]byte{}, - 0) + uint(batchHeader.ReferenceBlockNumber), + batchHeader.BlobHeadersRoot, + core.QuorumID(0)) if err != nil { fmt.Println("Error reading blob:", err) // TODO diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 8fdba18345..1ac89ff207 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -93,9 +93,10 @@ func (writer *BlobWriter) run() { // getRandomData returns a slice of random data to be used for a blob. func (writer *BlobWriter) getRandomData() *[]byte { - if *writer.fixedRandomData != nil { - return writer.fixedRandomData - } + // TODO + //if *writer.fixedRandomData != nil { + // return writer.fixedRandomData + //} data := make([]byte, writer.generator.Config.DataSize) _, err := rand.Read(data) diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index dae8b644c0..9641d1639e 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -9,6 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" "github.com/Layr-Labs/eigenda/indexer" + retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" @@ -80,7 +81,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi } // buildRetriever creates a retriever client for the traffic generator. -func (g *TrafficGenerator) buildRetriever() clients.RetrievalClient { +func (g *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { loggerConfig := common.LoggerConfig{ Format: "text", @@ -91,7 +92,8 @@ func (g *TrafficGenerator) buildRetriever() clients.RetrievalClient { } ethClientConfig := geth.EthClientConfig{ - RPCURLs: []string{"http://localhost:8545"}, + RPCURLs: []string{"http://localhost:8545"}, + NumRetries: 2, } gethClient, err := geth.NewMultiHomingClient(ethClientConfig, gethcommon.Address{}, logger) @@ -103,8 +105,18 @@ func (g *TrafficGenerator) buildRetriever() clients.RetrievalClient { cs := eth.NewChainState(tx, gethClient) - rpcClient, err := rpc.Dial("http://localhost:8545") + // ------------- + + // This is the indexer when config.UseGraph is true + //chainStateConfig := thegraph.Config{ + // Endpoint: "localhost:8000", + // PullInterval: 100 * time.Millisecond, + // MaxRetries: 5, + //} + //chainState := thegraph.MakeIndexedChainState(chainStateConfig, cs, logger) + // This is the indexer when config.UseGraph is false. + rpcClient, err := rpc.Dial("http://localhost:8545") indexerConfig := indexer.Config{ PullInterval: time.Second, } @@ -112,7 +124,7 @@ func (g *TrafficGenerator) buildRetriever() clients.RetrievalClient { &indexerConfig, gethClient, rpcClient, - "0x851356ae760d987E095750cCeb3bC6014560891C", + "0x851356ae760d987E095750cCeb3bC6014560891C", // eigenDaServeManagerAddr logger, ) if err != nil { @@ -123,7 +135,7 @@ func (g *TrafficGenerator) buildRetriever() clients.RetrievalClient { panic(err) // TODO } - //chainState := thegraph.MakeIndexedChainState(chainStateConfig, cs, logger) + // ------------- var assignmentCoordinator core.AssignmentCoordinator = &core.StdAssignmentCoordinator{} @@ -157,7 +169,9 @@ func (g *TrafficGenerator) buildRetriever() clients.RetrievalClient { panic(err) // TODO } - return retriever + chainClient := retrivereth.NewChainClient(gethClient, logger) + + return retriever, chainClient } // Run instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. @@ -177,8 +191,10 @@ func (g *TrafficGenerator) Run() error { time.Sleep(g.Config.InstanceLaunchInterval) } + retriever, chainClient := g.buildRetriever() + // TODO start multiple readers - reader := NewBlobReader(&ctx, &wg, g.buildRetriever(), &table) + reader := NewBlobReader(&ctx, &wg, retriever, chainClient, &table) reader.Start() signals := make(chan os.Signal, 1) From 31cf1d55fc005176d528d16843f0aab7b40375be Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 23 Jul 2024 11:53:06 -0500 Subject: [PATCH 11/74] Checkin. Signed-off-by: Cody Littley --- tools/traffic/generator.go | 61 +++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 9641d1639e..621ad80c60 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -5,14 +5,12 @@ import ( "fmt" "github.com/Layr-Labs/eigenda/common/geth" "github.com/Layr-Labs/eigenda/core/eth" - coreindexer "github.com/Layr-Labs/eigenda/core/indexer" + "github.com/Layr-Labs/eigenda/core/thegraph" "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" - "github.com/Layr-Labs/eigenda/indexer" retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rpc" "os" "os/signal" "sync" @@ -83,9 +81,12 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi // buildRetriever creates a retriever client for the traffic generator. func (g *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { - loggerConfig := common.LoggerConfig{ - Format: "text", - } + //loggerConfig := common.LoggerConfig{ + // Format: "text", + // } + + loggerConfig := common.DefaultLoggerConfig() + logger, err := common.NewLogger(loggerConfig) if err != nil { panic(err) // TODO @@ -108,32 +109,32 @@ func (g *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retriveret // ------------- // This is the indexer when config.UseGraph is true - //chainStateConfig := thegraph.Config{ - // Endpoint: "localhost:8000", - // PullInterval: 100 * time.Millisecond, - // MaxRetries: 5, - //} - //chainState := thegraph.MakeIndexedChainState(chainStateConfig, cs, logger) + chainStateConfig := thegraph.Config{ + Endpoint: "localhost:8000", + PullInterval: 100 * time.Millisecond, + MaxRetries: 5, + } + chainState := thegraph.MakeIndexedChainState(chainStateConfig, cs, logger) // This is the indexer when config.UseGraph is false. - rpcClient, err := rpc.Dial("http://localhost:8545") - indexerConfig := indexer.Config{ - PullInterval: time.Second, - } - indexer, err := coreindexer.CreateNewIndexer( - &indexerConfig, - gethClient, - rpcClient, - "0x851356ae760d987E095750cCeb3bC6014560891C", // eigenDaServeManagerAddr - logger, - ) - if err != nil { - panic(err) // TODO - } - chainState, err := coreindexer.NewIndexedChainState(cs, indexer) - if err != nil { - panic(err) // TODO - } + //rpcClient, err := rpc.Dial("http://localhost:8545") + //indexerConfig := indexer.Config{ + // PullInterval: time.Second, + //} + //indexer, err := coreindexer.CreateNewIndexer( + // &indexerConfig, + // gethClient, + // rpcClient, + // "0x851356ae760d987E095750cCeb3bC6014560891C", // eigenDaServeManagerAddr + // logger, + //) + //if err != nil { + // panic(err) // TODO + //} + //chainState, err := coreindexer.NewIndexedChainState(cs, indexer) + //if err != nil { + // panic(err) // TODO + //} // ------------- From a04fef55bb5c1ca954cfd05cffa5a0b3b4008ecf Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 23 Jul 2024 14:02:25 -0500 Subject: [PATCH 12/74] Working Signed-off-by: Cody Littley --- tools/traffic/generator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 621ad80c60..a1ade175b7 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -110,7 +110,7 @@ func (g *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retriveret // This is the indexer when config.UseGraph is true chainStateConfig := thegraph.Config{ - Endpoint: "localhost:8000", + Endpoint: "http://localhost:8000/subgraphs/name/Layr-Labs/eigenda-operator-state", PullInterval: 100 * time.Millisecond, MaxRetries: 5, } From f57e6776c2e344ed00d67f53eb0b22eabe46e000 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 23 Jul 2024 14:53:50 -0500 Subject: [PATCH 13/74] Expose chunks in API. Signed-off-by: Cody Littley --- api/clients/mock/retrieval_client.go | 18 ++++++++ api/clients/retrieval_client.go | 64 ++++++++++++++++++++++++++-- tools/traffic/blob_reader.go | 9 +++- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/api/clients/mock/retrieval_client.go b/api/clients/mock/retrieval_client.go index 3a0e3ccf49..5dfd326083 100644 --- a/api/clients/mock/retrieval_client.go +++ b/api/clients/mock/retrieval_client.go @@ -35,3 +35,21 @@ func (c *MockRetrievalClient) RetrieveBlob( result := args.Get(0) return result.([]byte), args.Error(1) } + +func (c *MockRetrievalClient) RetrieveBlobChunks( + ctx context.Context, + batchHeaderHash [32]byte, + blobIndex uint32, + referenceBlockNumber uint, + batchRoot [32]byte, + quorumID core.QuorumID) (*clients.BlobChunks, error) { + + return nil, nil // TODO +} + +func (c *MockRetrievalClient) CombineChunks(chunks *clients.BlobChunks) ([]byte, error) { + args := c.Called() + + result := args.Get(0) + return result.([]byte), args.Error(1) +} diff --git a/api/clients/retrieval_client.go b/api/clients/retrieval_client.go index c164b58f5b..b7d45e4bbe 100644 --- a/api/clients/retrieval_client.go +++ b/api/clients/retrieval_client.go @@ -14,7 +14,11 @@ import ( "github.com/wealdtech/go-merkletree/keccak256" ) +// RetrievalClient is an object that can retrieve blobs from the network. type RetrievalClient interface { + + // RetrieveBlob fetches a blob from the network. This method is equivalent to calling + // RetrieveBlobChunks to get the chunks and then CombineChunks to recombine those chunks into the original blob. RetrieveBlob( ctx context.Context, batchHeaderHash [32]byte, @@ -22,6 +26,28 @@ type RetrievalClient interface { referenceBlockNumber uint, batchRoot [32]byte, quorumID core.QuorumID) ([]byte, error) + + // RetrieveBlobChunks downloads the chunks of a blob from the network but do not recombine them. + RetrieveBlobChunks( + ctx context.Context, + batchHeaderHash [32]byte, + blobIndex uint32, + referenceBlockNumber uint, + batchRoot [32]byte, + quorumID core.QuorumID) (*BlobChunks, error) + + // CombineChunks recombines the chunks into the original blob. + CombineChunks(chunks *BlobChunks) ([]byte, error) +} + +// BlobChunks is a collection of chunks retrieved from the network which can be recombined into a blob. +type BlobChunks struct { + chunks []*encoding.Frame + indices []encoding.ChunkNumber + encodingParams encoding.EncodingParams + blobHeaderLength uint + assignments map[core.OperatorID]core.Assignment + assignmentInfo core.AssignmentInfo } type retrievalClient struct { @@ -33,16 +59,17 @@ type retrievalClient struct { numConnections int } +// TODO can this line be deleted? var _ RetrievalClient = (*retrievalClient)(nil) +// NewRetrievalClient creates a new retrieval client. func NewRetrievalClient( logger logging.Logger, chainState core.IndexedChainState, assignmentCoordinator core.AssignmentCoordinator, nodeClient NodeClient, verifier encoding.Verifier, - numConnections int, -) (RetrievalClient, error) { + numConnections int) (RetrievalClient, error) { return &retrievalClient{ logger: logger.With("component", "RetrievalClient"), @@ -54,6 +81,7 @@ func NewRetrievalClient( }, nil } +// RetrieveBlob retrieves a blob from the network. func (r *retrievalClient) RetrieveBlob( ctx context.Context, batchHeaderHash [32]byte, @@ -62,6 +90,22 @@ func (r *retrievalClient) RetrieveBlob( batchRoot [32]byte, quorumID core.QuorumID) ([]byte, error) { + chunks, err := r.RetrieveBlobChunks(ctx, batchHeaderHash, blobIndex, referenceBlockNumber, batchRoot, quorumID) + if err != nil { + return nil, err + } + + return r.CombineChunks(chunks) +} + +// RetrieveBlobChunks retrieves the chunks of a blob from the network but does not recombine them. +func (r *retrievalClient) RetrieveBlobChunks(ctx context.Context, + batchHeaderHash [32]byte, + blobIndex uint32, + referenceBlockNumber uint, + batchRoot [32]byte, + quorumID core.QuorumID) (*BlobChunks, error) { + fmt.Printf(" getting indexed operator state, referenceBlockNumber: %d, quorumId: %d\n", referenceBlockNumber, quorumID) // TODO indexedOperatorState, err := r.indexedChainState.GetIndexedOperatorState(ctx, referenceBlockNumber, []core.QuorumID{quorumID}) if err != nil { @@ -176,5 +220,19 @@ func (r *retrievalClient) RetrieveBlob( indices = append(indices, assignment.GetIndices()...) } - return r.verifier.Decode(chunks, indices, encodingParams, uint64(blobHeader.Length)*encoding.BYTES_PER_SYMBOL) + return &BlobChunks{ + chunks: chunks, + indices: indices, + encodingParams: encodingParams, + blobHeaderLength: blobHeader.Length, + }, nil +} + +// CombineChunks recombines the chunks into the original blob. +func (r *retrievalClient) CombineChunks(chunks *BlobChunks) ([]byte, error) { + return r.verifier.Decode( + chunks.chunks, + chunks.indices, + chunks.encodingParams, + uint64(chunks.blobHeaderLength)*encoding.BYTES_PER_SYMBOL) } diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index c7740517ba..4bd45a4263 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -96,7 +96,7 @@ func (reader *BlobReader) randomRead() { copy(batchHeaderHash[:], *metadata.batchHeaderHash) fmt.Printf("attempting to read blob, header hash: %x, index: %d\n, batch header: %+v\n", *metadata.batchHeaderHash, metadata.blobIndex, batchHeader) - data, err := reader.retriever.RetrieveBlob( + chunks, err := reader.retriever.RetrieveBlobChunks( *reader.ctx, batchHeaderHash, metadata.blobIndex, @@ -109,6 +109,13 @@ func (reader *BlobReader) randomRead() { return } + data, err := reader.retriever.CombineChunks(chunks) + + if err != nil { + fmt.Println("Error combining chunks:", err) // TODO + return + } + // TODO verify blob data fmt.Println("Read blob:", data) // TODO From 3c956d8573bd25bd300c0ff9ad1c996193036063 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 24 Jul 2024 08:06:02 -0500 Subject: [PATCH 14/74] Extracting needed data. Signed-off-by: Cody Littley --- api/clients/retrieval_client.go | 33 ++++++++++++++++++--------------- tools/traffic/blob_reader.go | 26 +++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/api/clients/retrieval_client.go b/api/clients/retrieval_client.go index b7d45e4bbe..60b9c42b13 100644 --- a/api/clients/retrieval_client.go +++ b/api/clients/retrieval_client.go @@ -27,7 +27,8 @@ type RetrievalClient interface { batchRoot [32]byte, quorumID core.QuorumID) ([]byte, error) - // RetrieveBlobChunks downloads the chunks of a blob from the network but do not recombine them. + // RetrieveBlobChunks downloads the chunks of a blob from the network but do not recombine them. Use this method + // if detailed information about which node returned which chunk is needed. Otherwise, use RetrieveBlob. RetrieveBlobChunks( ctx context.Context, batchHeaderHash [32]byte, @@ -42,12 +43,12 @@ type RetrievalClient interface { // BlobChunks is a collection of chunks retrieved from the network which can be recombined into a blob. type BlobChunks struct { - chunks []*encoding.Frame - indices []encoding.ChunkNumber - encodingParams encoding.EncodingParams - blobHeaderLength uint - assignments map[core.OperatorID]core.Assignment - assignmentInfo core.AssignmentInfo + Chunks []*encoding.Frame + Indices []encoding.ChunkNumber + EncodingParams encoding.EncodingParams + BlobHeaderLength uint + Assignments map[core.OperatorID]core.Assignment + AssignmentInfo core.AssignmentInfo } type retrievalClient struct { @@ -221,18 +222,20 @@ func (r *retrievalClient) RetrieveBlobChunks(ctx context.Context, } return &BlobChunks{ - chunks: chunks, - indices: indices, - encodingParams: encodingParams, - blobHeaderLength: blobHeader.Length, + Chunks: chunks, + Indices: indices, + EncodingParams: encodingParams, + BlobHeaderLength: blobHeader.Length, + Assignments: assignments, + AssignmentInfo: info, }, nil } // CombineChunks recombines the chunks into the original blob. func (r *retrievalClient) CombineChunks(chunks *BlobChunks) ([]byte, error) { return r.verifier.Decode( - chunks.chunks, - chunks.indices, - chunks.encodingParams, - uint64(chunks.blobHeaderLength)*encoding.BYTES_PER_SYMBOL) + chunks.Chunks, + chunks.Indices, + chunks.EncodingParams, + uint64(chunks.BlobHeaderLength)*encoding.BYTES_PER_SYMBOL) } diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 4bd45a4263..de24e8422b 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/retriever/eth" gcommon "github.com/ethereum/go-ethereum/common" "math/big" @@ -109,6 +110,11 @@ func (reader *BlobReader) randomRead() { return } + chunkCount := chunks.AssignmentInfo.TotalChunks + + var assignments map[core.OperatorID]core.Assignment + assignments = chunks.Assignments + data, err := reader.retriever.CombineChunks(chunks) if err != nil { @@ -118,5 +124,23 @@ func (reader *BlobReader) randomRead() { // TODO verify blob data - fmt.Println("Read blob:", data) // TODO + fmt.Printf("=====================================\nRead blob. Total chunk count = %d\nRetrieved chunk count = %d\nData length = %d\n", + chunkCount, len(chunks.Chunks), len(data)) // TODO + + indexSet := make(map[encoding.ChunkNumber]bool) + for index := range chunks.Indices { + indexSet[chunks.Indices[index]] = true + } + + for id, assignment := range assignments { + fmt.Printf(" - Operator ID: %d, Start Index: %d, Num Chunks: %d\n", id, assignment.StartIndex, assignment.NumChunks) + for index := assignment.StartIndex; index < assignment.StartIndex+assignment.NumChunks; index++ { + if !indexSet[index] { + fmt.Printf(" - Missing chunk: %d\n", index) + } else { + fmt.Printf(" - Chunk found: %d\n", index) + } + } + } + } From 9c296bf5aabd65552e00d30ca37d1613d0882605 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 24 Jul 2024 10:18:08 -0500 Subject: [PATCH 15/74] Added a few metrics. Signed-off-by: Cody Littley --- node/node.go | 4 +- tools/traffic/blob_writer.go | 33 ++++++-- tools/traffic/generator.go | 7 +- tools/traffic/metrics.go | 128 +++++++++++++++++++++++++++++++ tools/traffic/status_verifier.go | 52 ++++++++++--- 5 files changed, 201 insertions(+), 23 deletions(-) create mode 100644 tools/traffic/metrics.go diff --git a/node/node.go b/node/node.go index 59a4f9c331..572080d51c 100644 --- a/node/node.go +++ b/node/node.go @@ -300,7 +300,7 @@ func (n *Node) ProcessBatch(ctx context.Context, header *core.BatchHeader, blobs return nil, errors.New("number of parsed blobs must be the same as number of blobs from protobuf request") } - // Measure num batches received and its size in bytes + // Invoke num batches received and its size in bytes batchSize := uint64(0) for _, blob := range blobs { for quorumID, bundle := range blob.Bundles { @@ -326,7 +326,7 @@ func (n *Node) ProcessBatch(ctx context.Context, header *core.BatchHeader, blobs // Defined only if the batch not already exists and gets stored to database successfully. keys *[][]byte - // Latency to store the batch. + // latency to store the batch. // Defined only if the batch not already exists and gets stored to database successfully. latency time.Duration } diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 1ac89ff207..4c9410e173 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -27,6 +27,15 @@ type BlobWriter struct { // fixedRandomData contains random data for blobs if RandomizeBlobs is false, and nil otherwise. fixedRandomData *[]byte + + // writeLatencyMetric is used to record latency for write requests. + writeLatencyMetric LatencyMetric + + // writeSuccessMetric is used to record the number of successful write requests. + writeSuccessMetric CountMetric + + // writeFailureMetric is used to record the number of failed write requests. + writeFailureMetric CountMetric } // NewBlobWriter creates a new BlobWriter instance. @@ -34,7 +43,8 @@ func NewBlobWriter( ctx *context.Context, waitGroup *sync.WaitGroup, generator *TrafficGenerator, - verifier *StatusVerifier) BlobWriter { + verifier *StatusVerifier, + metrics *Metrics) BlobWriter { var fixedRandomData []byte if generator.Config.RandomizeBlobs { @@ -51,11 +61,14 @@ func NewBlobWriter( } return BlobWriter{ - ctx: ctx, - waitGroup: waitGroup, - generator: generator, - verifier: verifier, - fixedRandomData: &fixedRandomData, + ctx: ctx, + waitGroup: waitGroup, + generator: generator, + verifier: verifier, + fixedRandomData: &fixedRandomData, + writeLatencyMetric: metrics.NewLatencyMetric("write"), + writeSuccessMetric: metrics.NewCountMetric("write_success"), + writeFailureMetric: metrics.NewCountMetric("write_failure"), } } @@ -78,12 +91,16 @@ func (writer *BlobWriter) run() { return case <-ticker.C: data := writer.getRandomData() - key, err := writer.sendRequest(*data) - + key, err := InvokeAndReportLatency(&writer.writeLatencyMetric, func() ([]byte, error) { + return writer.sendRequest(*data) + }) if err != nil { + writer.writeFailureMetric.Increment() writer.generator.Logger.Error("failed to send blob request", "err:", err) continue } + + writer.writeSuccessMetric.Increment() fmt.Println("Sent blob with length", len(*data)) // TODO remove writer.verifier.AddUnconfirmedKey(&key) diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index a1ade175b7..67b2f0c7d7 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -179,15 +179,18 @@ func (g *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retriveret func (g *TrafficGenerator) Run() error { ctx, cancel := context.WithCancel(context.Background()) + metrics := NewMetrics("9101", g.Logger) // TODO config + metrics.Start(ctx) + // TODO add configuration table := NewBlobTable() - verifier := NewStatusVerifier(&table, &g.DisperserClient, -1) + verifier := NewStatusVerifier(&table, &g.DisperserClient, -1, metrics) verifier.Start(ctx, time.Second) var wg sync.WaitGroup for i := 0; i < int(g.Config.NumWriteInstances); i++ { - writer := NewBlobWriter(&ctx, &wg, g, &verifier) + writer := NewBlobWriter(&ctx, &wg, g, &verifier, metrics) writer.Start() time.Sleep(g.Config.InstanceLaunchInterval) } diff --git a/tools/traffic/metrics.go b/tools/traffic/metrics.go new file mode 100644 index 0000000000..cd3bfd4f3f --- /dev/null +++ b/tools/traffic/metrics.go @@ -0,0 +1,128 @@ +package traffic + +import ( + "context" + "fmt" + "github.com/Layr-Labs/eigensdk-go/logging" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" + "time" +) + +// Metrics encapsulates metrics for the traffic generator. +type Metrics struct { + registry *prometheus.Registry + + count *prometheus.CounterVec + latency *prometheus.SummaryVec + + httpPort string + logger logging.Logger +} + +// LatencyMetric tracks the latency of an operation. +type LatencyMetric struct { + metrics *Metrics + description string +} + +// CountMetric tracks the count of a type of event. +type CountMetric struct { + metrics *Metrics + description string +} + +// NewMetrics creates a new Metrics instance. +func NewMetrics(httpPort string, logger logging.Logger) *Metrics { + namespace := "eigenda_generator" + reg := prometheus.NewRegistry() + reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + reg.MustRegister(collectors.NewGoCollector()) + + metrics := &Metrics{ + count: promauto.With(reg).NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "event_count", + Help: "the count of various types of events", + }, + []string{"event"}, + ), + latency: promauto.With(reg).NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: namespace, + Name: "latency_s", + Help: "latency in seconds", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.01, 0.99: 0.001}, + }, + []string{"operation"}, + ), + registry: reg, + httpPort: httpPort, + logger: logger.With("component", "GeneratorMetrics"), + } + return metrics +} + +// Start starts the metrics server. +func (metrics *Metrics) Start(ctx context.Context) { // TODO context? + metrics.logger.Info("Starting metrics server at ", "port", metrics.httpPort) + addr := fmt.Sprintf(":%s", metrics.httpPort) + go func() { + log := metrics.logger + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor( + metrics.registry, + promhttp.HandlerOpts{}, + )) + err := http.ListenAndServe(addr, mux) + log.Error("Prometheus server failed", "err", err) + }() +} + +// NewLatencyMetric creates a new LatencyMetric instance. +func (metrics *Metrics) NewLatencyMetric(description string) LatencyMetric { + return LatencyMetric{ + metrics: metrics, + description: description, + } +} + +// ReportLatency reports the latency of an operation. +func (metric *LatencyMetric) ReportLatency(latency time.Duration) { + metric.metrics.latency.WithLabelValues(metric.description).Observe(latency.Seconds()) +} + +// InvokeAndReportLatency performs an operation. If the operation does not produce an error, then the latency +// of the operation is reported to the metrics framework. +func InvokeAndReportLatency[T any](metric *LatencyMetric, operation func() (T, error)) (T, error) { + start := time.Now() + + t, err := operation() + + if err == nil { + end := time.Now() + duration := end.Sub(start) + metric.ReportLatency(duration) + } + + return t, err +} + +// NewCountMetric creates a new CountMetric instance. +func (metrics *Metrics) NewCountMetric(description string) CountMetric { + return CountMetric{ + metrics: metrics, + description: description, + } +} + +// Increment increments the count of a type of event. +func (metric *CountMetric) Increment() { + metric.metrics.count.With(prometheus.Labels{ + "event": metric.description, + }).Inc() +} diff --git a/tools/traffic/status_verifier.go b/tools/traffic/status_verifier.go index 88182fb97e..472fcb9569 100644 --- a/tools/traffic/status_verifier.go +++ b/tools/traffic/status_verifier.go @@ -28,20 +28,42 @@ type StatusVerifier struct { // Newly added keys that require verification. keyChannel chan *[]byte + + getStatusLatencyMetric LatencyMetric + getStatusErrorCountMetric CountMetric + unknownCountMetric CountMetric + processingCountMetric CountMetric + dispersingCountMetric CountMetric + failedCountMetric CountMetric + insufficientSignaturesCountMetric CountMetric + confirmedCountMetric CountMetric + finalizedCountMetric CountMetric + + // TODO metric for average time for blob to become confirmed } // NewStatusVerifier creates a new StatusVerifier instance. func NewStatusVerifier( table *BlobTable, disperser *clients.DisperserClient, - blobReadLimit int32) StatusVerifier { + blobReadLimit int32, + metrics *Metrics) StatusVerifier { return StatusVerifier{ - table: table, - blobReadLimit: blobReadLimit, - dispenser: disperser, - unconfirmedKeys: make([]*[]byte, 0), - keyChannel: make(chan *[]byte), + table: table, + blobReadLimit: blobReadLimit, + dispenser: disperser, + unconfirmedKeys: make([]*[]byte, 0), + keyChannel: make(chan *[]byte), + getStatusLatencyMetric: metrics.NewLatencyMetric("getStatus"), + getStatusErrorCountMetric: metrics.NewCountMetric("getStatusError"), + unknownCountMetric: metrics.NewCountMetric("getStatusUnknown"), + processingCountMetric: metrics.NewCountMetric("getStatusProcessing"), + dispersingCountMetric: metrics.NewCountMetric("getStatusDispersing"), + failedCountMetric: metrics.NewCountMetric("getStatusFailed"), + insufficientSignaturesCountMetric: metrics.NewCountMetric("getStatusInsufficientSignatures"), + confirmedCountMetric: metrics.NewCountMetric("getStatusConfirmed"), + finalizedCountMetric: metrics.NewCountMetric("getStatusFinalized"), } } @@ -93,28 +115,36 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]b if err != nil { fmt.Println("Error getting blob status:", err) // TODO is this proper? + verifier.getStatusErrorCountMetric.Increment() return false } switch status.GetStatus() { case disperser.BlobStatus_UNKNOWN: - fallthrough + verifier.unknownCountMetric.Increment() + return false case disperser.BlobStatus_PROCESSING: - fallthrough + verifier.processingCountMetric.Increment() + return false case disperser.BlobStatus_DISPERSING: - // Final status is not yet known. Check it again later. + verifier.dispersingCountMetric.Increment() return false case disperser.BlobStatus_FAILED: - fallthrough + verifier.failedCountMetric.Increment() + fmt.Println("Blob dispersal failed:", status.GetStatus()) // TODO use logger + return true case disperser.BlobStatus_INSUFFICIENT_SIGNATURES: + verifier.insufficientSignaturesCountMetric.Increment() fmt.Println("Blob dispersal failed:", status.GetStatus()) // TODO use logger return true case disperser.BlobStatus_CONFIRMED: - fallthrough + verifier.confirmedCountMetric.Increment() + return true case disperser.BlobStatus_FINALIZED: + verifier.finalizedCountMetric.Increment() batchHeaderHash := status.GetInfo().BlobVerificationProof.BatchMetadata.BatchHeaderHash blobIndex := status.GetInfo().BlobVerificationProof.GetBlobIndex() From 1307405790d19168348133cc6833f39f9f7fabf1 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 24 Jul 2024 11:26:10 -0500 Subject: [PATCH 16/74] Added a bunch of new metrics. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 114 +++++++++++++++++++++++-------- tools/traffic/blob_writer.go | 5 -- tools/traffic/generator.go | 2 +- tools/traffic/metrics.go | 34 +++++++-- tools/traffic/status_verifier.go | 13 +++- 5 files changed, 124 insertions(+), 44 deletions(-) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index de24e8422b..801dca21fa 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/Layr-Labs/eigenda/api/clients" + contractEigenDAServiceManager "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDAServiceManager" "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/retriever/eth" @@ -29,6 +30,20 @@ type BlobReader struct { // table of blobs to read from. table *BlobTable + + metrics *Metrics + + fetchBatchHeaderMetric LatencyMetric + fetchBatchHeaderSuccess CountMetric + fetchBatchHeaderFailure CountMetric + readLatencyMetric LatencyMetric + readSuccessMetric CountMetric + readFailureMetric CountMetric + recombinationSuccess CountMetric + recombinationFailure CountMetric + operatorSuccessMetrics map[core.OperatorID]CountMetric + operatorFailureMetrics map[core.OperatorID]CountMetric + candidatePoolSize GaugeMetric } // NewBlobReader creates a new BlobReader instance. @@ -37,14 +52,27 @@ func NewBlobReader( waitGroup *sync.WaitGroup, retriever clients.RetrievalClient, chainClient eth.ChainClient, - table *BlobTable) BlobReader { + table *BlobTable, + metrics *Metrics) BlobReader { return BlobReader{ - ctx: ctx, - waitGroup: waitGroup, - retriever: retriever, - chainClient: chainClient, - table: table, + ctx: ctx, + waitGroup: waitGroup, + retriever: retriever, + chainClient: chainClient, + table: table, + metrics: metrics, + fetchBatchHeaderMetric: metrics.NewLatencyMetric("fetch_batch_header"), + fetchBatchHeaderSuccess: metrics.NewCountMetric("fetch_batch_header_success"), + fetchBatchHeaderFailure: metrics.NewCountMetric("fetch_batch_header_failure"), + recombinationSuccess: metrics.NewCountMetric("recombination_success"), + recombinationFailure: metrics.NewCountMetric("recombination_failure"), + readLatencyMetric: metrics.NewLatencyMetric("read"), + readSuccessMetric: metrics.NewCountMetric("read_success"), + readFailureMetric: metrics.NewCountMetric("read_failure"), + operatorSuccessMetrics: make(map[core.OperatorID]CountMetric), + operatorFailureMetrics: make(map[core.OperatorID]CountMetric), + candidatePoolSize: metrics.NewGaugeMetric("candidate_pool_size"), } } @@ -73,42 +101,49 @@ func (reader *BlobReader) run() { // randomRead reads a random blob. func (reader *BlobReader) randomRead() { + reader.candidatePoolSize.Set(float64(reader.table.Size())) + metadata := reader.table.GetRandom(true) if metadata == nil { // There are no blobs to read, do nothing. return } - // TODO use NodeClient - - batchHeader, err := reader.chainClient.FetchBatchHeader( - *reader.ctx, - gcommon.HexToAddress("0x851356ae760d987E095750cCeb3bC6014560891C"), - *metadata.batchHeaderHash, - big.NewInt(int64(0)), - nil) + batchHeader, err := InvokeAndReportLatency(&reader.fetchBatchHeaderMetric, + func() (*contractEigenDAServiceManager.IEigenDAServiceManagerBatchHeader, error) { + return reader.chainClient.FetchBatchHeader( + *reader.ctx, + gcommon.HexToAddress("0x851356ae760d987E095750cCeb3bC6014560891C"), + *metadata.batchHeaderHash, + big.NewInt(int64(0)), + nil) + }) if err != nil { - fmt.Println("Error fetching batch header:", err) // TODO + // TODO log + reader.fetchBatchHeaderFailure.Increment() return } + reader.fetchBatchHeaderSuccess.Increment() - // TODO is this correct? var batchHeaderHash [32]byte copy(batchHeaderHash[:], *metadata.batchHeaderHash) - fmt.Printf("attempting to read blob, header hash: %x, index: %d\n, batch header: %+v\n", *metadata.batchHeaderHash, metadata.blobIndex, batchHeader) - chunks, err := reader.retriever.RetrieveBlobChunks( - *reader.ctx, - batchHeaderHash, - metadata.blobIndex, - uint(batchHeader.ReferenceBlockNumber), - batchHeader.BlobHeadersRoot, - core.QuorumID(0)) + chunks, err := InvokeAndReportLatency(&reader.readLatencyMetric, func() (*clients.BlobChunks, error) { + return reader.retriever.RetrieveBlobChunks( + *reader.ctx, + batchHeaderHash, + metadata.blobIndex, + uint(batchHeader.ReferenceBlockNumber), + batchHeader.BlobHeadersRoot, + core.QuorumID(0)) + }) if err != nil { - fmt.Println("Error reading blob:", err) // TODO + // TODO log + reader.readFailureMetric.Increment() return } + reader.readFailureMetric.Increment() chunkCount := chunks.AssignmentInfo.TotalChunks @@ -119,8 +154,10 @@ func (reader *BlobReader) randomRead() { if err != nil { fmt.Println("Error combining chunks:", err) // TODO + reader.recombinationFailure.Increment() return } + reader.recombinationSuccess.Increment() // TODO verify blob data @@ -135,12 +172,33 @@ func (reader *BlobReader) randomRead() { for id, assignment := range assignments { fmt.Printf(" - Operator ID: %d, Start Index: %d, Num Chunks: %d\n", id, assignment.StartIndex, assignment.NumChunks) for index := assignment.StartIndex; index < assignment.StartIndex+assignment.NumChunks; index++ { - if !indexSet[index] { - fmt.Printf(" - Missing chunk: %d\n", index) + if indexSet[index] { + reader.reportChunk(id) } else { - fmt.Printf(" - Chunk found: %d\n", index) + reader.reportMissingChunk(id) } } } +} + +// reportChunk reports a successful chunk read. +func (reader *BlobReader) reportChunk(operatorId core.OperatorID) { + metric, exists := reader.operatorSuccessMetrics[operatorId] + if !exists { + metric = reader.metrics.NewCountMetric(fmt.Sprintf("operator_%x_returned_chunk", operatorId)) + reader.operatorSuccessMetrics[operatorId] = metric + } + + metric.Increment() +} + +// reportMissingChunk reports a missing chunk. +func (reader *BlobReader) reportMissingChunk(operatorId core.OperatorID) { + metric, exists := reader.operatorFailureMetrics[operatorId] + if !exists { + metric = reader.metrics.NewCountMetric(fmt.Sprintf("operator_%x_witheld_chunk", operatorId)) + reader.operatorFailureMetrics[operatorId] = metric + } + metric.Increment() } diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 4c9410e173..87630be099 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -3,14 +3,11 @@ package traffic import ( "context" "crypto/rand" - "fmt" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "sync" "time" ) -// TODO document - // BlobWriter sends blobs to a disperser at a configured rate. type BlobWriter struct { // The context for the generator. All work should cease when this context is cancelled. @@ -101,8 +98,6 @@ func (writer *BlobWriter) run() { } writer.writeSuccessMetric.Increment() - fmt.Println("Sent blob with length", len(*data)) // TODO remove - writer.verifier.AddUnconfirmedKey(&key) } } diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 67b2f0c7d7..2e508fc5ec 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -198,7 +198,7 @@ func (g *TrafficGenerator) Run() error { retriever, chainClient := g.buildRetriever() // TODO start multiple readers - reader := NewBlobReader(&ctx, &wg, retriever, chainClient, &table) + reader := NewBlobReader(&ctx, &wg, retriever, chainClient, &table, metrics) reader.Start() signals := make(chan os.Signal, 1) diff --git a/tools/traffic/metrics.go b/tools/traffic/metrics.go index cd3bfd4f3f..f8fa88f138 100644 --- a/tools/traffic/metrics.go +++ b/tools/traffic/metrics.go @@ -18,6 +18,7 @@ type Metrics struct { count *prometheus.CounterVec latency *prometheus.SummaryVec + gauge *prometheus.GaugeVec httpPort string logger logging.Logger @@ -35,6 +36,11 @@ type CountMetric struct { description string } +type GaugeMetric struct { + metrics *Metrics + description string +} + // NewMetrics creates a new Metrics instance. func NewMetrics(httpPort string, logger logging.Logger) *Metrics { namespace := "eigenda_generator" @@ -47,19 +53,22 @@ func NewMetrics(httpPort string, logger logging.Logger) *Metrics { prometheus.CounterOpts{ Namespace: namespace, Name: "event_count", - Help: "the count of various types of events", }, - []string{"event"}, + []string{"label"}, ), latency: promauto.With(reg).NewSummaryVec( prometheus.SummaryOpts{ Namespace: namespace, Name: "latency_s", - Help: "latency in seconds", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.01, 0.99: 0.001}, }, - []string{"operation"}, + []string{"label"}, ), + gauge: promauto.With(reg).NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "gauge", + }, []string{"label"}), registry: reg, httpPort: httpPort, logger: logger.With("component", "GeneratorMetrics"), @@ -122,7 +131,18 @@ func (metrics *Metrics) NewCountMetric(description string) CountMetric { // Increment increments the count of a type of event. func (metric *CountMetric) Increment() { - metric.metrics.count.With(prometheus.Labels{ - "event": metric.description, - }).Inc() + metric.metrics.count.WithLabelValues(metric.description).Inc() +} + +// NewGaugeMetric creates a new GaugeMetric instance. +func (metrics *Metrics) NewGaugeMetric(description string) GaugeMetric { + return GaugeMetric{ + metrics: metrics, + description: description, + } +} + +// Set sets the value of a gauge metric. +func (metric GaugeMetric) Set(value float64) { + metric.metrics.gauge.WithLabelValues(metric.description).Set(value) } diff --git a/tools/traffic/status_verifier.go b/tools/traffic/status_verifier.go index 472fcb9569..01a2d7600c 100644 --- a/tools/traffic/status_verifier.go +++ b/tools/traffic/status_verifier.go @@ -29,6 +29,7 @@ type StatusVerifier struct { // Newly added keys that require verification. keyChannel chan *[]byte + blobsInFlightMetric GaugeMetric getStatusLatencyMetric LatencyMetric getStatusErrorCountMetric CountMetric unknownCountMetric CountMetric @@ -55,6 +56,7 @@ func NewStatusVerifier( dispenser: disperser, unconfirmedKeys: make([]*[]byte, 0), keyChannel: make(chan *[]byte), + blobsInFlightMetric: metrics.NewGaugeMetric("blobsInFlight"), getStatusLatencyMetric: metrics.NewLatencyMetric("getStatus"), getStatusErrorCountMetric: metrics.NewCountMetric("getStatusError"), unknownCountMetric: metrics.NewCountMetric("getStatusUnknown"), @@ -107,11 +109,18 @@ func (verifier *StatusVerifier) poll(ctx context.Context) { } } verifier.unconfirmedKeys = unconfirmedKeys + verifier.blobsInFlightMetric.Set(float64(len(verifier.unconfirmedKeys))) } // checkStatusForBlob checks the status of a blob. Returns true if the final blob status is known, false otherwise. func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]byte) bool { - status, err := (*verifier.dispenser).GetBlobStatus(ctx, *key) + + // TODO add timeout + + status, err := InvokeAndReportLatency[*disperser.BlobStatusReply](&verifier.getStatusLatencyMetric, + func() (*disperser.BlobStatusReply, error) { + return (*verifier.dispenser).GetBlobStatus(ctx, *key) + }) if err != nil { fmt.Println("Error getting blob status:", err) // TODO is this proper? @@ -133,11 +142,9 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]b case disperser.BlobStatus_FAILED: verifier.failedCountMetric.Increment() - fmt.Println("Blob dispersal failed:", status.GetStatus()) // TODO use logger return true case disperser.BlobStatus_INSUFFICIENT_SIGNATURES: verifier.insufficientSignaturesCountMetric.Increment() - fmt.Println("Blob dispersal failed:", status.GetStatus()) // TODO use logger return true case disperser.BlobStatus_CONFIRMED: From 6fcc82551c5434c06574a59d4e03c10d2522db43 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 24 Jul 2024 11:39:07 -0500 Subject: [PATCH 17/74] Fix bug. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 13 ++++++++++--- tools/traffic/status_verifier.go | 26 +++++++++++++++----------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 801dca21fa..05cb9444cb 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -106,18 +106,22 @@ func (reader *BlobReader) randomRead() { metadata := reader.table.GetRandom(true) if metadata == nil { // There are no blobs to read, do nothing. + fmt.Println("no blobs to read") return } + // TODO add timeout config + ctxTimeout, cancel := context.WithTimeout(*reader.ctx, time.Second*5) batchHeader, err := InvokeAndReportLatency(&reader.fetchBatchHeaderMetric, func() (*contractEigenDAServiceManager.IEigenDAServiceManagerBatchHeader, error) { return reader.chainClient.FetchBatchHeader( - *reader.ctx, + ctxTimeout, gcommon.HexToAddress("0x851356ae760d987E095750cCeb3bC6014560891C"), *metadata.batchHeaderHash, big.NewInt(int64(0)), nil) }) + cancel() if err != nil { // TODO log reader.fetchBatchHeaderFailure.Increment() @@ -128,22 +132,25 @@ func (reader *BlobReader) randomRead() { var batchHeaderHash [32]byte copy(batchHeaderHash[:], *metadata.batchHeaderHash) + // TODO add timeout config + ctxTimeout, cancel = context.WithTimeout(*reader.ctx, time.Second*5) chunks, err := InvokeAndReportLatency(&reader.readLatencyMetric, func() (*clients.BlobChunks, error) { return reader.retriever.RetrieveBlobChunks( - *reader.ctx, + ctxTimeout, batchHeaderHash, metadata.blobIndex, uint(batchHeader.ReferenceBlockNumber), batchHeader.BlobHeadersRoot, core.QuorumID(0)) }) + cancel() if err != nil { // TODO log reader.readFailureMetric.Increment() return } - reader.readFailureMetric.Increment() + reader.readSuccessMetric.Increment() chunkCount := chunks.AssignmentInfo.TotalChunks diff --git a/tools/traffic/status_verifier.go b/tools/traffic/status_verifier.go index 01a2d7600c..e2bf5e5b6f 100644 --- a/tools/traffic/status_verifier.go +++ b/tools/traffic/status_verifier.go @@ -2,7 +2,6 @@ package traffic import ( "context" - "encoding/base64" "fmt" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/api/grpc/disperser" @@ -115,11 +114,13 @@ func (verifier *StatusVerifier) poll(ctx context.Context) { // checkStatusForBlob checks the status of a blob. Returns true if the final blob status is known, false otherwise. func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]byte) bool { - // TODO add timeout + // TODO add timeout config + ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() status, err := InvokeAndReportLatency[*disperser.BlobStatusReply](&verifier.getStatusLatencyMetric, func() (*disperser.BlobStatusReply, error) { - return (*verifier.dispenser).GetBlobStatus(ctx, *key) + return (*verifier.dispenser).GetBlobStatus(ctxTimeout, *key) }) if err != nil { @@ -149,17 +150,11 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]b case disperser.BlobStatus_CONFIRMED: verifier.confirmedCountMetric.Increment() + verifier.forwardToReader(key, status) return true case disperser.BlobStatus_FINALIZED: verifier.finalizedCountMetric.Increment() - batchHeaderHash := status.GetInfo().BlobVerificationProof.BatchMetadata.BatchHeaderHash - blobIndex := status.GetInfo().BlobVerificationProof.GetBlobIndex() - - blobMetadata := NewBlobMetadata(key, &batchHeaderHash, blobIndex, -1) // TODO permits - verifier.table.Add(blobMetadata) - fmt.Printf("Confirmed blob, batch header hash: %s, blobIndex %d\n", base64.StdEncoding.EncodeToString(batchHeaderHash), blobIndex) - //fmt.Println("Confirmed blob") - + verifier.forwardToReader(key, status) return true default: @@ -167,3 +162,12 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]b return true } } + +// forwardToReader forwards a blob to the reader. Only called once the blob is ready to be read. +func (verifier *StatusVerifier) forwardToReader(key *[]byte, status *disperser.BlobStatusReply) { + batchHeaderHash := status.GetInfo().BlobVerificationProof.BatchMetadata.BatchHeaderHash + blobIndex := status.GetInfo().BlobVerificationProof.GetBlobIndex() + + blobMetadata := NewBlobMetadata(key, &batchHeaderHash, blobIndex, -1) // TODO permits + verifier.table.Add(blobMetadata) +} From 68a6a172d5bfedd96d7551773f385256edd8ac54 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 24 Jul 2024 13:33:15 -0500 Subject: [PATCH 18/74] Added confirmation metric. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 1 - tools/traffic/blob_writer.go | 14 ------------- tools/traffic/generator.go | 24 ---------------------- tools/traffic/status_verifier.go | 35 +++++++++++++++++++++++--------- 4 files changed, 25 insertions(+), 49 deletions(-) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 05cb9444cb..70adeffed9 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -106,7 +106,6 @@ func (reader *BlobReader) randomRead() { metadata := reader.table.GetRandom(true) if metadata == nil { // There are no blobs to read, do nothing. - fmt.Println("no blobs to read") return } diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 87630be099..71f888d336 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -118,25 +118,11 @@ func (writer *BlobWriter) getRandomData() *[]byte { // TODO: get explanation why this is necessary data = codec.ConvertByPaddingEmptyByte(data) - // TODO remove - //if !fft.IsPowerOfTwo(uint64(len(data))) { - // p := uint32(math.Log2(float64(len(data)))) + 1 - // newSize := 1 << p - // bytesToAdd := newSize - len(data) - // for i := 0; i < bytesToAdd; i++ { - // data = append(data, 0) - // } - //} - return &data } // sendRequest sends a blob to a disperser. func (writer *BlobWriter) sendRequest(data []byte) ([]byte /* key */, error) { - // TODO remove - //if !fft.IsPowerOfTwo(uint64(len(data))) { - // return nil, fmt.Errorf("data length must be a power of two, data size = %d", len(data)) - //} // TODO add timeout to other types of requests ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.generator.Config.Timeout) diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 2e508fc5ec..1bde196357 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -106,8 +106,6 @@ func (g *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retriveret cs := eth.NewChainState(tx, gethClient) - // ------------- - // This is the indexer when config.UseGraph is true chainStateConfig := thegraph.Config{ Endpoint: "http://localhost:8000/subgraphs/name/Layr-Labs/eigenda-operator-state", @@ -116,28 +114,6 @@ func (g *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retriveret } chainState := thegraph.MakeIndexedChainState(chainStateConfig, cs, logger) - // This is the indexer when config.UseGraph is false. - //rpcClient, err := rpc.Dial("http://localhost:8545") - //indexerConfig := indexer.Config{ - // PullInterval: time.Second, - //} - //indexer, err := coreindexer.CreateNewIndexer( - // &indexerConfig, - // gethClient, - // rpcClient, - // "0x851356ae760d987E095750cCeb3bC6014560891C", // eigenDaServeManagerAddr - // logger, - //) - //if err != nil { - // panic(err) // TODO - //} - //chainState, err := coreindexer.NewIndexedChainState(cs, indexer) - //if err != nil { - // panic(err) // TODO - //} - - // ------------- - var assignmentCoordinator core.AssignmentCoordinator = &core.StdAssignmentCoordinator{} nodeClient := clients.NewNodeClient(10 * time.Second) diff --git a/tools/traffic/status_verifier.go b/tools/traffic/status_verifier.go index e2bf5e5b6f..317d7b1208 100644 --- a/tools/traffic/status_verifier.go +++ b/tools/traffic/status_verifier.go @@ -8,6 +8,12 @@ import ( "time" ) +// unconfirmedKey is a key that has not yet been confirmed by the disperser service. +type unconfirmedKey struct { + key *[]byte + submissionTime time.Time +} + // StatusVerifier periodically polls the disperser service to verify the status of blobs that were recently written. // When blobs become confirmed, the status verifier updates the blob table accordingly. // This is a thread safe data structure. @@ -23,13 +29,14 @@ type StatusVerifier struct { dispenser *clients.DisperserClient // The keys of blobs that have not yet been confirmed by the disperser service. - unconfirmedKeys []*[]byte + unconfirmedKeys []*unconfirmedKey // Newly added keys that require verification. - keyChannel chan *[]byte + keyChannel chan *unconfirmedKey blobsInFlightMetric GaugeMetric getStatusLatencyMetric LatencyMetric + confirmationLatencyMetric LatencyMetric getStatusErrorCountMetric CountMetric unknownCountMetric CountMetric processingCountMetric CountMetric @@ -53,10 +60,11 @@ func NewStatusVerifier( table: table, blobReadLimit: blobReadLimit, dispenser: disperser, - unconfirmedKeys: make([]*[]byte, 0), - keyChannel: make(chan *[]byte), + unconfirmedKeys: make([]*unconfirmedKey, 0), + keyChannel: make(chan *unconfirmedKey), blobsInFlightMetric: metrics.NewGaugeMetric("blobsInFlight"), getStatusLatencyMetric: metrics.NewLatencyMetric("getStatus"), + confirmationLatencyMetric: metrics.NewLatencyMetric("confirmation"), getStatusErrorCountMetric: metrics.NewCountMetric("getStatusError"), unknownCountMetric: metrics.NewCountMetric("getStatusUnknown"), processingCountMetric: metrics.NewCountMetric("getStatusProcessing"), @@ -70,7 +78,10 @@ func NewStatusVerifier( // AddUnconfirmedKey adds a key to the list of unconfirmed keys. func (verifier *StatusVerifier) AddUnconfirmedKey(key *[]byte) { - verifier.keyChannel <- key + verifier.keyChannel <- &unconfirmedKey{ + key: key, + submissionTime: time.Now(), + } } // Start begins the status goroutine, which periodically polls @@ -100,7 +111,7 @@ func (verifier *StatusVerifier) poll(ctx context.Context) { // TODO If the number of unconfirmed blobs is high and the time to confirm his high, this is not efficient. - unconfirmedKeys := make([]*[]byte, 0) + unconfirmedKeys := make([]*unconfirmedKey, 0) for _, key := range verifier.unconfirmedKeys { confirmed := verifier.checkStatusForBlob(ctx, key) if !confirmed { @@ -112,7 +123,7 @@ func (verifier *StatusVerifier) poll(ctx context.Context) { } // checkStatusForBlob checks the status of a blob. Returns true if the final blob status is known, false otherwise. -func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]byte) bool { +func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *unconfirmedKey) bool { // TODO add timeout config ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*5) @@ -120,7 +131,7 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]b status, err := InvokeAndReportLatency[*disperser.BlobStatusReply](&verifier.getStatusLatencyMetric, func() (*disperser.BlobStatusReply, error) { - return (*verifier.dispenser).GetBlobStatus(ctxTimeout, *key) + return (*verifier.dispenser).GetBlobStatus(ctxTimeout, *key.key) }) if err != nil { @@ -164,10 +175,14 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *[]b } // forwardToReader forwards a blob to the reader. Only called once the blob is ready to be read. -func (verifier *StatusVerifier) forwardToReader(key *[]byte, status *disperser.BlobStatusReply) { +func (verifier *StatusVerifier) forwardToReader(key *unconfirmedKey, status *disperser.BlobStatusReply) { batchHeaderHash := status.GetInfo().BlobVerificationProof.BatchMetadata.BatchHeaderHash blobIndex := status.GetInfo().BlobVerificationProof.GetBlobIndex() - blobMetadata := NewBlobMetadata(key, &batchHeaderHash, blobIndex, -1) // TODO permits + confirmationTime := time.Now() + confirmationLatency := confirmationTime.Sub(key.submissionTime) + verifier.confirmationLatencyMetric.ReportLatency(confirmationLatency) + + blobMetadata := NewBlobMetadata(key.key, &batchHeaderHash, blobIndex, -1) // TODO permits verifier.table.Add(blobMetadata) } From c098c352b3a42dba52fe3e600aec20dc99e8ba84 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 24 Jul 2024 14:28:08 -0500 Subject: [PATCH 19/74] Cleanup. Signed-off-by: Cody Littley --- api/clients/eigenda_client_e2e_test.go | 2 +- disperser/apiserver/server.go | 2 +- disperser/cmd/dataapi/main.go | 2 +- node/node.go | 2 +- node/plugin/cmd/main.go | 2 +- retriever/cmd/main.go | 2 +- test/synthetic-test/synthetic_client_test.go | 8 +- tools/traffic/blob_reader.go | 32 +++---- tools/traffic/blob_writer.go | 78 ++++++++++-------- tools/traffic/cmd/main.go | 2 +- tools/traffic/generator.go | 87 +++++++++++++------- tools/traffic/generator_test.go | 12 +-- tools/traffic/status_verifier.go | 44 ++++++---- 13 files changed, 166 insertions(+), 109 deletions(-) diff --git a/api/clients/eigenda_client_e2e_test.go b/api/clients/eigenda_client_e2e_test.go index f10a6aec91..235f1200ad 100644 --- a/api/clients/eigenda_client_e2e_test.go +++ b/api/clients/eigenda_client_e2e_test.go @@ -15,7 +15,7 @@ import ( var runTestnetIntegrationTests bool func init() { - flag.BoolVar(&runTestnetIntegrationTests, "testnet-integration", false, "Run testnet-based integration tests") + flag.BoolVar(&runTestnetIntegrationTests, "testnet-integration", false, "Start testnet-based integration tests") } func TestClientUsingTestnet(t *testing.T) { diff --git a/disperser/apiserver/server.go b/disperser/apiserver/server.go index 753b7c55a8..cfe1f861e3 100644 --- a/disperser/apiserver/server.go +++ b/disperser/apiserver/server.go @@ -158,7 +158,7 @@ func (s *DispersalServer) DisperseBlobAuthenticated(stream pb.Disperser_Disperse resultCh := make(chan *pb.AuthenticatedRequest) errCh := make(chan error) - // Run stream.Recv() in a goroutine + // Start stream.Recv() in a goroutine go func() { in, err := stream.Recv() if err != nil { diff --git a/disperser/cmd/dataapi/main.go b/disperser/cmd/dataapi/main.go index abddf5b1ea..7b9bcd1750 100644 --- a/disperser/cmd/dataapi/main.go +++ b/disperser/cmd/dataapi/main.go @@ -129,7 +129,7 @@ func RunDataApi(ctx *cli.Context) error { // catch SIGINT (Ctrl+C) and SIGTERM (e.g., from `kill`) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - // Run server in a separate goroutine so that it doesn't block. + // Start server in a separate goroutine so that it doesn't block. go func() { if err := server.Start(); err != nil { logger.Fatalf("Failed to start server: %v", err) diff --git a/node/node.go b/node/node.go index 572080d51c..b9b1da490a 100644 --- a/node/node.go +++ b/node/node.go @@ -314,7 +314,7 @@ func (n *Node) ProcessBatch(ctx context.Context, header *core.BatchHeader, blobs log.Debug("Start processing a batch", "batchHeaderHash", batchHeaderHashHex, "batchSize (in bytes)", batchSize, "num of blobs", len(blobs), "referenceBlockNumber", header.ReferenceBlockNumber) // Store the batch. - // Run this in a goroutine so we can parallelize the batch storing and batch + // Start this in a goroutine so we can parallelize the batch storing and batch // verifaction work. // This should be able to improve latency without needing more CPUs, because batch // storing is an IO operation. diff --git a/node/plugin/cmd/main.go b/node/plugin/cmd/main.go index c50aa88b61..3e80679d0d 100644 --- a/node/plugin/cmd/main.go +++ b/node/plugin/cmd/main.go @@ -37,7 +37,7 @@ func main() { } app.Name = "eigenda-node-plugin" app.Usage = "EigenDA Node Plugin" - app.Description = "Run one time operations like avs opt-in/opt-out for EigenDA Node" + app.Description = "Start one time operations like avs opt-in/opt-out for EigenDA Node" app.Action = pluginOps err := app.Run(os.Args) if err != nil { diff --git a/retriever/cmd/main.go b/retriever/cmd/main.go index aa58c3dc5f..7065123479 100644 --- a/retriever/cmd/main.go +++ b/retriever/cmd/main.go @@ -64,7 +64,7 @@ func RetrieverMain(ctx *cli.Context) error { grpc.ChainUnaryInterceptor( // TODO(ian-shim): Add interceptors // correlation.UnaryServerInterceptor(), - // logger.UnaryServerInterceptor(*s.logger.Logger), + // logger.UnaryServerInterceptor(*s.logger.logger), ), ) diff --git a/test/synthetic-test/synthetic_client_test.go b/test/synthetic-test/synthetic_client_test.go index ab868c557a..0d99b7afc8 100644 --- a/test/synthetic-test/synthetic_client_test.go +++ b/test/synthetic-test/synthetic_client_test.go @@ -51,7 +51,7 @@ type GrpcClient struct { Hostname string GrpcPort string Timeout time.Duration - Client interface{} // This can be a specific client type (disperser_rpc.DisperserClient, retriever_rpc.RetrieverClient, etc.) + Client interface{} // This can be a specific client type (disperser_rpc.disperserClient, retriever_rpc.RetrieverClient, etc.) } type RetrieverClientConfig struct { @@ -178,7 +178,7 @@ func TestMain(m *testing.M) { retrieverCachePath := os.Getenv("RETRIEVER_CACHE_PATH") batcherPullInterval := os.Getenv("BATCHER_PULL_INTERVAL") - // Retriever Config + // Retriever config retrieverClientConfig := &RetrieverClientConfig{ Bls_Operator_State_Retriever: blsOperatorStateRetriever, EigenDA_ServiceManager_Retriever: eigenDAServiceManagerRetreiever, @@ -204,9 +204,9 @@ func TestMain(m *testing.M) { logger.Println("Retriever Client Enabled...", isRetrieverClientEnabled) logger.Println("Running Test Client...") - // Run the tests and get the exit code + // Start the tests and get the exit code exitCode := m.Run() - logger.Printf("Exiting Test Client Run with Code:%d", exitCode) + logger.Printf("Exiting Test Client Start with Code:%d", exitCode) // Exit with the test result code os.Exit(exitCode) } diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 70adeffed9..2258475f1c 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -8,6 +8,7 @@ import ( "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/retriever/eth" + "github.com/Layr-Labs/eigensdk-go/logging" gcommon "github.com/ethereum/go-ethereum/common" "math/big" "sync" @@ -22,7 +23,10 @@ type BlobReader struct { ctx *context.Context // Tracks the number of active goroutines within the generator. - waitGroup *sync.WaitGroup + waitGroup *sync.WaitGroup // TODO other things should use this too + + // All logs should be written using this logger. + logger logging.Logger // TODO use code from this class retriever clients.RetrievalClient @@ -31,8 +35,7 @@ type BlobReader struct { // table of blobs to read from. table *BlobTable - metrics *Metrics - + metrics *Metrics fetchBatchHeaderMetric LatencyMetric fetchBatchHeaderSuccess CountMetric fetchBatchHeaderFailure CountMetric @@ -50,6 +53,7 @@ type BlobReader struct { func NewBlobReader( ctx *context.Context, waitGroup *sync.WaitGroup, + logger logging.Logger, retriever clients.RetrievalClient, chainClient eth.ChainClient, table *BlobTable, @@ -58,6 +62,7 @@ func NewBlobReader( return BlobReader{ ctx: ctx, waitGroup: waitGroup, + logger: logger, retriever: retriever, chainClient: chainClient, table: table, @@ -122,7 +127,7 @@ func (reader *BlobReader) randomRead() { }) cancel() if err != nil { - // TODO log + reader.logger.Error("failed to get batch header", "err:", err) reader.fetchBatchHeaderFailure.Increment() return } @@ -143,32 +148,25 @@ func (reader *BlobReader) randomRead() { core.QuorumID(0)) }) cancel() - if err != nil { - // TODO log + reader.logger.Error("failed to read chunks", "err:", err) reader.readFailureMetric.Increment() return } reader.readSuccessMetric.Increment() - chunkCount := chunks.AssignmentInfo.TotalChunks - var assignments map[core.OperatorID]core.Assignment assignments = chunks.Assignments data, err := reader.retriever.CombineChunks(chunks) - if err != nil { - fmt.Println("Error combining chunks:", err) // TODO + reader.logger.Error("failed to combine chunks", "err:", err) reader.recombinationFailure.Increment() return } reader.recombinationSuccess.Increment() - // TODO verify blob data - - fmt.Printf("=====================================\nRead blob. Total chunk count = %d\nRetrieved chunk count = %d\nData length = %d\n", - chunkCount, len(chunks.Chunks), len(data)) // TODO + reader.verifyBlob(data) indexSet := make(map[encoding.ChunkNumber]bool) for index := range chunks.Indices { @@ -176,7 +174,6 @@ func (reader *BlobReader) randomRead() { } for id, assignment := range assignments { - fmt.Printf(" - Operator ID: %d, Start Index: %d, Num Chunks: %d\n", id, assignment.StartIndex, assignment.NumChunks) for index := assignment.StartIndex; index < assignment.StartIndex+assignment.NumChunks; index++ { if indexSet[index] { reader.reportChunk(id) @@ -208,3 +205,8 @@ func (reader *BlobReader) reportMissingChunk(operatorId core.OperatorID) { metric.Increment() } + +// verifyBlob performs sanity checks on the blob. +func (reader *BlobReader) verifyBlob(blob []byte) { + // TODO +} diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 71f888d336..b83c6342ce 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -3,7 +3,10 @@ package traffic import ( "context" "crypto/rand" + "fmt" + "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/Layr-Labs/eigensdk-go/logging" "sync" "time" ) @@ -16,8 +19,14 @@ type BlobWriter struct { // Tracks the number of active goroutines within the generator. waitGroup *sync.WaitGroup - // TODO this type should be refactored maybe - generator *TrafficGenerator + // All logs should be written using this logger. + logger logging.Logger + + // Config contains the configuration for the generator. + config *Config + + // disperser is the client used to send blobs to the disperser. + disperser *clients.DisperserClient // Responsible for polling on the status of a recently written blob until it becomes confirmed. verifier *StatusVerifier @@ -39,20 +48,22 @@ type BlobWriter struct { func NewBlobWriter( ctx *context.Context, waitGroup *sync.WaitGroup, - generator *TrafficGenerator, + logger logging.Logger, + config *Config, + disperser *clients.DisperserClient, verifier *StatusVerifier, metrics *Metrics) BlobWriter { var fixedRandomData []byte - if generator.Config.RandomizeBlobs { + if config.RandomizeBlobs { // New random data will be generated for each blob. fixedRandomData = nil } else { // Use this random data for each blob. - fixedRandomData := make([]byte, generator.Config.DataSize) + fixedRandomData := make([]byte, config.DataSize) _, err := rand.Read(fixedRandomData) if err != nil { - panic(err) + panic(fmt.Sprintf("unable to read random data: %s", err)) } fixedRandomData = codec.ConvertByPaddingEmptyByte(fixedRandomData) } @@ -60,7 +71,9 @@ func NewBlobWriter( return BlobWriter{ ctx: ctx, waitGroup: waitGroup, - generator: generator, + logger: logger, + config: config, + disperser: disperser, verifier: verifier, fixedRandomData: &fixedRandomData, writeLatencyMetric: metrics.NewLatencyMetric("write"), @@ -81,7 +94,7 @@ func (writer *BlobWriter) Start() { // run sends blobs to a disperser at a configured rate. // Continues and dues not return until the context is cancelled. func (writer *BlobWriter) run() { - ticker := time.NewTicker(writer.generator.Config.WriteRequestInterval) + ticker := time.NewTicker(writer.config.WriteRequestInterval) for { select { case <-(*writer.ctx).Done(): @@ -93,7 +106,7 @@ func (writer *BlobWriter) run() { }) if err != nil { writer.writeFailureMetric.Increment() - writer.generator.Logger.Error("failed to send blob request", "err:", err) + writer.logger.Error("failed to send blob request", "err:", err) continue } @@ -105,17 +118,15 @@ func (writer *BlobWriter) run() { // getRandomData returns a slice of random data to be used for a blob. func (writer *BlobWriter) getRandomData() *[]byte { - // TODO - //if *writer.fixedRandomData != nil { - // return writer.fixedRandomData - //} + if *writer.fixedRandomData != nil { + return writer.fixedRandomData + } - data := make([]byte, writer.generator.Config.DataSize) + data := make([]byte, writer.config.DataSize) _, err := rand.Read(data) if err != nil { - panic(err) + panic(fmt.Sprintf("unable to read random data: %s", err)) } - // TODO: get explanation why this is necessary data = codec.ConvertByPaddingEmptyByte(data) return &data @@ -125,27 +136,24 @@ func (writer *BlobWriter) getRandomData() *[]byte { func (writer *BlobWriter) sendRequest(data []byte) ([]byte /* key */, error) { // TODO add timeout to other types of requests - ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.generator.Config.Timeout) + ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.config.Timeout) defer cancel() - if writer.generator.Config.SignerPrivateKey != "" { - _, key, err := - writer.generator.DisperserClient.DisperseBlobAuthenticated(ctxTimeout, data, writer.generator.Config.CustomQuorums) - if err != nil { - return nil, err - } - - // TODO - //writer.generator.Logger.Info("successfully dispersed new blob", "authenticated", true, "key", hex.EncodeToString(key), "status", blobStatus.String()) - return key, nil + var key []byte + var err error + if writer.config.SignerPrivateKey != "" { + _, key, err = (*writer.disperser).DisperseBlobAuthenticated( + ctxTimeout, + data, + writer.config.CustomQuorums) } else { - _, key, err := writer.generator.DisperserClient.DisperseBlob(ctxTimeout, data, writer.generator.Config.CustomQuorums) - if err != nil { - return nil, err - } - - // TODO - //writer.generator.Logger.Info("successfully dispersed new blob", "authenticated", false, "key", hex.EncodeToString(key), "status", blobStatus.String()) - return key, nil + _, key, err = (*writer.disperser).DisperseBlob( + ctxTimeout, + data, + writer.config.CustomQuorums) + } + if err != nil { + return nil, err } + return key, nil } diff --git a/tools/traffic/cmd/main.go b/tools/traffic/cmd/main.go index 160b04e931..aa401fdcc5 100644 --- a/tools/traffic/cmd/main.go +++ b/tools/traffic/cmd/main.go @@ -48,5 +48,5 @@ func trafficGeneratorMain(ctx *cli.Context) error { panic(fmt.Sprintf("failed to create new traffic generator\n%s", err)) } - return generator.Run() + return generator.Start() } diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 1bde196357..ba30d145d9 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -9,6 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" + "github.com/Layr-Labs/eigensdk-go/logging" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "os" @@ -20,9 +21,10 @@ import ( "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/core" - "github.com/Layr-Labs/eigensdk-go/logging" ) +// TODO use consistent snake case on metrics + // TrafficGenerator simulates read/write traffic to the DA service. // // ┌------------┐ ┌------------┐ @@ -39,10 +41,14 @@ import ( // When the verifier observes that a blob has been confirmed, it sends information about the blob // to the readers. The readers only attempt to read blobs that have been confirmed by the verifier. type TrafficGenerator struct { - Logger logging.Logger - DisperserClient clients.DisperserClient - EigenDAClient *clients.EigenDAClient - Config *Config + ctx *context.Context + cancel *context.CancelFunc + waitGroup *sync.WaitGroup + metrics *Metrics + logger *logging.Logger + disperserClient clients.DisperserClient + eigenDAClient *clients.EigenDAClient + config *Config } func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*TrafficGenerator, error) { @@ -70,16 +76,26 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi return nil, err } + ctx, cancel := context.WithCancel(context.Background()) + waitGroup := sync.WaitGroup{} + + metrics := NewMetrics("9101", logger) // TODO config + metrics.Start(ctx) + return &TrafficGenerator{ - Logger: logger, - DisperserClient: clients.NewDisperserClient(&config.Config, signer), - EigenDAClient: client, - Config: config, + ctx: &ctx, + cancel: &cancel, + waitGroup: &waitGroup, + metrics: metrics, + logger: &logger, + disperserClient: clients.NewDisperserClient(&config.Config, signer), + eigenDAClient: client, + config: config, }, nil } // buildRetriever creates a retriever client for the traffic generator. -func (g *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { +func (generator *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { //loggerConfig := common.LoggerConfig{ // Format: "text", @@ -151,37 +167,52 @@ func (g *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retriveret return retriever, chainClient } -// Run instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. -func (g *TrafficGenerator) Run() error { - ctx, cancel := context.WithCancel(context.Background()) - - metrics := NewMetrics("9101", g.Logger) // TODO config - metrics.Start(ctx) +// Start instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. +func (generator *TrafficGenerator) Start() error { // TODO add configuration table := NewBlobTable() - verifier := NewStatusVerifier(&table, &g.DisperserClient, -1, metrics) - verifier.Start(ctx, time.Second) - - var wg sync.WaitGroup - - for i := 0; i < int(g.Config.NumWriteInstances); i++ { - writer := NewBlobWriter(&ctx, &wg, g, &verifier, metrics) + statusVerifier := NewStatusVerifier( + generator.ctx, + generator.waitGroup, + *generator.logger, + &table, + &generator.disperserClient, + -1, + generator.metrics) + statusVerifier.Start(time.Second) + + for i := 0; i < int(generator.config.NumWriteInstances); i++ { + writer := NewBlobWriter( + generator.ctx, + generator.waitGroup, + *generator.logger, + generator.config, + &generator.disperserClient, + &statusVerifier, + generator.metrics) writer.Start() - time.Sleep(g.Config.InstanceLaunchInterval) + time.Sleep(generator.config.InstanceLaunchInterval) } - retriever, chainClient := g.buildRetriever() + retriever, chainClient := generator.buildRetriever() // TODO start multiple readers - reader := NewBlobReader(&ctx, &wg, retriever, chainClient, &table, metrics) + reader := NewBlobReader( + generator.ctx, + generator.waitGroup, + *generator.logger, + retriever, + chainClient, + &table, + generator.metrics) reader.Start() signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt, syscall.SIGTERM) <-signals - cancel() - wg.Wait() + (*generator.cancel)() + generator.waitGroup.Wait() return nil } diff --git a/tools/traffic/generator_test.go b/tools/traffic/generator_test.go index 05983e9b30..39307fa6b3 100644 --- a/tools/traffic/generator_test.go +++ b/tools/traffic/generator_test.go @@ -18,15 +18,15 @@ func TestTrafficGenerator(t *testing.T) { disperserClient := clientsmock.NewMockDisperserClient() logger := logging.NewNoopLogger() trafficGenerator := &traffic.TrafficGenerator{ - Logger: logger, - Config: &traffic.Config{ + logger: logger, + config: &traffic.Config{ Config: clients.Config{ Timeout: 1 * time.Second, }, DataSize: 1000_000, WriteRequestInterval: 2 * time.Second, }, - DisperserClient: disperserClient, + disperserClient: disperserClient, } processing := disperser.Processing @@ -46,8 +46,8 @@ func TestTrafficGeneratorAuthenticated(t *testing.T) { logger := logging.NewNoopLogger() trafficGenerator := &traffic.TrafficGenerator{ - Logger: logger, - Config: &traffic.Config{ + logger: logger, + config: &traffic.Config{ Config: clients.Config{ Timeout: 1 * time.Second, }, @@ -55,7 +55,7 @@ func TestTrafficGeneratorAuthenticated(t *testing.T) { WriteRequestInterval: 2 * time.Second, SignerPrivateKey: "Hi", }, - DisperserClient: disperserClient, + disperserClient: disperserClient, } processing := disperser.Processing diff --git a/tools/traffic/status_verifier.go b/tools/traffic/status_verifier.go index 317d7b1208..5cf1e98720 100644 --- a/tools/traffic/status_verifier.go +++ b/tools/traffic/status_verifier.go @@ -2,9 +2,10 @@ package traffic import ( "context" - "fmt" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/api/grpc/disperser" + "github.com/Layr-Labs/eigensdk-go/logging" + "sync" "time" ) @@ -19,6 +20,15 @@ type unconfirmedKey struct { // This is a thread safe data structure. type StatusVerifier struct { + // The context for the generator. All work should cease when this context is cancelled. + ctx *context.Context + + // Tracks the number of active goroutines within the generator. + waitGroup *sync.WaitGroup + + // All logs should be written using this logger. + logger logging.Logger + // A table of confirmed blobs. Blobs are added here when they are confirmed by the disperser service. table *BlobTable @@ -45,18 +55,22 @@ type StatusVerifier struct { insufficientSignaturesCountMetric CountMetric confirmedCountMetric CountMetric finalizedCountMetric CountMetric - - // TODO metric for average time for blob to become confirmed } // NewStatusVerifier creates a new StatusVerifier instance. func NewStatusVerifier( + ctx *context.Context, + waitGroup *sync.WaitGroup, + logger logging.Logger, table *BlobTable, disperser *clients.DisperserClient, blobReadLimit int32, metrics *Metrics) StatusVerifier { return StatusVerifier{ + ctx: ctx, + waitGroup: waitGroup, + logger: logger, table: table, blobReadLimit: blobReadLimit, dispenser: disperser, @@ -86,34 +100,36 @@ func (verifier *StatusVerifier) AddUnconfirmedKey(key *[]byte) { // Start begins the status goroutine, which periodically polls // the disperser service to verify the status of blobs. -func (verifier *StatusVerifier) Start(ctx context.Context, period time.Duration) { - go verifier.monitor(ctx, period) +func (verifier *StatusVerifier) Start(period time.Duration) { + verifier.waitGroup.Add(1) + go verifier.monitor(period) } // monitor periodically polls the disperser service to verify the status of blobs. -func (verifier *StatusVerifier) monitor(ctx context.Context, period time.Duration) { +func (verifier *StatusVerifier) monitor(period time.Duration) { ticker := time.NewTicker(period) for { select { - case <-ctx.Done(): + case <-(*verifier.ctx).Done(): + verifier.waitGroup.Done() return case key := <-verifier.keyChannel: verifier.unconfirmedKeys = append(verifier.unconfirmedKeys, key) case <-ticker.C: - verifier.poll(ctx) + verifier.poll() } } } // poll checks all unconfirmed keys to see if they have been confirmed by the disperser service. // If a key is confirmed, it is added to the blob table and removed from the list of unconfirmed keys. -func (verifier *StatusVerifier) poll(ctx context.Context) { +func (verifier *StatusVerifier) poll() { // TODO If the number of unconfirmed blobs is high and the time to confirm his high, this is not efficient. unconfirmedKeys := make([]*unconfirmedKey, 0) for _, key := range verifier.unconfirmedKeys { - confirmed := verifier.checkStatusForBlob(ctx, key) + confirmed := verifier.checkStatusForBlob(key) if !confirmed { unconfirmedKeys = append(unconfirmedKeys, key) } @@ -123,10 +139,10 @@ func (verifier *StatusVerifier) poll(ctx context.Context) { } // checkStatusForBlob checks the status of a blob. Returns true if the final blob status is known, false otherwise. -func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *unconfirmedKey) bool { +func (verifier *StatusVerifier) checkStatusForBlob(key *unconfirmedKey) bool { // TODO add timeout config - ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*5) + ctxTimeout, cancel := context.WithTimeout(*verifier.ctx, time.Second*5) defer cancel() status, err := InvokeAndReportLatency[*disperser.BlobStatusReply](&verifier.getStatusLatencyMetric, @@ -135,7 +151,7 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *unc }) if err != nil { - fmt.Println("Error getting blob status:", err) // TODO is this proper? + verifier.logger.Error("failed check blob status", "err:", err) verifier.getStatusErrorCountMetric.Increment() return false } @@ -169,7 +185,7 @@ func (verifier *StatusVerifier) checkStatusForBlob(ctx context.Context, key *unc return true default: - fmt.Println("Unknown blob status:", status.GetStatus()) // TODO use logger + verifier.logger.Error("unknown blob status", "status:", status.GetStatus()) return true } } From de4c7a7a871dacacd36df5aaebe016e2108e808d Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 24 Jul 2024 15:47:32 -0500 Subject: [PATCH 20/74] Cleanup. Signed-off-by: Cody Littley --- tools/traffic/generator.go | 92 ++++++++++++++++++++++++-------------- tools/traffic/metrics.go | 3 +- 2 files changed, 60 insertions(+), 35 deletions(-) diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index ba30d145d9..fc7ba32f45 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -49,6 +49,10 @@ type TrafficGenerator struct { disperserClient clients.DisperserClient eigenDAClient *clients.EigenDAClient config *Config + + writers []*BlobWriter + verifier *StatusVerifier + readers []*BlobReader } func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*TrafficGenerator, error) { @@ -80,7 +84,49 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi waitGroup := sync.WaitGroup{} metrics := NewMetrics("9101", logger) // TODO config - metrics.Start(ctx) + + // TODO add configuration + + table := NewBlobTable() + + disperserClient := clients.NewDisperserClient(&config.Config, signer) + statusVerifier := NewStatusVerifier( + &ctx, + &waitGroup, + logger, + &table, + &disperserClient, + -1, + metrics) + + writers := make([]*BlobWriter, 0) + for i := 0; i < int(config.NumWriteInstances); i++ { + writer := NewBlobWriter( + &ctx, + &waitGroup, + logger, + config, + &disperserClient, + &statusVerifier, + metrics) + writers = append(writers, &writer) + } + + retriever, chainClient := buildRetriever() + + readers := make([]*BlobReader, 0) + //int(config.NumReadInstances) // TODO + for i := 0; i < 1; i++ { + reader := NewBlobReader( + &ctx, + &waitGroup, + logger, + retriever, + chainClient, + &table, + metrics) + readers = append(readers, &reader) + } return &TrafficGenerator{ ctx: &ctx, @@ -91,11 +137,14 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi disperserClient: clients.NewDisperserClient(&config.Config, signer), eigenDAClient: client, config: config, + writers: writers, + verifier: &statusVerifier, + readers: readers, }, nil } // buildRetriever creates a retriever client for the traffic generator. -func (generator *TrafficGenerator) buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { +func buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { //loggerConfig := common.LoggerConfig{ // Format: "text", @@ -171,42 +220,19 @@ func (generator *TrafficGenerator) buildRetriever() (clients.RetrievalClient, re func (generator *TrafficGenerator) Start() error { // TODO add configuration - table := NewBlobTable() - statusVerifier := NewStatusVerifier( - generator.ctx, - generator.waitGroup, - *generator.logger, - &table, - &generator.disperserClient, - -1, - generator.metrics) - statusVerifier.Start(time.Second) - for i := 0; i < int(generator.config.NumWriteInstances); i++ { - writer := NewBlobWriter( - generator.ctx, - generator.waitGroup, - *generator.logger, - generator.config, - &generator.disperserClient, - &statusVerifier, - generator.metrics) + generator.metrics.Start(*generator.ctx) // TODO put context into metrics constructor + generator.verifier.Start(time.Second) + + for _, writer := range generator.writers { writer.Start() time.Sleep(generator.config.InstanceLaunchInterval) } - retriever, chainClient := generator.buildRetriever() - - // TODO start multiple readers - reader := NewBlobReader( - generator.ctx, - generator.waitGroup, - *generator.logger, - retriever, - chainClient, - &table, - generator.metrics) - reader.Start() + for _, reader := range generator.readers { + reader.Start() + time.Sleep(generator.config.InstanceLaunchInterval) + } signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt, syscall.SIGTERM) diff --git a/tools/traffic/metrics.go b/tools/traffic/metrics.go index f8fa88f138..d5a1798838 100644 --- a/tools/traffic/metrics.go +++ b/tools/traffic/metrics.go @@ -81,14 +81,13 @@ func (metrics *Metrics) Start(ctx context.Context) { // TODO context? metrics.logger.Info("Starting metrics server at ", "port", metrics.httpPort) addr := fmt.Sprintf(":%s", metrics.httpPort) go func() { - log := metrics.logger mux := http.NewServeMux() mux.Handle("/metrics", promhttp.HandlerFor( metrics.registry, promhttp.HandlerOpts{}, )) err := http.ListenAndServe(addr, mux) - log.Error("Prometheus server failed", "err", err) + panic(fmt.Sprintf("Prometheus server failed: %s", err)) }() } From c9de527c6199422fc853d785c2ba2251d88f6f27 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 07:19:34 -0500 Subject: [PATCH 21/74] Renamed verifier. Signed-off-by: Cody Littley --- tools/traffic/blob_writer.go | 4 ++-- tools/traffic/generator.go | 2 +- tools/traffic/status_verifier.go | 22 +++++++++++----------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index b83c6342ce..322a1a96d7 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -29,7 +29,7 @@ type BlobWriter struct { disperser *clients.DisperserClient // Responsible for polling on the status of a recently written blob until it becomes confirmed. - verifier *StatusVerifier + verifier *BlobVerifier // fixedRandomData contains random data for blobs if RandomizeBlobs is false, and nil otherwise. fixedRandomData *[]byte @@ -51,7 +51,7 @@ func NewBlobWriter( logger logging.Logger, config *Config, disperser *clients.DisperserClient, - verifier *StatusVerifier, + verifier *BlobVerifier, metrics *Metrics) BlobWriter { var fixedRandomData []byte diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index fc7ba32f45..6be125285e 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -51,7 +51,7 @@ type TrafficGenerator struct { config *Config writers []*BlobWriter - verifier *StatusVerifier + verifier *BlobVerifier readers []*BlobReader } diff --git a/tools/traffic/status_verifier.go b/tools/traffic/status_verifier.go index 5cf1e98720..7a63278899 100644 --- a/tools/traffic/status_verifier.go +++ b/tools/traffic/status_verifier.go @@ -15,10 +15,10 @@ type unconfirmedKey struct { submissionTime time.Time } -// StatusVerifier periodically polls the disperser service to verify the status of blobs that were recently written. +// BlobVerifier periodically polls the disperser service to verify the status of blobs that were recently written. // When blobs become confirmed, the status verifier updates the blob table accordingly. // This is a thread safe data structure. -type StatusVerifier struct { +type BlobVerifier struct { // The context for the generator. All work should cease when this context is cancelled. ctx *context.Context @@ -57,7 +57,7 @@ type StatusVerifier struct { finalizedCountMetric CountMetric } -// NewStatusVerifier creates a new StatusVerifier instance. +// NewStatusVerifier creates a new BlobVerifier instance. func NewStatusVerifier( ctx *context.Context, waitGroup *sync.WaitGroup, @@ -65,9 +65,9 @@ func NewStatusVerifier( table *BlobTable, disperser *clients.DisperserClient, blobReadLimit int32, - metrics *Metrics) StatusVerifier { + metrics *Metrics) BlobVerifier { - return StatusVerifier{ + return BlobVerifier{ ctx: ctx, waitGroup: waitGroup, logger: logger, @@ -91,7 +91,7 @@ func NewStatusVerifier( } // AddUnconfirmedKey adds a key to the list of unconfirmed keys. -func (verifier *StatusVerifier) AddUnconfirmedKey(key *[]byte) { +func (verifier *BlobVerifier) AddUnconfirmedKey(key *[]byte) { verifier.keyChannel <- &unconfirmedKey{ key: key, submissionTime: time.Now(), @@ -100,13 +100,13 @@ func (verifier *StatusVerifier) AddUnconfirmedKey(key *[]byte) { // Start begins the status goroutine, which periodically polls // the disperser service to verify the status of blobs. -func (verifier *StatusVerifier) Start(period time.Duration) { +func (verifier *BlobVerifier) Start(period time.Duration) { verifier.waitGroup.Add(1) go verifier.monitor(period) } // monitor periodically polls the disperser service to verify the status of blobs. -func (verifier *StatusVerifier) monitor(period time.Duration) { +func (verifier *BlobVerifier) monitor(period time.Duration) { ticker := time.NewTicker(period) for { select { @@ -123,7 +123,7 @@ func (verifier *StatusVerifier) monitor(period time.Duration) { // poll checks all unconfirmed keys to see if they have been confirmed by the disperser service. // If a key is confirmed, it is added to the blob table and removed from the list of unconfirmed keys. -func (verifier *StatusVerifier) poll() { +func (verifier *BlobVerifier) poll() { // TODO If the number of unconfirmed blobs is high and the time to confirm his high, this is not efficient. @@ -139,7 +139,7 @@ func (verifier *StatusVerifier) poll() { } // checkStatusForBlob checks the status of a blob. Returns true if the final blob status is known, false otherwise. -func (verifier *StatusVerifier) checkStatusForBlob(key *unconfirmedKey) bool { +func (verifier *BlobVerifier) checkStatusForBlob(key *unconfirmedKey) bool { // TODO add timeout config ctxTimeout, cancel := context.WithTimeout(*verifier.ctx, time.Second*5) @@ -191,7 +191,7 @@ func (verifier *StatusVerifier) checkStatusForBlob(key *unconfirmedKey) bool { } // forwardToReader forwards a blob to the reader. Only called once the blob is ready to be read. -func (verifier *StatusVerifier) forwardToReader(key *unconfirmedKey, status *disperser.BlobStatusReply) { +func (verifier *BlobVerifier) forwardToReader(key *unconfirmedKey, status *disperser.BlobStatusReply) { batchHeaderHash := status.GetInfo().BlobVerificationProof.BatchMetadata.BatchHeaderHash blobIndex := status.GetInfo().BlobVerificationProof.GetBlobIndex() From 64c7ae974fb956c90b76c1f4f0c983342feba5e0 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 08:14:54 -0500 Subject: [PATCH 22/74] Added some settings to the reader. Signed-off-by: Cody Littley --- api/clients/retrieval_client.go | 2 - tools/traffic/blob_reader.go | 7 +- .../{status_verifier.go => blob_verifier.go} | 35 ++++-- tools/traffic/config.go | 56 +++++----- tools/traffic/flags/flags.go | 102 ++++++++++++------ tools/traffic/generator.go | 6 +- 6 files changed, 139 insertions(+), 69 deletions(-) rename tools/traffic/{status_verifier.go => blob_verifier.go} (90%) diff --git a/api/clients/retrieval_client.go b/api/clients/retrieval_client.go index 60b9c42b13..a00ae9f0ae 100644 --- a/api/clients/retrieval_client.go +++ b/api/clients/retrieval_client.go @@ -107,12 +107,10 @@ func (r *retrievalClient) RetrieveBlobChunks(ctx context.Context, batchRoot [32]byte, quorumID core.QuorumID) (*BlobChunks, error) { - fmt.Printf(" getting indexed operator state, referenceBlockNumber: %d, quorumId: %d\n", referenceBlockNumber, quorumID) // TODO indexedOperatorState, err := r.indexedChainState.GetIndexedOperatorState(ctx, referenceBlockNumber, []core.QuorumID{quorumID}) if err != nil { return nil, err } - fmt.Println("getting operators") // TODO operators, ok := indexedOperatorState.Operators[quorumID] if !ok { return nil, fmt.Errorf(" no quorum with ID: %d", quorumID) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 2258475f1c..29ee361ec8 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -28,6 +28,9 @@ type BlobReader struct { // All logs should be written using this logger. logger logging.Logger + // config contains the configuration for the generator. + config *Config + // TODO use code from this class retriever clients.RetrievalClient chainClient eth.ChainClient @@ -54,6 +57,7 @@ func NewBlobReader( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, + config *Config, retriever clients.RetrievalClient, chainClient eth.ChainClient, table *BlobTable, @@ -63,6 +67,7 @@ func NewBlobReader( ctx: ctx, waitGroup: waitGroup, logger: logger, + config: config, retriever: retriever, chainClient: chainClient, table: table, @@ -92,7 +97,7 @@ func (reader *BlobReader) Start() { // run periodically performs reads on blobs. func (reader *BlobReader) run() { - ticker := time.NewTicker(time.Second) // TODO setting + ticker := time.NewTicker(reader.config.ReadRequestInterval) for { select { case <-(*reader.ctx).Done(): diff --git a/tools/traffic/status_verifier.go b/tools/traffic/blob_verifier.go similarity index 90% rename from tools/traffic/status_verifier.go rename to tools/traffic/blob_verifier.go index 7a63278899..180d983d7e 100644 --- a/tools/traffic/status_verifier.go +++ b/tools/traffic/blob_verifier.go @@ -5,6 +5,7 @@ import ( "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/api/grpc/disperser" "github.com/Layr-Labs/eigensdk-go/logging" + "math/rand" "sync" "time" ) @@ -29,12 +30,12 @@ type BlobVerifier struct { // All logs should be written using this logger. logger logging.Logger + // config contains the configuration for the generator. + config *Config + // A table of confirmed blobs. Blobs are added here when they are confirmed by the disperser service. table *BlobTable - // The maximum number of reads permitted against an individual blob, or -1 if unlimited. - blobReadLimit int32 - // The disperser client used to monitor the disperser service. dispenser *clients.DisperserClient @@ -62,17 +63,17 @@ func NewStatusVerifier( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, + config *Config, table *BlobTable, disperser *clients.DisperserClient, - blobReadLimit int32, metrics *Metrics) BlobVerifier { return BlobVerifier{ ctx: ctx, waitGroup: waitGroup, logger: logger, + config: config, table: table, - blobReadLimit: blobReadLimit, dispenser: disperser, unconfirmedKeys: make([]*unconfirmedKey, 0), keyChannel: make(chan *unconfirmedKey), @@ -199,6 +200,28 @@ func (verifier *BlobVerifier) forwardToReader(key *unconfirmedKey, status *dispe confirmationLatency := confirmationTime.Sub(key.submissionTime) verifier.confirmationLatencyMetric.ReportLatency(confirmationLatency) - blobMetadata := NewBlobMetadata(key.key, &batchHeaderHash, blobIndex, -1) // TODO permits + requiredDownloads := verifier.config.RequiredDownloads + var downloadCount int32 + if requiredDownloads <= 0 { + // Allow unlimited downloads. + downloadCount = -1 + } else if requiredDownloads == 0 { + // Do not download blob. + return + } else if requiredDownloads < 1 { + // Download blob with probability equal to requiredDownloads. + if rand.Float64() < requiredDownloads { + // Download the blob once. + downloadCount = 1 + } else { + // Do not download blob. + return + } + } else { + // Download blob requiredDownloads times. + downloadCount = int32(requiredDownloads) + } + + blobMetadata := NewBlobMetadata(key.key, &batchHeaderHash, blobIndex, downloadCount) verifier.table.Add(blobMetadata) } diff --git a/tools/traffic/config.go b/tools/traffic/config.go index 66bb754249..036a23b29a 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -12,19 +12,15 @@ import ( // Config configures a traffic generator. type Config struct { + // Configures logging for the traffic generator. + LoggingConfig common.LoggerConfig + // Configures the DA clients. clients.Config + SignerPrivateKey string + CustomQuorums []uint8 - // TODO add to flags.go - // The number of worker threads that generate read traffic. - NumReadInstances uint - // The period of the submission rate of read requests for each read worker thread. - ReadRequestInterval time.Duration - // For each blob, how many times should it be downloaded? If between 0.0 and 1.0, blob will be downloaded - // 0 or 1 times with the specified probability (e.g. 0.2 means each blob has a 20% chance of being downloaded). - // If greater than 1.0, then each blob will be downloaded the specified number of times. - DownloadRate float64 - // The minimum amount of time that must pass after a blob is written prior to the first read attempt being made. - ReadDelay time.Duration + // The amount of time to sleep after launching each worker thread. + InstanceLaunchInterval time.Duration // The number of worker threads that generate write traffic. NumWriteInstances uint @@ -36,13 +32,17 @@ type Config struct { // will be dispersed for each blob by a particular worker thread. RandomizeBlobs bool - // Configures logging for the traffic generator. - LoggingConfig common.LoggerConfig - // The amount of time to sleep after launching each worker thread. - InstanceLaunchInterval time.Duration - - SignerPrivateKey string - CustomQuorums []uint8 + // TODO add to flags.go + // The number of worker threads that generate read traffic. + NumReadInstances uint + // The period of the submission rate of read requests for each read worker thread. + ReadRequestInterval time.Duration + // For each blob, how many times should it be downloaded? If between 0.0 and 1.0, blob will be downloaded + // 0 or 1 times with the specified probability (e.g. 0.2 means each blob has a 20% chance of being downloaded). + // If greater than 1.0, then each blob will be downloaded the specified number of times. + RequiredDownloads float64 + // The minimum amount of time that must pass after a blob is written prior to the first read attempt being made. + ReadDelay time.Duration } func NewConfig(ctx *cli.Context) (*Config, error) { @@ -59,19 +59,25 @@ func NewConfig(ctx *cli.Context) (*Config, error) { customQuorumsUint8[i] = uint8(q) } return &Config{ + LoggingConfig: *loggerConfig, Config: *clients.NewConfig( ctx.GlobalString(flags.HostnameFlag.Name), ctx.GlobalString(flags.GrpcPortFlag.Name), ctx.Duration(flags.TimeoutFlag.Name), ctx.GlobalBool(flags.UseSecureGrpcFlag.Name), ), - NumWriteInstances: ctx.GlobalUint(flags.NumWriteInstancesFlag.Name), - WriteRequestInterval: ctx.Duration(flags.WriteRequestIntervalFlag.Name), - DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), - LoggingConfig: *loggerConfig, - RandomizeBlobs: ctx.GlobalBool(flags.RandomizeBlobsFlag.Name), + SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), + CustomQuorums: customQuorumsUint8, + InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), - SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), - CustomQuorums: customQuorumsUint8, + + NumWriteInstances: ctx.GlobalUint(flags.NumWriteInstancesFlag.Name), + WriteRequestInterval: ctx.Duration(flags.WriteRequestIntervalFlag.Name), + DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), + RandomizeBlobs: ctx.GlobalBool(flags.RandomizeBlobsFlag.Name), + + NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), + ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), + RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), }, nil } diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index 57ee87cba3..447c83c75e 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -13,7 +13,7 @@ const ( ) var ( - /* Required Flags */ + /* Configuration for DA clients. */ HostnameFlag = cli.StringFlag{ Name: common.PrefixFlag(FlagPrefix, "disperser-hostname"), @@ -31,67 +31,99 @@ var ( Name: common.PrefixFlag(FlagPrefix, "timeout"), Usage: "Amount of time to wait for GPRC", Required: false, - EnvVar: common.PrefixEnvVar(envPrefix, "TIMEOUT"), Value: 10 * time.Second, + EnvVar: common.PrefixEnvVar(envPrefix, "TIMEOUT"), + } + UseSecureGrpcFlag = cli.BoolFlag{ + Name: common.PrefixFlag(FlagPrefix, "use-secure-grpc"), + Usage: "Whether to use secure grpc", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "USE_SECURE_GRPC"), + } + SignerPrivateKeyFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "signer-private-key-hex"), + Usage: "Private key to use for signing requests", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "SIGNER_PRIVATE_KEY_HEX"), + } + CustomQuorumNumbersFlag = cli.IntSliceFlag{ + Name: common.PrefixFlag(FlagPrefix, "custom-quorum-numbers"), + Usage: "Custom quorum numbers to use for the traffic generator", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "CUSTOM_QUORUM_NUMBERS"), } + + /* Common Configuration. */ + + InstanceLaunchIntervalFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "instance-launch-interva"), + Usage: "Duration between generator instance launches", + Required: false, + Value: 1 * time.Second, + EnvVar: common.PrefixEnvVar(envPrefix, "INSTANCE_LAUNCH_INTERVAL"), + } + + /* Configuration for the blob writer. */ + NumWriteInstancesFlag = cli.UintFlag{ Name: common.PrefixFlag(FlagPrefix, "num-write-instances"), - Usage: "Number of generator instances producing write traffic to run in parallel", - Required: true, + Usage: "Number of writer instances producing traffic to run in parallel", + Required: false, + Value: 1, EnvVar: common.PrefixEnvVar(envPrefix, "NUM_WRITE_INSTANCES"), } WriteRequestIntervalFlag = cli.DurationFlag{ Name: common.PrefixFlag(FlagPrefix, "write-request-interval"), - Usage: "Duration between write requests", - Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "WRITE_REQUEST_INTERVAL"), + Usage: "Time between write requests", + Required: false, Value: 30 * time.Second, + EnvVar: common.PrefixEnvVar(envPrefix, "WRITE_REQUEST_INTERVAL"), } DataSizeFlag = cli.Uint64Flag{ Name: common.PrefixFlag(FlagPrefix, "data-size"), Usage: "Size of the data blob", - Required: true, + Required: false, + Value: 1024, EnvVar: common.PrefixEnvVar(envPrefix, "DATA_SIZE"), } - RandomizeBlobsFlag = cli.BoolFlag{ + RandomizeBlobsFlag = cli.BoolFlag{ // TODO invert flag Name: common.PrefixFlag(FlagPrefix, "randomize-blobs"), - Usage: "Whether to randomzie blob data", + Usage: "Whether to randomize each blob data", Required: false, EnvVar: common.PrefixEnvVar(envPrefix, "RANDOMIZE_BLOBS"), } - InstanceLaunchIntervalFlag = cli.DurationFlag{ - Name: common.PrefixFlag(FlagPrefix, "instance-launch-interva"), - Usage: "Duration between generator instance launches", - Required: false, - EnvVar: common.PrefixEnvVar(envPrefix, "INSTANCE_LAUNCH_INTERVAL"), - Value: 1 * time.Second, - } - UseSecureGrpcFlag = cli.BoolFlag{ - Name: common.PrefixFlag(FlagPrefix, "use-secure-grpc"), - Usage: "Whether to use secure grpc", + + /* Configuration for the blob validator. */ + + /* Configuration for the blob reader. */ + + NumReadInstancesFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "num-read-instances"), + Usage: "Number of reader instances producing traffic to run in parallel", Required: false, - EnvVar: common.PrefixEnvVar(envPrefix, "USE_SECURE_GRPC"), + Value: 1, + EnvVar: common.PrefixEnvVar(envPrefix, "NUM_READ_INSTANCES"), } - SignerPrivateKeyFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "signer-private-key-hex"), - Usage: "Private key to use for signing requests", + ReadRequestIntervalFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "read-request-interval"), + Usage: "Time between read requests", Required: false, - EnvVar: common.PrefixEnvVar(envPrefix, "SIGNER_PRIVATE_KEY_HEX"), + Value: time.Second, + EnvVar: common.PrefixEnvVar(envPrefix, "READ_REQUEST_INTERVAL"), } - CustomQuorumNumbersFlag = cli.IntSliceFlag{ - Name: common.PrefixFlag(FlagPrefix, "custom-quorum-numbers"), - Usage: "Custom quorum numbers to use for the traffic generator", + RequiredDownloadsFlag = cli.Float64Flag{ + Name: common.PrefixFlag(FlagPrefix, "required-downloads"), + Usage: "Number of required downloads. Numbers between 0.0 and 1.0 are treated as probabilities, " + + "numbers greater than 1.0 are treated as the number of downloads. -1 allows unlimited downloads.", Required: false, - EnvVar: common.PrefixEnvVar(envPrefix, "CUSTOM_QUORUM_NUMBERS"), + Value: 3.0, + EnvVar: common.PrefixEnvVar(envPrefix, "REQUIRED_DOWNLOADS"), } ) var requiredFlags = []cli.Flag{ HostnameFlag, GrpcPortFlag, - NumWriteInstancesFlag, - WriteRequestIntervalFlag, - DataSizeFlag, } var optionalFlags = []cli.Flag{ @@ -101,6 +133,12 @@ var optionalFlags = []cli.Flag{ UseSecureGrpcFlag, SignerPrivateKeyFlag, CustomQuorumNumbersFlag, + NumWriteInstancesFlag, + WriteRequestIntervalFlag, + DataSizeFlag, + NumReadInstancesFlag, + ReadRequestIntervalFlag, + RequiredDownloadsFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 6be125285e..21c4944112 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -94,9 +94,9 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi &ctx, &waitGroup, logger, + config, &table, &disperserClient, - -1, metrics) writers := make([]*BlobWriter, 0) @@ -115,12 +115,12 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi retriever, chainClient := buildRetriever() readers := make([]*BlobReader, 0) - //int(config.NumReadInstances) // TODO - for i := 0; i < 1; i++ { + for i := 0; i < int(config.NumReadInstances); i++ { reader := NewBlobReader( &ctx, &waitGroup, logger, + config, retriever, chainClient, &table, From 85b542d9f18bd1e38b31fdc3323f08f5fb87a3bf Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 08:38:45 -0500 Subject: [PATCH 23/74] Incremental progress on config. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 1 + tools/traffic/config.go | 8 +++--- tools/traffic/flags/flags.go | 48 +++++++++++++++++++++++------------- tools/traffic/generator.go | 20 +++++---------- tools/traffic/metrics.go | 2 ++ 5 files changed, 45 insertions(+), 34 deletions(-) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 35b13bc6af..ddc7a2c30d 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -15,6 +15,7 @@ run: build TRAFFIC_GENERATOR_DATA_SIZE=1000 \ TRAFFIC_GENERATOR_RANDOMIZE_BLOBS=true \ TRAFFIC_GENERATOR_SIGNER_PRIVATE_KEY_HEX=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \ + TRAFFIC_GENERATOR_METRICS_HTTP_PORT=9101 \ ./bin/server # TODO do not merge change TRAFFIC_GENERATOR_GRPC_PORT=32003 \ No newline at end of file diff --git a/tools/traffic/config.go b/tools/traffic/config.go index 036a23b29a..364f34e297 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -12,12 +12,12 @@ import ( // Config configures a traffic generator. type Config struct { - // Configures logging for the traffic generator. LoggingConfig common.LoggerConfig - // Configures the DA clients. clients.Config SignerPrivateKey string CustomQuorums []uint8 + DisableTlS bool + MetricsHTTPPort string // The amount of time to sleep after launching each worker thread. InstanceLaunchInterval time.Duration @@ -68,13 +68,15 @@ func NewConfig(ctx *cli.Context) (*Config, error) { ), SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), CustomQuorums: customQuorumsUint8, + DisableTlS: ctx.GlobalBool(flags.DisableTLSFlag.Name), + MetricsHTTPPort: ctx.GlobalString(flags.MetricsHTTPPortFlag.Name), InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), NumWriteInstances: ctx.GlobalUint(flags.NumWriteInstancesFlag.Name), WriteRequestInterval: ctx.Duration(flags.WriteRequestIntervalFlag.Name), DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), - RandomizeBlobs: ctx.GlobalBool(flags.RandomizeBlobsFlag.Name), + RandomizeBlobs: !ctx.GlobalBool(flags.UniformBlobsFlag.Name), NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index 447c83c75e..1696358ccb 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -17,47 +17,59 @@ var ( HostnameFlag = cli.StringFlag{ Name: common.PrefixFlag(FlagPrefix, "disperser-hostname"), - Usage: "Hostname at which disperser service is available", + Usage: "Hostname at which disperser service is available.", Required: true, EnvVar: common.PrefixEnvVar(envPrefix, "HOSTNAME"), } GrpcPortFlag = cli.StringFlag{ Name: common.PrefixFlag(FlagPrefix, "disperser-port"), - Usage: "Port at which a disperser listens for grpc calls", + Usage: "Port at which a disperser listens for grpc calls.", Required: true, EnvVar: common.PrefixEnvVar(envPrefix, "GRPC_PORT"), } TimeoutFlag = cli.DurationFlag{ Name: common.PrefixFlag(FlagPrefix, "timeout"), - Usage: "Amount of time to wait for GPRC", + Usage: "Amount of time to wait for grpc.", Required: false, Value: 10 * time.Second, EnvVar: common.PrefixEnvVar(envPrefix, "TIMEOUT"), } UseSecureGrpcFlag = cli.BoolFlag{ Name: common.PrefixFlag(FlagPrefix, "use-secure-grpc"), - Usage: "Whether to use secure grpc", + Usage: "Whether to use secure grpc.", Required: false, EnvVar: common.PrefixEnvVar(envPrefix, "USE_SECURE_GRPC"), } SignerPrivateKeyFlag = cli.StringFlag{ Name: common.PrefixFlag(FlagPrefix, "signer-private-key-hex"), - Usage: "Private key to use for signing requests", + Usage: "Private key to use for signing requests.", Required: false, EnvVar: common.PrefixEnvVar(envPrefix, "SIGNER_PRIVATE_KEY_HEX"), } CustomQuorumNumbersFlag = cli.IntSliceFlag{ Name: common.PrefixFlag(FlagPrefix, "custom-quorum-numbers"), - Usage: "Custom quorum numbers to use for the traffic generator", + Usage: "Custom quorum numbers to use for the traffic generator.", Required: false, EnvVar: common.PrefixEnvVar(envPrefix, "CUSTOM_QUORUM_NUMBERS"), } + DisableTLSFlag = cli.BoolFlag{ + Name: common.PrefixFlag(FlagPrefix, "disable-tls"), + Usage: "Whether to disable TLS for an insecure connection.", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "DISABLE_TLS"), + } + MetricsHTTPPortFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "metrics-http-port"), + Usage: "Port at which to expose metrics.", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "METRICS_HTTP_PORT"), + } /* Common Configuration. */ InstanceLaunchIntervalFlag = cli.DurationFlag{ Name: common.PrefixFlag(FlagPrefix, "instance-launch-interva"), - Usage: "Duration between generator instance launches", + Usage: "Duration between generator instance launches.", Required: false, Value: 1 * time.Second, EnvVar: common.PrefixEnvVar(envPrefix, "INSTANCE_LAUNCH_INTERVAL"), @@ -67,30 +79,30 @@ var ( NumWriteInstancesFlag = cli.UintFlag{ Name: common.PrefixFlag(FlagPrefix, "num-write-instances"), - Usage: "Number of writer instances producing traffic to run in parallel", + Usage: "Number of writer instances producing traffic to run in parallel.", Required: false, Value: 1, EnvVar: common.PrefixEnvVar(envPrefix, "NUM_WRITE_INSTANCES"), } WriteRequestIntervalFlag = cli.DurationFlag{ Name: common.PrefixFlag(FlagPrefix, "write-request-interval"), - Usage: "Time between write requests", + Usage: "Time between write requests.", Required: false, Value: 30 * time.Second, EnvVar: common.PrefixEnvVar(envPrefix, "WRITE_REQUEST_INTERVAL"), } DataSizeFlag = cli.Uint64Flag{ Name: common.PrefixFlag(FlagPrefix, "data-size"), - Usage: "Size of the data blob", + Usage: "Size of the data blob.", Required: false, Value: 1024, EnvVar: common.PrefixEnvVar(envPrefix, "DATA_SIZE"), } - RandomizeBlobsFlag = cli.BoolFlag{ // TODO invert flag - Name: common.PrefixFlag(FlagPrefix, "randomize-blobs"), - Usage: "Whether to randomize each blob data", + UniformBlobsFlag = cli.BoolFlag{ + Name: common.PrefixFlag(FlagPrefix, "uniform-blobs"), + Usage: "If set, do not randomize blobs.", Required: false, - EnvVar: common.PrefixEnvVar(envPrefix, "RANDOMIZE_BLOBS"), + EnvVar: common.PrefixEnvVar(envPrefix, "UNIFORM_BLOBS"), } /* Configuration for the blob validator. */ @@ -99,14 +111,14 @@ var ( NumReadInstancesFlag = cli.UintFlag{ Name: common.PrefixFlag(FlagPrefix, "num-read-instances"), - Usage: "Number of reader instances producing traffic to run in parallel", + Usage: "Number of reader instances producing traffic to run in parallel.", Required: false, Value: 1, EnvVar: common.PrefixEnvVar(envPrefix, "NUM_READ_INSTANCES"), } ReadRequestIntervalFlag = cli.DurationFlag{ Name: common.PrefixFlag(FlagPrefix, "read-request-interval"), - Usage: "Time between read requests", + Usage: "Time between read requests.", Required: false, Value: time.Second, EnvVar: common.PrefixEnvVar(envPrefix, "READ_REQUEST_INTERVAL"), @@ -128,7 +140,7 @@ var requiredFlags = []cli.Flag{ var optionalFlags = []cli.Flag{ TimeoutFlag, - RandomizeBlobsFlag, + UniformBlobsFlag, InstanceLaunchIntervalFlag, UseSecureGrpcFlag, SignerPrivateKeyFlag, @@ -139,6 +151,8 @@ var optionalFlags = []cli.Flag{ NumReadInstancesFlag, ReadRequestIntervalFlag, RequiredDownloadsFlag, + DisableTLSFlag, + MetricsHTTPPortFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 21c4944112..9079b502c1 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -65,8 +65,8 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi fmt.Printf("Signer private key: '%s', length: %d\n", config.SignerPrivateKey, len(config.SignerPrivateKey)) clientConfig := clients.EigenDAClientConfig{ - RPC: "localhost:32003", // TODO make this configurable - DisableTLS: true, // TODO config + RPC: config.Config.Hostname + ":" + config.Config.Port, + DisableTLS: config.DisableTlS, SignerPrivateKeyHex: config.SignerPrivateKey, } err = clientConfig.CheckAndSetDefaults() @@ -83,9 +83,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi ctx, cancel := context.WithCancel(context.Background()) waitGroup := sync.WaitGroup{} - metrics := NewMetrics("9101", logger) // TODO config - - // TODO add configuration + metrics := NewMetrics(config.MetricsHTTPPort, logger) table := NewBlobTable() @@ -145,16 +143,11 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi // buildRetriever creates a retriever client for the traffic generator. func buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { - - //loggerConfig := common.LoggerConfig{ - // Format: "text", - // } - loggerConfig := common.DefaultLoggerConfig() logger, err := common.NewLogger(loggerConfig) if err != nil { - panic(err) // TODO + panic(fmt.Sprintf("Unable to instantiate logger: %s", err)) } ethClientConfig := geth.EthClientConfig{ @@ -193,12 +186,11 @@ func buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { } v, err := verifier.NewVerifier(&encoderConfig, true) if err != nil { - panic(err) // TODO + panic(fmt.Sprintf("Unable to build verifier: %s", err)) } numConnections := 20 - //var retriever *clients.RetrievalClient retriever, err := clients.NewRetrievalClient( logger, chainState, @@ -208,7 +200,7 @@ func buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { numConnections) if err != nil { - panic(err) // TODO + panic(fmt.Sprintf("Unable to build retriever: %s", err)) } chainClient := retrivereth.NewChainClient(gethClient, logger) diff --git a/tools/traffic/metrics.go b/tools/traffic/metrics.go index d5a1798838..c1421dd1ef 100644 --- a/tools/traffic/metrics.go +++ b/tools/traffic/metrics.go @@ -41,6 +41,8 @@ type GaugeMetric struct { description string } +// TODO don't start metrics if httpPort is empty + // NewMetrics creates a new Metrics instance. func NewMetrics(httpPort string, logger logging.Logger) *Metrics { namespace := "eigenda_generator" From 47a3b7fa971f4913700d9e3d9207b8a55b87c330 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 08:57:11 -0500 Subject: [PATCH 24/74] Add more config. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 4 ++++ tools/traffic/config.go | 24 ++++++++++++++++-------- tools/traffic/flags/flags.go | 30 +++++++++++++++++++++++++++++- tools/traffic/generator.go | 10 +++++----- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index ddc7a2c30d..599cc66c93 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -9,6 +9,8 @@ build: clean run: build TRAFFIC_GENERATOR_HOSTNAME=localhost \ TRAFFIC_GENERATOR_GRPC_PORT=32003 \ + TRAFFIC_GENERATOR_ETH_CLIENT_HOSTNAME=http://localhost \ + TRAFFIC_GENERATOR_ETH_CLIENT_PORT=8545 \ TRAFFIC_GENERATOR_TIMEOUT=10s \ TRAFFIC_GENERATOR_NUM_WRITE_INSTANCES=1 \ TRAFFIC_GENERATOR_WRITE_REQUEST_INTERVAL=1s \ @@ -16,6 +18,8 @@ run: build TRAFFIC_GENERATOR_RANDOMIZE_BLOBS=true \ TRAFFIC_GENERATOR_SIGNER_PRIVATE_KEY_HEX=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \ TRAFFIC_GENERATOR_METRICS_HTTP_PORT=9101 \ + TRAFFIC_GENERATOR_BLS_OPERATOR_STATE_RETRIEVER=0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154 \ + TRAFFIC_GENERATOR_EIGENDA_SERVICE_MANAGER=0x851356ae760d987E095750cCeb3bC6014560891C \ ./bin/server # TODO do not merge change TRAFFIC_GENERATOR_GRPC_PORT=32003 \ No newline at end of file diff --git a/tools/traffic/config.go b/tools/traffic/config.go index 364f34e297..df2c61023c 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -14,10 +14,14 @@ import ( type Config struct { LoggingConfig common.LoggerConfig clients.Config - SignerPrivateKey string - CustomQuorums []uint8 - DisableTlS bool - MetricsHTTPPort string + SignerPrivateKey string + CustomQuorums []uint8 + DisableTlS bool + MetricsHTTPPort string + EthClientHostname string + EthClientPort string + BlsOperatorStateRetriever string + EigenDAServiceManager string // The amount of time to sleep after launching each worker thread. InstanceLaunchInterval time.Duration @@ -66,10 +70,14 @@ func NewConfig(ctx *cli.Context) (*Config, error) { ctx.Duration(flags.TimeoutFlag.Name), ctx.GlobalBool(flags.UseSecureGrpcFlag.Name), ), - SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), - CustomQuorums: customQuorumsUint8, - DisableTlS: ctx.GlobalBool(flags.DisableTLSFlag.Name), - MetricsHTTPPort: ctx.GlobalString(flags.MetricsHTTPPortFlag.Name), + SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), + CustomQuorums: customQuorumsUint8, + DisableTlS: ctx.GlobalBool(flags.DisableTLSFlag.Name), + MetricsHTTPPort: ctx.GlobalString(flags.MetricsHTTPPortFlag.Name), + EthClientHostname: ctx.GlobalString(flags.EthClientHostnameFlag.Name), + EthClientPort: ctx.GlobalString(flags.EthClientPortFlag.Name), + BlsOperatorStateRetriever: ctx.String(flags.BLSOperatorStateRetrieverFlag.Name), + EigenDAServiceManager: ctx.String(flags.EigenDAServiceManagerFlag.Name), InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index 1696358ccb..e3c7ae7737 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -64,6 +64,30 @@ var ( Required: false, EnvVar: common.PrefixEnvVar(envPrefix, "METRICS_HTTP_PORT"), } + EthClientHostnameFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "eth-client-hostname"), + Usage: "Hostname at which the Ethereum client is available.", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "ETH_CLIENT_HOSTNAME"), + } + EthClientPortFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "eth-client-port"), + Usage: "Port at which the Ethereum client is available.", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "ETH_CLIENT_PORT"), + } + BLSOperatorStateRetrieverFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "bls-operator-state-retriever"), + Usage: "Hex address of the BLS operator state retriever contract.", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "BLS_OPERATOR_STATE_RETRIEVER"), + } + EigenDAServiceManagerFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "eigenda-service-manager"), + Usage: "Hex address of the EigenDA service manager contract.", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "EIGENDA_SERVICE_MANAGER"), + } /* Common Configuration. */ @@ -120,7 +144,7 @@ var ( Name: common.PrefixFlag(FlagPrefix, "read-request-interval"), Usage: "Time between read requests.", Required: false, - Value: time.Second, + Value: time.Second / 3, EnvVar: common.PrefixEnvVar(envPrefix, "READ_REQUEST_INTERVAL"), } RequiredDownloadsFlag = cli.Float64Flag{ @@ -136,6 +160,10 @@ var ( var requiredFlags = []cli.Flag{ HostnameFlag, GrpcPortFlag, + EthClientHostnameFlag, + EthClientPortFlag, + BLSOperatorStateRetrieverFlag, + EigenDAServiceManagerFlag, } var optionalFlags = []cli.Flag{ diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 9079b502c1..5e6ee3a263 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -110,7 +110,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi writers = append(writers, &writer) } - retriever, chainClient := buildRetriever() + retriever, chainClient := buildRetriever(config) readers := make([]*BlobReader, 0) for i := 0; i < int(config.NumReadInstances); i++ { @@ -142,7 +142,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi } // buildRetriever creates a retriever client for the traffic generator. -func buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { +func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainClient) { loggerConfig := common.DefaultLoggerConfig() logger, err := common.NewLogger(loggerConfig) @@ -151,7 +151,7 @@ func buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { } ethClientConfig := geth.EthClientConfig{ - RPCURLs: []string{"http://localhost:8545"}, + RPCURLs: []string{config.EthClientHostname + ":" + config.EthClientPort}, NumRetries: 2, } gethClient, err := geth.NewMultiHomingClient(ethClientConfig, gethcommon.Address{}, logger) @@ -159,8 +159,8 @@ func buildRetriever() (clients.RetrievalClient, retrivereth.ChainClient) { tx, err := eth.NewTransactor( logger, gethClient, - "0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154", - "0x851356ae760d987E095750cCeb3bC6014560891C") + config.BlsOperatorStateRetriever, + config.EigenDAServiceManager) cs := eth.NewChainState(tx, gethClient) From 8fafa519853fd18bb221f7ec89e1e170e0d6a667 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 09:10:02 -0500 Subject: [PATCH 25/74] Added config for the graph. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 1 + tools/traffic/config.go | 38 ++++++++++++++++++++++++++++-------- tools/traffic/flags/flags.go | 31 +++++++++++++++++++++++++++++ tools/traffic/generator.go | 8 ++++---- 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 599cc66c93..850a45327e 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -20,6 +20,7 @@ run: build TRAFFIC_GENERATOR_METRICS_HTTP_PORT=9101 \ TRAFFIC_GENERATOR_BLS_OPERATOR_STATE_RETRIEVER=0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154 \ TRAFFIC_GENERATOR_EIGENDA_SERVICE_MANAGER=0x851356ae760d987E095750cCeb3bC6014560891C \ + TRAFFIC_GENERATOR_THE_GRAPH_URL=http://localhost:8000/subgraphs/name/Layr-Labs/eigenda-operator-state \ ./bin/server # TODO do not merge change TRAFFIC_GENERATOR_GRPC_PORT=32003 \ No newline at end of file diff --git a/tools/traffic/config.go b/tools/traffic/config.go index df2c61023c..e9ac52d1b7 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -12,16 +12,34 @@ import ( // Config configures a traffic generator. type Config struct { + // Logging configuration. LoggingConfig common.LoggerConfig - clients.Config - SignerPrivateKey string - CustomQuorums []uint8 - DisableTlS bool - MetricsHTTPPort string - EthClientHostname string - EthClientPort string + // Configuration for DA clients. + clients.Config // TODO for uniformity, don't use this nested config type + // The private key to use for signing requests. + SignerPrivateKey string + // Custom quorum numbers to use for the traffic generator. + CustomQuorums []uint8 + // Whether to disable TLS for an insecure connection. + DisableTlS bool + // The port at which the metrics server listens for HTTP requests. + MetricsHTTPPort string + // The hostname of the Ethereum client. + EthClientHostname string + // The port of the Ethereum client. + EthClientPort string + // The address of the BLS operator state retriever smart contract, in hex. BlsOperatorStateRetriever string - EigenDAServiceManager string + // The address of the EigenDA service manager smart contract, in hex. + EigenDAServiceManager string + // The number of times to retry an Ethereum client request. + EthClientRetries uint + // The URL of the subgraph instance. + TheGraphUrl string + // The interval at which to pull data from the subgraph. + TheGraphPullInterval time.Duration + // The number of times to retry a subgraph request. + TheGraphRetries uint // The amount of time to sleep after launching each worker thread. InstanceLaunchInterval time.Duration @@ -78,6 +96,10 @@ func NewConfig(ctx *cli.Context) (*Config, error) { EthClientPort: ctx.GlobalString(flags.EthClientPortFlag.Name), BlsOperatorStateRetriever: ctx.String(flags.BLSOperatorStateRetrieverFlag.Name), EigenDAServiceManager: ctx.String(flags.EigenDAServiceManagerFlag.Name), + EthClientRetries: ctx.Uint(flags.EthClientRetriesFlag.Name), + TheGraphUrl: ctx.String(flags.TheGraphUrlFlag.Name), + TheGraphPullInterval: ctx.Duration(flags.TheGraphPullIntervalFlag.Name), + TheGraphRetries: ctx.Uint(flags.TheGraphRetriesFlag.Name), InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index e3c7ae7737..cc09870651 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -88,6 +88,13 @@ var ( Required: true, EnvVar: common.PrefixEnvVar(envPrefix, "EIGENDA_SERVICE_MANAGER"), } + EthClientRetriesFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "eth-client-retries"), + Usage: "Number of times to retry an Ethereum client request.", + Required: false, + Value: 2, + EnvVar: common.PrefixEnvVar(envPrefix, "ETH_CLIENT_RETRIES"), + } /* Common Configuration. */ @@ -128,6 +135,26 @@ var ( Required: false, EnvVar: common.PrefixEnvVar(envPrefix, "UNIFORM_BLOBS"), } + TheGraphUrlFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "the-graph-url"), + Usage: "URL of the subgraph instance.", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "THE_GRAPH_URL"), + } + TheGraphPullIntervalFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "the-graph-pull-interval"), + Usage: "Interval at which to pull data from the subgraph.", + Required: false, + Value: 100 * time.Millisecond, + EnvVar: common.PrefixEnvVar(envPrefix, "THE_GRAPH_PULL_INTERVAL"), + } + TheGraphRetriesFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "the-graph-retries"), + Usage: "Number of times to retry a subgraph request.", + Required: false, + Value: 5, + EnvVar: common.PrefixEnvVar(envPrefix, "THE_GRAPH_RETRIES"), + } /* Configuration for the blob validator. */ @@ -164,6 +191,7 @@ var requiredFlags = []cli.Flag{ EthClientPortFlag, BLSOperatorStateRetrieverFlag, EigenDAServiceManagerFlag, + TheGraphUrlFlag, } var optionalFlags = []cli.Flag{ @@ -181,6 +209,9 @@ var optionalFlags = []cli.Flag{ RequiredDownloadsFlag, DisableTLSFlag, MetricsHTTPPortFlag, + EthClientRetriesFlag, + TheGraphPullIntervalFlag, + TheGraphRetriesFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 5e6ee3a263..21dfe1fff8 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -152,7 +152,7 @@ func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainC ethClientConfig := geth.EthClientConfig{ RPCURLs: []string{config.EthClientHostname + ":" + config.EthClientPort}, - NumRetries: 2, + NumRetries: int(config.EthClientRetries), } gethClient, err := geth.NewMultiHomingClient(ethClientConfig, gethcommon.Address{}, logger) @@ -166,9 +166,9 @@ func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainC // This is the indexer when config.UseGraph is true chainStateConfig := thegraph.Config{ - Endpoint: "http://localhost:8000/subgraphs/name/Layr-Labs/eigenda-operator-state", - PullInterval: 100 * time.Millisecond, - MaxRetries: 5, + Endpoint: config.TheGraphUrl, + PullInterval: config.TheGraphPullInterval, + MaxRetries: int(config.TheGraphRetries), } chainState := thegraph.MakeIndexedChainState(chainStateConfig, cs, logger) From 04468fc36c1c6ddf12fc5e75b830aa5f08e4ee59 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 09:20:09 -0500 Subject: [PATCH 26/74] Added flags for the encoder. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 3 +++ tools/traffic/config.go | 18 +++++++++++++++ tools/traffic/flags/flags.go | 44 ++++++++++++++++++++++++++++++++++++ tools/traffic/generator.go | 13 +++++------ 4 files changed, 71 insertions(+), 7 deletions(-) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 850a45327e..37ddf0a6a9 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -21,6 +21,9 @@ run: build TRAFFIC_GENERATOR_BLS_OPERATOR_STATE_RETRIEVER=0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154 \ TRAFFIC_GENERATOR_EIGENDA_SERVICE_MANAGER=0x851356ae760d987E095750cCeb3bC6014560891C \ TRAFFIC_GENERATOR_THE_GRAPH_URL=http://localhost:8000/subgraphs/name/Layr-Labs/eigenda-operator-state \ + TRAFFIC_GENERATOR_ENCODER_G1_PATH=../../inabox/resources/kzg/g1.point \ + TRAFFIC_GENERATOR_ENCODER_G2_PATH=../../inabox/resources/kzg/g2.point \ + TRAFFIC_GENERATOR_ENCODER_CACHE_DIR=../../inabox/resources/kzg/SRSTables \ ./bin/server # TODO do not merge change TRAFFIC_GENERATOR_GRPC_PORT=32003 \ No newline at end of file diff --git a/tools/traffic/config.go b/tools/traffic/config.go index e9ac52d1b7..e9f3d14d90 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -40,6 +40,18 @@ type Config struct { TheGraphPullInterval time.Duration // The number of times to retry a subgraph request. TheGraphRetries uint + // The path to the encoder G1 binary. + EncoderG1Path string + // The path to the encoder G2 binary. + EncoderG2Path string + // The path to the encoder cache directory. + EncoderCacheDir string + // The SRS order to use for the encoder. + EncoderSRSOrder uint64 + // The SRS number to load for the encoder. + EncoderSRSNumberToLoad uint64 + // The number of worker threads to use for the encoder. + EncoderNumWorkers uint64 // The amount of time to sleep after launching each worker thread. InstanceLaunchInterval time.Duration @@ -100,6 +112,12 @@ func NewConfig(ctx *cli.Context) (*Config, error) { TheGraphUrl: ctx.String(flags.TheGraphUrlFlag.Name), TheGraphPullInterval: ctx.Duration(flags.TheGraphPullIntervalFlag.Name), TheGraphRetries: ctx.Uint(flags.TheGraphRetriesFlag.Name), + EncoderG1Path: ctx.String(flags.EncoderG1PathFlag.Name), + EncoderG2Path: ctx.String(flags.EncoderG2PathFlag.Name), + EncoderCacheDir: ctx.String(flags.EncoderCacheDirFlag.Name), + EncoderSRSOrder: ctx.Uint64(flags.EncoderSRSOrderFlag.Name), + EncoderSRSNumberToLoad: ctx.Uint64(flags.EncoderSRSNumberToLoadFlag.Name), + EncoderNumWorkers: ctx.Uint64(flags.EncoderNumWorkersFlag.Name), InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index cc09870651..111614d10e 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -155,6 +155,44 @@ var ( Value: 5, EnvVar: common.PrefixEnvVar(envPrefix, "THE_GRAPH_RETRIES"), } + EncoderG1PathFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "encoder-g1-path"), + Usage: "Path to the encoder G1 binary.", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "ENCODER_G1_PATH"), + } + EncoderG2PathFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "encoder-g2-path"), + Usage: "Path to the encoder G2 binary.", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "ENCODER_G2_PATH"), + } + EncoderCacheDirFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "encoder-cache-dir"), + Usage: "Path to the encoder cache directory.", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "ENCODER_CACHE_DIR"), + } + EncoderSRSOrderFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "encoder-srs-order"), + Usage: "The SRS order to use for the encoder.", + Required: false, + Value: 3000, + EnvVar: common.PrefixEnvVar(envPrefix, "ENCODER_SRS_ORDER"), + } + EncoderSRSNumberToLoadFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "encoder-srs-number-to-load"), + Usage: "The SRS number to load for the encoder.", + Required: false, + Value: 3000, + EnvVar: common.PrefixEnvVar(envPrefix, "ENCODER_SRS_NUMBER_TO_LOAD"), + } + EncoderNumWorkersFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "encoder-num-workers"), + Usage: "The number of worker threads to use for the encoder.", + Required: false, + Value: 4, + } /* Configuration for the blob validator. */ @@ -192,6 +230,9 @@ var requiredFlags = []cli.Flag{ BLSOperatorStateRetrieverFlag, EigenDAServiceManagerFlag, TheGraphUrlFlag, + EncoderG1PathFlag, + EncoderG2PathFlag, + EncoderCacheDirFlag, } var optionalFlags = []cli.Flag{ @@ -212,6 +253,9 @@ var optionalFlags = []cli.Flag{ EthClientRetriesFlag, TheGraphPullIntervalFlag, TheGraphRetriesFlag, + EncoderSRSOrderFlag, + EncoderSRSNumberToLoadFlag, + EncoderNumWorkersFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 21dfe1fff8..a6c0582e74 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -177,12 +177,12 @@ func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainC nodeClient := clients.NewNodeClient(10 * time.Second) encoderConfig := kzg.KzgConfig{ - G1Path: "../../inabox/resources/kzg/g1.point", - G2Path: "../../inabox/resources/kzg/g2.point", - CacheDir: "../../inabox/resources/kzg/SRSTables", - SRSOrder: 3000, - SRSNumberToLoad: 3000, - NumWorker: 12, + G1Path: config.EncoderG1Path, + G2Path: config.EncoderG2Path, + CacheDir: config.EncoderCacheDir, + SRSOrder: config.EncoderSRSOrder, + SRSNumberToLoad: config.EncoderSRSNumberToLoad, + NumWorker: config.EncoderNumWorkers, } v, err := verifier.NewVerifier(&encoderConfig, true) if err != nil { @@ -190,7 +190,6 @@ func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainC } numConnections := 20 - retriever, err := clients.NewRetrievalClient( logger, chainState, From c0214a43684b130ef6946c467a08c30f88e32459 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 09:34:00 -0500 Subject: [PATCH 27/74] No more adhoc config in generator.go. Signed-off-by: Cody Littley --- tools/traffic/blob_verifier.go | 4 ++-- tools/traffic/config.go | 12 +++++++++++- tools/traffic/flags/flags.go | 25 +++++++++++++++++++++++++ tools/traffic/generator.go | 13 ++++--------- tools/traffic/metrics.go | 4 ++-- 5 files changed, 44 insertions(+), 14 deletions(-) diff --git a/tools/traffic/blob_verifier.go b/tools/traffic/blob_verifier.go index 180d983d7e..3cbdb2a465 100644 --- a/tools/traffic/blob_verifier.go +++ b/tools/traffic/blob_verifier.go @@ -101,9 +101,9 @@ func (verifier *BlobVerifier) AddUnconfirmedKey(key *[]byte) { // Start begins the status goroutine, which periodically polls // the disperser service to verify the status of blobs. -func (verifier *BlobVerifier) Start(period time.Duration) { +func (verifier *BlobVerifier) Start() { verifier.waitGroup.Add(1) - go verifier.monitor(period) + go verifier.monitor(verifier.config.VerifierInterval) } // monitor periodically polls the disperser service to verify the status of blobs. diff --git a/tools/traffic/config.go b/tools/traffic/config.go index e9f3d14d90..b2ae3a3740 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -52,6 +52,10 @@ type Config struct { EncoderSRSNumberToLoad uint64 // The number of worker threads to use for the encoder. EncoderNumWorkers uint64 + // The number of connections to use for the retriever. + RetrieverNumConnections uint + // The timeout for the node client. + NodeClientTimeout time.Duration // The amount of time to sleep after launching each worker thread. InstanceLaunchInterval time.Duration @@ -66,7 +70,9 @@ type Config struct { // will be dispersed for each blob by a particular worker thread. RandomizeBlobs bool - // TODO add to flags.go + // The amount of time between attempts by the verifier to confirm the status of blobs. + VerifierInterval time.Duration + // The number of worker threads that generate read traffic. NumReadInstances uint // The period of the submission rate of read requests for each read worker thread. @@ -118,6 +124,8 @@ func NewConfig(ctx *cli.Context) (*Config, error) { EncoderSRSOrder: ctx.Uint64(flags.EncoderSRSOrderFlag.Name), EncoderSRSNumberToLoad: ctx.Uint64(flags.EncoderSRSNumberToLoadFlag.Name), EncoderNumWorkers: ctx.Uint64(flags.EncoderNumWorkersFlag.Name), + RetrieverNumConnections: ctx.Uint(flags.RetrieverNumConnectionsFlag.Name), + NodeClientTimeout: ctx.Duration(flags.NodeClientTimeoutFlag.Name), InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), @@ -126,6 +134,8 @@ func NewConfig(ctx *cli.Context) (*Config, error) { DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), RandomizeBlobs: !ctx.GlobalBool(flags.UniformBlobsFlag.Name), + VerifierInterval: ctx.Duration(flags.VerifierIntervalFlag.Name), + NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index 111614d10e..e74cb51e16 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -193,9 +193,31 @@ var ( Required: false, Value: 4, } + RetrieverNumConnectionsFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "retriever-num-connections"), + Usage: "The number of connections to use for the retriever.", + Required: false, + Value: 20, + EnvVar: common.PrefixEnvVar(envPrefix, "RETRIEVER_NUM_CONNECTIONS"), + } + NodeClientTimeoutFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "node-client-timeout"), + Usage: "The timeout for the node client.", + Required: false, + Value: 10 * time.Second, + EnvVar: common.PrefixEnvVar(envPrefix, "NODE_CLIENT_TIMEOUT"), + } /* Configuration for the blob validator. */ + VerifierIntervalFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "verifier-interval"), + Usage: "Amount of time between verifier checks.", + Required: false, + Value: time.Second, + EnvVar: common.PrefixEnvVar(envPrefix, "VERIFIER_INTERVAL"), + } + /* Configuration for the blob reader. */ NumReadInstancesFlag = cli.UintFlag{ @@ -256,6 +278,9 @@ var optionalFlags = []cli.Flag{ EncoderSRSOrderFlag, EncoderSRSNumberToLoadFlag, EncoderNumWorkersFlag, + RetrieverNumConnectionsFlag, + VerifierIntervalFlag, + NodeClientTimeoutFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index a6c0582e74..e3bbcae75d 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -23,8 +23,6 @@ import ( "github.com/Layr-Labs/eigenda/core" ) -// TODO use consistent snake case on metrics - // TrafficGenerator simulates read/write traffic to the DA service. // // ┌------------┐ ┌------------┐ @@ -174,7 +172,7 @@ func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainC var assignmentCoordinator core.AssignmentCoordinator = &core.StdAssignmentCoordinator{} - nodeClient := clients.NewNodeClient(10 * time.Second) + nodeClient := clients.NewNodeClient(config.NodeClientTimeout) encoderConfig := kzg.KzgConfig{ G1Path: config.EncoderG1Path, @@ -189,14 +187,13 @@ func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainC panic(fmt.Sprintf("Unable to build verifier: %s", err)) } - numConnections := 20 retriever, err := clients.NewRetrievalClient( logger, chainState, assignmentCoordinator, nodeClient, v, - numConnections) + int(config.RetrieverNumConnections)) if err != nil { panic(fmt.Sprintf("Unable to build retriever: %s", err)) @@ -210,10 +207,8 @@ func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainC // Start instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. func (generator *TrafficGenerator) Start() error { - // TODO add configuration - - generator.metrics.Start(*generator.ctx) // TODO put context into metrics constructor - generator.verifier.Start(time.Second) + generator.metrics.Start() + generator.verifier.Start() for _, writer := range generator.writers { writer.Start() diff --git a/tools/traffic/metrics.go b/tools/traffic/metrics.go index c1421dd1ef..078fc3592a 100644 --- a/tools/traffic/metrics.go +++ b/tools/traffic/metrics.go @@ -1,7 +1,6 @@ package traffic import ( - "context" "fmt" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/prometheus/client_golang/prometheus" @@ -42,6 +41,7 @@ type GaugeMetric struct { } // TODO don't start metrics if httpPort is empty +// TODO use consistent snake case on metrics // NewMetrics creates a new Metrics instance. func NewMetrics(httpPort string, logger logging.Logger) *Metrics { @@ -79,7 +79,7 @@ func NewMetrics(httpPort string, logger logging.Logger) *Metrics { } // Start starts the metrics server. -func (metrics *Metrics) Start(ctx context.Context) { // TODO context? +func (metrics *Metrics) Start() { metrics.logger.Info("Starting metrics server at ", "port", metrics.httpPort) addr := fmt.Sprintf(":%s", metrics.httpPort) go func() { From 8784124695f3ca3ef8ea2d2c842b9322c5a60f03 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 10:27:38 -0500 Subject: [PATCH 28/74] Moar config. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 4 +-- tools/traffic/blob_reader.go | 3 +- tools/traffic/blob_writer.go | 3 +- tools/traffic/config.go | 64 +++++++++++++++++++----------------- tools/traffic/flags/flags.go | 8 ++--- tools/traffic/generator.go | 17 ++++++---- 6 files changed, 53 insertions(+), 46 deletions(-) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 37ddf0a6a9..ea919bac73 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -7,8 +7,8 @@ build: clean go build -o ./bin/server ./cmd run: build - TRAFFIC_GENERATOR_HOSTNAME=localhost \ - TRAFFIC_GENERATOR_GRPC_PORT=32003 \ + TRAFFIC_GENERATOR_DISPERSER_HOSTNAME=localhost \ + TRAFFIC_GENERATOR_DISPERSER_PORT=32003 \ TRAFFIC_GENERATOR_ETH_CLIENT_HOSTNAME=http://localhost \ TRAFFIC_GENERATOR_ETH_CLIENT_PORT=8545 \ TRAFFIC_GENERATOR_TIMEOUT=10s \ diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 29ee361ec8..53e4217544 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -23,7 +23,7 @@ type BlobReader struct { ctx *context.Context // Tracks the number of active goroutines within the generator. - waitGroup *sync.WaitGroup // TODO other things should use this too + waitGroup *sync.WaitGroup // All logs should be written using this logger. logger logging.Logger @@ -31,7 +31,6 @@ type BlobReader struct { // config contains the configuration for the generator. config *Config - // TODO use code from this class retriever clients.RetrievalClient chainClient eth.ChainClient diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 322a1a96d7..8b40ebada1 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -135,8 +135,7 @@ func (writer *BlobWriter) getRandomData() *[]byte { // sendRequest sends a blob to a disperser. func (writer *BlobWriter) sendRequest(data []byte) ([]byte /* key */, error) { - // TODO add timeout to other types of requests - ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.config.Timeout) + ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.config.DisperserTimeout) // TODO use different config defer cancel() var key []byte diff --git a/tools/traffic/config.go b/tools/traffic/config.go index b2ae3a3740..9b376fd07b 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -4,7 +4,6 @@ import ( "errors" "time" - "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/tools/traffic/flags" "github.com/urfave/cli" @@ -14,8 +13,14 @@ import ( type Config struct { // Logging configuration. LoggingConfig common.LoggerConfig - // Configuration for DA clients. - clients.Config // TODO for uniformity, don't use this nested config type + // The hostname of the disperser. + DisperserHostname string + // The port of the disperser. + DisperserPort string + // The timeout for the disperser. + DisperserTimeout time.Duration + // Whether to use a secure gRPC connection to the disperser. + DisperserUseSecureGrpcFlag bool // The private key to use for signing requests. SignerPrivateKey string // Custom quorum numbers to use for the traffic generator. @@ -98,34 +103,33 @@ func NewConfig(ctx *cli.Context) (*Config, error) { } customQuorumsUint8[i] = uint8(q) } + return &Config{ - LoggingConfig: *loggerConfig, - Config: *clients.NewConfig( - ctx.GlobalString(flags.HostnameFlag.Name), - ctx.GlobalString(flags.GrpcPortFlag.Name), - ctx.Duration(flags.TimeoutFlag.Name), - ctx.GlobalBool(flags.UseSecureGrpcFlag.Name), - ), - SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), - CustomQuorums: customQuorumsUint8, - DisableTlS: ctx.GlobalBool(flags.DisableTLSFlag.Name), - MetricsHTTPPort: ctx.GlobalString(flags.MetricsHTTPPortFlag.Name), - EthClientHostname: ctx.GlobalString(flags.EthClientHostnameFlag.Name), - EthClientPort: ctx.GlobalString(flags.EthClientPortFlag.Name), - BlsOperatorStateRetriever: ctx.String(flags.BLSOperatorStateRetrieverFlag.Name), - EigenDAServiceManager: ctx.String(flags.EigenDAServiceManagerFlag.Name), - EthClientRetries: ctx.Uint(flags.EthClientRetriesFlag.Name), - TheGraphUrl: ctx.String(flags.TheGraphUrlFlag.Name), - TheGraphPullInterval: ctx.Duration(flags.TheGraphPullIntervalFlag.Name), - TheGraphRetries: ctx.Uint(flags.TheGraphRetriesFlag.Name), - EncoderG1Path: ctx.String(flags.EncoderG1PathFlag.Name), - EncoderG2Path: ctx.String(flags.EncoderG2PathFlag.Name), - EncoderCacheDir: ctx.String(flags.EncoderCacheDirFlag.Name), - EncoderSRSOrder: ctx.Uint64(flags.EncoderSRSOrderFlag.Name), - EncoderSRSNumberToLoad: ctx.Uint64(flags.EncoderSRSNumberToLoadFlag.Name), - EncoderNumWorkers: ctx.Uint64(flags.EncoderNumWorkersFlag.Name), - RetrieverNumConnections: ctx.Uint(flags.RetrieverNumConnectionsFlag.Name), - NodeClientTimeout: ctx.Duration(flags.NodeClientTimeoutFlag.Name), + LoggingConfig: *loggerConfig, + DisperserHostname: ctx.GlobalString(flags.HostnameFlag.Name), + DisperserPort: ctx.GlobalString(flags.GrpcPortFlag.Name), + DisperserTimeout: ctx.Duration(flags.TimeoutFlag.Name), + DisperserUseSecureGrpcFlag: ctx.GlobalBool(flags.UseSecureGrpcFlag.Name), + SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), + CustomQuorums: customQuorumsUint8, + DisableTlS: ctx.GlobalBool(flags.DisableTLSFlag.Name), + MetricsHTTPPort: ctx.GlobalString(flags.MetricsHTTPPortFlag.Name), + EthClientHostname: ctx.GlobalString(flags.EthClientHostnameFlag.Name), + EthClientPort: ctx.GlobalString(flags.EthClientPortFlag.Name), + BlsOperatorStateRetriever: ctx.String(flags.BLSOperatorStateRetrieverFlag.Name), + EigenDAServiceManager: ctx.String(flags.EigenDAServiceManagerFlag.Name), + EthClientRetries: ctx.Uint(flags.EthClientRetriesFlag.Name), + TheGraphUrl: ctx.String(flags.TheGraphUrlFlag.Name), + TheGraphPullInterval: ctx.Duration(flags.TheGraphPullIntervalFlag.Name), + TheGraphRetries: ctx.Uint(flags.TheGraphRetriesFlag.Name), + EncoderG1Path: ctx.String(flags.EncoderG1PathFlag.Name), + EncoderG2Path: ctx.String(flags.EncoderG2PathFlag.Name), + EncoderCacheDir: ctx.String(flags.EncoderCacheDirFlag.Name), + EncoderSRSOrder: ctx.Uint64(flags.EncoderSRSOrderFlag.Name), + EncoderSRSNumberToLoad: ctx.Uint64(flags.EncoderSRSNumberToLoadFlag.Name), + EncoderNumWorkers: ctx.Uint64(flags.EncoderNumWorkersFlag.Name), + RetrieverNumConnections: ctx.Uint(flags.RetrieverNumConnectionsFlag.Name), + NodeClientTimeout: ctx.Duration(flags.NodeClientTimeoutFlag.Name), InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index e74cb51e16..59864b84a2 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -19,26 +19,26 @@ var ( Name: common.PrefixFlag(FlagPrefix, "disperser-hostname"), Usage: "Hostname at which disperser service is available.", Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "HOSTNAME"), + EnvVar: common.PrefixEnvVar(envPrefix, "DISPERSER_HOSTNAME"), } GrpcPortFlag = cli.StringFlag{ Name: common.PrefixFlag(FlagPrefix, "disperser-port"), Usage: "Port at which a disperser listens for grpc calls.", Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "GRPC_PORT"), + EnvVar: common.PrefixEnvVar(envPrefix, "DISPERSER_PORT"), } TimeoutFlag = cli.DurationFlag{ Name: common.PrefixFlag(FlagPrefix, "timeout"), Usage: "Amount of time to wait for grpc.", Required: false, Value: 10 * time.Second, - EnvVar: common.PrefixEnvVar(envPrefix, "TIMEOUT"), + EnvVar: common.PrefixEnvVar(envPrefix, "DISPERSER_TIMEOUT"), } UseSecureGrpcFlag = cli.BoolFlag{ Name: common.PrefixFlag(FlagPrefix, "use-secure-grpc"), Usage: "Whether to use secure grpc.", Required: false, - EnvVar: common.PrefixEnvVar(envPrefix, "USE_SECURE_GRPC"), + EnvVar: common.PrefixEnvVar(envPrefix, "DISPERSER_USE_SECURE_GRPC"), } SignerPrivateKeyFlag = cli.StringFlag{ Name: common.PrefixFlag(FlagPrefix, "signer-private-key-hex"), diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index e3bbcae75d..588c0bcc59 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -54,8 +54,7 @@ type TrafficGenerator struct { } func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*TrafficGenerator, error) { - loggerConfig := common.DefaultLoggerConfig() - logger, err := common.NewLogger(loggerConfig) + logger, err := common.NewLogger(config.LoggingConfig) if err != nil { return nil, err } @@ -63,7 +62,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi fmt.Printf("Signer private key: '%s', length: %d\n", config.SignerPrivateKey, len(config.SignerPrivateKey)) clientConfig := clients.EigenDAClientConfig{ - RPC: config.Config.Hostname + ":" + config.Config.Port, + RPC: config.DisperserHostname + ":" + config.DisperserPort, DisableTLS: config.DisableTlS, SignerPrivateKeyHex: config.SignerPrivateKey, } @@ -72,7 +71,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi return nil, err } - logger2 := log.NewLogger(log.NewTerminalHandler(os.Stderr, true)) // TODO + logger2 := log.NewLogger(log.NewTerminalHandler(os.Stderr, true)) client, err := clients.NewEigenDAClient(logger2, clientConfig) if err != nil { return nil, err @@ -85,7 +84,13 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi table := NewBlobTable() - disperserClient := clients.NewDisperserClient(&config.Config, signer) + disperserConfig := clients.Config{ + Hostname: config.DisperserHostname, + Port: config.DisperserPort, + Timeout: config.DisperserTimeout, + UseSecureGrpcFlag: config.DisperserUseSecureGrpcFlag, + } + disperserClient := clients.NewDisperserClient(&disperserConfig, signer) statusVerifier := NewStatusVerifier( &ctx, &waitGroup, @@ -130,7 +135,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi waitGroup: &waitGroup, metrics: metrics, logger: &logger, - disperserClient: clients.NewDisperserClient(&config.Config, signer), + disperserClient: clients.NewDisperserClient(&disperserConfig, signer), eigenDAClient: client, config: config, writers: writers, From e2615e4ed1c95f7f5488020276ca9f68a360ffcb Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 10:44:04 -0500 Subject: [PATCH 29/74] Converted remaining constants into configuration. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 12 ++++-------- tools/traffic/blob_verifier.go | 3 +-- tools/traffic/blob_writer.go | 2 +- tools/traffic/config.go | 19 +++++++++++++------ tools/traffic/flags/flags.go | 24 ++++++++++++++++++++++++ tools/traffic/generator.go | 2 -- 6 files changed, 43 insertions(+), 19 deletions(-) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 53e4217544..a3ddaf4dab 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -15,8 +15,6 @@ import ( "time" ) -// TODO for all of these new types, decide if variables need to be pointers or not - // BlobReader reads blobs from a disperser at a configured rate. type BlobReader struct { // The context for the generator. All work should cease when this context is cancelled. @@ -118,13 +116,12 @@ func (reader *BlobReader) randomRead() { return } - // TODO add timeout config - ctxTimeout, cancel := context.WithTimeout(*reader.ctx, time.Second*5) + ctxTimeout, cancel := context.WithTimeout(*reader.ctx, reader.config.FetchBatchHeaderTimeout) batchHeader, err := InvokeAndReportLatency(&reader.fetchBatchHeaderMetric, func() (*contractEigenDAServiceManager.IEigenDAServiceManagerBatchHeader, error) { return reader.chainClient.FetchBatchHeader( ctxTimeout, - gcommon.HexToAddress("0x851356ae760d987E095750cCeb3bC6014560891C"), + gcommon.HexToAddress(reader.config.EigenDAServiceManager), *metadata.batchHeaderHash, big.NewInt(int64(0)), nil) @@ -140,8 +137,7 @@ func (reader *BlobReader) randomRead() { var batchHeaderHash [32]byte copy(batchHeaderHash[:], *metadata.batchHeaderHash) - // TODO add timeout config - ctxTimeout, cancel = context.WithTimeout(*reader.ctx, time.Second*5) + ctxTimeout, cancel = context.WithTimeout(*reader.ctx, reader.config.RetrieveBlobChunksTimeout) chunks, err := InvokeAndReportLatency(&reader.readLatencyMetric, func() (*clients.BlobChunks, error) { return reader.retriever.RetrieveBlobChunks( ctxTimeout, @@ -212,5 +208,5 @@ func (reader *BlobReader) reportMissingChunk(operatorId core.OperatorID) { // verifyBlob performs sanity checks on the blob. func (reader *BlobReader) verifyBlob(blob []byte) { - // TODO + // TODO: do sanity checks on the data returned } diff --git a/tools/traffic/blob_verifier.go b/tools/traffic/blob_verifier.go index 3cbdb2a465..0cfe444ba0 100644 --- a/tools/traffic/blob_verifier.go +++ b/tools/traffic/blob_verifier.go @@ -142,8 +142,7 @@ func (verifier *BlobVerifier) poll() { // checkStatusForBlob checks the status of a blob. Returns true if the final blob status is known, false otherwise. func (verifier *BlobVerifier) checkStatusForBlob(key *unconfirmedKey) bool { - // TODO add timeout config - ctxTimeout, cancel := context.WithTimeout(*verifier.ctx, time.Second*5) + ctxTimeout, cancel := context.WithTimeout(*verifier.ctx, verifier.config.GetBlobStatusTimeout) defer cancel() status, err := InvokeAndReportLatency[*disperser.BlobStatusReply](&verifier.getStatusLatencyMetric, diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 8b40ebada1..26181d5390 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -135,7 +135,7 @@ func (writer *BlobWriter) getRandomData() *[]byte { // sendRequest sends a blob to a disperser. func (writer *BlobWriter) sendRequest(data []byte) ([]byte /* key */, error) { - ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.config.DisperserTimeout) // TODO use different config + ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.config.DisperserTimeout) defer cancel() var key []byte diff --git a/tools/traffic/config.go b/tools/traffic/config.go index 9b376fd07b..7149172cba 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -77,6 +77,8 @@ type Config struct { // The amount of time between attempts by the verifier to confirm the status of blobs. VerifierInterval time.Duration + // The amount of time to wait for a blob status to be fetched. + GetBlobStatusTimeout time.Duration // The number of worker threads that generate read traffic. NumReadInstances uint @@ -86,8 +88,10 @@ type Config struct { // 0 or 1 times with the specified probability (e.g. 0.2 means each blob has a 20% chance of being downloaded). // If greater than 1.0, then each blob will be downloaded the specified number of times. RequiredDownloads float64 - // The minimum amount of time that must pass after a blob is written prior to the first read attempt being made. - ReadDelay time.Duration + // The amount of time to wait for a batch header to be fetched. + FetchBatchHeaderTimeout time.Duration + // The amount of time to wait for a blob to be retrieved. + RetrieveBlobChunksTimeout time.Duration } func NewConfig(ctx *cli.Context) (*Config, error) { @@ -138,10 +142,13 @@ func NewConfig(ctx *cli.Context) (*Config, error) { DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), RandomizeBlobs: !ctx.GlobalBool(flags.UniformBlobsFlag.Name), - VerifierInterval: ctx.Duration(flags.VerifierIntervalFlag.Name), + VerifierInterval: ctx.Duration(flags.VerifierIntervalFlag.Name), + GetBlobStatusTimeout: ctx.Duration(flags.GetBlobStatusTimeoutFlag.Name), - NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), - ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), - RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), + NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), + ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), + RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), + FetchBatchHeaderTimeout: ctx.Duration(flags.FetchBatchHeaderTimeoutFlag.Name), + RetrieveBlobChunksTimeout: ctx.Duration(flags.RetrieveBlobChunksTimeoutFlag.Name), }, nil } diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index 59864b84a2..6ac6ad9d89 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -217,6 +217,13 @@ var ( Value: time.Second, EnvVar: common.PrefixEnvVar(envPrefix, "VERIFIER_INTERVAL"), } + GetBlobStatusTimeoutFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "get-blob-status-timeout"), + Usage: "Amount of time to wait for a blob status to be fetched.", + Required: false, + Value: 5 * time.Second, + EnvVar: common.PrefixEnvVar(envPrefix, "GET_BLOB_STATUS_TIMEOUT"), + } /* Configuration for the blob reader. */ @@ -242,6 +249,20 @@ var ( Value: 3.0, EnvVar: common.PrefixEnvVar(envPrefix, "REQUIRED_DOWNLOADS"), } + FetchBatchHeaderTimeoutFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "fetch-batch-header-timeout"), + Usage: "Amount of time to wait for a batch header to be fetched.", + Required: false, + Value: 5 * time.Second, + EnvVar: common.PrefixEnvVar(envPrefix, "FETCH_BATCH_HEADER_TIMEOUT"), + } + RetrieveBlobChunksTimeoutFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "retrieve-blob-chunks-timeout"), + Usage: "Amount of time to wait for a blob to be retrieved.", + Required: false, + Value: 5 * time.Second, + EnvVar: common.PrefixEnvVar(envPrefix, "RETRIEVE_BLOB_CHUNKS_TIMEOUT"), + } ) var requiredFlags = []cli.Flag{ @@ -281,6 +302,9 @@ var optionalFlags = []cli.Flag{ RetrieverNumConnectionsFlag, VerifierIntervalFlag, NodeClientTimeoutFlag, + FetchBatchHeaderTimeoutFlag, + RetrieveBlobChunksTimeoutFlag, + GetBlobStatusTimeoutFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 588c0bcc59..e83ba1b895 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -59,8 +59,6 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi return nil, err } - fmt.Printf("Signer private key: '%s', length: %d\n", config.SignerPrivateKey, len(config.SignerPrivateKey)) - clientConfig := clients.EigenDAClientConfig{ RPC: config.DisperserHostname + ":" + config.DisperserPort, DisableTLS: config.DisableTlS, From 6d4559dd6ab5cb3c268f244a3a92f9f4911c7914 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 11:17:11 -0500 Subject: [PATCH 30/74] Added overflow table. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 34 ++++++++++++++++------ tools/traffic/blob_table.go | 53 +++++++++++++++++++++++++--------- tools/traffic/blob_verifier.go | 2 +- tools/traffic/config.go | 5 ++++ tools/traffic/flags/flags.go | 10 ++++++- 5 files changed, 80 insertions(+), 24 deletions(-) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index a3ddaf4dab..b98db5d178 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -32,8 +32,11 @@ type BlobReader struct { retriever clients.RetrievalClient chainClient eth.ChainClient - // table of blobs to read from. - table *BlobTable + // requiredReads blobs we are required to read a certain number of times. + requiredReads *BlobTable + + // optionalReads blobs we are not required to read, but can choose to read if we want. + optionalReads *BlobTable metrics *Metrics fetchBatchHeaderMetric LatencyMetric @@ -46,7 +49,8 @@ type BlobReader struct { recombinationFailure CountMetric operatorSuccessMetrics map[core.OperatorID]CountMetric operatorFailureMetrics map[core.OperatorID]CountMetric - candidatePoolSize GaugeMetric + requiredReadPoolSize GaugeMetric + optionalReadPoolSize GaugeMetric } // NewBlobReader creates a new BlobReader instance. @@ -60,6 +64,8 @@ func NewBlobReader( table *BlobTable, metrics *Metrics) BlobReader { + optionalReads := NewBlobTable() + return BlobReader{ ctx: ctx, waitGroup: waitGroup, @@ -67,7 +73,8 @@ func NewBlobReader( config: config, retriever: retriever, chainClient: chainClient, - table: table, + requiredReads: table, + optionalReads: &optionalReads, metrics: metrics, fetchBatchHeaderMetric: metrics.NewLatencyMetric("fetch_batch_header"), fetchBatchHeaderSuccess: metrics.NewCountMetric("fetch_batch_header_success"), @@ -79,7 +86,8 @@ func NewBlobReader( readFailureMetric: metrics.NewCountMetric("read_failure"), operatorSuccessMetrics: make(map[core.OperatorID]CountMetric), operatorFailureMetrics: make(map[core.OperatorID]CountMetric), - candidatePoolSize: metrics.NewGaugeMetric("candidate_pool_size"), + requiredReadPoolSize: metrics.NewGaugeMetric("required_read_pool_size"), + optionalReadPoolSize: metrics.NewGaugeMetric("optional_read_pool_size"), } } @@ -108,12 +116,20 @@ func (reader *BlobReader) run() { // randomRead reads a random blob. func (reader *BlobReader) randomRead() { - reader.candidatePoolSize.Set(float64(reader.table.Size())) + reader.requiredReadPoolSize.Set(float64(reader.requiredReads.Size())) - metadata := reader.table.GetRandom(true) + metadata, removed := reader.requiredReads.GetRandom(true) if metadata == nil { - // There are no blobs to read, do nothing. - return + // There are no blobs that we are required to read. Get a random blob from the optionalReads. + metadata, _ = reader.optionalReads.GetRandom(false) + if metadata == nil { + // No blobs to read. + return + } + } else if removed { + // We have removed a blob from the requiredReads. Add it to the optionalReads. + reader.optionalReads.addOrReplace(metadata, reader.config.ReadOverflowTableSize) + reader.optionalReadPoolSize.Set(float64(reader.optionalReads.Size())) } ctxTimeout, cancel := context.WithTimeout(*reader.ctx, reader.config.FetchBatchHeaderTimeout) diff --git a/tools/traffic/blob_table.go b/tools/traffic/blob_table.go index d5ef8e37a4..2ed53a1b6e 100644 --- a/tools/traffic/blob_table.go +++ b/tools/traffic/blob_table.go @@ -9,14 +9,14 @@ import ( // BlobTable tracks blobs written by the traffic generator. This is a thread safe data structure. type BlobTable struct { - // blobs contains all blobs currently tracked by the table. + // blobs contains all blobs currently tracked by the requiredReads. blobs []*BlobMetadata - // size describes the total number of blobs currently tracked by the table. + // size describes the total number of blobs currently tracked by the requiredReads. // size may be smaller than the capacity of the blobs slice. size uint32 - // lock is used to synchronize access to the table. + // lock is used to synchronize access to the requiredReads. lock sync.Mutex } @@ -28,7 +28,7 @@ func NewBlobTable() BlobTable { } } -// Size returns the total number of blobs currently tracked by the table. +// Size returns the total number of blobs currently tracked by the requiredReads. func (table *BlobTable) Size() uint32 { table.lock.Lock() defer table.lock.Unlock() @@ -36,13 +36,14 @@ func (table *BlobTable) Size() uint32 { return table.size } -// Add a blob to the table. +// Add a blob to the requiredReads. func (table *BlobTable) Add(blob *BlobMetadata) { table.lock.Lock() defer table.lock.Unlock() + // TODO this calculation is probably a little wrong if table.size == uint32(len(table.blobs)) { - panic(fmt.Sprintf("blob table is full, cannot add blob %x", blob.Key)) + panic(fmt.Sprintf("blob requiredReads is full, cannot add blob %x", blob.Key)) } blob.index = table.size @@ -50,33 +51,59 @@ func (table *BlobTable) Add(blob *BlobMetadata) { table.size++ } -// GetRandom returns a random blob currently tracked by the table. Returns nil if the table is empty. +// addOrReplace adds a blob to the requiredReads if there is capacity or replaces an existing blob at random +// if the requiredReads is full. This method is a no-op if maximumCapacity is 0. +func (table *BlobTable) addOrReplace(blob *BlobMetadata, maximumCapacity uint32) { + if maximumCapacity == 0 { + return + } + + table.lock.Lock() + defer table.lock.Unlock() + + if table.size >= maximumCapacity { + // replace random existing blob + index := rand.Int31n(int32(table.size)) + table.blobs[index] = blob + blob.index = uint32(index) + } else { + // add new blob + blob.index = table.size + table.blobs[table.size] = blob + table.size++ + } +} + +// GetRandom returns a random blob currently tracked by the requiredReads. Returns nil if the requiredReads is empty. // Optionally decrements the read permits of the blob if decrement is true. If the number of read permits -// reaches 0, the blob is removed from the table. -func (table *BlobTable) GetRandom(decrement bool) *BlobMetadata { +// reaches 0, the blob is removed from the requiredReads. Returns the blob metadata (if there is at least one blob +// in the table) and a boolean indicating whether the blob was removed from the table as a result of this operation. +func (table *BlobTable) GetRandom(decrement bool) (*BlobMetadata, bool) { table.lock.Lock() defer table.lock.Unlock() if table.size == 0 { - return nil + return nil, false } blob := table.blobs[rand.Int31n(int32(table.size))] // TODO make sure we can get items if we overflow an int32 + removed := false if decrement && blob.remainingReadPermits != -1 { blob.remainingReadPermits-- if blob.remainingReadPermits == 0 { table.remove(blob) + removed = true } } - return blob + return blob, removed } -// remove a blob from the table. +// remove a blob from the requiredReads. func (table *BlobTable) remove(blob *BlobMetadata) { if table.blobs[blob.index] != blob { - panic(fmt.Sprintf("blob %x is not not present in the table at index %d", blob.Key, blob.index)) + panic(fmt.Sprintf("blob %x is not not present in the requiredReads at index %d", blob.Key, blob.index)) } if table.size == 1 { diff --git a/tools/traffic/blob_verifier.go b/tools/traffic/blob_verifier.go index 0cfe444ba0..71f6a91765 100644 --- a/tools/traffic/blob_verifier.go +++ b/tools/traffic/blob_verifier.go @@ -17,7 +17,7 @@ type unconfirmedKey struct { } // BlobVerifier periodically polls the disperser service to verify the status of blobs that were recently written. -// When blobs become confirmed, the status verifier updates the blob table accordingly. +// When blobs become confirmed, the status verifier updates the blob requiredReads accordingly. // This is a thread safe data structure. type BlobVerifier struct { diff --git a/tools/traffic/config.go b/tools/traffic/config.go index 7149172cba..679ba6de2e 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -88,6 +88,10 @@ type Config struct { // 0 or 1 times with the specified probability (e.g. 0.2 means each blob has a 20% chance of being downloaded). // If greater than 1.0, then each blob will be downloaded the specified number of times. RequiredDownloads float64 + // The size of a table of blobs to optionally read when we run out of blobs that we are required to read. Blobs + // that are no longer required are added to this table, and when the table is at capacity they are randomly retired. + // Set this to 0 to disable this feature. + ReadOverflowTableSize uint32 // The amount of time to wait for a batch header to be fetched. FetchBatchHeaderTimeout time.Duration // The amount of time to wait for a blob to be retrieved. @@ -148,6 +152,7 @@ func NewConfig(ctx *cli.Context) (*Config, error) { NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), + ReadOverflowTableSize: uint32(ctx.Uint(flags.ReadOverflowTableSizeFlag.Name)), FetchBatchHeaderTimeout: ctx.Duration(flags.FetchBatchHeaderTimeoutFlag.Name), RetrieveBlobChunksTimeout: ctx.Duration(flags.RetrieveBlobChunksTimeoutFlag.Name), }, nil diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index 6ac6ad9d89..efdab33ba7 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -238,7 +238,7 @@ var ( Name: common.PrefixFlag(FlagPrefix, "read-request-interval"), Usage: "Time between read requests.", Required: false, - Value: time.Second / 3, + Value: time.Second / 5, EnvVar: common.PrefixEnvVar(envPrefix, "READ_REQUEST_INTERVAL"), } RequiredDownloadsFlag = cli.Float64Flag{ @@ -249,6 +249,13 @@ var ( Value: 3.0, EnvVar: common.PrefixEnvVar(envPrefix, "REQUIRED_DOWNLOADS"), } + ReadOverflowTableSizeFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "read-overflow-table-size"), + Usage: "Size of the overflow table for read requests.", + Required: false, + Value: 1024, + EnvVar: common.PrefixEnvVar(envPrefix, "READ_OVERFLOW_TABLE_SIZE"), + } FetchBatchHeaderTimeoutFlag = cli.DurationFlag{ Name: common.PrefixFlag(FlagPrefix, "fetch-batch-header-timeout"), Usage: "Amount of time to wait for a batch header to be fetched.", @@ -305,6 +312,7 @@ var optionalFlags = []cli.Flag{ FetchBatchHeaderTimeoutFlag, RetrieveBlobChunksTimeoutFlag, GetBlobStatusTimeoutFlag, + ReadOverflowTableSizeFlag, } // Flags contains the list of configuration options available to the binary. From 890c8ac0354273c912700c93531abb07a1671d79 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 11:20:23 -0500 Subject: [PATCH 31/74] Fix metric names. Signed-off-by: Cody Littley --- tools/traffic/blob_verifier.go | 20 ++++++++++---------- tools/traffic/metrics.go | 3 --- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/tools/traffic/blob_verifier.go b/tools/traffic/blob_verifier.go index 71f6a91765..62f9f716de 100644 --- a/tools/traffic/blob_verifier.go +++ b/tools/traffic/blob_verifier.go @@ -77,17 +77,17 @@ func NewStatusVerifier( dispenser: disperser, unconfirmedKeys: make([]*unconfirmedKey, 0), keyChannel: make(chan *unconfirmedKey), - blobsInFlightMetric: metrics.NewGaugeMetric("blobsInFlight"), - getStatusLatencyMetric: metrics.NewLatencyMetric("getStatus"), + blobsInFlightMetric: metrics.NewGaugeMetric("blobs_in_flight"), + getStatusLatencyMetric: metrics.NewLatencyMetric("get_status"), confirmationLatencyMetric: metrics.NewLatencyMetric("confirmation"), - getStatusErrorCountMetric: metrics.NewCountMetric("getStatusError"), - unknownCountMetric: metrics.NewCountMetric("getStatusUnknown"), - processingCountMetric: metrics.NewCountMetric("getStatusProcessing"), - dispersingCountMetric: metrics.NewCountMetric("getStatusDispersing"), - failedCountMetric: metrics.NewCountMetric("getStatusFailed"), - insufficientSignaturesCountMetric: metrics.NewCountMetric("getStatusInsufficientSignatures"), - confirmedCountMetric: metrics.NewCountMetric("getStatusConfirmed"), - finalizedCountMetric: metrics.NewCountMetric("getStatusFinalized"), + getStatusErrorCountMetric: metrics.NewCountMetric("get_status_ERROR"), + unknownCountMetric: metrics.NewCountMetric("get_status_UNKNOWN"), + processingCountMetric: metrics.NewCountMetric("get_status_PROCESSING"), + dispersingCountMetric: metrics.NewCountMetric("get_status_DISPERSING"), + failedCountMetric: metrics.NewCountMetric("get_status_FAILED"), + insufficientSignaturesCountMetric: metrics.NewCountMetric("get_status_INSUFFICIENT_SIGNATURES"), + confirmedCountMetric: metrics.NewCountMetric("get_status_CONFIRMED"), + finalizedCountMetric: metrics.NewCountMetric("get_status_FINALIZED"), } } diff --git a/tools/traffic/metrics.go b/tools/traffic/metrics.go index 078fc3592a..09b53327b3 100644 --- a/tools/traffic/metrics.go +++ b/tools/traffic/metrics.go @@ -40,9 +40,6 @@ type GaugeMetric struct { description string } -// TODO don't start metrics if httpPort is empty -// TODO use consistent snake case on metrics - // NewMetrics creates a new Metrics instance. func NewMetrics(httpPort string, logger logging.Logger) *Metrics { namespace := "eigenda_generator" From 75b160a67fdbbfb1031554fa231d00626427accb Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 11:55:26 -0500 Subject: [PATCH 32/74] Compute checksum, data returned does not match. Signed-off-by: Cody Littley --- tools/traffic/blob_metadata.go | 7 ++- tools/traffic/blob_reader.go | 98 +++++++++++++++++++--------------- tools/traffic/blob_table.go | 4 +- tools/traffic/blob_verifier.go | 11 ++-- tools/traffic/blob_writer.go | 5 +- 5 files changed, 75 insertions(+), 50 deletions(-) diff --git a/tools/traffic/blob_metadata.go b/tools/traffic/blob_metadata.go index 10153b8395..d3e07ec455 100644 --- a/tools/traffic/blob_metadata.go +++ b/tools/traffic/blob_metadata.go @@ -5,6 +5,9 @@ type BlobMetadata struct { // key of the blob, set when the blob is initially uploaded. key *[]byte + // checksum of the blob. + checksum *[16]byte + // batchHeaderHash of the blob. batchHeaderHash *[]byte @@ -23,12 +26,14 @@ type BlobMetadata struct { // remaining reads permitted against this blob. If -1 then an unlimited number of reads are permitted. func NewBlobMetadata( key *[]byte, + checksum *[16]byte, batchHeaderHash *[]byte, blobIndex uint32, readPermits int32) *BlobMetadata { return &BlobMetadata{ key: key, + checksum: checksum, batchHeaderHash: batchHeaderHash, blobIndex: blobIndex, remainingReadPermits: readPermits, @@ -50,5 +55,3 @@ func (blob *BlobMetadata) BatchHeaderHash() *[]byte { func (blob *BlobMetadata) BlobIndex() uint32 { return blob.blobIndex } - -// TODO method for decrementing read permits diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index b98db5d178..e1ea38c323 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -2,11 +2,13 @@ package traffic import ( "context" + "crypto/md5" "fmt" "github.com/Layr-Labs/eigenda/api/clients" contractEigenDAServiceManager "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDAServiceManager" "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/Layr-Labs/eigenda/retriever/eth" "github.com/Layr-Labs/eigensdk-go/logging" gcommon "github.com/ethereum/go-ethereum/common" @@ -38,19 +40,21 @@ type BlobReader struct { // optionalReads blobs we are not required to read, but can choose to read if we want. optionalReads *BlobTable - metrics *Metrics - fetchBatchHeaderMetric LatencyMetric - fetchBatchHeaderSuccess CountMetric - fetchBatchHeaderFailure CountMetric - readLatencyMetric LatencyMetric - readSuccessMetric CountMetric - readFailureMetric CountMetric - recombinationSuccess CountMetric - recombinationFailure CountMetric - operatorSuccessMetrics map[core.OperatorID]CountMetric - operatorFailureMetrics map[core.OperatorID]CountMetric - requiredReadPoolSize GaugeMetric - optionalReadPoolSize GaugeMetric + metrics *Metrics + fetchBatchHeaderMetric LatencyMetric + fetchBatchHeaderSuccess CountMetric + fetchBatchHeaderFailure CountMetric + readLatencyMetric LatencyMetric + readSuccessMetric CountMetric + readFailureMetric CountMetric + recombinationSuccessMetric CountMetric + recombinationFailureMetric CountMetric + validBlobMetric CountMetric + invalidBlobMetric CountMetric + operatorSuccessMetrics map[core.OperatorID]CountMetric + operatorFailureMetrics map[core.OperatorID]CountMetric + requiredReadPoolSizeMetric GaugeMetric + optionalReadPoolSizeMetric GaugeMetric } // NewBlobReader creates a new BlobReader instance. @@ -67,27 +71,29 @@ func NewBlobReader( optionalReads := NewBlobTable() return BlobReader{ - ctx: ctx, - waitGroup: waitGroup, - logger: logger, - config: config, - retriever: retriever, - chainClient: chainClient, - requiredReads: table, - optionalReads: &optionalReads, - metrics: metrics, - fetchBatchHeaderMetric: metrics.NewLatencyMetric("fetch_batch_header"), - fetchBatchHeaderSuccess: metrics.NewCountMetric("fetch_batch_header_success"), - fetchBatchHeaderFailure: metrics.NewCountMetric("fetch_batch_header_failure"), - recombinationSuccess: metrics.NewCountMetric("recombination_success"), - recombinationFailure: metrics.NewCountMetric("recombination_failure"), - readLatencyMetric: metrics.NewLatencyMetric("read"), - readSuccessMetric: metrics.NewCountMetric("read_success"), - readFailureMetric: metrics.NewCountMetric("read_failure"), - operatorSuccessMetrics: make(map[core.OperatorID]CountMetric), - operatorFailureMetrics: make(map[core.OperatorID]CountMetric), - requiredReadPoolSize: metrics.NewGaugeMetric("required_read_pool_size"), - optionalReadPoolSize: metrics.NewGaugeMetric("optional_read_pool_size"), + ctx: ctx, + waitGroup: waitGroup, + logger: logger, + config: config, + retriever: retriever, + chainClient: chainClient, + requiredReads: table, + optionalReads: &optionalReads, + metrics: metrics, + fetchBatchHeaderMetric: metrics.NewLatencyMetric("fetch_batch_header"), + fetchBatchHeaderSuccess: metrics.NewCountMetric("fetch_batch_header_success"), + fetchBatchHeaderFailure: metrics.NewCountMetric("fetch_batch_header_failure"), + recombinationSuccessMetric: metrics.NewCountMetric("recombination_success"), + recombinationFailureMetric: metrics.NewCountMetric("recombination_failure"), + readLatencyMetric: metrics.NewLatencyMetric("read"), + validBlobMetric: metrics.NewCountMetric("valid_blob"), + invalidBlobMetric: metrics.NewCountMetric("invalid_blob"), + readSuccessMetric: metrics.NewCountMetric("read_success"), + readFailureMetric: metrics.NewCountMetric("read_failure"), + operatorSuccessMetrics: make(map[core.OperatorID]CountMetric), + operatorFailureMetrics: make(map[core.OperatorID]CountMetric), + requiredReadPoolSizeMetric: metrics.NewGaugeMetric("required_read_pool_size"), + optionalReadPoolSizeMetric: metrics.NewGaugeMetric("optional_read_pool_size"), } } @@ -116,7 +122,7 @@ func (reader *BlobReader) run() { // randomRead reads a random blob. func (reader *BlobReader) randomRead() { - reader.requiredReadPoolSize.Set(float64(reader.requiredReads.Size())) + reader.requiredReadPoolSizeMetric.Set(float64(reader.requiredReads.Size())) metadata, removed := reader.requiredReads.GetRandom(true) if metadata == nil { @@ -128,8 +134,8 @@ func (reader *BlobReader) randomRead() { } } else if removed { // We have removed a blob from the requiredReads. Add it to the optionalReads. - reader.optionalReads.addOrReplace(metadata, reader.config.ReadOverflowTableSize) - reader.optionalReadPoolSize.Set(float64(reader.optionalReads.Size())) + reader.optionalReads.AddOrReplace(metadata, reader.config.ReadOverflowTableSize) + reader.optionalReadPoolSizeMetric.Set(float64(reader.optionalReads.Size())) } ctxTimeout, cancel := context.WithTimeout(*reader.ctx, reader.config.FetchBatchHeaderTimeout) @@ -177,12 +183,12 @@ func (reader *BlobReader) randomRead() { data, err := reader.retriever.CombineChunks(chunks) if err != nil { reader.logger.Error("failed to combine chunks", "err:", err) - reader.recombinationFailure.Increment() + reader.recombinationFailureMetric.Increment() return } - reader.recombinationSuccess.Increment() + reader.recombinationSuccessMetric.Increment() - reader.verifyBlob(data) + reader.verifyBlob(metadata, data) indexSet := make(map[encoding.ChunkNumber]bool) for index := range chunks.Indices { @@ -223,6 +229,14 @@ func (reader *BlobReader) reportMissingChunk(operatorId core.OperatorID) { } // verifyBlob performs sanity checks on the blob. -func (reader *BlobReader) verifyBlob(blob []byte) { - // TODO: do sanity checks on the data returned +func (reader *BlobReader) verifyBlob(metadata *BlobMetadata, blob []byte) { + recomputedChecksum := md5.Sum(codec.RemoveEmptyByteFromPaddedBytes(blob)) + + fmt.Printf("metadata.checksum: %x, recomputed checksum: %x\n", *metadata.checksum, recomputedChecksum) + + if *metadata.checksum == recomputedChecksum { + reader.validBlobMetric.Increment() + } else { + reader.invalidBlobMetric.Increment() + } } diff --git a/tools/traffic/blob_table.go b/tools/traffic/blob_table.go index 2ed53a1b6e..22cb7c658d 100644 --- a/tools/traffic/blob_table.go +++ b/tools/traffic/blob_table.go @@ -51,9 +51,9 @@ func (table *BlobTable) Add(blob *BlobMetadata) { table.size++ } -// addOrReplace adds a blob to the requiredReads if there is capacity or replaces an existing blob at random +// AddOrReplace adds a blob to the requiredReads if there is capacity or replaces an existing blob at random // if the requiredReads is full. This method is a no-op if maximumCapacity is 0. -func (table *BlobTable) addOrReplace(blob *BlobMetadata, maximumCapacity uint32) { +func (table *BlobTable) AddOrReplace(blob *BlobMetadata, maximumCapacity uint32) { if maximumCapacity == 0 { return } diff --git a/tools/traffic/blob_verifier.go b/tools/traffic/blob_verifier.go index 62f9f716de..3954011df6 100644 --- a/tools/traffic/blob_verifier.go +++ b/tools/traffic/blob_verifier.go @@ -12,7 +12,11 @@ import ( // unconfirmedKey is a key that has not yet been confirmed by the disperser service. type unconfirmedKey struct { - key *[]byte + // The key of the blob. + key *[]byte + // The checksum of the blob. + checksum *[16]byte + // The time the blob was submitted to the disperser service. submissionTime time.Time } @@ -92,9 +96,10 @@ func NewStatusVerifier( } // AddUnconfirmedKey adds a key to the list of unconfirmed keys. -func (verifier *BlobVerifier) AddUnconfirmedKey(key *[]byte) { +func (verifier *BlobVerifier) AddUnconfirmedKey(key *[]byte, checksum *[16]byte) { verifier.keyChannel <- &unconfirmedKey{ key: key, + checksum: checksum, submissionTime: time.Now(), } } @@ -221,6 +226,6 @@ func (verifier *BlobVerifier) forwardToReader(key *unconfirmedKey, status *dispe downloadCount = int32(requiredDownloads) } - blobMetadata := NewBlobMetadata(key.key, &batchHeaderHash, blobIndex, downloadCount) + blobMetadata := NewBlobMetadata(key.key, key.checksum, &batchHeaderHash, blobIndex, downloadCount) verifier.table.Add(blobMetadata) } diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 26181d5390..004723ee0b 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -2,6 +2,7 @@ package traffic import ( "context" + "crypto/md5" "crypto/rand" "fmt" "github.com/Layr-Labs/eigenda/api/clients" @@ -111,7 +112,9 @@ func (writer *BlobWriter) run() { } writer.writeSuccessMetric.Increment() - writer.verifier.AddUnconfirmedKey(&key) + + checksum := md5.Sum(*data) + writer.verifier.AddUnconfirmedKey(&key, &checksum) } } } From 9972344c83c701468f641e50ae2f6615c9c2e884 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 12:55:07 -0500 Subject: [PATCH 33/74] Fix checksum. Signed-off-by: Cody Littley --- api/clients/mock/retrieval_client.go | 2 +- tools/traffic/blob_metadata.go | 5 +++++ tools/traffic/blob_reader.go | 11 +++++------ tools/traffic/blob_verifier.go | 10 +++++++--- tools/traffic/blob_writer.go | 4 +++- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/api/clients/mock/retrieval_client.go b/api/clients/mock/retrieval_client.go index 5dfd326083..3c54612573 100644 --- a/api/clients/mock/retrieval_client.go +++ b/api/clients/mock/retrieval_client.go @@ -44,7 +44,7 @@ func (c *MockRetrievalClient) RetrieveBlobChunks( batchRoot [32]byte, quorumID core.QuorumID) (*clients.BlobChunks, error) { - return nil, nil // TODO + return nil, nil } func (c *MockRetrievalClient) CombineChunks(chunks *clients.BlobChunks) ([]byte, error) { diff --git a/tools/traffic/blob_metadata.go b/tools/traffic/blob_metadata.go index d3e07ec455..2d71749210 100644 --- a/tools/traffic/blob_metadata.go +++ b/tools/traffic/blob_metadata.go @@ -8,6 +8,9 @@ type BlobMetadata struct { // checksum of the blob. checksum *[16]byte + // batchHeaderHash of the blob in bytes. + size uint + // batchHeaderHash of the blob. batchHeaderHash *[]byte @@ -27,6 +30,7 @@ type BlobMetadata struct { func NewBlobMetadata( key *[]byte, checksum *[16]byte, + size uint, batchHeaderHash *[]byte, blobIndex uint32, readPermits int32) *BlobMetadata { @@ -34,6 +38,7 @@ func NewBlobMetadata( return &BlobMetadata{ key: key, checksum: checksum, + size: size, batchHeaderHash: batchHeaderHash, blobIndex: blobIndex, remainingReadPermits: readPermits, diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index e1ea38c323..ad051492b7 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -8,7 +8,6 @@ import ( contractEigenDAServiceManager "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDAServiceManager" "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" - "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/Layr-Labs/eigenda/retriever/eth" "github.com/Layr-Labs/eigensdk-go/logging" gcommon "github.com/ethereum/go-ethereum/common" @@ -188,7 +187,7 @@ func (reader *BlobReader) randomRead() { } reader.recombinationSuccessMetric.Increment() - reader.verifyBlob(metadata, data) + reader.verifyBlob(metadata, &data) indexSet := make(map[encoding.ChunkNumber]bool) for index := range chunks.Indices { @@ -229,10 +228,10 @@ func (reader *BlobReader) reportMissingChunk(operatorId core.OperatorID) { } // verifyBlob performs sanity checks on the blob. -func (reader *BlobReader) verifyBlob(metadata *BlobMetadata, blob []byte) { - recomputedChecksum := md5.Sum(codec.RemoveEmptyByteFromPaddedBytes(blob)) - - fmt.Printf("metadata.checksum: %x, recomputed checksum: %x\n", *metadata.checksum, recomputedChecksum) +func (reader *BlobReader) verifyBlob(metadata *BlobMetadata, blob *[]byte) { + // Trim off the padding. + truncatedBlob := (*blob)[:metadata.size] + recomputedChecksum := md5.Sum(truncatedBlob) if *metadata.checksum == recomputedChecksum { reader.validBlobMetric.Increment() diff --git a/tools/traffic/blob_verifier.go b/tools/traffic/blob_verifier.go index 3954011df6..b3fbc1c50f 100644 --- a/tools/traffic/blob_verifier.go +++ b/tools/traffic/blob_verifier.go @@ -14,6 +14,8 @@ import ( type unconfirmedKey struct { // The key of the blob. key *[]byte + // The size of the blob in bytes. + size uint // The checksum of the blob. checksum *[16]byte // The time the blob was submitted to the disperser service. @@ -96,10 +98,11 @@ func NewStatusVerifier( } // AddUnconfirmedKey adds a key to the list of unconfirmed keys. -func (verifier *BlobVerifier) AddUnconfirmedKey(key *[]byte, checksum *[16]byte) { +func (verifier *BlobVerifier) AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) { verifier.keyChannel <- &unconfirmedKey{ key: key, checksum: checksum, + size: size, submissionTime: time.Now(), } } @@ -131,7 +134,8 @@ func (verifier *BlobVerifier) monitor(period time.Duration) { // If a key is confirmed, it is added to the blob table and removed from the list of unconfirmed keys. func (verifier *BlobVerifier) poll() { - // TODO If the number of unconfirmed blobs is high and the time to confirm his high, this is not efficient. + // FUTURE WORK If the number of unconfirmed blobs is high and the time to confirm is high, this is not efficient. + // Revisit this method if there are performance problems. unconfirmedKeys := make([]*unconfirmedKey, 0) for _, key := range verifier.unconfirmedKeys { @@ -226,6 +230,6 @@ func (verifier *BlobVerifier) forwardToReader(key *unconfirmedKey, status *dispe downloadCount = int32(requiredDownloads) } - blobMetadata := NewBlobMetadata(key.key, key.checksum, &batchHeaderHash, blobIndex, downloadCount) + blobMetadata := NewBlobMetadata(key.key, key.checksum, key.size, &batchHeaderHash, blobIndex, downloadCount) verifier.table.Add(blobMetadata) } diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 004723ee0b..05b4c5aa53 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -113,8 +113,10 @@ func (writer *BlobWriter) run() { writer.writeSuccessMetric.Increment() + fmt.Printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Data size = %d\n", len(*data)) + checksum := md5.Sum(*data) - writer.verifier.AddUnconfirmedKey(&key, &checksum) + writer.verifier.AddUnconfirmedKey(&key, &checksum, uint(len(*data))) } } } From 1f64d67c50babe44dfd8e34e9bbd5430d7290431 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 12:59:42 -0500 Subject: [PATCH 34/74] Revert some files. Signed-off-by: Cody Littley --- api/clients/eigenda_client_e2e_test.go | 2 +- api/clients/retrieval_client.go | 3 --- core/indexer/state.go | 9 ++++----- disperser/cmd/dataapi/main.go | 2 +- node/node.go | 6 +++--- node/plugin/cmd/main.go | 2 +- retriever/cmd/main.go | 5 +---- test/synthetic-test/synthetic_client_test.go | 8 ++++---- tools/traffic/blob_writer.go | 2 -- 9 files changed, 15 insertions(+), 24 deletions(-) diff --git a/api/clients/eigenda_client_e2e_test.go b/api/clients/eigenda_client_e2e_test.go index 235f1200ad..f10a6aec91 100644 --- a/api/clients/eigenda_client_e2e_test.go +++ b/api/clients/eigenda_client_e2e_test.go @@ -15,7 +15,7 @@ import ( var runTestnetIntegrationTests bool func init() { - flag.BoolVar(&runTestnetIntegrationTests, "testnet-integration", false, "Start testnet-based integration tests") + flag.BoolVar(&runTestnetIntegrationTests, "testnet-integration", false, "Run testnet-based integration tests") } func TestClientUsingTestnet(t *testing.T) { diff --git a/api/clients/retrieval_client.go b/api/clients/retrieval_client.go index a00ae9f0ae..07896baa08 100644 --- a/api/clients/retrieval_client.go +++ b/api/clients/retrieval_client.go @@ -60,9 +60,6 @@ type retrievalClient struct { numConnections int } -// TODO can this line be deleted? -var _ RetrievalClient = (*retrievalClient)(nil) - // NewRetrievalClient creates a new retrieval client. func NewRetrievalClient( logger logging.Logger, diff --git a/core/indexer/state.go b/core/indexer/state.go index 2078167ae1..28c1f823f6 100644 --- a/core/indexer/state.go +++ b/core/indexer/state.go @@ -3,7 +3,6 @@ package indexer import ( "context" "errors" - "fmt" "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/indexer" @@ -36,12 +35,12 @@ func (ics *IndexedChainState) GetIndexedOperatorState(ctx context.Context, block pubkeys, sockets, err := ics.getObjects(blockNumber) if err != nil { - return nil, fmt.Errorf("unable to complete ics.getObjects(%d): %s)", blockNumber, err) + return nil, err } operatorState, err := ics.ChainState.GetOperatorState(ctx, blockNumber, quorums) if err != nil { - return nil, fmt.Errorf("unable to complete ics.ChainState.GetOperatorState(%d, %v): %s", blockNumber, quorums, err) + return nil, err } ops := make(map[core.OperatorID]*core.IndexedOperatorInfo, len(pubkeys.Operators)) @@ -93,7 +92,7 @@ func (ics *IndexedChainState) getObjects(blockNumber uint) (*OperatorPubKeys, Op obj, err := ics.Indexer.GetObject(queryHeader, 0) if err != nil { - return nil, nil, fmt.Errorf("(1) unable to call Indexer.GetObject({Number = %d}, 0): %s", blockNumber, err) + return nil, nil, err } pubkeys, ok := obj.(*OperatorPubKeys) @@ -103,7 +102,7 @@ func (ics *IndexedChainState) getObjects(blockNumber uint) (*OperatorPubKeys, Op obj, err = ics.Indexer.GetObject(queryHeader, 1) if err != nil { - return nil, nil, fmt.Errorf("(2) unable to call Indexer.GetObject(%v, 1): %s", queryHeader, err) + return nil, nil, err } sockets, ok := obj.(OperatorSockets) diff --git a/disperser/cmd/dataapi/main.go b/disperser/cmd/dataapi/main.go index 7b9bcd1750..abddf5b1ea 100644 --- a/disperser/cmd/dataapi/main.go +++ b/disperser/cmd/dataapi/main.go @@ -129,7 +129,7 @@ func RunDataApi(ctx *cli.Context) error { // catch SIGINT (Ctrl+C) and SIGTERM (e.g., from `kill`) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - // Start server in a separate goroutine so that it doesn't block. + // Run server in a separate goroutine so that it doesn't block. go func() { if err := server.Start(); err != nil { logger.Fatalf("Failed to start server: %v", err) diff --git a/node/node.go b/node/node.go index b9b1da490a..59a4f9c331 100644 --- a/node/node.go +++ b/node/node.go @@ -300,7 +300,7 @@ func (n *Node) ProcessBatch(ctx context.Context, header *core.BatchHeader, blobs return nil, errors.New("number of parsed blobs must be the same as number of blobs from protobuf request") } - // Invoke num batches received and its size in bytes + // Measure num batches received and its size in bytes batchSize := uint64(0) for _, blob := range blobs { for quorumID, bundle := range blob.Bundles { @@ -314,7 +314,7 @@ func (n *Node) ProcessBatch(ctx context.Context, header *core.BatchHeader, blobs log.Debug("Start processing a batch", "batchHeaderHash", batchHeaderHashHex, "batchSize (in bytes)", batchSize, "num of blobs", len(blobs), "referenceBlockNumber", header.ReferenceBlockNumber) // Store the batch. - // Start this in a goroutine so we can parallelize the batch storing and batch + // Run this in a goroutine so we can parallelize the batch storing and batch // verifaction work. // This should be able to improve latency without needing more CPUs, because batch // storing is an IO operation. @@ -326,7 +326,7 @@ func (n *Node) ProcessBatch(ctx context.Context, header *core.BatchHeader, blobs // Defined only if the batch not already exists and gets stored to database successfully. keys *[][]byte - // latency to store the batch. + // Latency to store the batch. // Defined only if the batch not already exists and gets stored to database successfully. latency time.Duration } diff --git a/node/plugin/cmd/main.go b/node/plugin/cmd/main.go index 3e80679d0d..c50aa88b61 100644 --- a/node/plugin/cmd/main.go +++ b/node/plugin/cmd/main.go @@ -37,7 +37,7 @@ func main() { } app.Name = "eigenda-node-plugin" app.Usage = "EigenDA Node Plugin" - app.Description = "Start one time operations like avs opt-in/opt-out for EigenDA Node" + app.Description = "Run one time operations like avs opt-in/opt-out for EigenDA Node" app.Action = pluginOps err := app.Run(os.Args) if err != nil { diff --git a/retriever/cmd/main.go b/retriever/cmd/main.go index 7065123479..ab03ec6eff 100644 --- a/retriever/cmd/main.go +++ b/retriever/cmd/main.go @@ -64,7 +64,7 @@ func RetrieverMain(ctx *cli.Context) error { grpc.ChainUnaryInterceptor( // TODO(ian-shim): Add interceptors // correlation.UnaryServerInterceptor(), - // logger.UnaryServerInterceptor(*s.logger.logger), + // logger.UnaryServerInterceptor(*s.logger.Logger), ), ) @@ -72,9 +72,6 @@ func RetrieverMain(ctx *cli.Context) error { if err != nil { log.Fatalf("failed to parse the command line flags: %v", err) } - - fmt.Printf(">>>>>>>>>>> Configuration\n%+v\n", config) - logger, err := common.NewLogger(config.LoggerConfig) if err != nil { log.Fatalf("failed to create logger: %v", err) diff --git a/test/synthetic-test/synthetic_client_test.go b/test/synthetic-test/synthetic_client_test.go index 0d99b7afc8..ab868c557a 100644 --- a/test/synthetic-test/synthetic_client_test.go +++ b/test/synthetic-test/synthetic_client_test.go @@ -51,7 +51,7 @@ type GrpcClient struct { Hostname string GrpcPort string Timeout time.Duration - Client interface{} // This can be a specific client type (disperser_rpc.disperserClient, retriever_rpc.RetrieverClient, etc.) + Client interface{} // This can be a specific client type (disperser_rpc.DisperserClient, retriever_rpc.RetrieverClient, etc.) } type RetrieverClientConfig struct { @@ -178,7 +178,7 @@ func TestMain(m *testing.M) { retrieverCachePath := os.Getenv("RETRIEVER_CACHE_PATH") batcherPullInterval := os.Getenv("BATCHER_PULL_INTERVAL") - // Retriever config + // Retriever Config retrieverClientConfig := &RetrieverClientConfig{ Bls_Operator_State_Retriever: blsOperatorStateRetriever, EigenDA_ServiceManager_Retriever: eigenDAServiceManagerRetreiever, @@ -204,9 +204,9 @@ func TestMain(m *testing.M) { logger.Println("Retriever Client Enabled...", isRetrieverClientEnabled) logger.Println("Running Test Client...") - // Start the tests and get the exit code + // Run the tests and get the exit code exitCode := m.Run() - logger.Printf("Exiting Test Client Start with Code:%d", exitCode) + logger.Printf("Exiting Test Client Run with Code:%d", exitCode) // Exit with the test result code os.Exit(exitCode) } diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 05b4c5aa53..724c4cf622 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -113,8 +113,6 @@ func (writer *BlobWriter) run() { writer.writeSuccessMetric.Increment() - fmt.Printf(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Data size = %d\n", len(*data)) - checksum := md5.Sum(*data) writer.verifier.AddUnconfirmedKey(&key, &checksum, uint(len(*data))) } From a9d783c170d9ea2adcd1f8e71959d2628022b56d Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 13:03:23 -0500 Subject: [PATCH 35/74] Fixed a few remaining issues. Signed-off-by: Cody Littley --- retriever/Makefile | 10 ++++------ tools/traffic/Makefile | 2 +- tools/traffic/blob_table.go | 7 +------ tools/traffic/generator.go | 10 +++++----- tools/traffic/generator_test.go | 4 ++-- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/retriever/Makefile b/retriever/Makefile index 7219fe79b3..f71e1e49d9 100644 --- a/retriever/Makefile +++ b/retriever/Makefile @@ -12,15 +12,13 @@ run: build DA_RETRIEVER_TIMEOUT=10s \ ./bin/server \ --retriever.hostname localhost \ - --retriever.grpc-port 32099 \ + --retriever.grpc-port 32011 \ --retriever.timeout 10s \ - --retriever.bls-operator-state-retriever 0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154 \ - --retriever.eigenda-service-manager 0x851356ae760d987E095750cCeb3bC6014560891C \ + --retriever.bls-operator-state-retriever 0x9d4454B023096f34B160D6B654540c56A1F81688 \ + --retriever.eigenda-service-manager 0x67d269191c92Caf3cD7723F116c85e6E9bf55933 \ --kzg.g1-path ../inabox/resources/kzg/g1.point \ --kzg.g2-path ../inabox/resources/kzg/g2.point \ --kzg.cache-path ../inabox/resources/kzg/SRSTables \ - --kzg.srs-load 3000 \ --kzg.srs-order 3000 \ --chain.rpc http://localhost:8545 \ - --chain.private-key="" \ - --thegraph.endpoint localhost:8000 \ No newline at end of file + --chain.private-key="" \ No newline at end of file diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index ea919bac73..56f866255a 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -26,4 +26,4 @@ run: build TRAFFIC_GENERATOR_ENCODER_CACHE_DIR=../../inabox/resources/kzg/SRSTables \ ./bin/server -# TODO do not merge change TRAFFIC_GENERATOR_GRPC_PORT=32003 \ No newline at end of file +# TODO figure out what the appropriate values are for this prior to merging. These are the values that worked for me locally. \ No newline at end of file diff --git a/tools/traffic/blob_table.go b/tools/traffic/blob_table.go index 22cb7c658d..dcb1039093 100644 --- a/tools/traffic/blob_table.go +++ b/tools/traffic/blob_table.go @@ -41,11 +41,6 @@ func (table *BlobTable) Add(blob *BlobMetadata) { table.lock.Lock() defer table.lock.Unlock() - // TODO this calculation is probably a little wrong - if table.size == uint32(len(table.blobs)) { - panic(fmt.Sprintf("blob requiredReads is full, cannot add blob %x", blob.Key)) - } - blob.index = table.size table.blobs[table.size] = blob table.size++ @@ -86,7 +81,7 @@ func (table *BlobTable) GetRandom(decrement bool) (*BlobMetadata, bool) { return nil, false } - blob := table.blobs[rand.Int31n(int32(table.size))] // TODO make sure we can get items if we overflow an int32 + blob := table.blobs[rand.Int31n(int32(table.size))] removed := false if decrement && blob.remainingReadPermits != -1 { diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index e83ba1b895..a157f389d5 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -23,7 +23,7 @@ import ( "github.com/Layr-Labs/eigenda/core" ) -// TrafficGenerator simulates read/write traffic to the DA service. +// Generator simulates read/write traffic to the DA service. // // ┌------------┐ ┌------------┐ // | writer |-┐ ┌------------┐ | reader |-┐ @@ -38,7 +38,7 @@ import ( // When a writer finishes writing a blob, it sends information about that blob to the verifier. // When the verifier observes that a blob has been confirmed, it sends information about the blob // to the readers. The readers only attempt to read blobs that have been confirmed by the verifier. -type TrafficGenerator struct { +type Generator struct { ctx *context.Context cancel *context.CancelFunc waitGroup *sync.WaitGroup @@ -53,7 +53,7 @@ type TrafficGenerator struct { readers []*BlobReader } -func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*TrafficGenerator, error) { +func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Generator, error) { logger, err := common.NewLogger(config.LoggingConfig) if err != nil { return nil, err @@ -127,7 +127,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Traffi readers = append(readers, &reader) } - return &TrafficGenerator{ + return &Generator{ ctx: &ctx, cancel: &cancel, waitGroup: &waitGroup, @@ -208,7 +208,7 @@ func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainC } // Start instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. -func (generator *TrafficGenerator) Start() error { +func (generator *Generator) Start() error { generator.metrics.Start() generator.verifier.Start() diff --git a/tools/traffic/generator_test.go b/tools/traffic/generator_test.go index 39307fa6b3..74a6016492 100644 --- a/tools/traffic/generator_test.go +++ b/tools/traffic/generator_test.go @@ -17,7 +17,7 @@ import ( func TestTrafficGenerator(t *testing.T) { disperserClient := clientsmock.NewMockDisperserClient() logger := logging.NewNoopLogger() - trafficGenerator := &traffic.TrafficGenerator{ + trafficGenerator := &traffic.Generator{ logger: logger, config: &traffic.Config{ Config: clients.Config{ @@ -45,7 +45,7 @@ func TestTrafficGeneratorAuthenticated(t *testing.T) { disperserClient := clientsmock.NewMockDisperserClient() logger := logging.NewNoopLogger() - trafficGenerator := &traffic.TrafficGenerator{ + trafficGenerator := &traffic.Generator{ logger: logger, config: &traffic.Config{ Config: clients.Config{ From b23479468a96bb1210d6b02dd9ca1f892873ad75 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 13:06:51 -0500 Subject: [PATCH 36/74] Log formatting. Signed-off-by: Cody Littley --- api/clients/retrieval_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/clients/retrieval_client.go b/api/clients/retrieval_client.go index 07896baa08..5edf141874 100644 --- a/api/clients/retrieval_client.go +++ b/api/clients/retrieval_client.go @@ -110,7 +110,7 @@ func (r *retrievalClient) RetrieveBlobChunks(ctx context.Context, } operators, ok := indexedOperatorState.Operators[quorumID] if !ok { - return nil, fmt.Errorf(" no quorum with ID: %d", quorumID) + return nil, fmt.Errorf("no quorum with ID: %d", quorumID) } // Get blob header from any operator From 9e6a3d2846bbea725d6955d53995f82b885983c9 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 13:19:56 -0500 Subject: [PATCH 37/74] Comment out old test, replace before merging. Signed-off-by: Cody Littley --- tools/traffic/generator_test.go | 138 ++++++++++++++++---------------- 1 file changed, 70 insertions(+), 68 deletions(-) diff --git a/tools/traffic/generator_test.go b/tools/traffic/generator_test.go index 74a6016492..c94c06882a 100644 --- a/tools/traffic/generator_test.go +++ b/tools/traffic/generator_test.go @@ -1,71 +1,73 @@ package traffic_test -import ( - "context" - "testing" - "time" +// TODO reimplement this test - "github.com/Layr-Labs/eigenda/api/clients" - clientsmock "github.com/Layr-Labs/eigenda/api/clients/mock" - "github.com/Layr-Labs/eigenda/disperser" - "github.com/Layr-Labs/eigenda/tools/traffic" - "github.com/Layr-Labs/eigensdk-go/logging" - - "github.com/stretchr/testify/mock" -) - -func TestTrafficGenerator(t *testing.T) { - disperserClient := clientsmock.NewMockDisperserClient() - logger := logging.NewNoopLogger() - trafficGenerator := &traffic.Generator{ - logger: logger, - config: &traffic.Config{ - Config: clients.Config{ - Timeout: 1 * time.Second, - }, - DataSize: 1000_000, - WriteRequestInterval: 2 * time.Second, - }, - disperserClient: disperserClient, - } - - processing := disperser.Processing - disperserClient.On("DisperseBlob", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(&processing, []byte{1}, nil) - ctx, cancel := context.WithCancel(context.Background()) - go func() { - _ = trafficGenerator.StartBlobWriter(ctx) - }() - time.Sleep(5 * time.Second) - cancel() - disperserClient.AssertNumberOfCalls(t, "DisperseBlob", 2) -} - -func TestTrafficGeneratorAuthenticated(t *testing.T) { - disperserClient := clientsmock.NewMockDisperserClient() - logger := logging.NewNoopLogger() - - trafficGenerator := &traffic.Generator{ - logger: logger, - config: &traffic.Config{ - Config: clients.Config{ - Timeout: 1 * time.Second, - }, - DataSize: 1000_000, - WriteRequestInterval: 2 * time.Second, - SignerPrivateKey: "Hi", - }, - disperserClient: disperserClient, - } - - processing := disperser.Processing - disperserClient.On("DisperseBlobAuthenticated", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(&processing, []byte{1}, nil) - ctx, cancel := context.WithCancel(context.Background()) - go func() { - _ = trafficGenerator.StartBlobWriter(ctx) - }() - time.Sleep(5 * time.Second) - cancel() - disperserClient.AssertNumberOfCalls(t, "DisperseBlobAuthenticated", 2) -} +//import ( +// "context" +// "testing" +// "time" +// +// "github.com/Layr-Labs/eigenda/api/clients" +// clientsmock "github.com/Layr-Labs/eigenda/api/clients/mock" +// "github.com/Layr-Labs/eigenda/disperser" +// "github.com/Layr-Labs/eigenda/tools/traffic" +// "github.com/Layr-Labs/eigensdk-go/logging" +// +// "github.com/stretchr/testify/mock" +//) +// +//func TestTrafficGenerator(t *testing.T) { +// disperserClient := clientsmock.NewMockDisperserClient() +// logger := logging.NewNoopLogger() +// trafficGenerator := &traffic.Generator{ +// logger: logger, +// config: &traffic.Config{ +// Config: clients.Config{ +// Timeout: 1 * time.Second, +// }, +// DataSize: 1000_000, +// WriteRequestInterval: 2 * time.Second, +// }, +// disperserClient: disperserClient, +// } +// +// processing := disperser.Processing +// disperserClient.On("DisperseBlob", mock.Anything, mock.Anything, mock.Anything, mock.Anything). +// Return(&processing, []byte{1}, nil) +// ctx, cancel := context.WithCancel(context.Background()) +// go func() { +// _ = trafficGenerator.StartBlobWriter(ctx) +// }() +// time.Sleep(5 * time.Second) +// cancel() +// disperserClient.AssertNumberOfCalls(t, "DisperseBlob", 2) +//} +// +//func TestTrafficGeneratorAuthenticated(t *testing.T) { +// disperserClient := clientsmock.NewMockDisperserClient() +// logger := logging.NewNoopLogger() +// +// trafficGenerator := &traffic.Generator{ +// logger: logger, +// config: &traffic.Config{ +// Config: clients.Config{ +// Timeout: 1 * time.Second, +// }, +// DataSize: 1000_000, +// WriteRequestInterval: 2 * time.Second, +// SignerPrivateKey: "Hi", +// }, +// disperserClient: disperserClient, +// } +// +// processing := disperser.Processing +// disperserClient.On("DisperseBlobAuthenticated", mock.Anything, mock.Anything, mock.Anything, mock.Anything). +// Return(&processing, []byte{1}, nil) +// ctx, cancel := context.WithCancel(context.Background()) +// go func() { +// _ = trafficGenerator.StartBlobWriter(ctx) +// }() +// time.Sleep(5 * time.Second) +// cancel() +// disperserClient.AssertNumberOfCalls(t, "DisperseBlobAuthenticated", 2) +//} From c305f00db7fb876a484bafb1f4f5c2eb9b897f33 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 25 Jul 2024 14:38:50 -0500 Subject: [PATCH 38/74] Fix malformed sprintf Signed-off-by: Cody Littley --- tools/traffic/blob_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/traffic/blob_table.go b/tools/traffic/blob_table.go index dcb1039093..d6fb5ce5d1 100644 --- a/tools/traffic/blob_table.go +++ b/tools/traffic/blob_table.go @@ -98,7 +98,7 @@ func (table *BlobTable) GetRandom(decrement bool) (*BlobMetadata, bool) { // remove a blob from the requiredReads. func (table *BlobTable) remove(blob *BlobMetadata) { if table.blobs[blob.index] != blob { - panic(fmt.Sprintf("blob %x is not not present in the requiredReads at index %d", blob.Key, blob.index)) + panic(fmt.Sprintf("blob %x is not not present in the requiredReads at index %d", blob.Key(), blob.index)) } if table.size == 1 { From 8245ab00d72bb25c16aed110a08f109e4a647ce5 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 07:44:45 -0500 Subject: [PATCH 39/74] Fixed linter issues. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 3 +-- tools/traffic/blob_writer.go | 2 +- tools/traffic/generator.go | 6 ++++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index ad051492b7..907294db05 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -176,8 +176,7 @@ func (reader *BlobReader) randomRead() { } reader.readSuccessMetric.Increment() - var assignments map[core.OperatorID]core.Assignment - assignments = chunks.Assignments + assignments := chunks.Assignments data, err := reader.retriever.CombineChunks(chunks) if err != nil { diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 724c4cf622..6b41d237cd 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -61,7 +61,7 @@ func NewBlobWriter( fixedRandomData = nil } else { // Use this random data for each blob. - fixedRandomData := make([]byte, config.DataSize) + fixedRandomData = make([]byte, config.DataSize) _, err := rand.Read(fixedRandomData) if err != nil { panic(fmt.Sprintf("unable to read random data: %s", err)) diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index a157f389d5..885c19f238 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -156,12 +156,18 @@ func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainC NumRetries: int(config.EthClientRetries), } gethClient, err := geth.NewMultiHomingClient(ethClientConfig, gethcommon.Address{}, logger) + if err != nil { + panic(fmt.Sprintf("Unable to instantiate geth client: %s", err)) + } tx, err := eth.NewTransactor( logger, gethClient, config.BlsOperatorStateRetriever, config.EigenDAServiceManager) + if err != nil { + panic(fmt.Sprintf("Unable to instantiate transactor: %s", err)) + } cs := eth.NewChainState(tx, gethClient) From 508e78896cd252ddc06543aaadd8f4ac222a1ad9 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 08:03:35 -0500 Subject: [PATCH 40/74] Move table to new package. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 23 +++++++++++----------- tools/traffic/blob_verifier.go | 7 ++++--- tools/traffic/generator.go | 7 ++++--- tools/traffic/{ => table}/blob_metadata.go | 12 ++++++++++- tools/traffic/{ => table}/blob_table.go | 2 +- 5 files changed, 32 insertions(+), 19 deletions(-) rename tools/traffic/{ => table}/blob_metadata.go (87%) rename tools/traffic/{ => table}/blob_table.go (99%) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 907294db05..4d2eec8d54 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -9,6 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/retriever/eth" + "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigensdk-go/logging" gcommon "github.com/ethereum/go-ethereum/common" "math/big" @@ -34,10 +35,10 @@ type BlobReader struct { chainClient eth.ChainClient // requiredReads blobs we are required to read a certain number of times. - requiredReads *BlobTable + requiredReads *table.BlobTable // optionalReads blobs we are not required to read, but can choose to read if we want. - optionalReads *BlobTable + optionalReads *table.BlobTable metrics *Metrics fetchBatchHeaderMetric LatencyMetric @@ -64,10 +65,10 @@ func NewBlobReader( config *Config, retriever clients.RetrievalClient, chainClient eth.ChainClient, - table *BlobTable, + blobTable *table.BlobTable, metrics *Metrics) BlobReader { - optionalReads := NewBlobTable() + optionalReads := table.NewBlobTable() return BlobReader{ ctx: ctx, @@ -76,7 +77,7 @@ func NewBlobReader( config: config, retriever: retriever, chainClient: chainClient, - requiredReads: table, + requiredReads: blobTable, optionalReads: &optionalReads, metrics: metrics, fetchBatchHeaderMetric: metrics.NewLatencyMetric("fetch_batch_header"), @@ -143,7 +144,7 @@ func (reader *BlobReader) randomRead() { return reader.chainClient.FetchBatchHeader( ctxTimeout, gcommon.HexToAddress(reader.config.EigenDAServiceManager), - *metadata.batchHeaderHash, + *metadata.BatchHeaderHash(), big.NewInt(int64(0)), nil) }) @@ -156,14 +157,14 @@ func (reader *BlobReader) randomRead() { reader.fetchBatchHeaderSuccess.Increment() var batchHeaderHash [32]byte - copy(batchHeaderHash[:], *metadata.batchHeaderHash) + copy(batchHeaderHash[:], *metadata.BatchHeaderHash()) ctxTimeout, cancel = context.WithTimeout(*reader.ctx, reader.config.RetrieveBlobChunksTimeout) chunks, err := InvokeAndReportLatency(&reader.readLatencyMetric, func() (*clients.BlobChunks, error) { return reader.retriever.RetrieveBlobChunks( ctxTimeout, batchHeaderHash, - metadata.blobIndex, + metadata.BlobIndex(), uint(batchHeader.ReferenceBlockNumber), batchHeader.BlobHeadersRoot, core.QuorumID(0)) @@ -227,12 +228,12 @@ func (reader *BlobReader) reportMissingChunk(operatorId core.OperatorID) { } // verifyBlob performs sanity checks on the blob. -func (reader *BlobReader) verifyBlob(metadata *BlobMetadata, blob *[]byte) { +func (reader *BlobReader) verifyBlob(metadata *table.BlobMetadata, blob *[]byte) { // Trim off the padding. - truncatedBlob := (*blob)[:metadata.size] + truncatedBlob := (*blob)[:metadata.Size()] recomputedChecksum := md5.Sum(truncatedBlob) - if *metadata.checksum == recomputedChecksum { + if *metadata.Checksum() == recomputedChecksum { reader.validBlobMetric.Increment() } else { reader.invalidBlobMetric.Increment() diff --git a/tools/traffic/blob_verifier.go b/tools/traffic/blob_verifier.go index b3fbc1c50f..03a1f2026d 100644 --- a/tools/traffic/blob_verifier.go +++ b/tools/traffic/blob_verifier.go @@ -4,6 +4,7 @@ import ( "context" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/api/grpc/disperser" + "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigensdk-go/logging" "math/rand" "sync" @@ -40,7 +41,7 @@ type BlobVerifier struct { config *Config // A table of confirmed blobs. Blobs are added here when they are confirmed by the disperser service. - table *BlobTable + table *table.BlobTable // The disperser client used to monitor the disperser service. dispenser *clients.DisperserClient @@ -70,7 +71,7 @@ func NewStatusVerifier( waitGroup *sync.WaitGroup, logger logging.Logger, config *Config, - table *BlobTable, + table *table.BlobTable, disperser *clients.DisperserClient, metrics *Metrics) BlobVerifier { @@ -230,6 +231,6 @@ func (verifier *BlobVerifier) forwardToReader(key *unconfirmedKey, status *dispe downloadCount = int32(requiredDownloads) } - blobMetadata := NewBlobMetadata(key.key, key.checksum, key.size, &batchHeaderHash, blobIndex, downloadCount) + blobMetadata := table.NewBlobMetadata(key.key, key.checksum, key.size, &batchHeaderHash, blobIndex, downloadCount) verifier.table.Add(blobMetadata) } diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 885c19f238..7af0efa492 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -9,6 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" + "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigensdk-go/logging" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" @@ -80,7 +81,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera metrics := NewMetrics(config.MetricsHTTPPort, logger) - table := NewBlobTable() + blobTable := table.NewBlobTable() disperserConfig := clients.Config{ Hostname: config.DisperserHostname, @@ -94,7 +95,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera &waitGroup, logger, config, - &table, + &blobTable, &disperserClient, metrics) @@ -122,7 +123,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera config, retriever, chainClient, - &table, + &blobTable, metrics) readers = append(readers, &reader) } diff --git a/tools/traffic/blob_metadata.go b/tools/traffic/table/blob_metadata.go similarity index 87% rename from tools/traffic/blob_metadata.go rename to tools/traffic/table/blob_metadata.go index 2d71749210..056e76d126 100644 --- a/tools/traffic/blob_metadata.go +++ b/tools/traffic/table/blob_metadata.go @@ -1,4 +1,4 @@ -package traffic +package table // BlobMetadata encapsulates various information about a blob written by the traffic generator. type BlobMetadata struct { @@ -51,6 +51,16 @@ func (blob *BlobMetadata) Key() *[]byte { return blob.key } +// Checksum returns the checksum of the blob. +func (blob *BlobMetadata) Checksum() *[16]byte { + return blob.checksum +} + +// Size returns the size of the blob, in bytes. +func (blob *BlobMetadata) Size() uint { + return blob.size +} + // BatchHeaderHash returns the batchHeaderHash of the blob. func (blob *BlobMetadata) BatchHeaderHash() *[]byte { return blob.batchHeaderHash diff --git a/tools/traffic/blob_table.go b/tools/traffic/table/blob_table.go similarity index 99% rename from tools/traffic/blob_table.go rename to tools/traffic/table/blob_table.go index d6fb5ce5d1..6c4a1bfdaf 100644 --- a/tools/traffic/blob_table.go +++ b/tools/traffic/table/blob_table.go @@ -1,4 +1,4 @@ -package traffic +package table import ( "fmt" From eadce73022209e2658d03182723c0d769d0a62dc Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 08:20:47 -0500 Subject: [PATCH 41/74] Moved metrics into package. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 41 +++++++------- tools/traffic/blob_verifier.go | 27 ++++----- tools/traffic/blob_writer.go | 11 ++-- tools/traffic/generator.go | 5 +- tools/traffic/metrics/count_metric.go | 28 +++++++++ tools/traffic/metrics/gauge_metric.go | 27 +++++++++ tools/traffic/metrics/latency_metric.go | 46 +++++++++++++++ tools/traffic/{ => metrics}/metrics.go | 75 ++----------------------- 8 files changed, 149 insertions(+), 111 deletions(-) create mode 100644 tools/traffic/metrics/count_metric.go create mode 100644 tools/traffic/metrics/gauge_metric.go create mode 100644 tools/traffic/metrics/latency_metric.go rename tools/traffic/{ => metrics}/metrics.go (51%) diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index 4d2eec8d54..ae88527f1a 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -9,6 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/retriever/eth" + metrics2 "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigensdk-go/logging" gcommon "github.com/ethereum/go-ethereum/common" @@ -40,21 +41,21 @@ type BlobReader struct { // optionalReads blobs we are not required to read, but can choose to read if we want. optionalReads *table.BlobTable - metrics *Metrics - fetchBatchHeaderMetric LatencyMetric - fetchBatchHeaderSuccess CountMetric - fetchBatchHeaderFailure CountMetric - readLatencyMetric LatencyMetric - readSuccessMetric CountMetric - readFailureMetric CountMetric - recombinationSuccessMetric CountMetric - recombinationFailureMetric CountMetric - validBlobMetric CountMetric - invalidBlobMetric CountMetric - operatorSuccessMetrics map[core.OperatorID]CountMetric - operatorFailureMetrics map[core.OperatorID]CountMetric - requiredReadPoolSizeMetric GaugeMetric - optionalReadPoolSizeMetric GaugeMetric + metrics *metrics2.Metrics + fetchBatchHeaderMetric metrics2.LatencyMetric + fetchBatchHeaderSuccess metrics2.CountMetric + fetchBatchHeaderFailure metrics2.CountMetric + readLatencyMetric metrics2.LatencyMetric + readSuccessMetric metrics2.CountMetric + readFailureMetric metrics2.CountMetric + recombinationSuccessMetric metrics2.CountMetric + recombinationFailureMetric metrics2.CountMetric + validBlobMetric metrics2.CountMetric + invalidBlobMetric metrics2.CountMetric + operatorSuccessMetrics map[core.OperatorID]metrics2.CountMetric + operatorFailureMetrics map[core.OperatorID]metrics2.CountMetric + requiredReadPoolSizeMetric metrics2.GaugeMetric + optionalReadPoolSizeMetric metrics2.GaugeMetric } // NewBlobReader creates a new BlobReader instance. @@ -66,7 +67,7 @@ func NewBlobReader( retriever clients.RetrievalClient, chainClient eth.ChainClient, blobTable *table.BlobTable, - metrics *Metrics) BlobReader { + metrics *metrics2.Metrics) BlobReader { optionalReads := table.NewBlobTable() @@ -90,8 +91,8 @@ func NewBlobReader( invalidBlobMetric: metrics.NewCountMetric("invalid_blob"), readSuccessMetric: metrics.NewCountMetric("read_success"), readFailureMetric: metrics.NewCountMetric("read_failure"), - operatorSuccessMetrics: make(map[core.OperatorID]CountMetric), - operatorFailureMetrics: make(map[core.OperatorID]CountMetric), + operatorSuccessMetrics: make(map[core.OperatorID]metrics2.CountMetric), + operatorFailureMetrics: make(map[core.OperatorID]metrics2.CountMetric), requiredReadPoolSizeMetric: metrics.NewGaugeMetric("required_read_pool_size"), optionalReadPoolSizeMetric: metrics.NewGaugeMetric("optional_read_pool_size"), } @@ -139,7 +140,7 @@ func (reader *BlobReader) randomRead() { } ctxTimeout, cancel := context.WithTimeout(*reader.ctx, reader.config.FetchBatchHeaderTimeout) - batchHeader, err := InvokeAndReportLatency(&reader.fetchBatchHeaderMetric, + batchHeader, err := metrics2.InvokeAndReportLatency(&reader.fetchBatchHeaderMetric, func() (*contractEigenDAServiceManager.IEigenDAServiceManagerBatchHeader, error) { return reader.chainClient.FetchBatchHeader( ctxTimeout, @@ -160,7 +161,7 @@ func (reader *BlobReader) randomRead() { copy(batchHeaderHash[:], *metadata.BatchHeaderHash()) ctxTimeout, cancel = context.WithTimeout(*reader.ctx, reader.config.RetrieveBlobChunksTimeout) - chunks, err := InvokeAndReportLatency(&reader.readLatencyMetric, func() (*clients.BlobChunks, error) { + chunks, err := metrics2.InvokeAndReportLatency(&reader.readLatencyMetric, func() (*clients.BlobChunks, error) { return reader.retriever.RetrieveBlobChunks( ctxTimeout, batchHeaderHash, diff --git a/tools/traffic/blob_verifier.go b/tools/traffic/blob_verifier.go index 03a1f2026d..11963e180b 100644 --- a/tools/traffic/blob_verifier.go +++ b/tools/traffic/blob_verifier.go @@ -4,6 +4,7 @@ import ( "context" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/api/grpc/disperser" + "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigensdk-go/logging" "math/rand" @@ -52,17 +53,17 @@ type BlobVerifier struct { // Newly added keys that require verification. keyChannel chan *unconfirmedKey - blobsInFlightMetric GaugeMetric - getStatusLatencyMetric LatencyMetric - confirmationLatencyMetric LatencyMetric - getStatusErrorCountMetric CountMetric - unknownCountMetric CountMetric - processingCountMetric CountMetric - dispersingCountMetric CountMetric - failedCountMetric CountMetric - insufficientSignaturesCountMetric CountMetric - confirmedCountMetric CountMetric - finalizedCountMetric CountMetric + blobsInFlightMetric metrics.GaugeMetric + getStatusLatencyMetric metrics.LatencyMetric + confirmationLatencyMetric metrics.LatencyMetric + getStatusErrorCountMetric metrics.CountMetric + unknownCountMetric metrics.CountMetric + processingCountMetric metrics.CountMetric + dispersingCountMetric metrics.CountMetric + failedCountMetric metrics.CountMetric + insufficientSignaturesCountMetric metrics.CountMetric + confirmedCountMetric metrics.CountMetric + finalizedCountMetric metrics.CountMetric } // NewStatusVerifier creates a new BlobVerifier instance. @@ -73,7 +74,7 @@ func NewStatusVerifier( config *Config, table *table.BlobTable, disperser *clients.DisperserClient, - metrics *Metrics) BlobVerifier { + metrics *metrics.Metrics) BlobVerifier { return BlobVerifier{ ctx: ctx, @@ -155,7 +156,7 @@ func (verifier *BlobVerifier) checkStatusForBlob(key *unconfirmedKey) bool { ctxTimeout, cancel := context.WithTimeout(*verifier.ctx, verifier.config.GetBlobStatusTimeout) defer cancel() - status, err := InvokeAndReportLatency[*disperser.BlobStatusReply](&verifier.getStatusLatencyMetric, + status, err := metrics.InvokeAndReportLatency[*disperser.BlobStatusReply](&verifier.getStatusLatencyMetric, func() (*disperser.BlobStatusReply, error) { return (*verifier.dispenser).GetBlobStatus(ctxTimeout, *key.key) }) diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index 6b41d237cd..f605581768 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigensdk-go/logging" "sync" "time" @@ -36,13 +37,13 @@ type BlobWriter struct { fixedRandomData *[]byte // writeLatencyMetric is used to record latency for write requests. - writeLatencyMetric LatencyMetric + writeLatencyMetric metrics.LatencyMetric // writeSuccessMetric is used to record the number of successful write requests. - writeSuccessMetric CountMetric + writeSuccessMetric metrics.CountMetric // writeFailureMetric is used to record the number of failed write requests. - writeFailureMetric CountMetric + writeFailureMetric metrics.CountMetric } // NewBlobWriter creates a new BlobWriter instance. @@ -53,7 +54,7 @@ func NewBlobWriter( config *Config, disperser *clients.DisperserClient, verifier *BlobVerifier, - metrics *Metrics) BlobWriter { + metrics *metrics.Metrics) BlobWriter { var fixedRandomData []byte if config.RandomizeBlobs { @@ -102,7 +103,7 @@ func (writer *BlobWriter) run() { return case <-ticker.C: data := writer.getRandomData() - key, err := InvokeAndReportLatency(&writer.writeLatencyMetric, func() ([]byte, error) { + key, err := metrics.InvokeAndReportLatency(&writer.writeLatencyMetric, func() ([]byte, error) { return writer.sendRequest(*data) }) if err != nil { diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 7af0efa492..1d70fb2c55 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -9,6 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" + metrics2 "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigensdk-go/logging" gethcommon "github.com/ethereum/go-ethereum/common" @@ -43,7 +44,7 @@ type Generator struct { ctx *context.Context cancel *context.CancelFunc waitGroup *sync.WaitGroup - metrics *Metrics + metrics *metrics2.Metrics logger *logging.Logger disperserClient clients.DisperserClient eigenDAClient *clients.EigenDAClient @@ -79,7 +80,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera ctx, cancel := context.WithCancel(context.Background()) waitGroup := sync.WaitGroup{} - metrics := NewMetrics(config.MetricsHTTPPort, logger) + metrics := metrics2.NewMetrics(config.MetricsHTTPPort, logger) blobTable := table.NewBlobTable() diff --git a/tools/traffic/metrics/count_metric.go b/tools/traffic/metrics/count_metric.go new file mode 100644 index 0000000000..fed55f5d37 --- /dev/null +++ b/tools/traffic/metrics/count_metric.go @@ -0,0 +1,28 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// CountMetric tracks the count of a type of event. +type CountMetric struct { + metrics *Metrics + description string +} + +// Increment increments the count of a type of event. +func (metric *CountMetric) Increment() { + metric.metrics.count.WithLabelValues(metric.description).Inc() +} + +// NewCountMetric creates a new prometheus collector for counting metrics. +func buildCounterCollector(namespace string, registry *prometheus.Registry) *prometheus.CounterVec { + return promauto.With(registry).NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "event_count", + }, + []string{"label"}, + ) +} diff --git a/tools/traffic/metrics/gauge_metric.go b/tools/traffic/metrics/gauge_metric.go new file mode 100644 index 0000000000..64d055709b --- /dev/null +++ b/tools/traffic/metrics/gauge_metric.go @@ -0,0 +1,27 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// GaugeMetric allows values to be reported. +type GaugeMetric struct { + metrics *Metrics + description string +} + +// Set sets the value of a gauge metric. +func (metric GaugeMetric) Set(value float64) { + metric.metrics.gauge.WithLabelValues(metric.description).Set(value) +} + +// NewGaugeMetric creates a collector for gauge metrics. +func buildGaugeCollector(namespace string, registry *prometheus.Registry) *prometheus.GaugeVec { + return promauto.With(registry).NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "gauge", + }, []string{"label"}, + ) +} diff --git a/tools/traffic/metrics/latency_metric.go b/tools/traffic/metrics/latency_metric.go new file mode 100644 index 0000000000..ec2b692c0f --- /dev/null +++ b/tools/traffic/metrics/latency_metric.go @@ -0,0 +1,46 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "time" +) + +// LatencyMetric tracks the latency of an operation. +type LatencyMetric struct { + metrics *Metrics + description string +} + +// ReportLatency reports the latency of an operation. +func (metric *LatencyMetric) ReportLatency(latency time.Duration) { + metric.metrics.latency.WithLabelValues(metric.description).Observe(latency.Seconds()) +} + +// InvokeAndReportLatency performs an operation. If the operation does not produce an error, then the latency +// of the operation is reported to the metrics framework. +func InvokeAndReportLatency[T any](metric *LatencyMetric, operation func() (T, error)) (T, error) { + start := time.Now() + + t, err := operation() + + if err == nil { + end := time.Now() + duration := end.Sub(start) + metric.ReportLatency(duration) + } + + return t, err +} + +// NewLatencyMetric creates a new prometheus collector for latency metrics. +func buildLatencyCollector(namespace string, registry *prometheus.Registry) *prometheus.SummaryVec { + return promauto.With(registry).NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: namespace, + Name: "latency_s", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.01, 0.99: 0.001}, + }, + []string{"label"}, + ) +} diff --git a/tools/traffic/metrics.go b/tools/traffic/metrics/metrics.go similarity index 51% rename from tools/traffic/metrics.go rename to tools/traffic/metrics/metrics.go index 09b53327b3..a1d878255e 100644 --- a/tools/traffic/metrics.go +++ b/tools/traffic/metrics/metrics.go @@ -1,14 +1,12 @@ -package traffic +package metrics import ( "fmt" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" "net/http" - "time" ) // Metrics encapsulates metrics for the traffic generator. @@ -23,23 +21,6 @@ type Metrics struct { logger logging.Logger } -// LatencyMetric tracks the latency of an operation. -type LatencyMetric struct { - metrics *Metrics - description string -} - -// CountMetric tracks the count of a type of event. -type CountMetric struct { - metrics *Metrics - description string -} - -type GaugeMetric struct { - metrics *Metrics - description string -} - // NewMetrics creates a new Metrics instance. func NewMetrics(httpPort string, logger logging.Logger) *Metrics { namespace := "eigenda_generator" @@ -48,26 +29,9 @@ func NewMetrics(httpPort string, logger logging.Logger) *Metrics { reg.MustRegister(collectors.NewGoCollector()) metrics := &Metrics{ - count: promauto.With(reg).NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "event_count", - }, - []string{"label"}, - ), - latency: promauto.With(reg).NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: namespace, - Name: "latency_s", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.01, 0.99: 0.001}, - }, - []string{"label"}, - ), - gauge: promauto.With(reg).NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "gauge", - }, []string{"label"}), + count: buildCounterCollector(namespace, reg), + latency: buildLatencyCollector(namespace, reg), + gauge: buildGaugeCollector(namespace, reg), registry: reg, httpPort: httpPort, logger: logger.With("component", "GeneratorMetrics"), @@ -98,27 +62,6 @@ func (metrics *Metrics) NewLatencyMetric(description string) LatencyMetric { } } -// ReportLatency reports the latency of an operation. -func (metric *LatencyMetric) ReportLatency(latency time.Duration) { - metric.metrics.latency.WithLabelValues(metric.description).Observe(latency.Seconds()) -} - -// InvokeAndReportLatency performs an operation. If the operation does not produce an error, then the latency -// of the operation is reported to the metrics framework. -func InvokeAndReportLatency[T any](metric *LatencyMetric, operation func() (T, error)) (T, error) { - start := time.Now() - - t, err := operation() - - if err == nil { - end := time.Now() - duration := end.Sub(start) - metric.ReportLatency(duration) - } - - return t, err -} - // NewCountMetric creates a new CountMetric instance. func (metrics *Metrics) NewCountMetric(description string) CountMetric { return CountMetric{ @@ -127,11 +70,6 @@ func (metrics *Metrics) NewCountMetric(description string) CountMetric { } } -// Increment increments the count of a type of event. -func (metric *CountMetric) Increment() { - metric.metrics.count.WithLabelValues(metric.description).Inc() -} - // NewGaugeMetric creates a new GaugeMetric instance. func (metrics *Metrics) NewGaugeMetric(description string) GaugeMetric { return GaugeMetric{ @@ -139,8 +77,3 @@ func (metrics *Metrics) NewGaugeMetric(description string) GaugeMetric { description: description, } } - -// Set sets the value of a gauge metric. -func (metric GaugeMetric) Set(value float64) { - metric.metrics.gauge.WithLabelValues(metric.description).Set(value) -} From 5b9b3ac40ab640de54e13da3053d83f08889455d Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 08:31:14 -0500 Subject: [PATCH 42/74] Split worker config into new struct. Signed-off-by: Cody Littley --- tools/traffic/blob_reader.go | 4 +-- tools/traffic/blob_verifier.go | 4 +-- tools/traffic/blob_writer.go | 6 ++-- tools/traffic/config.go | 64 +++++++++++----------------------- tools/traffic/flags/flags.go | 8 +++++ tools/traffic/generator.go | 10 +++--- tools/traffic/worker_config.go | 47 +++++++++++++++++++++++++ 7 files changed, 88 insertions(+), 55 deletions(-) create mode 100644 tools/traffic/worker_config.go diff --git a/tools/traffic/blob_reader.go b/tools/traffic/blob_reader.go index ae88527f1a..85427f7d42 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/blob_reader.go @@ -30,7 +30,7 @@ type BlobReader struct { logger logging.Logger // config contains the configuration for the generator. - config *Config + config *WorkerConfig retriever clients.RetrievalClient chainClient eth.ChainClient @@ -63,7 +63,7 @@ func NewBlobReader( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, - config *Config, + config *WorkerConfig, retriever clients.RetrievalClient, chainClient eth.ChainClient, blobTable *table.BlobTable, diff --git a/tools/traffic/blob_verifier.go b/tools/traffic/blob_verifier.go index 11963e180b..3fc93ef754 100644 --- a/tools/traffic/blob_verifier.go +++ b/tools/traffic/blob_verifier.go @@ -39,7 +39,7 @@ type BlobVerifier struct { logger logging.Logger // config contains the configuration for the generator. - config *Config + config *WorkerConfig // A table of confirmed blobs. Blobs are added here when they are confirmed by the disperser service. table *table.BlobTable @@ -71,7 +71,7 @@ func NewStatusVerifier( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, - config *Config, + config *WorkerConfig, table *table.BlobTable, disperser *clients.DisperserClient, metrics *metrics.Metrics) BlobVerifier { diff --git a/tools/traffic/blob_writer.go b/tools/traffic/blob_writer.go index f605581768..ed15fbf0c6 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/blob_writer.go @@ -25,7 +25,7 @@ type BlobWriter struct { logger logging.Logger // Config contains the configuration for the generator. - config *Config + config *WorkerConfig // disperser is the client used to send blobs to the disperser. disperser *clients.DisperserClient @@ -51,7 +51,7 @@ func NewBlobWriter( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, - config *Config, + config *WorkerConfig, disperser *clients.DisperserClient, verifier *BlobVerifier, metrics *metrics.Metrics) BlobWriter { @@ -139,7 +139,7 @@ func (writer *BlobWriter) getRandomData() *[]byte { // sendRequest sends a blob to a disperser. func (writer *BlobWriter) sendRequest(data []byte) ([]byte /* key */, error) { - ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.config.DisperserTimeout) + ctxTimeout, cancel := context.WithTimeout(*writer.ctx, writer.config.WriteTimeout) defer cancel() var key []byte diff --git a/tools/traffic/config.go b/tools/traffic/config.go index 679ba6de2e..d06c17aa84 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -65,37 +65,8 @@ type Config struct { // The amount of time to sleep after launching each worker thread. InstanceLaunchInterval time.Duration - // The number of worker threads that generate write traffic. - NumWriteInstances uint - // The period of the submission rate of new blobs for each write worker thread. - WriteRequestInterval time.Duration - // The Size of each blob dispersed, in bytes. - DataSize uint64 - // If true, then each blob will contain unique random data. If false, the same random data - // will be dispersed for each blob by a particular worker thread. - RandomizeBlobs bool - - // The amount of time between attempts by the verifier to confirm the status of blobs. - VerifierInterval time.Duration - // The amount of time to wait for a blob status to be fetched. - GetBlobStatusTimeout time.Duration - - // The number of worker threads that generate read traffic. - NumReadInstances uint - // The period of the submission rate of read requests for each read worker thread. - ReadRequestInterval time.Duration - // For each blob, how many times should it be downloaded? If between 0.0 and 1.0, blob will be downloaded - // 0 or 1 times with the specified probability (e.g. 0.2 means each blob has a 20% chance of being downloaded). - // If greater than 1.0, then each blob will be downloaded the specified number of times. - RequiredDownloads float64 - // The size of a table of blobs to optionally read when we run out of blobs that we are required to read. Blobs - // that are no longer required are added to this table, and when the table is at capacity they are randomly retired. - // Set this to 0 to disable this feature. - ReadOverflowTableSize uint32 - // The amount of time to wait for a batch header to be fetched. - FetchBatchHeaderTimeout time.Duration - // The amount of time to wait for a blob to be retrieved. - RetrieveBlobChunksTimeout time.Duration + // Configures the traffic generator workers. + WorkerConfig WorkerConfig } func NewConfig(ctx *cli.Context) (*Config, error) { @@ -141,19 +112,26 @@ func NewConfig(ctx *cli.Context) (*Config, error) { InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), - NumWriteInstances: ctx.GlobalUint(flags.NumWriteInstancesFlag.Name), - WriteRequestInterval: ctx.Duration(flags.WriteRequestIntervalFlag.Name), - DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), - RandomizeBlobs: !ctx.GlobalBool(flags.UniformBlobsFlag.Name), + WorkerConfig: WorkerConfig{ + NumWriteInstances: ctx.GlobalUint(flags.NumWriteInstancesFlag.Name), + WriteRequestInterval: ctx.Duration(flags.WriteRequestIntervalFlag.Name), + DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), + RandomizeBlobs: !ctx.GlobalBool(flags.UniformBlobsFlag.Name), + WriteTimeout: ctx.Duration(flags.WriteTimeoutFlag.Name), + + VerifierInterval: ctx.Duration(flags.VerifierIntervalFlag.Name), + GetBlobStatusTimeout: ctx.Duration(flags.GetBlobStatusTimeoutFlag.Name), - VerifierInterval: ctx.Duration(flags.VerifierIntervalFlag.Name), - GetBlobStatusTimeout: ctx.Duration(flags.GetBlobStatusTimeoutFlag.Name), + NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), + ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), + RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), + ReadOverflowTableSize: uint32(ctx.Uint(flags.ReadOverflowTableSizeFlag.Name)), + FetchBatchHeaderTimeout: ctx.Duration(flags.FetchBatchHeaderTimeoutFlag.Name), + RetrieveBlobChunksTimeout: ctx.Duration(flags.RetrieveBlobChunksTimeoutFlag.Name), - NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), - ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), - RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), - ReadOverflowTableSize: uint32(ctx.Uint(flags.ReadOverflowTableSizeFlag.Name)), - FetchBatchHeaderTimeout: ctx.Duration(flags.FetchBatchHeaderTimeoutFlag.Name), - RetrieveBlobChunksTimeout: ctx.Duration(flags.RetrieveBlobChunksTimeoutFlag.Name), + EigenDAServiceManager: ctx.String(flags.EigenDAServiceManagerFlag.Name), + SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), + CustomQuorums: customQuorumsUint8, + }, }, nil } diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index efdab33ba7..385687b08d 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -135,6 +135,13 @@ var ( Required: false, EnvVar: common.PrefixEnvVar(envPrefix, "UNIFORM_BLOBS"), } + WriteTimeoutFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "write-timeout"), + Usage: "Amount of time to wait for a blob to be written.", + Required: false, + Value: 10 * time.Second, + EnvVar: common.PrefixEnvVar(envPrefix, "WRITE_TIMEOUT"), + } TheGraphUrlFlag = cli.StringFlag{ Name: common.PrefixFlag(FlagPrefix, "the-graph-url"), Usage: "URL of the subgraph instance.", @@ -313,6 +320,7 @@ var optionalFlags = []cli.Flag{ RetrieveBlobChunksTimeoutFlag, GetBlobStatusTimeoutFlag, ReadOverflowTableSizeFlag, + WriteTimeoutFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 1d70fb2c55..a16d5912b4 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -95,18 +95,18 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera &ctx, &waitGroup, logger, - config, + &config.WorkerConfig, &blobTable, &disperserClient, metrics) writers := make([]*BlobWriter, 0) - for i := 0; i < int(config.NumWriteInstances); i++ { + for i := 0; i < int(config.WorkerConfig.NumWriteInstances); i++ { writer := NewBlobWriter( &ctx, &waitGroup, logger, - config, + &config.WorkerConfig, &disperserClient, &statusVerifier, metrics) @@ -116,12 +116,12 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera retriever, chainClient := buildRetriever(config) readers := make([]*BlobReader, 0) - for i := 0; i < int(config.NumReadInstances); i++ { + for i := 0; i < int(config.WorkerConfig.NumReadInstances); i++ { reader := NewBlobReader( &ctx, &waitGroup, logger, - config, + &config.WorkerConfig, retriever, chainClient, &blobTable, diff --git a/tools/traffic/worker_config.go b/tools/traffic/worker_config.go new file mode 100644 index 0000000000..298d208180 --- /dev/null +++ b/tools/traffic/worker_config.go @@ -0,0 +1,47 @@ +package traffic + +import "time" + +// WorkerConfig configures the traffic generator workers. +type WorkerConfig struct { + // The number of worker threads that generate write traffic. + NumWriteInstances uint + // The period of the submission rate of new blobs for each write worker thread. + WriteRequestInterval time.Duration + // The Size of each blob dispersed, in bytes. + DataSize uint64 + // If true, then each blob will contain unique random data. If false, the same random data + // will be dispersed for each blob by a particular worker thread. + RandomizeBlobs bool + // The amount of time to wait for a blob to be written. + WriteTimeout time.Duration + + // The amount of time between attempts by the verifier to confirm the status of blobs. + VerifierInterval time.Duration + // The amount of time to wait for a blob status to be fetched. + GetBlobStatusTimeout time.Duration + + // The number of worker threads that generate read traffic. + NumReadInstances uint + // The period of the submission rate of read requests for each read worker thread. + ReadRequestInterval time.Duration + // For each blob, how many times should it be downloaded? If between 0.0 and 1.0, blob will be downloaded + // 0 or 1 times with the specified probability (e.g. 0.2 means each blob has a 20% chance of being downloaded). + // If greater than 1.0, then each blob will be downloaded the specified number of times. + RequiredDownloads float64 + // The size of a table of blobs to optionally read when we run out of blobs that we are required to read. Blobs + // that are no longer required are added to this table, and when the table is at capacity they are randomly retired. + // Set this to 0 to disable this feature. + ReadOverflowTableSize uint32 + // The amount of time to wait for a batch header to be fetched. + FetchBatchHeaderTimeout time.Duration + // The amount of time to wait for a blob to be retrieved. + RetrieveBlobChunksTimeout time.Duration + + // The address of the EigenDA service manager smart contract, in hex. + EigenDAServiceManager string + // The private key to use for signing requests. + SignerPrivateKey string + // Custom quorum numbers to use for the traffic generator. + CustomQuorums []uint8 +} From 088095d556f683e3cfc3820f4cb4761ba7354304 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 08:32:33 -0500 Subject: [PATCH 43/74] Move workers to new package. Signed-off-by: Cody Littley --- tools/traffic/config.go | 5 +++-- tools/traffic/generator.go | 17 +++++++++-------- tools/traffic/{ => workers}/blob_reader.go | 2 +- tools/traffic/{ => workers}/blob_verifier.go | 2 +- tools/traffic/{ => workers}/blob_writer.go | 2 +- tools/traffic/{ => workers}/worker_config.go | 2 +- 6 files changed, 16 insertions(+), 14 deletions(-) rename tools/traffic/{ => workers}/blob_reader.go (99%) rename tools/traffic/{ => workers}/blob_verifier.go (99%) rename tools/traffic/{ => workers}/blob_writer.go (99%) rename tools/traffic/{ => workers}/worker_config.go (99%) diff --git a/tools/traffic/config.go b/tools/traffic/config.go index d06c17aa84..1a0919ebee 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -2,6 +2,7 @@ package traffic import ( "errors" + "github.com/Layr-Labs/eigenda/tools/traffic/workers" "time" "github.com/Layr-Labs/eigenda/common" @@ -66,7 +67,7 @@ type Config struct { InstanceLaunchInterval time.Duration // Configures the traffic generator workers. - WorkerConfig WorkerConfig + WorkerConfig workers.WorkerConfig } func NewConfig(ctx *cli.Context) (*Config, error) { @@ -112,7 +113,7 @@ func NewConfig(ctx *cli.Context) (*Config, error) { InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), - WorkerConfig: WorkerConfig{ + WorkerConfig: workers.WorkerConfig{ NumWriteInstances: ctx.GlobalUint(flags.NumWriteInstancesFlag.Name), WriteRequestInterval: ctx.Duration(flags.WriteRequestIntervalFlag.Name), DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index a16d5912b4..0645169026 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -11,6 +11,7 @@ import ( retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" metrics2 "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" + "github.com/Layr-Labs/eigenda/tools/traffic/workers" "github.com/Layr-Labs/eigensdk-go/logging" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" @@ -50,9 +51,9 @@ type Generator struct { eigenDAClient *clients.EigenDAClient config *Config - writers []*BlobWriter - verifier *BlobVerifier - readers []*BlobReader + writers []*workers.BlobWriter + verifier *workers.BlobVerifier + readers []*workers.BlobReader } func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Generator, error) { @@ -91,7 +92,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera UseSecureGrpcFlag: config.DisperserUseSecureGrpcFlag, } disperserClient := clients.NewDisperserClient(&disperserConfig, signer) - statusVerifier := NewStatusVerifier( + statusVerifier := workers.NewStatusVerifier( &ctx, &waitGroup, logger, @@ -100,9 +101,9 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera &disperserClient, metrics) - writers := make([]*BlobWriter, 0) + writers := make([]*workers.BlobWriter, 0) for i := 0; i < int(config.WorkerConfig.NumWriteInstances); i++ { - writer := NewBlobWriter( + writer := workers.NewBlobWriter( &ctx, &waitGroup, logger, @@ -115,9 +116,9 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera retriever, chainClient := buildRetriever(config) - readers := make([]*BlobReader, 0) + readers := make([]*workers.BlobReader, 0) for i := 0; i < int(config.WorkerConfig.NumReadInstances); i++ { - reader := NewBlobReader( + reader := workers.NewBlobReader( &ctx, &waitGroup, logger, diff --git a/tools/traffic/blob_reader.go b/tools/traffic/workers/blob_reader.go similarity index 99% rename from tools/traffic/blob_reader.go rename to tools/traffic/workers/blob_reader.go index 85427f7d42..116ef60bcc 100644 --- a/tools/traffic/blob_reader.go +++ b/tools/traffic/workers/blob_reader.go @@ -1,4 +1,4 @@ -package traffic +package workers import ( "context" diff --git a/tools/traffic/blob_verifier.go b/tools/traffic/workers/blob_verifier.go similarity index 99% rename from tools/traffic/blob_verifier.go rename to tools/traffic/workers/blob_verifier.go index 3fc93ef754..0d626c5f1d 100644 --- a/tools/traffic/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -1,4 +1,4 @@ -package traffic +package workers import ( "context" diff --git a/tools/traffic/blob_writer.go b/tools/traffic/workers/blob_writer.go similarity index 99% rename from tools/traffic/blob_writer.go rename to tools/traffic/workers/blob_writer.go index ed15fbf0c6..07f1556968 100644 --- a/tools/traffic/blob_writer.go +++ b/tools/traffic/workers/blob_writer.go @@ -1,4 +1,4 @@ -package traffic +package workers import ( "context" diff --git a/tools/traffic/worker_config.go b/tools/traffic/workers/worker_config.go similarity index 99% rename from tools/traffic/worker_config.go rename to tools/traffic/workers/worker_config.go index 298d208180..0a938f7322 100644 --- a/tools/traffic/worker_config.go +++ b/tools/traffic/workers/worker_config.go @@ -1,4 +1,4 @@ -package traffic +package workers import "time" From b687785f6273c724400d0608d9aa0ccf3ebdcc77 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 08:38:17 -0500 Subject: [PATCH 44/74] Fixed worker config name. Signed-off-by: Cody Littley --- tools/traffic/config.go | 4 ++-- tools/traffic/workers/blob_reader.go | 4 ++-- tools/traffic/workers/blob_verifier.go | 4 ++-- tools/traffic/workers/blob_writer.go | 4 ++-- tools/traffic/workers/worker_config.go | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/traffic/config.go b/tools/traffic/config.go index 1a0919ebee..84c4f97b08 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -67,7 +67,7 @@ type Config struct { InstanceLaunchInterval time.Duration // Configures the traffic generator workers. - WorkerConfig workers.WorkerConfig + WorkerConfig workers.Config } func NewConfig(ctx *cli.Context) (*Config, error) { @@ -113,7 +113,7 @@ func NewConfig(ctx *cli.Context) (*Config, error) { InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), - WorkerConfig: workers.WorkerConfig{ + WorkerConfig: workers.Config{ NumWriteInstances: ctx.GlobalUint(flags.NumWriteInstancesFlag.Name), WriteRequestInterval: ctx.Duration(flags.WriteRequestIntervalFlag.Name), DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), diff --git a/tools/traffic/workers/blob_reader.go b/tools/traffic/workers/blob_reader.go index 116ef60bcc..f9bfc6b5ff 100644 --- a/tools/traffic/workers/blob_reader.go +++ b/tools/traffic/workers/blob_reader.go @@ -30,7 +30,7 @@ type BlobReader struct { logger logging.Logger // config contains the configuration for the generator. - config *WorkerConfig + config *Config retriever clients.RetrievalClient chainClient eth.ChainClient @@ -63,7 +63,7 @@ func NewBlobReader( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, - config *WorkerConfig, + config *Config, retriever clients.RetrievalClient, chainClient eth.ChainClient, blobTable *table.BlobTable, diff --git a/tools/traffic/workers/blob_verifier.go b/tools/traffic/workers/blob_verifier.go index 0d626c5f1d..1689bd9ceb 100644 --- a/tools/traffic/workers/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -39,7 +39,7 @@ type BlobVerifier struct { logger logging.Logger // config contains the configuration for the generator. - config *WorkerConfig + config *Config // A table of confirmed blobs. Blobs are added here when they are confirmed by the disperser service. table *table.BlobTable @@ -71,7 +71,7 @@ func NewStatusVerifier( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, - config *WorkerConfig, + config *Config, table *table.BlobTable, disperser *clients.DisperserClient, metrics *metrics.Metrics) BlobVerifier { diff --git a/tools/traffic/workers/blob_writer.go b/tools/traffic/workers/blob_writer.go index 07f1556968..cb9c229950 100644 --- a/tools/traffic/workers/blob_writer.go +++ b/tools/traffic/workers/blob_writer.go @@ -25,7 +25,7 @@ type BlobWriter struct { logger logging.Logger // Config contains the configuration for the generator. - config *WorkerConfig + config *Config // disperser is the client used to send blobs to the disperser. disperser *clients.DisperserClient @@ -51,7 +51,7 @@ func NewBlobWriter( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, - config *WorkerConfig, + config *Config, disperser *clients.DisperserClient, verifier *BlobVerifier, metrics *metrics.Metrics) BlobWriter { diff --git a/tools/traffic/workers/worker_config.go b/tools/traffic/workers/worker_config.go index 0a938f7322..2abbb9ef6b 100644 --- a/tools/traffic/workers/worker_config.go +++ b/tools/traffic/workers/worker_config.go @@ -2,8 +2,8 @@ package workers import "time" -// WorkerConfig configures the traffic generator workers. -type WorkerConfig struct { +// Config configures the traffic generator workers. +type Config struct { // The number of worker threads that generate write traffic. NumWriteInstances uint // The period of the submission rate of new blobs for each write worker thread. From 79e5274a71e8bc6a8988d5f05bc827e026129237 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 08:38:37 -0500 Subject: [PATCH 45/74] Fix file name. Signed-off-by: Cody Littley --- tools/traffic/workers/{worker_config.go => config.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/traffic/workers/{worker_config.go => config.go} (100%) diff --git a/tools/traffic/workers/worker_config.go b/tools/traffic/workers/config.go similarity index 100% rename from tools/traffic/workers/worker_config.go rename to tools/traffic/workers/config.go From 0dfd5fd66b17ab785a7c73a940f7b7389fe33f62 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 10:05:22 -0500 Subject: [PATCH 46/74] Added tests for blob table. Signed-off-by: Cody Littley --- tools/traffic/table/blob_metadata.go | 12 +- tools/traffic/table/blob_table.go | 20 ++- tools/traffic/table/blob_table_test.go | 192 +++++++++++++++++++++++++ tools/traffic/workers/blob_reader.go | 4 +- tools/traffic/workers/blob_verifier.go | 2 +- 5 files changed, 217 insertions(+), 13 deletions(-) create mode 100644 tools/traffic/table/blob_table_test.go diff --git a/tools/traffic/table/blob_metadata.go b/tools/traffic/table/blob_metadata.go index 056e76d126..1eecede347 100644 --- a/tools/traffic/table/blob_metadata.go +++ b/tools/traffic/table/blob_metadata.go @@ -15,14 +15,14 @@ type BlobMetadata struct { batchHeaderHash *[]byte // blobIndex of the blob. - blobIndex uint32 + blobIndex uint // remainingReadPermits describes the maximum number of remaining reads permitted against this blob. // If -1 then an unlimited number of reads are permitted. - remainingReadPermits int32 + remainingReadPermits int // index describes the position of this blob within the blobTable. - index uint32 + index uint } // NewBlobMetadata creates a new BlobMetadata instance. The readPermits parameter describes the maximum number of @@ -32,8 +32,8 @@ func NewBlobMetadata( checksum *[16]byte, size uint, batchHeaderHash *[]byte, - blobIndex uint32, - readPermits int32) *BlobMetadata { + blobIndex uint, + readPermits int) *BlobMetadata { return &BlobMetadata{ key: key, @@ -67,6 +67,6 @@ func (blob *BlobMetadata) BatchHeaderHash() *[]byte { } // BlobIndex returns the blobIndex of the blob. -func (blob *BlobMetadata) BlobIndex() uint32 { +func (blob *BlobMetadata) BlobIndex() uint { return blob.blobIndex } diff --git a/tools/traffic/table/blob_table.go b/tools/traffic/table/blob_table.go index 6c4a1bfdaf..70f181cd51 100644 --- a/tools/traffic/table/blob_table.go +++ b/tools/traffic/table/blob_table.go @@ -14,7 +14,7 @@ type BlobTable struct { // size describes the total number of blobs currently tracked by the requiredReads. // size may be smaller than the capacity of the blobs slice. - size uint32 + size uint // lock is used to synchronize access to the requiredReads. lock sync.Mutex @@ -29,13 +29,25 @@ func NewBlobTable() BlobTable { } // Size returns the total number of blobs currently tracked by the requiredReads. -func (table *BlobTable) Size() uint32 { +func (table *BlobTable) Size() uint { table.lock.Lock() defer table.lock.Unlock() return table.size } +// Get returns the blob at the specified index. Returns nil if the index is out of bounds. +func (table *BlobTable) Get(index uint) *BlobMetadata { + table.lock.Lock() + defer table.lock.Unlock() + + if index >= table.size { + return nil + } + + return table.blobs[index] +} + // Add a blob to the requiredReads. func (table *BlobTable) Add(blob *BlobMetadata) { table.lock.Lock() @@ -48,7 +60,7 @@ func (table *BlobTable) Add(blob *BlobMetadata) { // AddOrReplace adds a blob to the requiredReads if there is capacity or replaces an existing blob at random // if the requiredReads is full. This method is a no-op if maximumCapacity is 0. -func (table *BlobTable) AddOrReplace(blob *BlobMetadata, maximumCapacity uint32) { +func (table *BlobTable) AddOrReplace(blob *BlobMetadata, maximumCapacity uint) { if maximumCapacity == 0 { return } @@ -60,7 +72,7 @@ func (table *BlobTable) AddOrReplace(blob *BlobMetadata, maximumCapacity uint32) // replace random existing blob index := rand.Int31n(int32(table.size)) table.blobs[index] = blob - blob.index = uint32(index) + blob.index = uint(index) } else { // add new blob blob.index = table.size diff --git a/tools/traffic/table/blob_table_test.go b/tools/traffic/table/blob_table_test.go new file mode 100644 index 0000000000..f072931862 --- /dev/null +++ b/tools/traffic/table/blob_table_test.go @@ -0,0 +1,192 @@ +package table + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "golang.org/x/exp/rand" + "testing" + "time" +) + +// initializeRandom initializes the random number generator. Prints the seed so that the test can be rerun +// deterministically. Replace a call to this method with a call to initializeRandomWithSeed to rerun a test +// with a specific seed. +func initializeRandom() { + rand.Seed(uint64(time.Now().UnixNano())) + seed := rand.Uint64() + fmt.Printf("Random seed: %d\n", seed) + rand.Seed(seed) +} + +// initializeRandomWithSeed initializes the random number generator with a specific seed. +func initializeRandomWithSeed(seed uint64) { + fmt.Printf("Random seed: %d\n", seed) + rand.Seed(seed) +} + +// randomMetadata generates a random BlobMetadata instance. +func randomMetadata(permits int) *BlobMetadata { + key := make([]byte, 32) + batchHeaderHash := make([]byte, 32) + checksum := [16]byte{} + _, _ = rand.Read(key) + _, _ = rand.Read(checksum[:]) + _, _ = rand.Read(batchHeaderHash) + return NewBlobMetadata(&key, &checksum, 1024, &batchHeaderHash, 0, permits) +} + +// TestBasicOperation tests basic operations of the BlobTable. Adds blobs and iterates over them. +func TestBasicOperation(t *testing.T) { + initializeRandom() + + table := NewBlobTable() + assert.Equal(t, uint(0), table.Size()) + + size := 1024 + expectedMetadata := make([]*BlobMetadata, 0) + for i := 0; i < size; i++ { + metadata := randomMetadata(1) + table.Add(metadata) + expectedMetadata = append(expectedMetadata, metadata) + assert.Equal(t, uint(i+1), table.Size()) + } + + for i := 0; i < size; i++ { + assert.Equal(t, expectedMetadata[i], table.Get(uint(i))) + } + + // Requesting an index that is out of bounds should return nil. + assert.Nil(t, table.Get(uint(size))) +} + +// TestGetRandomWithRemoval tests getting a random blob data, but where the number of permits per blob is unlimited. +func TestGetRandomNoRemovalByConfiguration(t *testing.T) { + initializeRandom() + + table := NewBlobTable() + assert.Equal(t, uint(0), table.Size()) + + // Requesting a random element from an empty table should return nil. + element, _ := table.GetRandom(true) + assert.Nil(t, element) + + expectedMetadata := make([]*BlobMetadata, 0) + size := 128 + for i := 0; i < size; i++ { + metadata := randomMetadata(-1) // -1 == unlimited permits + table.Add(metadata) + expectedMetadata = append(expectedMetadata, metadata) + assert.Equal(t, uint(i+1), table.Size()) + } + + randomIndices := make(map[uint]bool) + + // Query more times than the number of blobs to ensure that blobs are not removed. + for i := 0; i < size*8; i++ { + // This parameter will be ignored given that the number of permits is unlimited. + // But not a bad thing to exercise the code path. + decrement := rand.Intn(2) == 1 + + metadata, removed := table.GetRandom(decrement) + assert.False(t, removed) + assert.NotNil(t, metadata) + assert.Equal(t, expectedMetadata[metadata.index], metadata) + + randomIndices[metadata.index] = true + } + + // Sanity check: ensure that at least 10 different blobs were returned. This check is attempting to verify + // that we are actually getting random blobs. The probability of this check failing is extremely low if + // the random number generator is working correctly. + assert.GreaterOrEqual(t, len(randomIndices), 10) +} + +// TestGetRandomWithRemoval tests getting a random blob data, where the number of permits per blob is limited. +func TestGetRandomWithRemoval(t *testing.T) { + initializeRandom() + + table := NewBlobTable() + assert.Equal(t, uint(0), table.Size()) + + // Requesting a random element from an empty table should return nil. + element, _ := table.GetRandom(true) + assert.Nil(t, element) + + permitCount := 2 + + size := 1024 + expectedMetadata := make(map[*[]byte]uint, 0) + for i := 0; i < size; i++ { + metadata := randomMetadata(permitCount) + table.Add(metadata) + expectedMetadata[metadata.Key()] = 0 + assert.Equal(t, uint(i+1), table.Size()) + } + + // Requesting random elements without decrementing should not remove any elements. + for i := 0; i < size; i++ { + metadata, removed := table.GetRandom(false) + assert.NotNil(t, metadata) + _, exists := expectedMetadata[metadata.Key()] + assert.True(t, exists) + assert.False(t, removed) + } + assert.Equal(t, uint(size), table.Size()) + + // Requesting elements a number of times equal to the size times the number of permits should completely + // drain the table and return all elements a number of times equal to the number of permits. + for i := 0; i < size*permitCount; i++ { + metadata, removed := table.GetRandom(true) + assert.NotNil(t, metadata) + + permitsUsed := expectedMetadata[metadata.Key()] + 1 + expectedMetadata[metadata.Key()] = permitsUsed + assert.LessOrEqual(t, permitsUsed, uint(permitCount)) + + if int(permitsUsed) == permitCount { + assert.True(t, removed) + } else { + assert.False(t, removed) + } + } + + assert.Equal(t, uint(0), table.Size()) +} + +func TestAddOrReplace(t *testing.T) { + initializeRandom() + + table := NewBlobTable() + assert.Equal(t, uint(0), table.Size()) + + // Adding data to a table with capacity 0 should be a no-op. + table.AddOrReplace(randomMetadata(1), 0) + assert.Equal(t, uint(0), table.Size()) + + randomIndices := make(map[uint]bool) + + size := 1024 + for i := 0; i < size*2; i++ { + metadata := randomMetadata(-1) // -1 == unlimited permits + + initialSize := table.Size() + table.AddOrReplace(metadata, uint(size)) + resultingSize := table.Size() + + assert.LessOrEqual(t, resultingSize, uint(size)) + if initialSize < uint(size) { + assert.Equal(t, initialSize+1, resultingSize) + } else { + randomIndices[metadata.index] = true + } + + // Verify that the metadata is in the table. + assert.Less(t, metadata.index, table.Size()) + assert.Equal(t, metadata, table.Get(metadata.index)) + } + + // Sanity check: ensure that replacements happened at least 10 different indices. This check is attempting to + // verify that we are actually replacing blobs. The probability of this check failing is extremely low if + // the random number generator is working correctly. + assert.GreaterOrEqual(t, len(randomIndices), 10) +} diff --git a/tools/traffic/workers/blob_reader.go b/tools/traffic/workers/blob_reader.go index f9bfc6b5ff..90ef82b228 100644 --- a/tools/traffic/workers/blob_reader.go +++ b/tools/traffic/workers/blob_reader.go @@ -135,7 +135,7 @@ func (reader *BlobReader) randomRead() { } } else if removed { // We have removed a blob from the requiredReads. Add it to the optionalReads. - reader.optionalReads.AddOrReplace(metadata, reader.config.ReadOverflowTableSize) + reader.optionalReads.AddOrReplace(metadata, uint(reader.config.ReadOverflowTableSize)) reader.optionalReadPoolSizeMetric.Set(float64(reader.optionalReads.Size())) } @@ -165,7 +165,7 @@ func (reader *BlobReader) randomRead() { return reader.retriever.RetrieveBlobChunks( ctxTimeout, batchHeaderHash, - metadata.BlobIndex(), + uint32(metadata.BlobIndex()), uint(batchHeader.ReferenceBlockNumber), batchHeader.BlobHeadersRoot, core.QuorumID(0)) diff --git a/tools/traffic/workers/blob_verifier.go b/tools/traffic/workers/blob_verifier.go index 1689bd9ceb..c3614f6022 100644 --- a/tools/traffic/workers/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -232,6 +232,6 @@ func (verifier *BlobVerifier) forwardToReader(key *unconfirmedKey, status *dispe downloadCount = int32(requiredDownloads) } - blobMetadata := table.NewBlobMetadata(key.key, key.checksum, key.size, &batchHeaderHash, blobIndex, downloadCount) + blobMetadata := table.NewBlobMetadata(key.key, key.checksum, key.size, &batchHeaderHash, uint(blobIndex), int(downloadCount)) verifier.table.Add(blobMetadata) } From 0c8de99d7ad5ed498a0d4a641b555c27ceda9f1b Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 10:29:29 -0500 Subject: [PATCH 47/74] Created metrics interface. Signed-off-by: Cody Littley --- tools/traffic/generator.go | 50 ++++++++--------- tools/traffic/metrics/count_metric.go | 2 +- tools/traffic/metrics/gauge_metric.go | 2 +- tools/traffic/metrics/latency_metric.go | 2 +- tools/traffic/metrics/metrics.go | 26 ++++++--- tools/traffic/table/blob_table_test.go | 1 + tools/traffic/workers/blob_reader.go | 72 ++++++++++++------------- tools/traffic/workers/blob_verifier.go | 24 ++++----- tools/traffic/workers/blob_writer.go | 8 +-- 9 files changed, 100 insertions(+), 87 deletions(-) diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 0645169026..22f4b58731 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -9,7 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" - metrics2 "github.com/Layr-Labs/eigenda/tools/traffic/metrics" + "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigenda/tools/traffic/workers" "github.com/Layr-Labs/eigensdk-go/logging" @@ -42,14 +42,14 @@ import ( // When the verifier observes that a blob has been confirmed, it sends information about the blob // to the readers. The readers only attempt to read blobs that have been confirmed by the verifier. type Generator struct { - ctx *context.Context - cancel *context.CancelFunc - waitGroup *sync.WaitGroup - metrics *metrics2.Metrics - logger *logging.Logger - disperserClient clients.DisperserClient - eigenDAClient *clients.EigenDAClient - config *Config + ctx *context.Context + cancel *context.CancelFunc + waitGroup *sync.WaitGroup + generatorMetrics metrics.Metrics + logger *logging.Logger + disperserClient clients.DisperserClient + eigenDAClient *clients.EigenDAClient + config *Config writers []*workers.BlobWriter verifier *workers.BlobVerifier @@ -81,7 +81,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera ctx, cancel := context.WithCancel(context.Background()) waitGroup := sync.WaitGroup{} - metrics := metrics2.NewMetrics(config.MetricsHTTPPort, logger) + generatorMetrics := metrics.NewMetrics(config.MetricsHTTPPort, logger) blobTable := table.NewBlobTable() @@ -99,7 +99,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera &config.WorkerConfig, &blobTable, &disperserClient, - metrics) + generatorMetrics) writers := make([]*workers.BlobWriter, 0) for i := 0; i < int(config.WorkerConfig.NumWriteInstances); i++ { @@ -110,7 +110,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera &config.WorkerConfig, &disperserClient, &statusVerifier, - metrics) + generatorMetrics) writers = append(writers, &writer) } @@ -126,22 +126,22 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera retriever, chainClient, &blobTable, - metrics) + generatorMetrics) readers = append(readers, &reader) } return &Generator{ - ctx: &ctx, - cancel: &cancel, - waitGroup: &waitGroup, - metrics: metrics, - logger: &logger, - disperserClient: clients.NewDisperserClient(&disperserConfig, signer), - eigenDAClient: client, - config: config, - writers: writers, - verifier: &statusVerifier, - readers: readers, + ctx: &ctx, + cancel: &cancel, + waitGroup: &waitGroup, + generatorMetrics: generatorMetrics, + logger: &logger, + disperserClient: clients.NewDisperserClient(&disperserConfig, signer), + eigenDAClient: client, + config: config, + writers: writers, + verifier: &statusVerifier, + readers: readers, }, nil } @@ -219,7 +219,7 @@ func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainC // Start instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. func (generator *Generator) Start() error { - generator.metrics.Start() + generator.generatorMetrics.Start() generator.verifier.Start() for _, writer := range generator.writers { diff --git a/tools/traffic/metrics/count_metric.go b/tools/traffic/metrics/count_metric.go index fed55f5d37..2c35b0c25e 100644 --- a/tools/traffic/metrics/count_metric.go +++ b/tools/traffic/metrics/count_metric.go @@ -7,7 +7,7 @@ import ( // CountMetric tracks the count of a type of event. type CountMetric struct { - metrics *Metrics + metrics *metrics description string } diff --git a/tools/traffic/metrics/gauge_metric.go b/tools/traffic/metrics/gauge_metric.go index 64d055709b..d0a4f8f14a 100644 --- a/tools/traffic/metrics/gauge_metric.go +++ b/tools/traffic/metrics/gauge_metric.go @@ -7,7 +7,7 @@ import ( // GaugeMetric allows values to be reported. type GaugeMetric struct { - metrics *Metrics + metrics *metrics description string } diff --git a/tools/traffic/metrics/latency_metric.go b/tools/traffic/metrics/latency_metric.go index ec2b692c0f..bd5f559094 100644 --- a/tools/traffic/metrics/latency_metric.go +++ b/tools/traffic/metrics/latency_metric.go @@ -8,7 +8,7 @@ import ( // LatencyMetric tracks the latency of an operation. type LatencyMetric struct { - metrics *Metrics + metrics *metrics description string } diff --git a/tools/traffic/metrics/metrics.go b/tools/traffic/metrics/metrics.go index a1d878255e..1a91637400 100644 --- a/tools/traffic/metrics/metrics.go +++ b/tools/traffic/metrics/metrics.go @@ -9,8 +9,20 @@ import ( "net/http" ) +// Metrics allows the creation of metrics for the traffic generator. +type Metrics interface { + // Start starts the metrics server. + Start() + // NewLatencyMetric creates a new LatencyMetric instance. Useful for reporting the latency of an operation. + NewLatencyMetric(description string) LatencyMetric + // NewCountMetric creates a new CountMetric instance. Useful for tracking the count of a type of event. + NewCountMetric(description string) CountMetric + // NewGaugeMetric creates a new GaugeMetric instance. Useful for reporting specific values. + NewGaugeMetric(description string) GaugeMetric +} + // Metrics encapsulates metrics for the traffic generator. -type Metrics struct { +type metrics struct { registry *prometheus.Registry count *prometheus.CounterVec @@ -22,13 +34,13 @@ type Metrics struct { } // NewMetrics creates a new Metrics instance. -func NewMetrics(httpPort string, logger logging.Logger) *Metrics { +func NewMetrics(httpPort string, logger logging.Logger) Metrics { namespace := "eigenda_generator" reg := prometheus.NewRegistry() reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) reg.MustRegister(collectors.NewGoCollector()) - metrics := &Metrics{ + metrics := &metrics{ count: buildCounterCollector(namespace, reg), latency: buildLatencyCollector(namespace, reg), gauge: buildGaugeCollector(namespace, reg), @@ -40,7 +52,7 @@ func NewMetrics(httpPort string, logger logging.Logger) *Metrics { } // Start starts the metrics server. -func (metrics *Metrics) Start() { +func (metrics *metrics) Start() { metrics.logger.Info("Starting metrics server at ", "port", metrics.httpPort) addr := fmt.Sprintf(":%s", metrics.httpPort) go func() { @@ -55,7 +67,7 @@ func (metrics *Metrics) Start() { } // NewLatencyMetric creates a new LatencyMetric instance. -func (metrics *Metrics) NewLatencyMetric(description string) LatencyMetric { +func (metrics *metrics) NewLatencyMetric(description string) LatencyMetric { return LatencyMetric{ metrics: metrics, description: description, @@ -63,7 +75,7 @@ func (metrics *Metrics) NewLatencyMetric(description string) LatencyMetric { } // NewCountMetric creates a new CountMetric instance. -func (metrics *Metrics) NewCountMetric(description string) CountMetric { +func (metrics *metrics) NewCountMetric(description string) CountMetric { return CountMetric{ metrics: metrics, description: description, @@ -71,7 +83,7 @@ func (metrics *Metrics) NewCountMetric(description string) CountMetric { } // NewGaugeMetric creates a new GaugeMetric instance. -func (metrics *Metrics) NewGaugeMetric(description string) GaugeMetric { +func (metrics *metrics) NewGaugeMetric(description string) GaugeMetric { return GaugeMetric{ metrics: metrics, description: description, diff --git a/tools/traffic/table/blob_table_test.go b/tools/traffic/table/blob_table_test.go index f072931862..cf091b2ab6 100644 --- a/tools/traffic/table/blob_table_test.go +++ b/tools/traffic/table/blob_table_test.go @@ -153,6 +153,7 @@ func TestGetRandomWithRemoval(t *testing.T) { assert.Equal(t, uint(0), table.Size()) } +// TestAddOrReplace tests adding blobs to a table with a maximum capacity. The table should replace blobs when full. func TestAddOrReplace(t *testing.T) { initializeRandom() diff --git a/tools/traffic/workers/blob_reader.go b/tools/traffic/workers/blob_reader.go index 90ef82b228..1a24b3e3fa 100644 --- a/tools/traffic/workers/blob_reader.go +++ b/tools/traffic/workers/blob_reader.go @@ -9,7 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/retriever/eth" - metrics2 "github.com/Layr-Labs/eigenda/tools/traffic/metrics" + "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigensdk-go/logging" gcommon "github.com/ethereum/go-ethereum/common" @@ -41,21 +41,21 @@ type BlobReader struct { // optionalReads blobs we are not required to read, but can choose to read if we want. optionalReads *table.BlobTable - metrics *metrics2.Metrics - fetchBatchHeaderMetric metrics2.LatencyMetric - fetchBatchHeaderSuccess metrics2.CountMetric - fetchBatchHeaderFailure metrics2.CountMetric - readLatencyMetric metrics2.LatencyMetric - readSuccessMetric metrics2.CountMetric - readFailureMetric metrics2.CountMetric - recombinationSuccessMetric metrics2.CountMetric - recombinationFailureMetric metrics2.CountMetric - validBlobMetric metrics2.CountMetric - invalidBlobMetric metrics2.CountMetric - operatorSuccessMetrics map[core.OperatorID]metrics2.CountMetric - operatorFailureMetrics map[core.OperatorID]metrics2.CountMetric - requiredReadPoolSizeMetric metrics2.GaugeMetric - optionalReadPoolSizeMetric metrics2.GaugeMetric + generatorMetrics metrics.Metrics + fetchBatchHeaderMetric metrics.LatencyMetric + fetchBatchHeaderSuccess metrics.CountMetric + fetchBatchHeaderFailure metrics.CountMetric + readLatencyMetric metrics.LatencyMetric + readSuccessMetric metrics.CountMetric + readFailureMetric metrics.CountMetric + recombinationSuccessMetric metrics.CountMetric + recombinationFailureMetric metrics.CountMetric + validBlobMetric metrics.CountMetric + invalidBlobMetric metrics.CountMetric + operatorSuccessMetrics map[core.OperatorID]metrics.CountMetric + operatorFailureMetrics map[core.OperatorID]metrics.CountMetric + requiredReadPoolSizeMetric metrics.GaugeMetric + optionalReadPoolSizeMetric metrics.GaugeMetric } // NewBlobReader creates a new BlobReader instance. @@ -67,7 +67,7 @@ func NewBlobReader( retriever clients.RetrievalClient, chainClient eth.ChainClient, blobTable *table.BlobTable, - metrics *metrics2.Metrics) BlobReader { + generatorMetrics metrics.Metrics) BlobReader { optionalReads := table.NewBlobTable() @@ -80,21 +80,21 @@ func NewBlobReader( chainClient: chainClient, requiredReads: blobTable, optionalReads: &optionalReads, - metrics: metrics, - fetchBatchHeaderMetric: metrics.NewLatencyMetric("fetch_batch_header"), - fetchBatchHeaderSuccess: metrics.NewCountMetric("fetch_batch_header_success"), - fetchBatchHeaderFailure: metrics.NewCountMetric("fetch_batch_header_failure"), - recombinationSuccessMetric: metrics.NewCountMetric("recombination_success"), - recombinationFailureMetric: metrics.NewCountMetric("recombination_failure"), - readLatencyMetric: metrics.NewLatencyMetric("read"), - validBlobMetric: metrics.NewCountMetric("valid_blob"), - invalidBlobMetric: metrics.NewCountMetric("invalid_blob"), - readSuccessMetric: metrics.NewCountMetric("read_success"), - readFailureMetric: metrics.NewCountMetric("read_failure"), - operatorSuccessMetrics: make(map[core.OperatorID]metrics2.CountMetric), - operatorFailureMetrics: make(map[core.OperatorID]metrics2.CountMetric), - requiredReadPoolSizeMetric: metrics.NewGaugeMetric("required_read_pool_size"), - optionalReadPoolSizeMetric: metrics.NewGaugeMetric("optional_read_pool_size"), + generatorMetrics: generatorMetrics, + fetchBatchHeaderMetric: generatorMetrics.NewLatencyMetric("fetch_batch_header"), + fetchBatchHeaderSuccess: generatorMetrics.NewCountMetric("fetch_batch_header_success"), + fetchBatchHeaderFailure: generatorMetrics.NewCountMetric("fetch_batch_header_failure"), + recombinationSuccessMetric: generatorMetrics.NewCountMetric("recombination_success"), + recombinationFailureMetric: generatorMetrics.NewCountMetric("recombination_failure"), + readLatencyMetric: generatorMetrics.NewLatencyMetric("read"), + validBlobMetric: generatorMetrics.NewCountMetric("valid_blob"), + invalidBlobMetric: generatorMetrics.NewCountMetric("invalid_blob"), + readSuccessMetric: generatorMetrics.NewCountMetric("read_success"), + readFailureMetric: generatorMetrics.NewCountMetric("read_failure"), + operatorSuccessMetrics: make(map[core.OperatorID]metrics.CountMetric), + operatorFailureMetrics: make(map[core.OperatorID]metrics.CountMetric), + requiredReadPoolSizeMetric: generatorMetrics.NewGaugeMetric("required_read_pool_size"), + optionalReadPoolSizeMetric: generatorMetrics.NewGaugeMetric("optional_read_pool_size"), } } @@ -140,7 +140,7 @@ func (reader *BlobReader) randomRead() { } ctxTimeout, cancel := context.WithTimeout(*reader.ctx, reader.config.FetchBatchHeaderTimeout) - batchHeader, err := metrics2.InvokeAndReportLatency(&reader.fetchBatchHeaderMetric, + batchHeader, err := metrics.InvokeAndReportLatency(&reader.fetchBatchHeaderMetric, func() (*contractEigenDAServiceManager.IEigenDAServiceManagerBatchHeader, error) { return reader.chainClient.FetchBatchHeader( ctxTimeout, @@ -161,7 +161,7 @@ func (reader *BlobReader) randomRead() { copy(batchHeaderHash[:], *metadata.BatchHeaderHash()) ctxTimeout, cancel = context.WithTimeout(*reader.ctx, reader.config.RetrieveBlobChunksTimeout) - chunks, err := metrics2.InvokeAndReportLatency(&reader.readLatencyMetric, func() (*clients.BlobChunks, error) { + chunks, err := metrics.InvokeAndReportLatency(&reader.readLatencyMetric, func() (*clients.BlobChunks, error) { return reader.retriever.RetrieveBlobChunks( ctxTimeout, batchHeaderHash, @@ -210,7 +210,7 @@ func (reader *BlobReader) randomRead() { func (reader *BlobReader) reportChunk(operatorId core.OperatorID) { metric, exists := reader.operatorSuccessMetrics[operatorId] if !exists { - metric = reader.metrics.NewCountMetric(fmt.Sprintf("operator_%x_returned_chunk", operatorId)) + metric = reader.generatorMetrics.NewCountMetric(fmt.Sprintf("operator_%x_returned_chunk", operatorId)) reader.operatorSuccessMetrics[operatorId] = metric } @@ -221,7 +221,7 @@ func (reader *BlobReader) reportChunk(operatorId core.OperatorID) { func (reader *BlobReader) reportMissingChunk(operatorId core.OperatorID) { metric, exists := reader.operatorFailureMetrics[operatorId] if !exists { - metric = reader.metrics.NewCountMetric(fmt.Sprintf("operator_%x_witheld_chunk", operatorId)) + metric = reader.generatorMetrics.NewCountMetric(fmt.Sprintf("operator_%x_witheld_chunk", operatorId)) reader.operatorFailureMetrics[operatorId] = metric } diff --git a/tools/traffic/workers/blob_verifier.go b/tools/traffic/workers/blob_verifier.go index c3614f6022..dea17fc347 100644 --- a/tools/traffic/workers/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -74,7 +74,7 @@ func NewStatusVerifier( config *Config, table *table.BlobTable, disperser *clients.DisperserClient, - metrics *metrics.Metrics) BlobVerifier { + generatorMetrics metrics.Metrics) BlobVerifier { return BlobVerifier{ ctx: ctx, @@ -85,17 +85,17 @@ func NewStatusVerifier( dispenser: disperser, unconfirmedKeys: make([]*unconfirmedKey, 0), keyChannel: make(chan *unconfirmedKey), - blobsInFlightMetric: metrics.NewGaugeMetric("blobs_in_flight"), - getStatusLatencyMetric: metrics.NewLatencyMetric("get_status"), - confirmationLatencyMetric: metrics.NewLatencyMetric("confirmation"), - getStatusErrorCountMetric: metrics.NewCountMetric("get_status_ERROR"), - unknownCountMetric: metrics.NewCountMetric("get_status_UNKNOWN"), - processingCountMetric: metrics.NewCountMetric("get_status_PROCESSING"), - dispersingCountMetric: metrics.NewCountMetric("get_status_DISPERSING"), - failedCountMetric: metrics.NewCountMetric("get_status_FAILED"), - insufficientSignaturesCountMetric: metrics.NewCountMetric("get_status_INSUFFICIENT_SIGNATURES"), - confirmedCountMetric: metrics.NewCountMetric("get_status_CONFIRMED"), - finalizedCountMetric: metrics.NewCountMetric("get_status_FINALIZED"), + blobsInFlightMetric: generatorMetrics.NewGaugeMetric("blobs_in_flight"), + getStatusLatencyMetric: generatorMetrics.NewLatencyMetric("get_status"), + confirmationLatencyMetric: generatorMetrics.NewLatencyMetric("confirmation"), + getStatusErrorCountMetric: generatorMetrics.NewCountMetric("get_status_ERROR"), + unknownCountMetric: generatorMetrics.NewCountMetric("get_status_UNKNOWN"), + processingCountMetric: generatorMetrics.NewCountMetric("get_status_PROCESSING"), + dispersingCountMetric: generatorMetrics.NewCountMetric("get_status_DISPERSING"), + failedCountMetric: generatorMetrics.NewCountMetric("get_status_FAILED"), + insufficientSignaturesCountMetric: generatorMetrics.NewCountMetric("get_status_INSUFFICIENT_SIGNATURES"), + confirmedCountMetric: generatorMetrics.NewCountMetric("get_status_CONFIRMED"), + finalizedCountMetric: generatorMetrics.NewCountMetric("get_status_FINALIZED"), } } diff --git a/tools/traffic/workers/blob_writer.go b/tools/traffic/workers/blob_writer.go index cb9c229950..9e37301f3b 100644 --- a/tools/traffic/workers/blob_writer.go +++ b/tools/traffic/workers/blob_writer.go @@ -54,7 +54,7 @@ func NewBlobWriter( config *Config, disperser *clients.DisperserClient, verifier *BlobVerifier, - metrics *metrics.Metrics) BlobWriter { + generatorMetrics metrics.Metrics) BlobWriter { var fixedRandomData []byte if config.RandomizeBlobs { @@ -78,9 +78,9 @@ func NewBlobWriter( disperser: disperser, verifier: verifier, fixedRandomData: &fixedRandomData, - writeLatencyMetric: metrics.NewLatencyMetric("write"), - writeSuccessMetric: metrics.NewCountMetric("write_success"), - writeFailureMetric: metrics.NewCountMetric("write_failure"), + writeLatencyMetric: generatorMetrics.NewLatencyMetric("write"), + writeSuccessMetric: generatorMetrics.NewCountMetric("write_success"), + writeFailureMetric: generatorMetrics.NewCountMetric("write_failure"), } } From 0de6b4fea15ea5a7977a7560044fdfe0667d5f63 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 10:35:33 -0500 Subject: [PATCH 48/74] Added interfaces for metric types. Signed-off-by: Cody Littley --- tools/traffic/metrics/count_metric.go | 11 ++++++++--- tools/traffic/metrics/gauge_metric.go | 12 +++++++++--- tools/traffic/metrics/latency_metric.go | 13 +++++++++---- tools/traffic/metrics/metrics.go | 8 ++++---- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/tools/traffic/metrics/count_metric.go b/tools/traffic/metrics/count_metric.go index 2c35b0c25e..0e9ba6e7d2 100644 --- a/tools/traffic/metrics/count_metric.go +++ b/tools/traffic/metrics/count_metric.go @@ -5,14 +5,19 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) -// CountMetric tracks the count of a type of event. -type CountMetric struct { +// CountMetric allows the count of a type of event to be tracked. +type CountMetric interface { + Increment() +} + +// countMetric a standard implementation of the CountMetric interface via prometheus. +type countMetric struct { metrics *metrics description string } // Increment increments the count of a type of event. -func (metric *CountMetric) Increment() { +func (metric *countMetric) Increment() { metric.metrics.count.WithLabelValues(metric.description).Inc() } diff --git a/tools/traffic/metrics/gauge_metric.go b/tools/traffic/metrics/gauge_metric.go index d0a4f8f14a..5cace773d5 100644 --- a/tools/traffic/metrics/gauge_metric.go +++ b/tools/traffic/metrics/gauge_metric.go @@ -5,14 +5,20 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) -// GaugeMetric allows values to be reported. -type GaugeMetric struct { +// GaugeMetric allows specific values to be reported. +type GaugeMetric interface { + // Set sets the value of a gauge metric. + Set(value float64) +} + +// gaugeMetric is a standard implementation of the GaugeMetric interface via prometheus. +type gaugeMetric struct { metrics *metrics description string } // Set sets the value of a gauge metric. -func (metric GaugeMetric) Set(value float64) { +func (metric *gaugeMetric) Set(value float64) { metric.metrics.gauge.WithLabelValues(metric.description).Set(value) } diff --git a/tools/traffic/metrics/latency_metric.go b/tools/traffic/metrics/latency_metric.go index bd5f559094..e2469d0d0b 100644 --- a/tools/traffic/metrics/latency_metric.go +++ b/tools/traffic/metrics/latency_metric.go @@ -6,20 +6,25 @@ import ( "time" ) -// LatencyMetric tracks the latency of an operation. -type LatencyMetric struct { +// LatencyMetric allows the latency of an operation to be tracked. +type LatencyMetric interface { + ReportLatency(latency time.Duration) +} + +// latencyMetric is a standard implementation of the LatencyMetric interface via prometheus. +type latencyMetric struct { metrics *metrics description string } // ReportLatency reports the latency of an operation. -func (metric *LatencyMetric) ReportLatency(latency time.Duration) { +func (metric *latencyMetric) ReportLatency(latency time.Duration) { metric.metrics.latency.WithLabelValues(metric.description).Observe(latency.Seconds()) } // InvokeAndReportLatency performs an operation. If the operation does not produce an error, then the latency // of the operation is reported to the metrics framework. -func InvokeAndReportLatency[T any](metric *LatencyMetric, operation func() (T, error)) (T, error) { +func InvokeAndReportLatency[T any](metric *latencyMetric, operation func() (T, error)) (T, error) { start := time.Now() t, err := operation() diff --git a/tools/traffic/metrics/metrics.go b/tools/traffic/metrics/metrics.go index 1a91637400..971dad2483 100644 --- a/tools/traffic/metrics/metrics.go +++ b/tools/traffic/metrics/metrics.go @@ -21,7 +21,7 @@ type Metrics interface { NewGaugeMetric(description string) GaugeMetric } -// Metrics encapsulates metrics for the traffic generator. +// metrics is a standard implementation of the Metrics interface via prometheus. type metrics struct { registry *prometheus.Registry @@ -68,7 +68,7 @@ func (metrics *metrics) Start() { // NewLatencyMetric creates a new LatencyMetric instance. func (metrics *metrics) NewLatencyMetric(description string) LatencyMetric { - return LatencyMetric{ + return &latencyMetric{ metrics: metrics, description: description, } @@ -76,7 +76,7 @@ func (metrics *metrics) NewLatencyMetric(description string) LatencyMetric { // NewCountMetric creates a new CountMetric instance. func (metrics *metrics) NewCountMetric(description string) CountMetric { - return CountMetric{ + return &countMetric{ metrics: metrics, description: description, } @@ -84,7 +84,7 @@ func (metrics *metrics) NewCountMetric(description string) CountMetric { // NewGaugeMetric creates a new GaugeMetric instance. func (metrics *metrics) NewGaugeMetric(description string) GaugeMetric { - return GaugeMetric{ + return &gaugeMetric{ metrics: metrics, description: description, } From c5b6e7063d45b794f5b764fe50d1bbc15e4879a8 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 10:44:48 -0500 Subject: [PATCH 49/74] Added mock metrics for testing. Signed-off-by: Cody Littley --- tools/traffic/metrics/latency_metric.go | 2 +- tools/traffic/metrics/mock_metrics.go | 101 ++++++++++++++++++++++++ tools/traffic/workers/blob_reader.go | 4 +- tools/traffic/workers/blob_verifier.go | 2 +- tools/traffic/workers/blob_writer.go | 2 +- 5 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 tools/traffic/metrics/mock_metrics.go diff --git a/tools/traffic/metrics/latency_metric.go b/tools/traffic/metrics/latency_metric.go index e2469d0d0b..37413bfc28 100644 --- a/tools/traffic/metrics/latency_metric.go +++ b/tools/traffic/metrics/latency_metric.go @@ -24,7 +24,7 @@ func (metric *latencyMetric) ReportLatency(latency time.Duration) { // InvokeAndReportLatency performs an operation. If the operation does not produce an error, then the latency // of the operation is reported to the metrics framework. -func InvokeAndReportLatency[T any](metric *latencyMetric, operation func() (T, error)) (T, error) { +func InvokeAndReportLatency[T any](metric LatencyMetric, operation func() (T, error)) (T, error) { start := time.Now() t, err := operation() diff --git a/tools/traffic/metrics/mock_metrics.go b/tools/traffic/metrics/mock_metrics.go new file mode 100644 index 0000000000..23801a3a8a --- /dev/null +++ b/tools/traffic/metrics/mock_metrics.go @@ -0,0 +1,101 @@ +package metrics + +import ( + "sync" + "time" +) + +// MockMetrics implements metrics, useful for testing. +type MockMetrics struct { + // A map from each count metric's description to its count. + counts map[string]float64 + // A map from each gauge metric's description to its value. + gauges map[string]float64 + // A map from each latency metric's description to its most recently reported latency. + latencies map[string]time.Duration + + // Used to ensure thread safety. + lock sync.Mutex +} + +// MockLatencyMetric implements LatencyMetric, useful for testing. +type MockLatencyMetric struct { + metrics *MockMetrics + description string +} + +// MockCountMetric implements CountMetric, useful for testing. +type MockCountMetric struct { + metrics *MockMetrics + description string +} + +// MockGaugeMetric implements GaugeMetric, useful for testing. +type MockGaugeMetric struct { + metrics *MockMetrics + description string +} + +// GetCount returns the count of a type of event. +func (m *MockMetrics) GetCount(description string) float64 { + m.lock.Lock() + defer m.lock.Unlock() + return m.counts[description] +} + +// GetGaugeValue returns the value of a gauge metric. +func (m *MockMetrics) GetGaugeValue(description string) float64 { + m.lock.Lock() + defer m.lock.Unlock() + return m.gauges[description] +} + +// GetLatency returns the most recently reported latency of an operation. +func (m *MockMetrics) GetLatency(description string) time.Duration { + m.lock.Lock() + defer m.lock.Unlock() + return m.latencies[description] +} + +func (m *MockMetrics) Start() { + // intentional no-op +} + +func (m *MockMetrics) NewLatencyMetric(description string) LatencyMetric { + return &MockLatencyMetric{ + metrics: m, + description: description, + } +} + +func (m *MockLatencyMetric) ReportLatency(latency time.Duration) { + m.metrics.lock.Lock() + m.metrics.latencies[m.description] = latency + m.metrics.lock.Unlock() +} + +func (m *MockMetrics) NewCountMetric(description string) CountMetric { + return &MockCountMetric{ + metrics: m, + description: description, + } +} + +func (m *MockCountMetric) Increment() { + m.metrics.lock.Lock() + m.metrics.counts[m.description]++ + m.metrics.lock.Unlock() +} + +func (m *MockMetrics) NewGaugeMetric(description string) GaugeMetric { + return &MockGaugeMetric{ + metrics: m, + description: description, + } +} + +func (m *MockGaugeMetric) Set(value float64) { + m.metrics.lock.Lock() + m.metrics.gauges[m.description] = value + m.metrics.lock.Unlock() +} diff --git a/tools/traffic/workers/blob_reader.go b/tools/traffic/workers/blob_reader.go index 1a24b3e3fa..f88d2d9463 100644 --- a/tools/traffic/workers/blob_reader.go +++ b/tools/traffic/workers/blob_reader.go @@ -140,7 +140,7 @@ func (reader *BlobReader) randomRead() { } ctxTimeout, cancel := context.WithTimeout(*reader.ctx, reader.config.FetchBatchHeaderTimeout) - batchHeader, err := metrics.InvokeAndReportLatency(&reader.fetchBatchHeaderMetric, + batchHeader, err := metrics.InvokeAndReportLatency(reader.fetchBatchHeaderMetric, func() (*contractEigenDAServiceManager.IEigenDAServiceManagerBatchHeader, error) { return reader.chainClient.FetchBatchHeader( ctxTimeout, @@ -161,7 +161,7 @@ func (reader *BlobReader) randomRead() { copy(batchHeaderHash[:], *metadata.BatchHeaderHash()) ctxTimeout, cancel = context.WithTimeout(*reader.ctx, reader.config.RetrieveBlobChunksTimeout) - chunks, err := metrics.InvokeAndReportLatency(&reader.readLatencyMetric, func() (*clients.BlobChunks, error) { + chunks, err := metrics.InvokeAndReportLatency(reader.readLatencyMetric, func() (*clients.BlobChunks, error) { return reader.retriever.RetrieveBlobChunks( ctxTimeout, batchHeaderHash, diff --git a/tools/traffic/workers/blob_verifier.go b/tools/traffic/workers/blob_verifier.go index dea17fc347..7ab1bcf29f 100644 --- a/tools/traffic/workers/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -156,7 +156,7 @@ func (verifier *BlobVerifier) checkStatusForBlob(key *unconfirmedKey) bool { ctxTimeout, cancel := context.WithTimeout(*verifier.ctx, verifier.config.GetBlobStatusTimeout) defer cancel() - status, err := metrics.InvokeAndReportLatency[*disperser.BlobStatusReply](&verifier.getStatusLatencyMetric, + status, err := metrics.InvokeAndReportLatency[*disperser.BlobStatusReply](verifier.getStatusLatencyMetric, func() (*disperser.BlobStatusReply, error) { return (*verifier.dispenser).GetBlobStatus(ctxTimeout, *key.key) }) diff --git a/tools/traffic/workers/blob_writer.go b/tools/traffic/workers/blob_writer.go index 9e37301f3b..6f86f3f2cf 100644 --- a/tools/traffic/workers/blob_writer.go +++ b/tools/traffic/workers/blob_writer.go @@ -103,7 +103,7 @@ func (writer *BlobWriter) run() { return case <-ticker.C: data := writer.getRandomData() - key, err := metrics.InvokeAndReportLatency(&writer.writeLatencyMetric, func() ([]byte, error) { + key, err := metrics.InvokeAndReportLatency(writer.writeLatencyMetric, func() ([]byte, error) { return writer.sendRequest(*data) }) if err != nil { From fd2d342afc9865de2e7548ff72f44d734382a929 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 11:13:25 -0500 Subject: [PATCH 50/74] Created interceptable ticker. Signed-off-by: Cody Littley --- tools/traffic/workers/blob_verifier.go | 4 +++ tools/traffic/workers/blob_writer.go | 28 +++++++-------- tools/traffic/workers/ticker.go | 50 ++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 tools/traffic/workers/ticker.go diff --git a/tools/traffic/workers/blob_verifier.go b/tools/traffic/workers/blob_verifier.go index 7ab1bcf29f..96f80a0bf5 100644 --- a/tools/traffic/workers/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -66,6 +66,10 @@ type BlobVerifier struct { finalizedCountMetric metrics.CountMetric } +type UnconfirmedKeyHandler interface { + AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) +} + // NewStatusVerifier creates a new BlobVerifier instance. func NewStatusVerifier( ctx *context.Context, diff --git a/tools/traffic/workers/blob_writer.go b/tools/traffic/workers/blob_writer.go index 6f86f3f2cf..fb6af96bc1 100644 --- a/tools/traffic/workers/blob_writer.go +++ b/tools/traffic/workers/blob_writer.go @@ -30,8 +30,8 @@ type BlobWriter struct { // disperser is the client used to send blobs to the disperser. disperser *clients.DisperserClient - // Responsible for polling on the status of a recently written blob until it becomes confirmed. - verifier *BlobVerifier + // Unconfirmed keys are sent here. + unconfirmedKeyHandler UnconfirmedKeyHandler // fixedRandomData contains random data for blobs if RandomizeBlobs is false, and nil otherwise. fixedRandomData *[]byte @@ -53,7 +53,7 @@ func NewBlobWriter( logger logging.Logger, config *Config, disperser *clients.DisperserClient, - verifier *BlobVerifier, + unconfirmedKeyHandler UnconfirmedKeyHandler, generatorMetrics metrics.Metrics) BlobWriter { var fixedRandomData []byte @@ -71,16 +71,16 @@ func NewBlobWriter( } return BlobWriter{ - ctx: ctx, - waitGroup: waitGroup, - logger: logger, - config: config, - disperser: disperser, - verifier: verifier, - fixedRandomData: &fixedRandomData, - writeLatencyMetric: generatorMetrics.NewLatencyMetric("write"), - writeSuccessMetric: generatorMetrics.NewCountMetric("write_success"), - writeFailureMetric: generatorMetrics.NewCountMetric("write_failure"), + ctx: ctx, + waitGroup: waitGroup, + logger: logger, + config: config, + disperser: disperser, + unconfirmedKeyHandler: unconfirmedKeyHandler, + fixedRandomData: &fixedRandomData, + writeLatencyMetric: generatorMetrics.NewLatencyMetric("write"), + writeSuccessMetric: generatorMetrics.NewCountMetric("write_success"), + writeFailureMetric: generatorMetrics.NewCountMetric("write_failure"), } } @@ -115,7 +115,7 @@ func (writer *BlobWriter) run() { writer.writeSuccessMetric.Increment() checksum := md5.Sum(*data) - writer.verifier.AddUnconfirmedKey(&key, &checksum, uint(len(*data))) + writer.unconfirmedKeyHandler.AddUnconfirmedKey(&key, &checksum, uint(len(*data))) } } } diff --git a/tools/traffic/workers/ticker.go b/tools/traffic/workers/ticker.go new file mode 100644 index 0000000000..ee90d27f0a --- /dev/null +++ b/tools/traffic/workers/ticker.go @@ -0,0 +1,50 @@ +package workers + +import "time" + +// InterceptableTicker is a wrapper around the time.Ticker struct. +// It allows for deterministic time passage to be simulated in tests. +type InterceptableTicker interface { + // getTimeChannel returns the channel that the ticker sends ticks on. Equivalent to time.Ticker.C. + getTimeChannel() <-chan time.Time +} + +// standardTicker behaves exactly like a time.Ticker, for use in production code. +type standardTicker struct { + ticker *time.Ticker +} + +// NewTicker creates a new InterceptableTicker that behaves like a time.Ticker. +func NewTicker(d time.Duration) InterceptableTicker { + return &standardTicker{ + ticker: time.NewTicker(d), + } +} + +func (s *standardTicker) getTimeChannel() <-chan time.Time { + return s.ticker.C +} + +// MockTicker is a deterministic implementation of the InterceptableTicker interface. +type MockTicker struct { + channel chan time.Time + now time.Time +} + +// NewMockTicker creates a new InterceptableTicker that can be deterministically controlled in tests. +func NewMockTicker(now time.Time) InterceptableTicker { + return &MockTicker{ + channel: make(chan time.Time), + now: now, + } +} + +func (m *MockTicker) getTimeChannel() <-chan time.Time { + return m.channel +} + +// Tick advances the ticker by the specified duration. +func (m *MockTicker) Tick(elapsedTime time.Duration) { + m.now = m.now.Add(elapsedTime) + m.channel <- m.now +} From ed51c91b834ec6aaf59b971dea5fbbb10362b670 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 26 Jul 2024 14:30:30 -0500 Subject: [PATCH 51/74] Incremental checkin. Signed-off-by: Cody Littley --- tools/traffic/generator.go | 1 + tools/traffic/metrics/mock_metrics.go | 69 +++++++------- tools/traffic/workers/blob_writer.go | 9 +- tools/traffic/workers/blob_writer_test.go | 105 ++++++++++++++++++++++ tools/traffic/workers/ticker.go | 2 +- 5 files changed, 152 insertions(+), 34 deletions(-) create mode 100644 tools/traffic/workers/blob_writer_test.go diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 22f4b58731..1884876689 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -107,6 +107,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera &ctx, &waitGroup, logger, + workers.NewTicker(config.WorkerConfig.WriteRequestInterval), &config.WorkerConfig, &disperserClient, &statusVerifier, diff --git a/tools/traffic/metrics/mock_metrics.go b/tools/traffic/metrics/mock_metrics.go index 23801a3a8a..88eb11447b 100644 --- a/tools/traffic/metrics/mock_metrics.go +++ b/tools/traffic/metrics/mock_metrics.go @@ -18,22 +18,13 @@ type MockMetrics struct { lock sync.Mutex } -// MockLatencyMetric implements LatencyMetric, useful for testing. -type MockLatencyMetric struct { - metrics *MockMetrics - description string -} - -// MockCountMetric implements CountMetric, useful for testing. -type MockCountMetric struct { - metrics *MockMetrics - description string -} - -// MockGaugeMetric implements GaugeMetric, useful for testing. -type MockGaugeMetric struct { - metrics *MockMetrics - description string +// NewMockMetrics creates a new MockMetrics instance. +func NewMockMetrics() *MockMetrics { + return &MockMetrics{ + counts: make(map[string]float64), + gauges: make(map[string]float64), + latencies: make(map[string]time.Duration), + } } // GetCount returns the count of a type of event. @@ -62,39 +53,57 @@ func (m *MockMetrics) Start() { } func (m *MockMetrics) NewLatencyMetric(description string) LatencyMetric { - return &MockLatencyMetric{ + return &mockLatencyMetric{ metrics: m, description: description, } } -func (m *MockLatencyMetric) ReportLatency(latency time.Duration) { - m.metrics.lock.Lock() - m.metrics.latencies[m.description] = latency - m.metrics.lock.Unlock() +func (m *MockMetrics) NewCountMetric(description string) CountMetric { + return &mockCountMetric{ + metrics: m, + description: description, + } } -func (m *MockMetrics) NewCountMetric(description string) CountMetric { - return &MockCountMetric{ +func (m *MockMetrics) NewGaugeMetric(description string) GaugeMetric { + return &mockGaugeMetric{ metrics: m, description: description, } } -func (m *MockCountMetric) Increment() { +// mockLatencyMetric implements LatencyMetric, useful for testing. +type mockLatencyMetric struct { + metrics *MockMetrics + description string +} + +func (m *mockLatencyMetric) ReportLatency(latency time.Duration) { + m.metrics.lock.Lock() + m.metrics.latencies[m.description] = latency + m.metrics.lock.Unlock() +} + +// mockCountMetric implements CountMetric, useful for testing. +type mockCountMetric struct { + metrics *MockMetrics + description string +} + +func (m *mockCountMetric) Increment() { m.metrics.lock.Lock() m.metrics.counts[m.description]++ m.metrics.lock.Unlock() } -func (m *MockMetrics) NewGaugeMetric(description string) GaugeMetric { - return &MockGaugeMetric{ - metrics: m, - description: description, - } +// mockGaugeMetric implements GaugeMetric, useful for testing. +type mockGaugeMetric struct { + metrics *MockMetrics + description string } -func (m *MockGaugeMetric) Set(value float64) { +func (m *mockGaugeMetric) Set(value float64) { m.metrics.lock.Lock() m.metrics.gauges[m.description] = value m.metrics.lock.Unlock() diff --git a/tools/traffic/workers/blob_writer.go b/tools/traffic/workers/blob_writer.go index fb6af96bc1..de50a5c1d5 100644 --- a/tools/traffic/workers/blob_writer.go +++ b/tools/traffic/workers/blob_writer.go @@ -10,7 +10,6 @@ import ( "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigensdk-go/logging" "sync" - "time" ) // BlobWriter sends blobs to a disperser at a configured rate. @@ -24,6 +23,9 @@ type BlobWriter struct { // All logs should be written using this logger. logger logging.Logger + // ticker is used to control the rate at which blobs are sent to the disperser. + ticker InterceptableTicker + // Config contains the configuration for the generator. config *Config @@ -51,6 +53,7 @@ func NewBlobWriter( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, + ticker InterceptableTicker, config *Config, disperser *clients.DisperserClient, unconfirmedKeyHandler UnconfirmedKeyHandler, @@ -74,6 +77,7 @@ func NewBlobWriter( ctx: ctx, waitGroup: waitGroup, logger: logger, + ticker: ticker, config: config, disperser: disperser, unconfirmedKeyHandler: unconfirmedKeyHandler, @@ -96,12 +100,11 @@ func (writer *BlobWriter) Start() { // run sends blobs to a disperser at a configured rate. // Continues and dues not return until the context is cancelled. func (writer *BlobWriter) run() { - ticker := time.NewTicker(writer.config.WriteRequestInterval) for { select { case <-(*writer.ctx).Done(): return - case <-ticker.C: + case <-writer.ticker.getTimeChannel(): data := writer.getRandomData() key, err := metrics.InvokeAndReportLatency(writer.writeLatencyMetric, func() ([]byte, error) { return writer.sendRequest(*data) diff --git a/tools/traffic/workers/blob_writer_test.go b/tools/traffic/workers/blob_writer_test.go new file mode 100644 index 0000000000..7e32ecf9ca --- /dev/null +++ b/tools/traffic/workers/blob_writer_test.go @@ -0,0 +1,105 @@ +package workers + +import ( + "context" + "fmt" + "github.com/Layr-Labs/eigenda/api/clients" + "github.com/Layr-Labs/eigenda/common" + "github.com/Layr-Labs/eigenda/tools/traffic/metrics" + "github.com/stretchr/testify/assert" + "golang.org/x/exp/rand" + "sync" + "testing" + "time" +) + +// TODO create test util package maybe + +// initializeRandom initializes the random number generator. Prints the seed so that the test can be rerun +// deterministically. Replace a call to this method with a call to initializeRandomWithSeed to rerun a test +// with a specific seed. +func initializeRandom() { + rand.Seed(uint64(time.Now().UnixNano())) + seed := rand.Uint64() + fmt.Printf("Random seed: %d\n", seed) + rand.Seed(seed) +} + +// initializeRandomWithSeed initializes the random number generator with a specific seed. +func initializeRandomWithSeed(seed uint64) { + fmt.Printf("Random seed: %d\n", seed) + rand.Seed(seed) +} + +// MockUnconfirmedKeyHandler is a stand-in for the blob verifier's UnconfirmedKeyHandler. +type MockUnconfirmedKeyHandler struct { + t *testing.T + key *[]byte + checksum *[16]byte + size uint +} + +func (m *MockUnconfirmedKeyHandler) AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) { + // Ensure that we have already verified the previous key. + assert.Nil(m.t, m.key) + + m.key = key + m.checksum = checksum + m.size = size +} + +// GetAndClear returns the unconfirmed key and clears the internal state. Must be called after each +// AddUnconfirmedKey call. +func (m *MockUnconfirmedKeyHandler) GetAndCLear() (*[]byte, *[16]byte, uint) { + defer func() { + m.key = nil + m.checksum = nil + m.size = 0 + }() + return m.key, m.checksum, m.size +} + +// TestBasicBehavior tests the basic behavior of the BlobWriter with no special cases. +func TestBasicBehavior(t *testing.T) { + initializeRandom() + + ctx, cancel := context.WithCancel(context.Background()) + waitGroup := sync.WaitGroup{} + logger, err := common.NewLogger(common.DefaultLoggerConfig()) + assert.Nil(t, err) + // TODO use deterministic start time + ticker := NewMockTicker(time.Now()) + + dataSize := rand.Uint64()%1024 + 64 + + config := &Config{ + DataSize: dataSize, + } + var disperser *clients.DisperserClient // TODO create mock + + unconfirmedKeyHandler := &MockUnconfirmedKeyHandler{ + t: t, + } + + generatorMetrics := metrics.NewMockMetrics() + + writer := NewBlobWriter( + &ctx, + &waitGroup, + logger, + ticker, + config, + disperser, + unconfirmedKeyHandler, + generatorMetrics) + writer.Start() + + ticker.Tick(1 * time.Second) + _, _, _ = unconfirmedKeyHandler.GetAndCLear() // TODO + + // ... + + cancel() + // TODO add timeout + waitGroup.Wait() +} diff --git a/tools/traffic/workers/ticker.go b/tools/traffic/workers/ticker.go index ee90d27f0a..56315574af 100644 --- a/tools/traffic/workers/ticker.go +++ b/tools/traffic/workers/ticker.go @@ -32,7 +32,7 @@ type MockTicker struct { } // NewMockTicker creates a new InterceptableTicker that can be deterministically controlled in tests. -func NewMockTicker(now time.Time) InterceptableTicker { +func NewMockTicker(now time.Time) *MockTicker { return &MockTicker{ channel: make(chan time.Time), now: now, From e68281edba3baf08a27e510ec93922a1267e896c Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Mon, 29 Jul 2024 10:15:13 -0500 Subject: [PATCH 52/74] Added unit test for writer. Signed-off-by: Cody Littley --- tools/traffic/config.go | 13 +- tools/traffic/flags/flags.go | 8 + tools/traffic/generator.go | 4 +- tools/traffic/workers/blob_verifier.go | 8 +- tools/traffic/workers/blob_writer.go | 8 +- tools/traffic/workers/blob_writer_test.go | 245 +++++++++++++++++++--- tools/traffic/workers/config.go | 2 + 7 files changed, 239 insertions(+), 49 deletions(-) diff --git a/tools/traffic/config.go b/tools/traffic/config.go index 84c4f97b08..ba289c1490 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -123,12 +123,13 @@ func NewConfig(ctx *cli.Context) (*Config, error) { VerifierInterval: ctx.Duration(flags.VerifierIntervalFlag.Name), GetBlobStatusTimeout: ctx.Duration(flags.GetBlobStatusTimeoutFlag.Name), - NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), - ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), - RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), - ReadOverflowTableSize: uint32(ctx.Uint(flags.ReadOverflowTableSizeFlag.Name)), - FetchBatchHeaderTimeout: ctx.Duration(flags.FetchBatchHeaderTimeoutFlag.Name), - RetrieveBlobChunksTimeout: ctx.Duration(flags.RetrieveBlobChunksTimeoutFlag.Name), + NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), + ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), + RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), + ReadOverflowTableSize: uint32(ctx.Uint(flags.ReadOverflowTableSizeFlag.Name)), + FetchBatchHeaderTimeout: ctx.Duration(flags.FetchBatchHeaderTimeoutFlag.Name), + RetrieveBlobChunksTimeout: ctx.Duration(flags.RetrieveBlobChunksTimeoutFlag.Name), + VerificationChannelCapacity: ctx.Uint(flags.VerificationChannelCapacityFlag.Name), EigenDAServiceManager: ctx.String(flags.EigenDAServiceManagerFlag.Name), SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go index 385687b08d..734332c60a 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/flags/flags.go @@ -231,6 +231,13 @@ var ( Value: 5 * time.Second, EnvVar: common.PrefixEnvVar(envPrefix, "GET_BLOB_STATUS_TIMEOUT"), } + VerificationChannelCapacityFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "verification-channel-capacity"), + Usage: "Size of the channel used to communicate between the writer and verifier.", + Required: false, + Value: 1000, + EnvVar: common.PrefixEnvVar(envPrefix, "VERIFICATION_CHANNEL_CAPACITY"), + } /* Configuration for the blob reader. */ @@ -321,6 +328,7 @@ var optionalFlags = []cli.Flag{ GetBlobStatusTimeoutFlag, ReadOverflowTableSizeFlag, WriteTimeoutFlag, + VerificationChannelCapacityFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 1884876689..03da44b6ab 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -98,7 +98,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera logger, &config.WorkerConfig, &blobTable, - &disperserClient, + disperserClient, generatorMetrics) writers := make([]*workers.BlobWriter, 0) @@ -109,7 +109,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera logger, workers.NewTicker(config.WorkerConfig.WriteRequestInterval), &config.WorkerConfig, - &disperserClient, + disperserClient, &statusVerifier, generatorMetrics) writers = append(writers, &writer) diff --git a/tools/traffic/workers/blob_verifier.go b/tools/traffic/workers/blob_verifier.go index 96f80a0bf5..2fe8ab1f65 100644 --- a/tools/traffic/workers/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -45,7 +45,7 @@ type BlobVerifier struct { table *table.BlobTable // The disperser client used to monitor the disperser service. - dispenser *clients.DisperserClient + dispenser clients.DisperserClient // The keys of blobs that have not yet been confirmed by the disperser service. unconfirmedKeys []*unconfirmedKey @@ -77,7 +77,7 @@ func NewStatusVerifier( logger logging.Logger, config *Config, table *table.BlobTable, - disperser *clients.DisperserClient, + disperser clients.DisperserClient, generatorMetrics metrics.Metrics) BlobVerifier { return BlobVerifier{ @@ -88,7 +88,7 @@ func NewStatusVerifier( table: table, dispenser: disperser, unconfirmedKeys: make([]*unconfirmedKey, 0), - keyChannel: make(chan *unconfirmedKey), + keyChannel: make(chan *unconfirmedKey, config.VerificationChannelCapacity), blobsInFlightMetric: generatorMetrics.NewGaugeMetric("blobs_in_flight"), getStatusLatencyMetric: generatorMetrics.NewLatencyMetric("get_status"), confirmationLatencyMetric: generatorMetrics.NewLatencyMetric("confirmation"), @@ -162,7 +162,7 @@ func (verifier *BlobVerifier) checkStatusForBlob(key *unconfirmedKey) bool { status, err := metrics.InvokeAndReportLatency[*disperser.BlobStatusReply](verifier.getStatusLatencyMetric, func() (*disperser.BlobStatusReply, error) { - return (*verifier.dispenser).GetBlobStatus(ctxTimeout, *key.key) + return verifier.dispenser.GetBlobStatus(ctxTimeout, *key.key) }) if err != nil { diff --git a/tools/traffic/workers/blob_writer.go b/tools/traffic/workers/blob_writer.go index de50a5c1d5..67ea414a77 100644 --- a/tools/traffic/workers/blob_writer.go +++ b/tools/traffic/workers/blob_writer.go @@ -30,7 +30,7 @@ type BlobWriter struct { config *Config // disperser is the client used to send blobs to the disperser. - disperser *clients.DisperserClient + disperser clients.DisperserClient // Unconfirmed keys are sent here. unconfirmedKeyHandler UnconfirmedKeyHandler @@ -55,7 +55,7 @@ func NewBlobWriter( logger logging.Logger, ticker InterceptableTicker, config *Config, - disperser *clients.DisperserClient, + disperser clients.DisperserClient, unconfirmedKeyHandler UnconfirmedKeyHandler, generatorMetrics metrics.Metrics) BlobWriter { @@ -148,12 +148,12 @@ func (writer *BlobWriter) sendRequest(data []byte) ([]byte /* key */, error) { var key []byte var err error if writer.config.SignerPrivateKey != "" { - _, key, err = (*writer.disperser).DisperseBlobAuthenticated( + _, key, err = writer.disperser.DisperseBlobAuthenticated( ctxTimeout, data, writer.config.CustomQuorums) } else { - _, key, err = (*writer.disperser).DisperseBlob( + _, key, err = writer.disperser.DisperseBlob( ctxTimeout, data, writer.config.CustomQuorums) diff --git a/tools/traffic/workers/blob_writer_test.go b/tools/traffic/workers/blob_writer_test.go index 7e32ecf9ca..90019aeec4 100644 --- a/tools/traffic/workers/blob_writer_test.go +++ b/tools/traffic/workers/blob_writer_test.go @@ -2,9 +2,12 @@ package workers import ( "context" + "crypto/md5" "fmt" - "github.com/Layr-Labs/eigenda/api/clients" + disperser_rpc "github.com/Layr-Labs/eigenda/api/grpc/disperser" "github.com/Layr-Labs/eigenda/common" + "github.com/Layr-Labs/eigenda/disperser" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/stretchr/testify/assert" "golang.org/x/exp/rand" @@ -31,35 +34,133 @@ func initializeRandomWithSeed(seed uint64) { rand.Seed(seed) } +// assertEventuallyTrue asserts that a condition is true within a given duration. Repeatably checks the condition. +func assertEventuallyTrue(t *testing.T, condition func() bool, duration time.Duration) { + start := time.Now() + for time.Since(start) < duration { + if condition() { + return + } + time.Sleep(1 * time.Millisecond) + } + assert.True(t, condition(), "Condition did not become true within the given duration") +} + +// executeWithTimeout executes a function with a timeout. +// Panics if the function does not complete within the given duration. +func executeWithTimeout(f func(), duration time.Duration) { + done := make(chan struct{}) + go func() { + f() + close(done) + }() + select { + case <-done: + case <-time.After(duration): + panic("function did not complete within the given duration") + } +} + // MockUnconfirmedKeyHandler is a stand-in for the blob verifier's UnconfirmedKeyHandler. type MockUnconfirmedKeyHandler struct { - t *testing.T - key *[]byte - checksum *[16]byte - size uint + t *testing.T + + // TODO rename + ProvidedKey []byte + ProvidedChecksum [16]byte + ProvidedSize uint + + // Incremented each time AddUnconfirmedKey is called. + Count uint + + lock *sync.Mutex +} + +func NewMockUnconfirmedKeyHandler(t *testing.T, lock *sync.Mutex) *MockUnconfirmedKeyHandler { + return &MockUnconfirmedKeyHandler{ + t: t, + lock: lock, + } } func (m *MockUnconfirmedKeyHandler) AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) { - // Ensure that we have already verified the previous key. - assert.Nil(m.t, m.key) + m.lock.Lock() + defer m.lock.Unlock() - m.key = key - m.checksum = checksum - m.size = size + m.ProvidedKey = *key + m.ProvidedChecksum = *checksum + m.ProvidedSize = size + + m.Count++ } -// GetAndClear returns the unconfirmed key and clears the internal state. Must be called after each -// AddUnconfirmedKey call. -func (m *MockUnconfirmedKeyHandler) GetAndCLear() (*[]byte, *[16]byte, uint) { - defer func() { - m.key = nil - m.checksum = nil - m.size = 0 - }() - return m.key, m.checksum, m.size +type MockDisperserClient struct { + t *testing.T + // if true, DisperseBlobAuthenticated is expected to be used, otherwise DisperseBlob is expected to be used + authenticated bool + + // The next status, key, and error to return from DisperseBlob or DisperseBlobAuthenticated + StatusToReturn disperser.BlobStatus + KeyToReturn []byte + ErrorToReturn error + + // The previous values passed to DisperseBlob or DisperseBlobAuthenticated + ProvidedData []byte + ProvidedQuorum []uint8 + + // Incremented each time DisperseBlob or DisperseBlobAuthenticated is called. + Count uint + + lock *sync.Mutex +} + +func NewMockDisperserClient(t *testing.T, lock *sync.Mutex, authenticated bool) *MockDisperserClient { + return &MockDisperserClient{ + t: t, + lock: lock, + authenticated: authenticated, + } +} + +func (m *MockDisperserClient) DisperseBlob( + ctx context.Context, + data []byte, + customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { + + m.lock.Lock() + defer m.lock.Unlock() + + assert.False(m.t, m.authenticated, "writer configured to use non-authenticated disperser method") + m.ProvidedData = data + m.ProvidedQuorum = customQuorums + m.Count++ + return &m.StatusToReturn, m.KeyToReturn, m.ErrorToReturn } -// TestBasicBehavior tests the basic behavior of the BlobWriter with no special cases. +func (m *MockDisperserClient) DisperseBlobAuthenticated( + ctx context.Context, + data []byte, + customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { + + m.lock.Lock() + defer m.lock.Unlock() + + assert.True(m.t, m.authenticated, "writer configured to use authenticated disperser method") + m.ProvidedData = data + m.ProvidedQuorum = customQuorums + m.Count++ + return &m.StatusToReturn, m.KeyToReturn, m.ErrorToReturn +} + +func (m *MockDisperserClient) GetBlobStatus(ctx context.Context, key []byte) (*disperser_rpc.BlobStatusReply, error) { + panic("this method should not be called in this test") +} + +func (m *MockDisperserClient) RetrieveBlob(ctx context.Context, batchHeaderHash []byte, blobIndex uint32) ([]byte, error) { + panic("this method should not be called in this test") +} + +// TestBasicBehavior tests the basic behavior of the BlobWriter. func TestBasicBehavior(t *testing.T) { initializeRandom() @@ -67,20 +168,37 @@ func TestBasicBehavior(t *testing.T) { waitGroup := sync.WaitGroup{} logger, err := common.NewLogger(common.DefaultLoggerConfig()) assert.Nil(t, err) - // TODO use deterministic start time - ticker := NewMockTicker(time.Now()) + startTime := time.Unix(rand.Int63()%2_000_000_000, 0) + ticker := NewMockTicker(startTime) dataSize := rand.Uint64()%1024 + 64 - config := &Config{ - DataSize: dataSize, + authenticated := rand.Intn(2) == 0 + var signerPrivateKey string + if authenticated { + signerPrivateKey = "asdf" + } + + randomizeBlobs := rand.Intn(2) == 0 + + useCustomQuorum := rand.Intn(2) == 0 + var customQuorum []uint8 + if useCustomQuorum { + customQuorum = []uint8{1, 2, 3} } - var disperser *clients.DisperserClient // TODO create mock - unconfirmedKeyHandler := &MockUnconfirmedKeyHandler{ - t: t, + config := &Config{ + DataSize: dataSize, + SignerPrivateKey: signerPrivateKey, + RandomizeBlobs: randomizeBlobs, + CustomQuorums: customQuorum, } + lock := sync.Mutex{} + + disperserClient := NewMockDisperserClient(t, &lock, authenticated) + unconfirmedKeyHandler := NewMockUnconfirmedKeyHandler(t, &lock) + generatorMetrics := metrics.NewMockMetrics() writer := NewBlobWriter( @@ -89,17 +207,78 @@ func TestBasicBehavior(t *testing.T) { logger, ticker, config, - disperser, + disperserClient, unconfirmedKeyHandler, generatorMetrics) writer.Start() - ticker.Tick(1 * time.Second) - _, _, _ = unconfirmedKeyHandler.GetAndCLear() // TODO + errorProbability := 0.1 + errorCount := 0 + + var previousData []byte + + for i := 0; i < 100; i++ { + if rand.Float64() < errorProbability { + disperserClient.ErrorToReturn = fmt.Errorf("intentional error for testing purposes") + errorCount++ + } else { + disperserClient.ErrorToReturn = nil + } + + // This is the key that will be assigned to the next blob. + disperserClient.KeyToReturn = make([]byte, 32) + _, err = rand.Read(disperserClient.KeyToReturn) + assert.Nil(t, err) + + // Move time forward, allowing the writer to attempt to send a blob. + ticker.Tick(1 * time.Second) + + // Wait until the writer finishes its work. + assertEventuallyTrue(t, func() bool { + lock.Lock() + defer lock.Unlock() + return int(disperserClient.Count) > i && int(unconfirmedKeyHandler.Count)+errorCount > i + }, time.Second) - // ... + // These methods should be called exactly once per tick if there are no errors. + // In the presence of errors, nothing should be passed to the unconfirmed key handler. + assert.Equal(t, uint(i+1), disperserClient.Count) + assert.Equal(t, uint(i+1-errorCount), unconfirmedKeyHandler.Count) + + if disperserClient.ErrorToReturn == nil { + assert.NotNil(t, disperserClient.ProvidedData) + assert.Equal(t, customQuorum, disperserClient.ProvidedQuorum) + + // Strip away the extra encoding bytes. We should have data of the expected size. + decodedData := codec.RemoveEmptyByteFromPaddedBytes(disperserClient.ProvidedData) + assert.Equal(t, dataSize, uint64(len(decodedData))) + + // Verify that the proper data was sent to the unconfirmed key handler. + assert.Equal(t, uint(len(disperserClient.ProvidedData)), unconfirmedKeyHandler.ProvidedSize) + checksum := md5.Sum(disperserClient.ProvidedData) + assert.Equal(t, checksum, unconfirmedKeyHandler.ProvidedChecksum) + assert.Equal(t, disperserClient.KeyToReturn, unconfirmedKeyHandler.ProvidedKey) + + // Verify that data has the proper amount of randomness. + if previousData != nil { + if randomizeBlobs { + // We expect each blob to be different. + assert.NotEqual(t, previousData, disperserClient.ProvidedData) + } else { + // We expect each blob to be the same. + assert.Equal(t, previousData, disperserClient.ProvidedData) + } + } + previousData = disperserClient.ProvidedData + } + + // Verify metrics. + assert.Equal(t, float64(i+1-errorCount), generatorMetrics.GetCount("write_success")) + assert.Equal(t, float64(errorCount), generatorMetrics.GetCount("write_failure")) + } cancel() - // TODO add timeout - waitGroup.Wait() + executeWithTimeout(func() { + waitGroup.Wait() + }, time.Second) } diff --git a/tools/traffic/workers/config.go b/tools/traffic/workers/config.go index 2abbb9ef6b..3bce555908 100644 --- a/tools/traffic/workers/config.go +++ b/tools/traffic/workers/config.go @@ -20,6 +20,8 @@ type Config struct { VerifierInterval time.Duration // The amount of time to wait for a blob status to be fetched. GetBlobStatusTimeout time.Duration + // The size of the channel used to communicate between the writer and verifier. + VerificationChannelCapacity uint // The number of worker threads that generate read traffic. NumReadInstances uint From 06055f0ab0a79e1dbd48eb049da4be8c101fd62d Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Mon, 29 Jul 2024 10:25:50 -0500 Subject: [PATCH 53/74] Moved test methods to utility package. Signed-off-by: Cody Littley --- common/testutils/test_utils.go | 52 +++++++++++++++ common/testutils/test_utils_test.go | 26 ++++++++ tools/traffic/table/blob_table_test.go | 27 ++------ tools/traffic/workers/blob_writer_test.go | 81 +++++------------------ 4 files changed, 101 insertions(+), 85 deletions(-) create mode 100644 common/testutils/test_utils.go create mode 100644 common/testutils/test_utils_test.go diff --git a/common/testutils/test_utils.go b/common/testutils/test_utils.go new file mode 100644 index 0000000000..73a031ef08 --- /dev/null +++ b/common/testutils/test_utils.go @@ -0,0 +1,52 @@ +package testutils + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "golang.org/x/exp/rand" + "testing" + "time" +) + +// InitializeRandom initializes the random number generator. Prints the seed so that the test can be rerun +// deterministically. Replace a call to this method with a call to initializeRandomWithSeed to rerun a test +// with a specific seed. +func InitializeRandom() { + rand.Seed(uint64(time.Now().UnixNano())) + seed := rand.Uint64() + fmt.Printf("Random seed: %d\n", seed) + rand.Seed(seed) +} + +// InitializeRandomWithSeed initializes the random number generator with a specific seed. +func InitializeRandomWithSeed(seed uint64) { + fmt.Printf("Random seed: %d\n", seed) + rand.Seed(seed) +} + +// AssertEventuallyTrue asserts that a condition is true within a given duration. Repeatably checks the condition. +func AssertEventuallyTrue(t *testing.T, condition func() bool, duration time.Duration) { + start := time.Now() + for time.Since(start) < duration { + if condition() { + return + } + time.Sleep(1 * time.Millisecond) + } + assert.True(t, condition(), "Condition did not become true within the given duration") +} + +// ExecuteWithTimeout executes a function with a timeout. +// Panics if the function does not complete within the given duration. +func ExecuteWithTimeout(f func(), duration time.Duration) { + done := make(chan struct{}) + go func() { + f() + close(done) + }() + select { + case <-done: + case <-time.After(duration): + panic("function did not complete within the given duration") + } +} diff --git a/common/testutils/test_utils_test.go b/common/testutils/test_utils_test.go new file mode 100644 index 0000000000..f0c9d9d385 --- /dev/null +++ b/common/testutils/test_utils_test.go @@ -0,0 +1,26 @@ +package testutils + +import ( + "github.com/stretchr/testify/assert" + "golang.org/x/exp/rand" + "testing" +) + +func TestRandomSetup(t *testing.T) { + InitializeRandom() + x := rand.Int() + + InitializeRandom() + y := rand.Int() + + assert.NotEqual(t, x, y) + + seed := uint64(rand.Int()) + InitializeRandomWithSeed(seed) + a := rand.Int() + + InitializeRandomWithSeed(seed) + b := rand.Int() + + assert.Equal(t, a, b) +} diff --git a/tools/traffic/table/blob_table_test.go b/tools/traffic/table/blob_table_test.go index cf091b2ab6..b60dd54bfb 100644 --- a/tools/traffic/table/blob_table_test.go +++ b/tools/traffic/table/blob_table_test.go @@ -1,29 +1,12 @@ package table import ( - "fmt" + tu "github.com/Layr-Labs/eigenda/common/testutils" "github.com/stretchr/testify/assert" "golang.org/x/exp/rand" "testing" - "time" ) -// initializeRandom initializes the random number generator. Prints the seed so that the test can be rerun -// deterministically. Replace a call to this method with a call to initializeRandomWithSeed to rerun a test -// with a specific seed. -func initializeRandom() { - rand.Seed(uint64(time.Now().UnixNano())) - seed := rand.Uint64() - fmt.Printf("Random seed: %d\n", seed) - rand.Seed(seed) -} - -// initializeRandomWithSeed initializes the random number generator with a specific seed. -func initializeRandomWithSeed(seed uint64) { - fmt.Printf("Random seed: %d\n", seed) - rand.Seed(seed) -} - // randomMetadata generates a random BlobMetadata instance. func randomMetadata(permits int) *BlobMetadata { key := make([]byte, 32) @@ -37,7 +20,7 @@ func randomMetadata(permits int) *BlobMetadata { // TestBasicOperation tests basic operations of the BlobTable. Adds blobs and iterates over them. func TestBasicOperation(t *testing.T) { - initializeRandom() + tu.InitializeRandom() table := NewBlobTable() assert.Equal(t, uint(0), table.Size()) @@ -61,7 +44,7 @@ func TestBasicOperation(t *testing.T) { // TestGetRandomWithRemoval tests getting a random blob data, but where the number of permits per blob is unlimited. func TestGetRandomNoRemovalByConfiguration(t *testing.T) { - initializeRandom() + tu.InitializeRandom() table := NewBlobTable() assert.Equal(t, uint(0), table.Size()) @@ -103,7 +86,7 @@ func TestGetRandomNoRemovalByConfiguration(t *testing.T) { // TestGetRandomWithRemoval tests getting a random blob data, where the number of permits per blob is limited. func TestGetRandomWithRemoval(t *testing.T) { - initializeRandom() + tu.InitializeRandom() table := NewBlobTable() assert.Equal(t, uint(0), table.Size()) @@ -155,7 +138,7 @@ func TestGetRandomWithRemoval(t *testing.T) { // TestAddOrReplace tests adding blobs to a table with a maximum capacity. The table should replace blobs when full. func TestAddOrReplace(t *testing.T) { - initializeRandom() + tu.InitializeRandom() table := NewBlobTable() assert.Equal(t, uint(0), table.Size()) diff --git a/tools/traffic/workers/blob_writer_test.go b/tools/traffic/workers/blob_writer_test.go index 90019aeec4..0a3dd1aadd 100644 --- a/tools/traffic/workers/blob_writer_test.go +++ b/tools/traffic/workers/blob_writer_test.go @@ -6,6 +6,7 @@ import ( "fmt" disperser_rpc "github.com/Layr-Labs/eigenda/api/grpc/disperser" "github.com/Layr-Labs/eigenda/common" + tu "github.com/Layr-Labs/eigenda/common/testutils" "github.com/Layr-Labs/eigenda/disperser" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" @@ -16,56 +17,10 @@ import ( "time" ) -// TODO create test util package maybe - -// initializeRandom initializes the random number generator. Prints the seed so that the test can be rerun -// deterministically. Replace a call to this method with a call to initializeRandomWithSeed to rerun a test -// with a specific seed. -func initializeRandom() { - rand.Seed(uint64(time.Now().UnixNano())) - seed := rand.Uint64() - fmt.Printf("Random seed: %d\n", seed) - rand.Seed(seed) -} - -// initializeRandomWithSeed initializes the random number generator with a specific seed. -func initializeRandomWithSeed(seed uint64) { - fmt.Printf("Random seed: %d\n", seed) - rand.Seed(seed) -} - -// assertEventuallyTrue asserts that a condition is true within a given duration. Repeatably checks the condition. -func assertEventuallyTrue(t *testing.T, condition func() bool, duration time.Duration) { - start := time.Now() - for time.Since(start) < duration { - if condition() { - return - } - time.Sleep(1 * time.Millisecond) - } - assert.True(t, condition(), "Condition did not become true within the given duration") -} - -// executeWithTimeout executes a function with a timeout. -// Panics if the function does not complete within the given duration. -func executeWithTimeout(f func(), duration time.Duration) { - done := make(chan struct{}) - go func() { - f() - close(done) - }() - select { - case <-done: - case <-time.After(duration): - panic("function did not complete within the given duration") - } -} - -// MockUnconfirmedKeyHandler is a stand-in for the blob verifier's UnconfirmedKeyHandler. -type MockUnconfirmedKeyHandler struct { +// mockUnconfirmedKeyHandler is a stand-in for the blob verifier's UnconfirmedKeyHandler. +type mockUnconfirmedKeyHandler struct { t *testing.T - // TODO rename ProvidedKey []byte ProvidedChecksum [16]byte ProvidedSize uint @@ -76,14 +31,14 @@ type MockUnconfirmedKeyHandler struct { lock *sync.Mutex } -func NewMockUnconfirmedKeyHandler(t *testing.T, lock *sync.Mutex) *MockUnconfirmedKeyHandler { - return &MockUnconfirmedKeyHandler{ +func newMockUnconfirmedKeyHandler(t *testing.T, lock *sync.Mutex) *mockUnconfirmedKeyHandler { + return &mockUnconfirmedKeyHandler{ t: t, lock: lock, } } -func (m *MockUnconfirmedKeyHandler) AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) { +func (m *mockUnconfirmedKeyHandler) AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) { m.lock.Lock() defer m.lock.Unlock() @@ -94,7 +49,7 @@ func (m *MockUnconfirmedKeyHandler) AddUnconfirmedKey(key *[]byte, checksum *[16 m.Count++ } -type MockDisperserClient struct { +type mockDisperserClient struct { t *testing.T // if true, DisperseBlobAuthenticated is expected to be used, otherwise DisperseBlob is expected to be used authenticated bool @@ -114,15 +69,15 @@ type MockDisperserClient struct { lock *sync.Mutex } -func NewMockDisperserClient(t *testing.T, lock *sync.Mutex, authenticated bool) *MockDisperserClient { - return &MockDisperserClient{ +func newMockDisperserClient(t *testing.T, lock *sync.Mutex, authenticated bool) *mockDisperserClient { + return &mockDisperserClient{ t: t, lock: lock, authenticated: authenticated, } } -func (m *MockDisperserClient) DisperseBlob( +func (m *mockDisperserClient) DisperseBlob( ctx context.Context, data []byte, customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { @@ -137,7 +92,7 @@ func (m *MockDisperserClient) DisperseBlob( return &m.StatusToReturn, m.KeyToReturn, m.ErrorToReturn } -func (m *MockDisperserClient) DisperseBlobAuthenticated( +func (m *mockDisperserClient) DisperseBlobAuthenticated( ctx context.Context, data []byte, customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { @@ -152,17 +107,17 @@ func (m *MockDisperserClient) DisperseBlobAuthenticated( return &m.StatusToReturn, m.KeyToReturn, m.ErrorToReturn } -func (m *MockDisperserClient) GetBlobStatus(ctx context.Context, key []byte) (*disperser_rpc.BlobStatusReply, error) { +func (m *mockDisperserClient) GetBlobStatus(ctx context.Context, key []byte) (*disperser_rpc.BlobStatusReply, error) { panic("this method should not be called in this test") } -func (m *MockDisperserClient) RetrieveBlob(ctx context.Context, batchHeaderHash []byte, blobIndex uint32) ([]byte, error) { +func (m *mockDisperserClient) RetrieveBlob(ctx context.Context, batchHeaderHash []byte, blobIndex uint32) ([]byte, error) { panic("this method should not be called in this test") } // TestBasicBehavior tests the basic behavior of the BlobWriter. func TestBasicBehavior(t *testing.T) { - initializeRandom() + tu.InitializeRandom() ctx, cancel := context.WithCancel(context.Background()) waitGroup := sync.WaitGroup{} @@ -196,8 +151,8 @@ func TestBasicBehavior(t *testing.T) { lock := sync.Mutex{} - disperserClient := NewMockDisperserClient(t, &lock, authenticated) - unconfirmedKeyHandler := NewMockUnconfirmedKeyHandler(t, &lock) + disperserClient := newMockDisperserClient(t, &lock, authenticated) + unconfirmedKeyHandler := newMockUnconfirmedKeyHandler(t, &lock) generatorMetrics := metrics.NewMockMetrics() @@ -234,7 +189,7 @@ func TestBasicBehavior(t *testing.T) { ticker.Tick(1 * time.Second) // Wait until the writer finishes its work. - assertEventuallyTrue(t, func() bool { + tu.AssertEventuallyTrue(t, func() bool { lock.Lock() defer lock.Unlock() return int(disperserClient.Count) > i && int(unconfirmedKeyHandler.Count)+errorCount > i @@ -278,7 +233,7 @@ func TestBasicBehavior(t *testing.T) { } cancel() - executeWithTimeout(func() { + tu.ExecuteWithTimeout(func() { waitGroup.Wait() }, time.Second) } From bc3d88763dda19913d571f3352ceddade241c344 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Mon, 29 Jul 2024 10:46:28 -0500 Subject: [PATCH 54/74] Reorganize code. Signed-off-by: Cody Littley --- tools/traffic/generator.go | 3 +- tools/traffic/workers/blob_verifier.go | 17 +-- tools/traffic/workers/blob_verifier_test.go | 48 +++++++++ tools/traffic/workers/blob_writer_test.go | 103 +------------------ tools/traffic/workers/mock.go | 108 ++++++++++++++++++++ 5 files changed, 170 insertions(+), 109 deletions(-) create mode 100644 tools/traffic/workers/blob_verifier_test.go create mode 100644 tools/traffic/workers/mock.go diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 03da44b6ab..fcf6376927 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -92,10 +92,11 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera UseSecureGrpcFlag: config.DisperserUseSecureGrpcFlag, } disperserClient := clients.NewDisperserClient(&disperserConfig, signer) - statusVerifier := workers.NewStatusVerifier( + statusVerifier := workers.NewBlobVerifier( &ctx, &waitGroup, logger, + workers.NewTicker(config.WorkerConfig.VerifierInterval), &config.WorkerConfig, &blobTable, disperserClient, diff --git a/tools/traffic/workers/blob_verifier.go b/tools/traffic/workers/blob_verifier.go index 2fe8ab1f65..f85f3f2e4d 100644 --- a/tools/traffic/workers/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -53,6 +53,9 @@ type BlobVerifier struct { // Newly added keys that require verification. keyChannel chan *unconfirmedKey + // ticker is used to control the rate at which blobs queried for status. + ticker InterceptableTicker + blobsInFlightMetric metrics.GaugeMetric getStatusLatencyMetric metrics.LatencyMetric confirmationLatencyMetric metrics.LatencyMetric @@ -70,11 +73,12 @@ type UnconfirmedKeyHandler interface { AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) } -// NewStatusVerifier creates a new BlobVerifier instance. -func NewStatusVerifier( +// NewBlobVerifier creates a new BlobVerifier instance. +func NewBlobVerifier( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, + ticker InterceptableTicker, config *Config, table *table.BlobTable, disperser clients.DisperserClient, @@ -84,6 +88,7 @@ func NewStatusVerifier( ctx: ctx, waitGroup: waitGroup, logger: logger, + ticker: ticker, config: config, table: table, dispenser: disperser, @@ -117,12 +122,12 @@ func (verifier *BlobVerifier) AddUnconfirmedKey(key *[]byte, checksum *[16]byte, // the disperser service to verify the status of blobs. func (verifier *BlobVerifier) Start() { verifier.waitGroup.Add(1) - go verifier.monitor(verifier.config.VerifierInterval) + go verifier.monitor() } // monitor periodically polls the disperser service to verify the status of blobs. -func (verifier *BlobVerifier) monitor(period time.Duration) { - ticker := time.NewTicker(period) +func (verifier *BlobVerifier) monitor() { + ticker := verifier.ticker.getTimeChannel() for { select { case <-(*verifier.ctx).Done(): @@ -130,7 +135,7 @@ func (verifier *BlobVerifier) monitor(period time.Duration) { return case key := <-verifier.keyChannel: verifier.unconfirmedKeys = append(verifier.unconfirmedKeys, key) - case <-ticker.C: + case <-ticker: verifier.poll() } } diff --git a/tools/traffic/workers/blob_verifier_test.go b/tools/traffic/workers/blob_verifier_test.go new file mode 100644 index 0000000000..60acdde799 --- /dev/null +++ b/tools/traffic/workers/blob_verifier_test.go @@ -0,0 +1,48 @@ +package workers + +import ( + "context" + "github.com/Layr-Labs/eigenda/common" + tu "github.com/Layr-Labs/eigenda/common/testutils" + "github.com/Layr-Labs/eigenda/tools/traffic/metrics" + "github.com/Layr-Labs/eigenda/tools/traffic/table" + "github.com/stretchr/testify/assert" + "golang.org/x/exp/rand" + "sync" + "testing" + "time" +) + +func TestBlobVerifier(t *testing.T) { + tu.InitializeRandom() + + ctx, cancel := context.WithCancel(context.Background()) + waitGroup := sync.WaitGroup{} + logger, err := common.NewLogger(common.DefaultLoggerConfig()) + assert.Nil(t, err) + startTime := time.Unix(rand.Int63()%2_000_000_000, 0) + ticker := NewMockTicker(startTime) + + config := &Config{} + + blobTable := table.NewBlobTable() + + verifierMetrics := metrics.NewMockMetrics() + + verifier := NewBlobVerifier( + &ctx, + &waitGroup, + logger, + ticker, + config, + &blobTable, + nil, + verifierMetrics) + + verifier.Start() + + cancel() + tu.ExecuteWithTimeout(func() { + waitGroup.Wait() + }, time.Second) +} diff --git a/tools/traffic/workers/blob_writer_test.go b/tools/traffic/workers/blob_writer_test.go index 0a3dd1aadd..51234acc73 100644 --- a/tools/traffic/workers/blob_writer_test.go +++ b/tools/traffic/workers/blob_writer_test.go @@ -4,10 +4,8 @@ import ( "context" "crypto/md5" "fmt" - disperser_rpc "github.com/Layr-Labs/eigenda/api/grpc/disperser" "github.com/Layr-Labs/eigenda/common" tu "github.com/Layr-Labs/eigenda/common/testutils" - "github.com/Layr-Labs/eigenda/disperser" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/stretchr/testify/assert" @@ -17,106 +15,7 @@ import ( "time" ) -// mockUnconfirmedKeyHandler is a stand-in for the blob verifier's UnconfirmedKeyHandler. -type mockUnconfirmedKeyHandler struct { - t *testing.T - - ProvidedKey []byte - ProvidedChecksum [16]byte - ProvidedSize uint - - // Incremented each time AddUnconfirmedKey is called. - Count uint - - lock *sync.Mutex -} - -func newMockUnconfirmedKeyHandler(t *testing.T, lock *sync.Mutex) *mockUnconfirmedKeyHandler { - return &mockUnconfirmedKeyHandler{ - t: t, - lock: lock, - } -} - -func (m *mockUnconfirmedKeyHandler) AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) { - m.lock.Lock() - defer m.lock.Unlock() - - m.ProvidedKey = *key - m.ProvidedChecksum = *checksum - m.ProvidedSize = size - - m.Count++ -} - -type mockDisperserClient struct { - t *testing.T - // if true, DisperseBlobAuthenticated is expected to be used, otherwise DisperseBlob is expected to be used - authenticated bool - - // The next status, key, and error to return from DisperseBlob or DisperseBlobAuthenticated - StatusToReturn disperser.BlobStatus - KeyToReturn []byte - ErrorToReturn error - - // The previous values passed to DisperseBlob or DisperseBlobAuthenticated - ProvidedData []byte - ProvidedQuorum []uint8 - - // Incremented each time DisperseBlob or DisperseBlobAuthenticated is called. - Count uint - - lock *sync.Mutex -} - -func newMockDisperserClient(t *testing.T, lock *sync.Mutex, authenticated bool) *mockDisperserClient { - return &mockDisperserClient{ - t: t, - lock: lock, - authenticated: authenticated, - } -} - -func (m *mockDisperserClient) DisperseBlob( - ctx context.Context, - data []byte, - customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { - - m.lock.Lock() - defer m.lock.Unlock() - - assert.False(m.t, m.authenticated, "writer configured to use non-authenticated disperser method") - m.ProvidedData = data - m.ProvidedQuorum = customQuorums - m.Count++ - return &m.StatusToReturn, m.KeyToReturn, m.ErrorToReturn -} - -func (m *mockDisperserClient) DisperseBlobAuthenticated( - ctx context.Context, - data []byte, - customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { - - m.lock.Lock() - defer m.lock.Unlock() - - assert.True(m.t, m.authenticated, "writer configured to use authenticated disperser method") - m.ProvidedData = data - m.ProvidedQuorum = customQuorums - m.Count++ - return &m.StatusToReturn, m.KeyToReturn, m.ErrorToReturn -} - -func (m *mockDisperserClient) GetBlobStatus(ctx context.Context, key []byte) (*disperser_rpc.BlobStatusReply, error) { - panic("this method should not be called in this test") -} - -func (m *mockDisperserClient) RetrieveBlob(ctx context.Context, batchHeaderHash []byte, blobIndex uint32) ([]byte, error) { - panic("this method should not be called in this test") -} - -// TestBasicBehavior tests the basic behavior of the BlobWriter. -func TestBasicBehavior(t *testing.T) { +func TestBlobWriter(t *testing.T) { tu.InitializeRandom() ctx, cancel := context.WithCancel(context.Background()) diff --git a/tools/traffic/workers/mock.go b/tools/traffic/workers/mock.go new file mode 100644 index 0000000000..6dc002e662 --- /dev/null +++ b/tools/traffic/workers/mock.go @@ -0,0 +1,108 @@ +package workers + +import ( + "context" + disperser_rpc "github.com/Layr-Labs/eigenda/api/grpc/disperser" + "github.com/Layr-Labs/eigenda/disperser" + "github.com/stretchr/testify/assert" + "sync" + "testing" +) + +// mockUnconfirmedKeyHandler is a stand-in for the blob verifier's UnconfirmedKeyHandler. +type mockUnconfirmedKeyHandler struct { + t *testing.T + + ProvidedKey []byte + ProvidedChecksum [16]byte + ProvidedSize uint + + // Incremented each time AddUnconfirmedKey is called. + Count uint + + lock *sync.Mutex +} + +func newMockUnconfirmedKeyHandler(t *testing.T, lock *sync.Mutex) *mockUnconfirmedKeyHandler { + return &mockUnconfirmedKeyHandler{ + t: t, + lock: lock, + } +} + +func (m *mockUnconfirmedKeyHandler) AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) { + m.lock.Lock() + defer m.lock.Unlock() + + m.ProvidedKey = *key + m.ProvidedChecksum = *checksum + m.ProvidedSize = size + + m.Count++ +} + +type mockDisperserClient struct { + t *testing.T + // if true, DisperseBlobAuthenticated is expected to be used, otherwise DisperseBlob is expected to be used + authenticated bool + + // The next status, key, and error to return from DisperseBlob or DisperseBlobAuthenticated + StatusToReturn disperser.BlobStatus + KeyToReturn []byte + ErrorToReturn error + + // The previous values passed to DisperseBlob or DisperseBlobAuthenticated + ProvidedData []byte + ProvidedQuorum []uint8 + + // Incremented each time DisperseBlob or DisperseBlobAuthenticated is called. + Count uint + + lock *sync.Mutex +} + +func newMockDisperserClient(t *testing.T, lock *sync.Mutex, authenticated bool) *mockDisperserClient { + return &mockDisperserClient{ + t: t, + lock: lock, + authenticated: authenticated, + } +} + +func (m *mockDisperserClient) DisperseBlob( + ctx context.Context, + data []byte, + customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { + + m.lock.Lock() + defer m.lock.Unlock() + + assert.False(m.t, m.authenticated, "writer configured to use non-authenticated disperser method") + m.ProvidedData = data + m.ProvidedQuorum = customQuorums + m.Count++ + return &m.StatusToReturn, m.KeyToReturn, m.ErrorToReturn +} + +func (m *mockDisperserClient) DisperseBlobAuthenticated( + ctx context.Context, + data []byte, + customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { + + m.lock.Lock() + defer m.lock.Unlock() + + assert.True(m.t, m.authenticated, "writer configured to use authenticated disperser method") + m.ProvidedData = data + m.ProvidedQuorum = customQuorums + m.Count++ + return &m.StatusToReturn, m.KeyToReturn, m.ErrorToReturn +} + +func (m *mockDisperserClient) GetBlobStatus(ctx context.Context, key []byte) (*disperser_rpc.BlobStatusReply, error) { + panic("this method should not be called in this test") +} + +func (m *mockDisperserClient) RetrieveBlob(ctx context.Context, batchHeaderHash []byte, blobIndex uint32) ([]byte, error) { + panic("this method should not be called in this test") +} From 3cb98118abeadfece2dadaa6a27b11b012616358 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Mon, 29 Jul 2024 10:58:53 -0500 Subject: [PATCH 55/74] Create test package. Signed-off-by: Cody Littley --- .../{workers => test}/blob_verifier_test.go | 9 ++-- .../{workers => test}/blob_writer_test.go | 11 ++-- .../mock.go => test/mock_dispatcher.go} | 34 +------------ .../traffic/test/mock_interceptable_ticker.go | 27 ++++++++++ tools/traffic/test/mock_key_handler.go | 38 ++++++++++++++ tools/traffic/workers/blob_verifier.go | 6 +-- tools/traffic/workers/blob_writer.go | 6 +-- tools/traffic/workers/interceptable_ticker.go | 26 ++++++++++ tools/traffic/workers/key_handler.go | 7 +++ tools/traffic/workers/ticker.go | 50 ------------------- 10 files changed, 114 insertions(+), 100 deletions(-) rename tools/traffic/{workers => test}/blob_verifier_test.go (83%) rename tools/traffic/{workers => test}/blob_writer_test.go (94%) rename tools/traffic/{workers/mock.go => test/mock_dispatcher.go} (75%) create mode 100644 tools/traffic/test/mock_interceptable_ticker.go create mode 100644 tools/traffic/test/mock_key_handler.go create mode 100644 tools/traffic/workers/interceptable_ticker.go create mode 100644 tools/traffic/workers/key_handler.go delete mode 100644 tools/traffic/workers/ticker.go diff --git a/tools/traffic/workers/blob_verifier_test.go b/tools/traffic/test/blob_verifier_test.go similarity index 83% rename from tools/traffic/workers/blob_verifier_test.go rename to tools/traffic/test/blob_verifier_test.go index 60acdde799..d6f95ff2e6 100644 --- a/tools/traffic/workers/blob_verifier_test.go +++ b/tools/traffic/test/blob_verifier_test.go @@ -1,4 +1,4 @@ -package workers +package test import ( "context" @@ -6,6 +6,7 @@ import ( tu "github.com/Layr-Labs/eigenda/common/testutils" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" + "github.com/Layr-Labs/eigenda/tools/traffic/workers" "github.com/stretchr/testify/assert" "golang.org/x/exp/rand" "sync" @@ -21,15 +22,15 @@ func TestBlobVerifier(t *testing.T) { logger, err := common.NewLogger(common.DefaultLoggerConfig()) assert.Nil(t, err) startTime := time.Unix(rand.Int63()%2_000_000_000, 0) - ticker := NewMockTicker(startTime) + ticker := newMockTicker(startTime) - config := &Config{} + config := &workers.Config{} blobTable := table.NewBlobTable() verifierMetrics := metrics.NewMockMetrics() - verifier := NewBlobVerifier( + verifier := workers.NewBlobVerifier( &ctx, &waitGroup, logger, diff --git a/tools/traffic/workers/blob_writer_test.go b/tools/traffic/test/blob_writer_test.go similarity index 94% rename from tools/traffic/workers/blob_writer_test.go rename to tools/traffic/test/blob_writer_test.go index 51234acc73..20a0d5d968 100644 --- a/tools/traffic/workers/blob_writer_test.go +++ b/tools/traffic/test/blob_writer_test.go @@ -1,4 +1,4 @@ -package workers +package test import ( "context" @@ -8,6 +8,7 @@ import ( tu "github.com/Layr-Labs/eigenda/common/testutils" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" + "github.com/Layr-Labs/eigenda/tools/traffic/workers" "github.com/stretchr/testify/assert" "golang.org/x/exp/rand" "sync" @@ -23,7 +24,7 @@ func TestBlobWriter(t *testing.T) { logger, err := common.NewLogger(common.DefaultLoggerConfig()) assert.Nil(t, err) startTime := time.Unix(rand.Int63()%2_000_000_000, 0) - ticker := NewMockTicker(startTime) + ticker := newMockTicker(startTime) dataSize := rand.Uint64()%1024 + 64 @@ -41,7 +42,7 @@ func TestBlobWriter(t *testing.T) { customQuorum = []uint8{1, 2, 3} } - config := &Config{ + config := &workers.Config{ DataSize: dataSize, SignerPrivateKey: signerPrivateKey, RandomizeBlobs: randomizeBlobs, @@ -51,11 +52,11 @@ func TestBlobWriter(t *testing.T) { lock := sync.Mutex{} disperserClient := newMockDisperserClient(t, &lock, authenticated) - unconfirmedKeyHandler := newMockUnconfirmedKeyHandler(t, &lock) + unconfirmedKeyHandler := newMockKeyHandler(t, &lock) generatorMetrics := metrics.NewMockMetrics() - writer := NewBlobWriter( + writer := workers.NewBlobWriter( &ctx, &waitGroup, logger, diff --git a/tools/traffic/workers/mock.go b/tools/traffic/test/mock_dispatcher.go similarity index 75% rename from tools/traffic/workers/mock.go rename to tools/traffic/test/mock_dispatcher.go index 6dc002e662..83c8e0f556 100644 --- a/tools/traffic/workers/mock.go +++ b/tools/traffic/test/mock_dispatcher.go @@ -1,4 +1,4 @@ -package workers +package test import ( "context" @@ -9,38 +9,6 @@ import ( "testing" ) -// mockUnconfirmedKeyHandler is a stand-in for the blob verifier's UnconfirmedKeyHandler. -type mockUnconfirmedKeyHandler struct { - t *testing.T - - ProvidedKey []byte - ProvidedChecksum [16]byte - ProvidedSize uint - - // Incremented each time AddUnconfirmedKey is called. - Count uint - - lock *sync.Mutex -} - -func newMockUnconfirmedKeyHandler(t *testing.T, lock *sync.Mutex) *mockUnconfirmedKeyHandler { - return &mockUnconfirmedKeyHandler{ - t: t, - lock: lock, - } -} - -func (m *mockUnconfirmedKeyHandler) AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) { - m.lock.Lock() - defer m.lock.Unlock() - - m.ProvidedKey = *key - m.ProvidedChecksum = *checksum - m.ProvidedSize = size - - m.Count++ -} - type mockDisperserClient struct { t *testing.T // if true, DisperseBlobAuthenticated is expected to be used, otherwise DisperseBlob is expected to be used diff --git a/tools/traffic/test/mock_interceptable_ticker.go b/tools/traffic/test/mock_interceptable_ticker.go new file mode 100644 index 0000000000..7179c37052 --- /dev/null +++ b/tools/traffic/test/mock_interceptable_ticker.go @@ -0,0 +1,27 @@ +package test + +import "time" + +// mockTicker is a deterministic implementation of the InterceptableTicker interface. +type mockTicker struct { + channel chan time.Time + now time.Time +} + +// newMockTicker creates a new InterceptableTicker that can be deterministically controlled in tests. +func newMockTicker(now time.Time) *mockTicker { + return &mockTicker{ + channel: make(chan time.Time), + now: now, + } +} + +func (m *mockTicker) GetTimeChannel() <-chan time.Time { + return m.channel +} + +// Tick advances the ticker by the specified duration. +func (m *mockTicker) Tick(elapsedTime time.Duration) { + m.now = m.now.Add(elapsedTime) + m.channel <- m.now +} diff --git a/tools/traffic/test/mock_key_handler.go b/tools/traffic/test/mock_key_handler.go new file mode 100644 index 0000000000..2d04671c72 --- /dev/null +++ b/tools/traffic/test/mock_key_handler.go @@ -0,0 +1,38 @@ +package test + +import ( + "sync" + "testing" +) + +// mockKeyHandler is a stand-in for the blob verifier's UnconfirmedKeyHandler. +type mockKeyHandler struct { + t *testing.T + + ProvidedKey []byte + ProvidedChecksum [16]byte + ProvidedSize uint + + // Incremented each time AddUnconfirmedKey is called. + Count uint + + lock *sync.Mutex +} + +func newMockKeyHandler(t *testing.T, lock *sync.Mutex) *mockKeyHandler { + return &mockKeyHandler{ + t: t, + lock: lock, + } +} + +func (m *mockKeyHandler) AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) { + m.lock.Lock() + defer m.lock.Unlock() + + m.ProvidedKey = *key + m.ProvidedChecksum = *checksum + m.ProvidedSize = size + + m.Count++ +} diff --git a/tools/traffic/workers/blob_verifier.go b/tools/traffic/workers/blob_verifier.go index f85f3f2e4d..a39803e700 100644 --- a/tools/traffic/workers/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -69,10 +69,6 @@ type BlobVerifier struct { finalizedCountMetric metrics.CountMetric } -type UnconfirmedKeyHandler interface { - AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) -} - // NewBlobVerifier creates a new BlobVerifier instance. func NewBlobVerifier( ctx *context.Context, @@ -127,7 +123,7 @@ func (verifier *BlobVerifier) Start() { // monitor periodically polls the disperser service to verify the status of blobs. func (verifier *BlobVerifier) monitor() { - ticker := verifier.ticker.getTimeChannel() + ticker := verifier.ticker.GetTimeChannel() for { select { case <-(*verifier.ctx).Done(): diff --git a/tools/traffic/workers/blob_writer.go b/tools/traffic/workers/blob_writer.go index 67ea414a77..f71e2bb67e 100644 --- a/tools/traffic/workers/blob_writer.go +++ b/tools/traffic/workers/blob_writer.go @@ -33,7 +33,7 @@ type BlobWriter struct { disperser clients.DisperserClient // Unconfirmed keys are sent here. - unconfirmedKeyHandler UnconfirmedKeyHandler + unconfirmedKeyHandler KeyHandler // fixedRandomData contains random data for blobs if RandomizeBlobs is false, and nil otherwise. fixedRandomData *[]byte @@ -56,7 +56,7 @@ func NewBlobWriter( ticker InterceptableTicker, config *Config, disperser clients.DisperserClient, - unconfirmedKeyHandler UnconfirmedKeyHandler, + unconfirmedKeyHandler KeyHandler, generatorMetrics metrics.Metrics) BlobWriter { var fixedRandomData []byte @@ -104,7 +104,7 @@ func (writer *BlobWriter) run() { select { case <-(*writer.ctx).Done(): return - case <-writer.ticker.getTimeChannel(): + case <-writer.ticker.GetTimeChannel(): data := writer.getRandomData() key, err := metrics.InvokeAndReportLatency(writer.writeLatencyMetric, func() ([]byte, error) { return writer.sendRequest(*data) diff --git a/tools/traffic/workers/interceptable_ticker.go b/tools/traffic/workers/interceptable_ticker.go new file mode 100644 index 0000000000..847f6acb25 --- /dev/null +++ b/tools/traffic/workers/interceptable_ticker.go @@ -0,0 +1,26 @@ +package workers + +import "time" + +// InterceptableTicker is a wrapper around the time.Ticker struct. +// It allows for deterministic time passage to be simulated in tests. +type InterceptableTicker interface { + // getTimeChannel returns the channel that the ticker sends ticks on. Equivalent to time.Ticker.C. + GetTimeChannel() <-chan time.Time +} + +// standardTicker behaves exactly like a time.Ticker, for use in production code. +type standardTicker struct { + ticker *time.Ticker +} + +// NewTicker creates a new InterceptableTicker that behaves like a time.Ticker. +func NewTicker(d time.Duration) InterceptableTicker { + return &standardTicker{ + ticker: time.NewTicker(d), + } +} + +func (s *standardTicker) GetTimeChannel() <-chan time.Time { + return s.ticker.C +} diff --git a/tools/traffic/workers/key_handler.go b/tools/traffic/workers/key_handler.go new file mode 100644 index 0000000000..3f6e1c2865 --- /dev/null +++ b/tools/traffic/workers/key_handler.go @@ -0,0 +1,7 @@ +package workers + +// KeyHandler is an interface describing an object that can accept unconfirmed keys. +type KeyHandler interface { + // AddUnconfirmedKey accepts an unconfirmed blob key, the checksum of the blob, and the size of the blob in bytes. + AddUnconfirmedKey(key *[]byte, checksum *[16]byte, size uint) +} diff --git a/tools/traffic/workers/ticker.go b/tools/traffic/workers/ticker.go deleted file mode 100644 index 56315574af..0000000000 --- a/tools/traffic/workers/ticker.go +++ /dev/null @@ -1,50 +0,0 @@ -package workers - -import "time" - -// InterceptableTicker is a wrapper around the time.Ticker struct. -// It allows for deterministic time passage to be simulated in tests. -type InterceptableTicker interface { - // getTimeChannel returns the channel that the ticker sends ticks on. Equivalent to time.Ticker.C. - getTimeChannel() <-chan time.Time -} - -// standardTicker behaves exactly like a time.Ticker, for use in production code. -type standardTicker struct { - ticker *time.Ticker -} - -// NewTicker creates a new InterceptableTicker that behaves like a time.Ticker. -func NewTicker(d time.Duration) InterceptableTicker { - return &standardTicker{ - ticker: time.NewTicker(d), - } -} - -func (s *standardTicker) getTimeChannel() <-chan time.Time { - return s.ticker.C -} - -// MockTicker is a deterministic implementation of the InterceptableTicker interface. -type MockTicker struct { - channel chan time.Time - now time.Time -} - -// NewMockTicker creates a new InterceptableTicker that can be deterministically controlled in tests. -func NewMockTicker(now time.Time) *MockTicker { - return &MockTicker{ - channel: make(chan time.Time), - now: now, - } -} - -func (m *MockTicker) getTimeChannel() <-chan time.Time { - return m.channel -} - -// Tick advances the ticker by the specified duration. -func (m *MockTicker) Tick(elapsedTime time.Duration) { - m.now = m.now.Add(elapsedTime) - m.channel <- m.now -} From 0d030cff1ab5f380649e9c7b026779a8fef5f0f0 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Mon, 29 Jul 2024 14:25:42 -0500 Subject: [PATCH 56/74] Added test for blob verifier. Signed-off-by: Cody Littley --- common/ratelimit/limiter_cli.go | 2 +- tools/traffic/table/blob_metadata.go | 5 + tools/traffic/test/blob_verifier_test.go | 157 +++++++++++++++++- tools/traffic/test/blob_writer_test.go | 13 +- .../{mock_dispatcher.go => mock_disperser.go} | 42 +++-- 5 files changed, 201 insertions(+), 18 deletions(-) rename tools/traffic/test/{mock_dispatcher.go => mock_disperser.go} (64%) diff --git a/common/ratelimit/limiter_cli.go b/common/ratelimit/limiter_cli.go index 33ff407a17..48f84fc75e 100644 --- a/common/ratelimit/limiter_cli.go +++ b/common/ratelimit/limiter_cli.go @@ -42,7 +42,7 @@ func RatelimiterCLIFlags(envPrefix string, flagPrefix string) []cli.Flag { }, cli.BoolFlag{ Name: common.PrefixFlag(flagPrefix, CountFailedFlagName), - Usage: "Count failed requests", + Usage: "DisperseCount failed requests", EnvVar: common.PrefixEnvVar(envPrefix, "COUNT_FAILED"), }, cli.IntFlag{ diff --git a/tools/traffic/table/blob_metadata.go b/tools/traffic/table/blob_metadata.go index 1eecede347..d365f541b2 100644 --- a/tools/traffic/table/blob_metadata.go +++ b/tools/traffic/table/blob_metadata.go @@ -70,3 +70,8 @@ func (blob *BlobMetadata) BatchHeaderHash() *[]byte { func (blob *BlobMetadata) BlobIndex() uint { return blob.blobIndex } + +// RemainingReadPermits returns the maximum number of remaining reads permitted against this blob. +func (blob *BlobMetadata) RemainingReadPermits() int { + return blob.remainingReadPermits +} diff --git a/tools/traffic/test/blob_verifier_test.go b/tools/traffic/test/blob_verifier_test.go index d6f95ff2e6..ac9a35507b 100644 --- a/tools/traffic/test/blob_verifier_test.go +++ b/tools/traffic/test/blob_verifier_test.go @@ -2,6 +2,8 @@ package test import ( "context" + "fmt" + disperser_rpc "github.com/Layr-Labs/eigenda/api/grpc/disperser" "github.com/Layr-Labs/eigenda/common" tu "github.com/Layr-Labs/eigenda/common/testutils" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" @@ -14,6 +16,43 @@ import ( "time" ) +func getRandomStatus() disperser_rpc.BlobStatus { + return disperser_rpc.BlobStatus(rand.Intn(7)) +} + +func isStatusTerminal(status disperser_rpc.BlobStatus) bool { + switch status { + case disperser_rpc.BlobStatus_UNKNOWN: + return false + case disperser_rpc.BlobStatus_PROCESSING: + return false + case disperser_rpc.BlobStatus_DISPERSING: + return false + + case disperser_rpc.BlobStatus_INSUFFICIENT_SIGNATURES: + return true + case disperser_rpc.BlobStatus_FAILED: + return true + case disperser_rpc.BlobStatus_FINALIZED: + return true + case disperser_rpc.BlobStatus_CONFIRMED: + return true + default: + panic("unknown status") + } +} + +func isStatusSuccess(status disperser_rpc.BlobStatus) bool { + switch status { + case disperser_rpc.BlobStatus_CONFIRMED: + return true + case disperser_rpc.BlobStatus_FINALIZED: + return true + default: + return false + } +} + func TestBlobVerifier(t *testing.T) { tu.InitializeRandom() @@ -24,12 +63,19 @@ func TestBlobVerifier(t *testing.T) { startTime := time.Unix(rand.Int63()%2_000_000_000, 0) ticker := newMockTicker(startTime) - config := &workers.Config{} + requiredDownloads := rand.Intn(10) + config := &workers.Config{ + RequiredDownloads: float64(requiredDownloads), + } blobTable := table.NewBlobTable() verifierMetrics := metrics.NewMockMetrics() + lock := sync.Mutex{} + + disperserClient := newMockDisperserClient(t, &lock, true) + verifier := workers.NewBlobVerifier( &ctx, &waitGroup, @@ -37,11 +83,118 @@ func TestBlobVerifier(t *testing.T) { ticker, config, &blobTable, - nil, + disperserClient, verifierMetrics) verifier.Start() + expectedGetStatusCount := 0 + statusCounts := make(map[disperser_rpc.BlobStatus]int) + checksums := make(map[string][16]byte) + sizes := make(map[string]uint) + + for i := 0; i < 100; i++ { + + // Add some new keys to track. + newKeys := rand.Intn(10) + for j := 0; j < newKeys; j++ { + key := make([]byte, 16) + checksum := [16]byte{} + size := rand.Uint32() + + _, err = rand.Read(key) + assert.Nil(t, err) + _, err = rand.Read(checksum[:]) + assert.Nil(t, err) + + checksums[string(key)] = checksum + sizes[string(key)] = uint(size) + + stringifiedKey := string(key) + disperserClient.StatusMap[stringifiedKey] = disperser_rpc.BlobStatus_UNKNOWN + + verifier.AddUnconfirmedKey(&key, &checksum, uint(size)) + } + + // Choose some new statuses to be returned. + // Count the number of status queries we expect to see in this iteration. + for key, status := range disperserClient.StatusMap { + if !isStatusTerminal(status) { + // Blobs in a non-terminal status will be queried again. + expectedGetStatusCount += 1 + // Set the next status to be returned. + newStatus := getRandomStatus() + disperserClient.StatusMap[key] = newStatus + statusCounts[newStatus] += 1 + } + } + + // Advance to the next cycle, allowing the verifier to process the new keys. + ticker.Tick(time.Second) + + // Data is inserted asynchronously, so we may need to wait for it to be processed. + tu.AssertEventuallyTrue(t, func() bool { + lock.Lock() + defer lock.Unlock() + + // Validate the number of calls made to the disperser client. + if int(disperserClient.GetStatusCount) < expectedGetStatusCount { + return false + } + + // Read the data in the table into a map for quick lookup. + tableData := make(map[string]*table.BlobMetadata) + for i := uint(0); i < blobTable.Size(); i++ { + metadata := blobTable.Get(i) + tableData[string(*metadata.Key())] = metadata + } + + blobsInFlight := 0 + for key, status := range disperserClient.StatusMap { + metadata, present := tableData[key] + + if !isStatusTerminal(status) { + blobsInFlight++ + } + + if isStatusSuccess(status) { + // Successful blobs should be in the table. + if !present { + // Blob might not yet be in table due to timing. + return false + } + } else { + // Non-successful blobs should not be in the table. + assert.False(t, present) + } + + // Verify metadata. + if present { + assert.Equal(t, checksums[key], *metadata.Checksum()) + assert.Equal(t, sizes[key], metadata.Size()) + assert.Equal(t, requiredDownloads, metadata.RemainingReadPermits()) + } + } + + // Verify metrics. + for status, count := range statusCounts { + metricName := fmt.Sprintf("get_status_%s", status.String()) + if float64(count) != verifierMetrics.GetCount(metricName) { + return false + } + } + if float64(blobsInFlight) != verifierMetrics.GetGaugeValue("blobs_in_flight") { + fmt.Printf("expected blobs_in_flight to be %d, got %f\n", blobsInFlight, verifierMetrics.GetCount("blobs_in_flight")) + return false + } + + return true + }, time.Second) + } + + assert.Equal(t, expectedGetStatusCount, int(disperserClient.GetStatusCount)) + assert.Equal(t, 0, int(disperserClient.DisperseCount)) + cancel() tu.ExecuteWithTimeout(func() { waitGroup.Wait() diff --git a/tools/traffic/test/blob_writer_test.go b/tools/traffic/test/blob_writer_test.go index 20a0d5d968..e67dfc7bdb 100644 --- a/tools/traffic/test/blob_writer_test.go +++ b/tools/traffic/test/blob_writer_test.go @@ -74,10 +74,10 @@ func TestBlobWriter(t *testing.T) { for i := 0; i < 100; i++ { if rand.Float64() < errorProbability { - disperserClient.ErrorToReturn = fmt.Errorf("intentional error for testing purposes") + disperserClient.DispenseErrorToReturn = fmt.Errorf("intentional error for testing purposes") errorCount++ } else { - disperserClient.ErrorToReturn = nil + disperserClient.DispenseErrorToReturn = nil } // This is the key that will be assigned to the next blob. @@ -92,15 +92,18 @@ func TestBlobWriter(t *testing.T) { tu.AssertEventuallyTrue(t, func() bool { lock.Lock() defer lock.Unlock() - return int(disperserClient.Count) > i && int(unconfirmedKeyHandler.Count)+errorCount > i + return int(disperserClient.DisperseCount) > i && int(unconfirmedKeyHandler.Count)+errorCount > i }, time.Second) // These methods should be called exactly once per tick if there are no errors. // In the presence of errors, nothing should be passed to the unconfirmed key handler. - assert.Equal(t, uint(i+1), disperserClient.Count) + assert.Equal(t, uint(i+1), disperserClient.DisperseCount) assert.Equal(t, uint(i+1-errorCount), unconfirmedKeyHandler.Count) - if disperserClient.ErrorToReturn == nil { + // This method should not be called in this test. + assert.Equal(t, uint(0), disperserClient.GetStatusCount) + + if disperserClient.DispenseErrorToReturn == nil { assert.NotNil(t, disperserClient.ProvidedData) assert.Equal(t, customQuorum, disperserClient.ProvidedQuorum) diff --git a/tools/traffic/test/mock_dispatcher.go b/tools/traffic/test/mock_disperser.go similarity index 64% rename from tools/traffic/test/mock_dispatcher.go rename to tools/traffic/test/mock_disperser.go index 83c8e0f556..bdfc6c2ddc 100644 --- a/tools/traffic/test/mock_dispatcher.go +++ b/tools/traffic/test/mock_disperser.go @@ -15,16 +15,22 @@ type mockDisperserClient struct { authenticated bool // The next status, key, and error to return from DisperseBlob or DisperseBlobAuthenticated - StatusToReturn disperser.BlobStatus - KeyToReturn []byte - ErrorToReturn error + StatusToReturn disperser.BlobStatus + KeyToReturn []byte + DispenseErrorToReturn error // The previous values passed to DisperseBlob or DisperseBlobAuthenticated ProvidedData []byte ProvidedQuorum []uint8 // Incremented each time DisperseBlob or DisperseBlobAuthenticated is called. - Count uint + DisperseCount uint + + // A map from key (in string form) to the status to return from GetBlobStatus. If nil, then an error is returned. + StatusMap map[string]disperser_rpc.BlobStatus + + // Incremented each time GetBlobStatus is called. + GetStatusCount uint lock *sync.Mutex } @@ -34,6 +40,7 @@ func newMockDisperserClient(t *testing.T, lock *sync.Mutex, authenticated bool) t: t, lock: lock, authenticated: authenticated, + StatusMap: make(map[string]disperser_rpc.BlobStatus), } } @@ -48,8 +55,8 @@ func (m *mockDisperserClient) DisperseBlob( assert.False(m.t, m.authenticated, "writer configured to use non-authenticated disperser method") m.ProvidedData = data m.ProvidedQuorum = customQuorums - m.Count++ - return &m.StatusToReturn, m.KeyToReturn, m.ErrorToReturn + m.DisperseCount++ + return &m.StatusToReturn, m.KeyToReturn, m.DispenseErrorToReturn } func (m *mockDisperserClient) DisperseBlobAuthenticated( @@ -63,14 +70,29 @@ func (m *mockDisperserClient) DisperseBlobAuthenticated( assert.True(m.t, m.authenticated, "writer configured to use authenticated disperser method") m.ProvidedData = data m.ProvidedQuorum = customQuorums - m.Count++ - return &m.StatusToReturn, m.KeyToReturn, m.ErrorToReturn + m.DisperseCount++ + return &m.StatusToReturn, m.KeyToReturn, m.DispenseErrorToReturn } func (m *mockDisperserClient) GetBlobStatus(ctx context.Context, key []byte) (*disperser_rpc.BlobStatusReply, error) { - panic("this method should not be called in this test") + m.lock.Lock() + defer m.lock.Unlock() + + status := m.StatusMap[string(key)] + m.GetStatusCount++ + + return &disperser_rpc.BlobStatusReply{ + Status: status, + Info: &disperser_rpc.BlobInfo{ + BlobVerificationProof: &disperser_rpc.BlobVerificationProof{ + BatchMetadata: &disperser_rpc.BatchMetadata{ + BatchHeaderHash: nil, // TODO + }, + }, + }, + }, nil } func (m *mockDisperserClient) RetrieveBlob(ctx context.Context, batchHeaderHash []byte, blobIndex uint32) ([]byte, error) { - panic("this method should not be called in this test") + panic("this method should not be called by the generator utility") } From a953f020aeee32c2587ebe86eb7b4e091d75d0e2 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 30 Jul 2024 09:11:56 -0500 Subject: [PATCH 57/74] Mostly finished tests for blob reader. Signed-off-by: Cody Littley --- common/testutils/test_utils.go | 22 +- tools/traffic/generator.go | 1 + tools/traffic/generator_test.go | 73 ----- tools/traffic/test/blob_reader_test.go | 288 ++++++++++++++++++++ tools/traffic/test/mock_chain_client.go | 36 +++ tools/traffic/test/mock_retrieval_client.go | 107 ++++++++ tools/traffic/workers/blob_reader.go | 17 +- tools/traffic/workers/config.go | 2 +- 8 files changed, 463 insertions(+), 83 deletions(-) delete mode 100644 tools/traffic/generator_test.go create mode 100644 tools/traffic/test/blob_reader_test.go create mode 100644 tools/traffic/test/mock_chain_client.go create mode 100644 tools/traffic/test/mock_retrieval_client.go diff --git a/common/testutils/test_utils.go b/common/testutils/test_utils.go index 73a031ef08..6cc53924ef 100644 --- a/common/testutils/test_utils.go +++ b/common/testutils/test_utils.go @@ -25,7 +25,7 @@ func InitializeRandomWithSeed(seed uint64) { } // AssertEventuallyTrue asserts that a condition is true within a given duration. Repeatably checks the condition. -func AssertEventuallyTrue(t *testing.T, condition func() bool, duration time.Duration) { +func AssertEventuallyTrue(t *testing.T, condition func() bool, duration time.Duration, debugInfo ...any) { start := time.Now() for time.Since(start) < duration { if condition() { @@ -33,7 +33,25 @@ func AssertEventuallyTrue(t *testing.T, condition func() bool, duration time.Dur } time.Sleep(1 * time.Millisecond) } - assert.True(t, condition(), "Condition did not become true within the given duration") + + if len(debugInfo) == 0 { + assert.True(t, condition(), "Condition did not become true within the given duration") // TODO use this elsewhere + } else { + assert.True(t, condition(), debugInfo...) + } +} + +// AssertEventuallyEquals asserts that a function returns a specific value within a given duration. +func AssertEventuallyEquals(t *testing.T, expected any, actual func() any, duration time.Duration, debugInfo ...any) { + if len(debugInfo) == 0 { + debugInfo = append(debugInfo, + "Expected value did not match actual value within the given duration. Expected: %v, Actual: %v", + expected, actual()) + } + + AssertEventuallyTrue(t, func() bool { + return expected == actual() + }, duration, debugInfo...) } // ExecuteWithTimeout executes a function with a timeout. diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index fcf6376927..ed0066f001 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -124,6 +124,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera &ctx, &waitGroup, logger, + workers.NewTicker(config.WorkerConfig.ReadRequestInterval), &config.WorkerConfig, retriever, chainClient, diff --git a/tools/traffic/generator_test.go b/tools/traffic/generator_test.go deleted file mode 100644 index c94c06882a..0000000000 --- a/tools/traffic/generator_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package traffic_test - -// TODO reimplement this test - -//import ( -// "context" -// "testing" -// "time" -// -// "github.com/Layr-Labs/eigenda/api/clients" -// clientsmock "github.com/Layr-Labs/eigenda/api/clients/mock" -// "github.com/Layr-Labs/eigenda/disperser" -// "github.com/Layr-Labs/eigenda/tools/traffic" -// "github.com/Layr-Labs/eigensdk-go/logging" -// -// "github.com/stretchr/testify/mock" -//) -// -//func TestTrafficGenerator(t *testing.T) { -// disperserClient := clientsmock.NewMockDisperserClient() -// logger := logging.NewNoopLogger() -// trafficGenerator := &traffic.Generator{ -// logger: logger, -// config: &traffic.Config{ -// Config: clients.Config{ -// Timeout: 1 * time.Second, -// }, -// DataSize: 1000_000, -// WriteRequestInterval: 2 * time.Second, -// }, -// disperserClient: disperserClient, -// } -// -// processing := disperser.Processing -// disperserClient.On("DisperseBlob", mock.Anything, mock.Anything, mock.Anything, mock.Anything). -// Return(&processing, []byte{1}, nil) -// ctx, cancel := context.WithCancel(context.Background()) -// go func() { -// _ = trafficGenerator.StartBlobWriter(ctx) -// }() -// time.Sleep(5 * time.Second) -// cancel() -// disperserClient.AssertNumberOfCalls(t, "DisperseBlob", 2) -//} -// -//func TestTrafficGeneratorAuthenticated(t *testing.T) { -// disperserClient := clientsmock.NewMockDisperserClient() -// logger := logging.NewNoopLogger() -// -// trafficGenerator := &traffic.Generator{ -// logger: logger, -// config: &traffic.Config{ -// Config: clients.Config{ -// Timeout: 1 * time.Second, -// }, -// DataSize: 1000_000, -// WriteRequestInterval: 2 * time.Second, -// SignerPrivateKey: "Hi", -// }, -// disperserClient: disperserClient, -// } -// -// processing := disperser.Processing -// disperserClient.On("DisperseBlobAuthenticated", mock.Anything, mock.Anything, mock.Anything, mock.Anything). -// Return(&processing, []byte{1}, nil) -// ctx, cancel := context.WithCancel(context.Background()) -// go func() { -// _ = trafficGenerator.StartBlobWriter(ctx) -// }() -// time.Sleep(5 * time.Second) -// cancel() -// disperserClient.AssertNumberOfCalls(t, "DisperseBlobAuthenticated", 2) -//} diff --git a/tools/traffic/test/blob_reader_test.go b/tools/traffic/test/blob_reader_test.go new file mode 100644 index 0000000000..5bf0a4519f --- /dev/null +++ b/tools/traffic/test/blob_reader_test.go @@ -0,0 +1,288 @@ +package test + +import ( + "context" + "crypto/md5" + "github.com/Layr-Labs/eigenda/common" + tu "github.com/Layr-Labs/eigenda/common/testutils" + "github.com/Layr-Labs/eigenda/tools/traffic/metrics" + "github.com/Layr-Labs/eigenda/tools/traffic/table" + "github.com/Layr-Labs/eigenda/tools/traffic/workers" + "github.com/stretchr/testify/assert" + "golang.org/x/exp/rand" + "sync" + "testing" + "time" +) + +// TestBlobReaderNoOptionalReads tests the BlobReader with only required reads. +func TestBlobReaderNoOverflow(t *testing.T) { + tu.InitializeRandom() + + ctx, cancel := context.WithCancel(context.Background()) + waitGroup := sync.WaitGroup{} + logger, err := common.NewLogger(common.DefaultLoggerConfig()) + assert.Nil(t, err) + startTime := time.Unix(rand.Int63()%2_000_000_000, 0) + ticker := newMockTicker(startTime) + + config := &workers.Config{ + ReadOverflowTableSize: 0, + } + + blobTable := table.NewBlobTable() + + readerMetrics := metrics.NewMockMetrics() + + lock := sync.Mutex{} + chainClient := newMockChainClient(&lock) + retrievalClient := newMockRetrievalClient(t, &lock) + + blobReader := workers.NewBlobReader( + &ctx, + &waitGroup, + logger, + ticker, + config, + retrievalClient, + chainClient, + &blobTable, + readerMetrics) + + blobSize := 1024 + readPermits := 2 + blobCount := 100 + + invalidBlobCount := 0 + + // Insert some blobs into the table. + for i := 0; i < blobCount; i++ { + + key := make([]byte, 32) + _, err = rand.Read(key) + assert.Nil(t, err) + + blobData := make([]byte, blobSize) + _, err = rand.Read(blobData) + + var checksum [16]byte + if i%10 == 0 { + // Simulate an invalid blob + invalidBlobCount++ + _, err = rand.Read(checksum[:]) + } else { + checksum = md5.Sum(blobData) + } + + batchHeaderHash := make([]byte, 32) + _, err = rand.Read(batchHeaderHash) + assert.Nil(t, err) + + blobMetadata := table.NewBlobMetadata( + &key, + &checksum, + uint(blobSize), + &batchHeaderHash, + uint(i), + readPermits) + + retrievalClient.AddBlob(blobMetadata, blobData) + + blobTable.Add(blobMetadata) + } + + blobReader.Start() + + // Do a bunch of reads. + expectedTotalReads := uint(readPermits * blobCount) + for i := uint(0); i < expectedTotalReads; i++ { + ticker.Tick(time.Second) + + tu.AssertEventuallyTrue(t, func() bool { + return retrievalClient.RetrieveBlobChunksCount == i+1 && + retrievalClient.CombineChunksCount == i+1 && + chainClient.Count == i+1 + }, time.Second) + + remainingPermits := uint(0) + for j := uint(0); j < blobTable.Size(); j++ { + blob := blobTable.Get(j) + if blob.RemainingReadPermits() != 2 { + } + remainingPermits += uint(blob.RemainingReadPermits()) + } + assert.Equal(t, remainingPermits, expectedTotalReads-i-1) + + tu.AssertEventuallyTrue(t, func() bool { + return uint(readerMetrics.GetCount("read_success")) == i+1 && + uint(readerMetrics.GetCount("fetch_batch_header_success")) == i+1 && + uint(readerMetrics.GetCount("recombination_success")) == i+1 + }, time.Second) + } + + expectedInvalidBlobs := uint(invalidBlobCount * readPermits) + expectedValidBlobs := expectedTotalReads - expectedInvalidBlobs + tu.AssertEventuallyEquals(t, expectedValidBlobs, + func() any { + return uint(readerMetrics.GetCount("valid_blob")) + }, time.Second) + tu.AssertEventuallyEquals(t, expectedInvalidBlobs, + func() any { + return uint(readerMetrics.GetCount("invalid_blob")) + }, time.Second) + + assert.Equal(t, uint(0), uint(readerMetrics.GetGaugeValue("required_read_pool_size"))) + assert.Equal(t, uint(0), uint(readerMetrics.GetGaugeValue("optional_read_pool_size"))) + + // Table is empty, so ticking time forward should not result in any reads. + ticker.Tick(time.Second) + // Give the system a moment to attempt to do work. This should not result in any reads. + time.Sleep(time.Second / 10) + assert.Equal(t, expectedTotalReads, uint(readerMetrics.GetCount("read_success"))) + assert.Equal(t, expectedTotalReads, uint(readerMetrics.GetCount("fetch_batch_header_success"))) + assert.Equal(t, expectedTotalReads, uint(readerMetrics.GetCount("recombination_success"))) + assert.Equal(t, expectedValidBlobs, uint(readerMetrics.GetCount("valid_blob"))) + assert.Equal(t, expectedInvalidBlobs, uint(readerMetrics.GetCount("invalid_blob"))) + + cancel() + tu.ExecuteWithTimeout(func() { + waitGroup.Wait() + }, time.Second) +} + +// TestBlobReaderWithOverflow tests the BlobReader with a non-zero sized overflow table. +func TestBlobReaderWithOverflow(t *testing.T) { + tu.InitializeRandom() + + ctx, cancel := context.WithCancel(context.Background()) + waitGroup := sync.WaitGroup{} + logger, err := common.NewLogger(common.DefaultLoggerConfig()) + assert.Nil(t, err) + startTime := time.Unix(rand.Int63()%2_000_000_000, 0) + ticker := newMockTicker(startTime) + + blobCount := 100 + overflowTableSize := uint(rand.Intn(blobCount-1) + 1) + + config := &workers.Config{ + ReadOverflowTableSize: overflowTableSize, + } + + blobTable := table.NewBlobTable() + + readerMetrics := metrics.NewMockMetrics() + + lock := sync.Mutex{} + chainClient := newMockChainClient(&lock) + retrievalClient := newMockRetrievalClient(t, &lock) + + blobReader := workers.NewBlobReader( + &ctx, + &waitGroup, + logger, + ticker, + config, + retrievalClient, + chainClient, + &blobTable, + readerMetrics) + + blobSize := 1024 + readPermits := 2 + + invalidBlobCount := 0 + + // Insert some blobs into the table. + for i := 0; i < blobCount; i++ { + + key := make([]byte, 32) + _, err = rand.Read(key) + assert.Nil(t, err) + + blobData := make([]byte, blobSize) + _, err = rand.Read(blobData) + + var checksum [16]byte + if i%10 == 0 { + // Simulate an invalid blob + invalidBlobCount++ + _, err = rand.Read(checksum[:]) + } else { + checksum = md5.Sum(blobData) + } + + batchHeaderHash := make([]byte, 32) + _, err = rand.Read(batchHeaderHash) + assert.Nil(t, err) + + blobMetadata := table.NewBlobMetadata( + &key, + &checksum, + uint(blobSize), + &batchHeaderHash, + uint(i), + readPermits) + + retrievalClient.AddBlob(blobMetadata, blobData) + + blobTable.Add(blobMetadata) + } + + blobReader.Start() + + // Do a bunch of reads. + expectedTotalReads := uint(readPermits * blobCount) + for i := uint(0); i < expectedTotalReads; i++ { + ticker.Tick(time.Second) + + tu.AssertEventuallyTrue(t, func() bool { + return retrievalClient.RetrieveBlobChunksCount == i+1 && + retrievalClient.CombineChunksCount == i+1 && + chainClient.Count == i+1 + }, time.Second) + + remainingPermits := uint(0) + for j := uint(0); j < blobTable.Size(); j++ { + blob := blobTable.Get(j) + if blob.RemainingReadPermits() != 2 { + } + remainingPermits += uint(blob.RemainingReadPermits()) + } + assert.Equal(t, remainingPermits, expectedTotalReads-i-1) + + tu.AssertEventuallyTrue(t, func() bool { + return uint(readerMetrics.GetCount("read_success")) == i+1 && + uint(readerMetrics.GetCount("fetch_batch_header_success")) == i+1 && + uint(readerMetrics.GetCount("recombination_success")) == i+1 + }, time.Second) + } + + expectedInvalidBlobs := uint(invalidBlobCount * readPermits) + expectedValidBlobs := expectedTotalReads - expectedInvalidBlobs + tu.AssertEventuallyEquals(t, expectedValidBlobs, + func() any { + return uint(readerMetrics.GetCount("valid_blob")) + }, time.Second) + tu.AssertEventuallyEquals(t, expectedInvalidBlobs, + func() any { + return uint(readerMetrics.GetCount("invalid_blob")) + }, time.Second) + + assert.Equal(t, uint(0), uint(readerMetrics.GetGaugeValue("required_read_pool_size"))) + assert.Equal(t, overflowTableSize, uint(readerMetrics.GetGaugeValue("optional_read_pool_size"))) + + // Do an additional read. We should be reading from the overflow table. + ticker.Tick(time.Second) + tu.AssertEventuallyEquals(t, expectedTotalReads+1, func() any { + return uint(readerMetrics.GetCount("read_success")) + }, time.Second) + tu.AssertEventuallyTrue(t, func() bool { + return uint(readerMetrics.GetCount("valid_blob")) == expectedValidBlobs+1 || + uint(readerMetrics.GetCount("invalid_blob")) == expectedInvalidBlobs+1 + }, time.Second) + + cancel() + tu.ExecuteWithTimeout(func() { + waitGroup.Wait() + }, time.Second) +} diff --git a/tools/traffic/test/mock_chain_client.go b/tools/traffic/test/mock_chain_client.go new file mode 100644 index 0000000000..793d9e3ee7 --- /dev/null +++ b/tools/traffic/test/mock_chain_client.go @@ -0,0 +1,36 @@ +package test + +import ( + "context" + binding "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDAServiceManager" + "github.com/ethereum/go-ethereum/common" + "math/big" + "sync" +) + +type mockChainClient struct { + lock *sync.Mutex + Count uint +} + +func newMockChainClient(lock *sync.Mutex) *mockChainClient { + return &mockChainClient{ + lock: lock, + } + +} + +func (m *mockChainClient) FetchBatchHeader( + ctx context.Context, + serviceManagerAddress common.Address, + batchHeaderHash []byte, + fromBlock *big.Int, + toBlock *big.Int) (*binding.IEigenDAServiceManagerBatchHeader, error) { + + m.lock.Lock() + defer m.lock.Unlock() + + m.Count++ + + return &binding.IEigenDAServiceManagerBatchHeader{}, nil +} diff --git a/tools/traffic/test/mock_retrieval_client.go b/tools/traffic/test/mock_retrieval_client.go new file mode 100644 index 0000000000..0d28d32a16 --- /dev/null +++ b/tools/traffic/test/mock_retrieval_client.go @@ -0,0 +1,107 @@ +package test + +import ( + "context" + "github.com/Layr-Labs/eigenda/api/clients" + "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/tools/traffic/table" + "github.com/stretchr/testify/assert" + "sync" + "testing" +) + +// mockRetrievalClient is a mock implementation of the clients.RetrievalClient interface. +type mockRetrievalClient struct { + t *testing.T + + lock *sync.Mutex + + // Since it isn't being used during this test, blob index field is used + // as a convenient unique identifier for the blob. + + // A map from blob index to the blob data. + blobData map[uint]*[]byte + + // A map from blob index to the blob metadata. + blobMetadata map[uint]*table.BlobMetadata + + // A map from blob index to the blob chunks corresponding to that blob. + blobChunks map[uint]*clients.BlobChunks + + RetrieveBlobChunksCount uint + CombineChunksCount uint +} + +func newMockRetrievalClient(t *testing.T, lock *sync.Mutex) *mockRetrievalClient { + return &mockRetrievalClient{ + t: t, + lock: lock, + blobData: make(map[uint]*[]byte), + blobMetadata: make(map[uint]*table.BlobMetadata), + blobChunks: make(map[uint]*clients.BlobChunks), + } +} + +// AddBlob adds a blob to the mock retrieval client. Once added, the retrieval client will act as if +// it is able to retrieve the blob. +func (m *mockRetrievalClient) AddBlob(metadata *table.BlobMetadata, data []byte) { + m.lock.Lock() + defer m.lock.Unlock() + + m.blobData[metadata.BlobIndex()] = &data + m.blobMetadata[metadata.BlobIndex()] = metadata + + // The blob index is used in this test as a convenient unique identifier for the blob. + + m.blobChunks[metadata.BlobIndex()] = &clients.BlobChunks{ + // Since it isn't otherwise used in this field, we can use it to store the unique identifier for the blob. + BlobHeaderLength: metadata.BlobIndex(), + } +} + +func (m *mockRetrievalClient) RetrieveBlob( + ctx context.Context, + batchHeaderHash [32]byte, + blobIndex uint32, + referenceBlockNumber uint, + batchRoot [32]byte, + quorumID core.QuorumID) ([]byte, error) { + panic("this method should not be called during this test") +} + +func (m *mockRetrievalClient) RetrieveBlobChunks( + ctx context.Context, + batchHeaderHash [32]byte, + blobIndex uint32, + referenceBlockNumber uint, + batchRoot [32]byte, + quorumID core.QuorumID) (*clients.BlobChunks, error) { + + m.lock.Lock() + defer m.lock.Unlock() + + m.RetrieveBlobChunksCount++ + + chunks, ok := m.blobChunks[uint(blobIndex)] + assert.True(m.t, ok, "blob not found") + + metadata := m.blobMetadata[uint(blobIndex)] + assert.Equal(m.t, metadata.BlobIndex(), uint(blobIndex)) + assert.Equal(m.t, (*metadata.BatchHeaderHash())[:32], batchHeaderHash[:32]) + + return chunks, nil + +} + +func (m *mockRetrievalClient) CombineChunks(chunks *clients.BlobChunks) ([]byte, error) { + m.lock.Lock() + defer m.lock.Unlock() + + m.CombineChunksCount++ + + blobIndex := chunks.BlobHeaderLength + data, ok := m.blobData[blobIndex] + assert.True(m.t, ok, "blob not found") + + return *data, nil +} diff --git a/tools/traffic/workers/blob_reader.go b/tools/traffic/workers/blob_reader.go index f88d2d9463..db8f2105d7 100644 --- a/tools/traffic/workers/blob_reader.go +++ b/tools/traffic/workers/blob_reader.go @@ -15,7 +15,6 @@ import ( gcommon "github.com/ethereum/go-ethereum/common" "math/big" "sync" - "time" ) // BlobReader reads blobs from a disperser at a configured rate. @@ -29,6 +28,9 @@ type BlobReader struct { // All logs should be written using this logger. logger logging.Logger + // ticker is used to control the rate at which blobs are read. + ticker InterceptableTicker + // config contains the configuration for the generator. config *Config @@ -63,6 +65,7 @@ func NewBlobReader( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, + ticker InterceptableTicker, config *Config, retriever clients.RetrievalClient, chainClient eth.ChainClient, @@ -75,6 +78,7 @@ func NewBlobReader( ctx: ctx, waitGroup: waitGroup, logger: logger, + ticker: ticker, config: config, retriever: retriever, chainClient: chainClient, @@ -109,12 +113,12 @@ func (reader *BlobReader) Start() { // run periodically performs reads on blobs. func (reader *BlobReader) run() { - ticker := time.NewTicker(reader.config.ReadRequestInterval) + ticker := reader.ticker.GetTimeChannel() for { select { case <-(*reader.ctx).Done(): return - case <-ticker.C: + case <-ticker: reader.randomRead() } } @@ -122,9 +126,6 @@ func (reader *BlobReader) run() { // randomRead reads a random blob. func (reader *BlobReader) randomRead() { - - reader.requiredReadPoolSizeMetric.Set(float64(reader.requiredReads.Size())) - metadata, removed := reader.requiredReads.GetRandom(true) if metadata == nil { // There are no blobs that we are required to read. Get a random blob from the optionalReads. @@ -135,10 +136,12 @@ func (reader *BlobReader) randomRead() { } } else if removed { // We have removed a blob from the requiredReads. Add it to the optionalReads. - reader.optionalReads.AddOrReplace(metadata, uint(reader.config.ReadOverflowTableSize)) + reader.optionalReads.AddOrReplace(metadata, reader.config.ReadOverflowTableSize) reader.optionalReadPoolSizeMetric.Set(float64(reader.optionalReads.Size())) } + reader.requiredReadPoolSizeMetric.Set(float64(reader.requiredReads.Size())) + ctxTimeout, cancel := context.WithTimeout(*reader.ctx, reader.config.FetchBatchHeaderTimeout) batchHeader, err := metrics.InvokeAndReportLatency(reader.fetchBatchHeaderMetric, func() (*contractEigenDAServiceManager.IEigenDAServiceManagerBatchHeader, error) { diff --git a/tools/traffic/workers/config.go b/tools/traffic/workers/config.go index 3bce555908..3326029f79 100644 --- a/tools/traffic/workers/config.go +++ b/tools/traffic/workers/config.go @@ -34,7 +34,7 @@ type Config struct { // The size of a table of blobs to optionally read when we run out of blobs that we are required to read. Blobs // that are no longer required are added to this table, and when the table is at capacity they are randomly retired. // Set this to 0 to disable this feature. - ReadOverflowTableSize uint32 + ReadOverflowTableSize uint // The amount of time to wait for a batch header to be fetched. FetchBatchHeaderTimeout time.Duration // The amount of time to wait for a blob to be retrieved. From e0b5d3a69e316337d951ffec3a08cdf783ba2852 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 30 Jul 2024 09:15:45 -0500 Subject: [PATCH 58/74] lint Signed-off-by: Cody Littley --- tools/traffic/config.go | 2 +- tools/traffic/test/blob_reader_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/traffic/config.go b/tools/traffic/config.go index ba289c1490..f1f58841bf 100644 --- a/tools/traffic/config.go +++ b/tools/traffic/config.go @@ -126,7 +126,7 @@ func NewConfig(ctx *cli.Context) (*Config, error) { NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), - ReadOverflowTableSize: uint32(ctx.Uint(flags.ReadOverflowTableSizeFlag.Name)), + ReadOverflowTableSize: ctx.Uint(flags.ReadOverflowTableSizeFlag.Name), FetchBatchHeaderTimeout: ctx.Duration(flags.FetchBatchHeaderTimeoutFlag.Name), RetrieveBlobChunksTimeout: ctx.Duration(flags.RetrieveBlobChunksTimeoutFlag.Name), VerificationChannelCapacity: ctx.Uint(flags.VerificationChannelCapacityFlag.Name), diff --git a/tools/traffic/test/blob_reader_test.go b/tools/traffic/test/blob_reader_test.go index 5bf0a4519f..065453d363 100644 --- a/tools/traffic/test/blob_reader_test.go +++ b/tools/traffic/test/blob_reader_test.go @@ -64,12 +64,14 @@ func TestBlobReaderNoOverflow(t *testing.T) { blobData := make([]byte, blobSize) _, err = rand.Read(blobData) + assert.Nil(t, err) var checksum [16]byte if i%10 == 0 { // Simulate an invalid blob invalidBlobCount++ _, err = rand.Read(checksum[:]) + assert.Nil(t, err) } else { checksum = md5.Sum(blobData) } @@ -107,8 +109,6 @@ func TestBlobReaderNoOverflow(t *testing.T) { remainingPermits := uint(0) for j := uint(0); j < blobTable.Size(); j++ { blob := blobTable.Get(j) - if blob.RemainingReadPermits() != 2 { - } remainingPermits += uint(blob.RemainingReadPermits()) } assert.Equal(t, remainingPermits, expectedTotalReads-i-1) @@ -201,12 +201,14 @@ func TestBlobReaderWithOverflow(t *testing.T) { blobData := make([]byte, blobSize) _, err = rand.Read(blobData) + assert.Nil(t, err) var checksum [16]byte if i%10 == 0 { // Simulate an invalid blob invalidBlobCount++ _, err = rand.Read(checksum[:]) + assert.Nil(t, err) } else { checksum = md5.Sum(blobData) } @@ -244,8 +246,6 @@ func TestBlobReaderWithOverflow(t *testing.T) { remainingPermits := uint(0) for j := uint(0); j < blobTable.Size(); j++ { blob := blobTable.Get(j) - if blob.RemainingReadPermits() != 2 { - } remainingPermits += uint(blob.RemainingReadPermits()) } assert.Equal(t, remainingPermits, expectedTotalReads-i-1) From 0bc3dcd3674ba4132ab29f967de3ffb5acd4cfd5 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 30 Jul 2024 09:18:02 -0500 Subject: [PATCH 59/74] Cleanup. Signed-off-by: Cody Littley --- common/testutils/test_utils.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/common/testutils/test_utils.go b/common/testutils/test_utils.go index 6cc53924ef..706574618f 100644 --- a/common/testutils/test_utils.go +++ b/common/testutils/test_utils.go @@ -35,7 +35,7 @@ func AssertEventuallyTrue(t *testing.T, condition func() bool, duration time.Dur } if len(debugInfo) == 0 { - assert.True(t, condition(), "Condition did not become true within the given duration") // TODO use this elsewhere + assert.True(t, condition(), "Condition did not become true within the given duration") } else { assert.True(t, condition(), debugInfo...) } @@ -56,7 +56,7 @@ func AssertEventuallyEquals(t *testing.T, expected any, actual func() any, durat // ExecuteWithTimeout executes a function with a timeout. // Panics if the function does not complete within the given duration. -func ExecuteWithTimeout(f func(), duration time.Duration) { +func ExecuteWithTimeout(f func(), duration time.Duration, debugInfo ...any) { done := make(chan struct{}) go func() { f() @@ -65,6 +65,10 @@ func ExecuteWithTimeout(f func(), duration time.Duration) { select { case <-done: case <-time.After(duration): + if len(debugInfo) > 0 { + panic(fmt.Sprintf(debugInfo[0].(string), debugInfo[1:]...)) + } + panic("function did not complete within the given duration") } } From d779162dd09b3788fcc203ef75e00ca41c6e294a Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 30 Jul 2024 09:19:03 -0500 Subject: [PATCH 60/74] Cleanup. Signed-off-by: Cody Littley --- tools/traffic/test/mock_disperser.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/traffic/test/mock_disperser.go b/tools/traffic/test/mock_disperser.go index bdfc6c2ddc..a03b3bb2ab 100644 --- a/tools/traffic/test/mock_disperser.go +++ b/tools/traffic/test/mock_disperser.go @@ -85,9 +85,7 @@ func (m *mockDisperserClient) GetBlobStatus(ctx context.Context, key []byte) (*d Status: status, Info: &disperser_rpc.BlobInfo{ BlobVerificationProof: &disperser_rpc.BlobVerificationProof{ - BatchMetadata: &disperser_rpc.BatchMetadata{ - BatchHeaderHash: nil, // TODO - }, + BatchMetadata: &disperser_rpc.BatchMetadata{}, }, }, }, nil From 2d8821961a17ae346f50ee77f6d7f05ce6bd3f7e Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 30 Jul 2024 09:48:03 -0500 Subject: [PATCH 61/74] Moved config and flags to same directory. Signed-off-by: Cody Littley --- tools/traffic/cmd/main.go | 12 +- tools/traffic/config.go | 139 ----------------------- tools/traffic/config/config.go | 138 ++++++++++++++++++++++ tools/traffic/{flags => config}/flags.go | 2 +- tools/traffic/generator.go | 7 +- 5 files changed, 149 insertions(+), 149 deletions(-) delete mode 100644 tools/traffic/config.go create mode 100644 tools/traffic/config/config.go rename tools/traffic/{flags => config}/flags.go (99%) diff --git a/tools/traffic/cmd/main.go b/tools/traffic/cmd/main.go index aa401fdcc5..02b8846a7a 100644 --- a/tools/traffic/cmd/main.go +++ b/tools/traffic/cmd/main.go @@ -8,7 +8,7 @@ import ( "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/core/auth" "github.com/Layr-Labs/eigenda/tools/traffic" - "github.com/Layr-Labs/eigenda/tools/traffic/flags" + "github.com/Layr-Labs/eigenda/tools/traffic/config" "github.com/urfave/cli" ) @@ -24,7 +24,7 @@ func main() { app.Name = "da-traffic-generator" app.Usage = "EigenDA Traffic Generator" app.Description = "Service for generating traffic to EigenDA disperser" - app.Flags = flags.Flags + app.Flags = config.Flags app.Action = trafficGeneratorMain if err := app.Run(os.Args); err != nil { log.Fatalf("application failed: %v", err) @@ -32,18 +32,18 @@ func main() { } func trafficGeneratorMain(ctx *cli.Context) error { - config, err := traffic.NewConfig(ctx) + generatorConfig, err := config.NewConfig(ctx) if err != nil { return err } var signer core.BlobRequestSigner - if config.SignerPrivateKey != "" { + if generatorConfig.SignerPrivateKey != "" { log.Println("Using signer private key") - signer = auth.NewLocalBlobRequestSigner(config.SignerPrivateKey) + signer = auth.NewLocalBlobRequestSigner(generatorConfig.SignerPrivateKey) } - generator, err := traffic.NewTrafficGenerator(config, signer) + generator, err := traffic.NewTrafficGenerator(generatorConfig, signer) if err != nil { panic(fmt.Sprintf("failed to create new traffic generator\n%s", err)) } diff --git a/tools/traffic/config.go b/tools/traffic/config.go deleted file mode 100644 index f1f58841bf..0000000000 --- a/tools/traffic/config.go +++ /dev/null @@ -1,139 +0,0 @@ -package traffic - -import ( - "errors" - "github.com/Layr-Labs/eigenda/tools/traffic/workers" - "time" - - "github.com/Layr-Labs/eigenda/common" - "github.com/Layr-Labs/eigenda/tools/traffic/flags" - "github.com/urfave/cli" -) - -// Config configures a traffic generator. -type Config struct { - // Logging configuration. - LoggingConfig common.LoggerConfig - // The hostname of the disperser. - DisperserHostname string - // The port of the disperser. - DisperserPort string - // The timeout for the disperser. - DisperserTimeout time.Duration - // Whether to use a secure gRPC connection to the disperser. - DisperserUseSecureGrpcFlag bool - // The private key to use for signing requests. - SignerPrivateKey string - // Custom quorum numbers to use for the traffic generator. - CustomQuorums []uint8 - // Whether to disable TLS for an insecure connection. - DisableTlS bool - // The port at which the metrics server listens for HTTP requests. - MetricsHTTPPort string - // The hostname of the Ethereum client. - EthClientHostname string - // The port of the Ethereum client. - EthClientPort string - // The address of the BLS operator state retriever smart contract, in hex. - BlsOperatorStateRetriever string - // The address of the EigenDA service manager smart contract, in hex. - EigenDAServiceManager string - // The number of times to retry an Ethereum client request. - EthClientRetries uint - // The URL of the subgraph instance. - TheGraphUrl string - // The interval at which to pull data from the subgraph. - TheGraphPullInterval time.Duration - // The number of times to retry a subgraph request. - TheGraphRetries uint - // The path to the encoder G1 binary. - EncoderG1Path string - // The path to the encoder G2 binary. - EncoderG2Path string - // The path to the encoder cache directory. - EncoderCacheDir string - // The SRS order to use for the encoder. - EncoderSRSOrder uint64 - // The SRS number to load for the encoder. - EncoderSRSNumberToLoad uint64 - // The number of worker threads to use for the encoder. - EncoderNumWorkers uint64 - // The number of connections to use for the retriever. - RetrieverNumConnections uint - // The timeout for the node client. - NodeClientTimeout time.Duration - - // The amount of time to sleep after launching each worker thread. - InstanceLaunchInterval time.Duration - - // Configures the traffic generator workers. - WorkerConfig workers.Config -} - -func NewConfig(ctx *cli.Context) (*Config, error) { - loggerConfig, err := common.ReadLoggerCLIConfig(ctx, flags.FlagPrefix) - if err != nil { - return nil, err - } - customQuorums := ctx.GlobalIntSlice(flags.CustomQuorumNumbersFlag.Name) - customQuorumsUint8 := make([]uint8, len(customQuorums)) - for i, q := range customQuorums { - if q < 0 || q > 255 { - return nil, errors.New("invalid custom quorum number") - } - customQuorumsUint8[i] = uint8(q) - } - - return &Config{ - LoggingConfig: *loggerConfig, - DisperserHostname: ctx.GlobalString(flags.HostnameFlag.Name), - DisperserPort: ctx.GlobalString(flags.GrpcPortFlag.Name), - DisperserTimeout: ctx.Duration(flags.TimeoutFlag.Name), - DisperserUseSecureGrpcFlag: ctx.GlobalBool(flags.UseSecureGrpcFlag.Name), - SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), - CustomQuorums: customQuorumsUint8, - DisableTlS: ctx.GlobalBool(flags.DisableTLSFlag.Name), - MetricsHTTPPort: ctx.GlobalString(flags.MetricsHTTPPortFlag.Name), - EthClientHostname: ctx.GlobalString(flags.EthClientHostnameFlag.Name), - EthClientPort: ctx.GlobalString(flags.EthClientPortFlag.Name), - BlsOperatorStateRetriever: ctx.String(flags.BLSOperatorStateRetrieverFlag.Name), - EigenDAServiceManager: ctx.String(flags.EigenDAServiceManagerFlag.Name), - EthClientRetries: ctx.Uint(flags.EthClientRetriesFlag.Name), - TheGraphUrl: ctx.String(flags.TheGraphUrlFlag.Name), - TheGraphPullInterval: ctx.Duration(flags.TheGraphPullIntervalFlag.Name), - TheGraphRetries: ctx.Uint(flags.TheGraphRetriesFlag.Name), - EncoderG1Path: ctx.String(flags.EncoderG1PathFlag.Name), - EncoderG2Path: ctx.String(flags.EncoderG2PathFlag.Name), - EncoderCacheDir: ctx.String(flags.EncoderCacheDirFlag.Name), - EncoderSRSOrder: ctx.Uint64(flags.EncoderSRSOrderFlag.Name), - EncoderSRSNumberToLoad: ctx.Uint64(flags.EncoderSRSNumberToLoadFlag.Name), - EncoderNumWorkers: ctx.Uint64(flags.EncoderNumWorkersFlag.Name), - RetrieverNumConnections: ctx.Uint(flags.RetrieverNumConnectionsFlag.Name), - NodeClientTimeout: ctx.Duration(flags.NodeClientTimeoutFlag.Name), - - InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), - - WorkerConfig: workers.Config{ - NumWriteInstances: ctx.GlobalUint(flags.NumWriteInstancesFlag.Name), - WriteRequestInterval: ctx.Duration(flags.WriteRequestIntervalFlag.Name), - DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), - RandomizeBlobs: !ctx.GlobalBool(flags.UniformBlobsFlag.Name), - WriteTimeout: ctx.Duration(flags.WriteTimeoutFlag.Name), - - VerifierInterval: ctx.Duration(flags.VerifierIntervalFlag.Name), - GetBlobStatusTimeout: ctx.Duration(flags.GetBlobStatusTimeoutFlag.Name), - - NumReadInstances: ctx.GlobalUint(flags.NumReadInstancesFlag.Name), - ReadRequestInterval: ctx.Duration(flags.ReadRequestIntervalFlag.Name), - RequiredDownloads: ctx.Float64(flags.RequiredDownloadsFlag.Name), - ReadOverflowTableSize: ctx.Uint(flags.ReadOverflowTableSizeFlag.Name), - FetchBatchHeaderTimeout: ctx.Duration(flags.FetchBatchHeaderTimeoutFlag.Name), - RetrieveBlobChunksTimeout: ctx.Duration(flags.RetrieveBlobChunksTimeoutFlag.Name), - VerificationChannelCapacity: ctx.Uint(flags.VerificationChannelCapacityFlag.Name), - - EigenDAServiceManager: ctx.String(flags.EigenDAServiceManagerFlag.Name), - SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), - CustomQuorums: customQuorumsUint8, - }, - }, nil -} diff --git a/tools/traffic/config/config.go b/tools/traffic/config/config.go new file mode 100644 index 0000000000..a097c0148e --- /dev/null +++ b/tools/traffic/config/config.go @@ -0,0 +1,138 @@ +package config + +import ( + "errors" + "github.com/Layr-Labs/eigenda/tools/traffic/workers" + "time" + + "github.com/Layr-Labs/eigenda/common" + "github.com/urfave/cli" +) + +// Config configures a traffic generator. +type Config struct { + // Logging configuration. + LoggingConfig common.LoggerConfig + // The hostname of the disperser. + DisperserHostname string + // The port of the disperser. + DisperserPort string + // The timeout for the disperser. + DisperserTimeout time.Duration + // Whether to use a secure gRPC connection to the disperser. + DisperserUseSecureGrpcFlag bool + // The private key to use for signing requests. + SignerPrivateKey string + // Custom quorum numbers to use for the traffic generator. + CustomQuorums []uint8 + // Whether to disable TLS for an insecure connection. + DisableTlS bool + // The port at which the metrics server listens for HTTP requests. + MetricsHTTPPort string + // The hostname of the Ethereum client. + EthClientHostname string + // The port of the Ethereum client. + EthClientPort string + // The address of the BLS operator state retriever smart contract, in hex. + BlsOperatorStateRetriever string + // The address of the EigenDA service manager smart contract, in hex. + EigenDAServiceManager string + // The number of times to retry an Ethereum client request. + EthClientRetries uint + // The URL of the subgraph instance. + TheGraphUrl string + // The interval at which to pull data from the subgraph. + TheGraphPullInterval time.Duration + // The number of times to retry a subgraph request. + TheGraphRetries uint + // The path to the encoder G1 binary. + EncoderG1Path string + // The path to the encoder G2 binary. + EncoderG2Path string + // The path to the encoder cache directory. + EncoderCacheDir string + // The SRS order to use for the encoder. + EncoderSRSOrder uint64 + // The SRS number to load for the encoder. + EncoderSRSNumberToLoad uint64 + // The number of worker threads to use for the encoder. + EncoderNumWorkers uint64 + // The number of connections to use for the retriever. + RetrieverNumConnections uint + // The timeout for the node client. + NodeClientTimeout time.Duration + + // The amount of time to sleep after launching each worker thread. + InstanceLaunchInterval time.Duration + + // Configures the traffic generator workers. + WorkerConfig workers.Config +} + +func NewConfig(ctx *cli.Context) (*Config, error) { + loggerConfig, err := common.ReadLoggerCLIConfig(ctx, FlagPrefix) + if err != nil { + return nil, err + } + customQuorums := ctx.GlobalIntSlice(CustomQuorumNumbersFlag.Name) + customQuorumsUint8 := make([]uint8, len(customQuorums)) + for i, q := range customQuorums { + if q < 0 || q > 255 { + return nil, errors.New("invalid custom quorum number") + } + customQuorumsUint8[i] = uint8(q) + } + + return &Config{ + LoggingConfig: *loggerConfig, + DisperserHostname: ctx.GlobalString(HostnameFlag.Name), + DisperserPort: ctx.GlobalString(GrpcPortFlag.Name), + DisperserTimeout: ctx.Duration(TimeoutFlag.Name), + DisperserUseSecureGrpcFlag: ctx.GlobalBool(UseSecureGrpcFlag.Name), + SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), + CustomQuorums: customQuorumsUint8, + DisableTlS: ctx.GlobalBool(DisableTLSFlag.Name), + MetricsHTTPPort: ctx.GlobalString(MetricsHTTPPortFlag.Name), + EthClientHostname: ctx.GlobalString(EthClientHostnameFlag.Name), + EthClientPort: ctx.GlobalString(EthClientPortFlag.Name), + BlsOperatorStateRetriever: ctx.String(BLSOperatorStateRetrieverFlag.Name), + EigenDAServiceManager: ctx.String(EigenDAServiceManagerFlag.Name), + EthClientRetries: ctx.Uint(EthClientRetriesFlag.Name), + TheGraphUrl: ctx.String(TheGraphUrlFlag.Name), + TheGraphPullInterval: ctx.Duration(TheGraphPullIntervalFlag.Name), + TheGraphRetries: ctx.Uint(TheGraphRetriesFlag.Name), + EncoderG1Path: ctx.String(EncoderG1PathFlag.Name), + EncoderG2Path: ctx.String(EncoderG2PathFlag.Name), + EncoderCacheDir: ctx.String(EncoderCacheDirFlag.Name), + EncoderSRSOrder: ctx.Uint64(EncoderSRSOrderFlag.Name), + EncoderSRSNumberToLoad: ctx.Uint64(EncoderSRSNumberToLoadFlag.Name), + EncoderNumWorkers: ctx.Uint64(EncoderNumWorkersFlag.Name), + RetrieverNumConnections: ctx.Uint(RetrieverNumConnectionsFlag.Name), + NodeClientTimeout: ctx.Duration(NodeClientTimeoutFlag.Name), + + InstanceLaunchInterval: ctx.Duration(InstanceLaunchIntervalFlag.Name), + + WorkerConfig: workers.Config{ + NumWriteInstances: ctx.GlobalUint(NumWriteInstancesFlag.Name), + WriteRequestInterval: ctx.Duration(WriteRequestIntervalFlag.Name), + DataSize: ctx.GlobalUint64(DataSizeFlag.Name), + RandomizeBlobs: !ctx.GlobalBool(UniformBlobsFlag.Name), + WriteTimeout: ctx.Duration(WriteTimeoutFlag.Name), + + VerifierInterval: ctx.Duration(VerifierIntervalFlag.Name), + GetBlobStatusTimeout: ctx.Duration(GetBlobStatusTimeoutFlag.Name), + + NumReadInstances: ctx.GlobalUint(NumReadInstancesFlag.Name), + ReadRequestInterval: ctx.Duration(ReadRequestIntervalFlag.Name), + RequiredDownloads: ctx.Float64(RequiredDownloadsFlag.Name), + ReadOverflowTableSize: ctx.Uint(ReadOverflowTableSizeFlag.Name), + FetchBatchHeaderTimeout: ctx.Duration(FetchBatchHeaderTimeoutFlag.Name), + RetrieveBlobChunksTimeout: ctx.Duration(RetrieveBlobChunksTimeoutFlag.Name), + VerificationChannelCapacity: ctx.Uint(VerificationChannelCapacityFlag.Name), + + EigenDAServiceManager: ctx.String(EigenDAServiceManagerFlag.Name), + SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), + CustomQuorums: customQuorumsUint8, + }, + }, nil +} diff --git a/tools/traffic/flags/flags.go b/tools/traffic/config/flags.go similarity index 99% rename from tools/traffic/flags/flags.go rename to tools/traffic/config/flags.go index 734332c60a..e890f732a1 100644 --- a/tools/traffic/flags/flags.go +++ b/tools/traffic/config/flags.go @@ -1,4 +1,4 @@ -package flags +package config import ( "time" diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index ed0066f001..5234819547 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -9,6 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" + "github.com/Layr-Labs/eigenda/tools/traffic/config" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigenda/tools/traffic/workers" @@ -49,14 +50,14 @@ type Generator struct { logger *logging.Logger disperserClient clients.DisperserClient eigenDAClient *clients.EigenDAClient - config *Config + config *config.Config writers []*workers.BlobWriter verifier *workers.BlobVerifier readers []*workers.BlobReader } -func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Generator, error) { +func NewTrafficGenerator(config *config.Config, signer core.BlobRequestSigner) (*Generator, error) { logger, err := common.NewLogger(config.LoggingConfig) if err != nil { return nil, err @@ -149,7 +150,7 @@ func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*Genera } // buildRetriever creates a retriever client for the traffic generator. -func buildRetriever(config *Config) (clients.RetrievalClient, retrivereth.ChainClient) { +func buildRetriever(config *config.Config) (clients.RetrievalClient, retrivereth.ChainClient) { loggerConfig := common.DefaultLoggerConfig() logger, err := common.NewLogger(loggerConfig) From ee85c8892a17c30db13d41ddca97806e50f854ab Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 30 Jul 2024 09:56:53 -0500 Subject: [PATCH 62/74] Move worker config into config directory. Signed-off-by: Cody Littley --- tools/traffic/config/config.go | 5 ++--- tools/traffic/{workers/config.go => config/worker_config.go} | 4 ++-- tools/traffic/test/blob_reader_test.go | 5 +++-- tools/traffic/test/blob_verifier_test.go | 3 ++- tools/traffic/test/blob_writer_test.go | 3 ++- tools/traffic/workers/blob_reader.go | 5 +++-- tools/traffic/workers/blob_verifier.go | 5 +++-- tools/traffic/workers/blob_writer.go | 5 +++-- 8 files changed, 20 insertions(+), 15 deletions(-) rename tools/traffic/{workers/config.go => config/worker_config.go} (98%) diff --git a/tools/traffic/config/config.go b/tools/traffic/config/config.go index a097c0148e..071931e5db 100644 --- a/tools/traffic/config/config.go +++ b/tools/traffic/config/config.go @@ -2,7 +2,6 @@ package config import ( "errors" - "github.com/Layr-Labs/eigenda/tools/traffic/workers" "time" "github.com/Layr-Labs/eigenda/common" @@ -66,7 +65,7 @@ type Config struct { InstanceLaunchInterval time.Duration // Configures the traffic generator workers. - WorkerConfig workers.Config + WorkerConfig WorkerConfig } func NewConfig(ctx *cli.Context) (*Config, error) { @@ -112,7 +111,7 @@ func NewConfig(ctx *cli.Context) (*Config, error) { InstanceLaunchInterval: ctx.Duration(InstanceLaunchIntervalFlag.Name), - WorkerConfig: workers.Config{ + WorkerConfig: WorkerConfig{ NumWriteInstances: ctx.GlobalUint(NumWriteInstancesFlag.Name), WriteRequestInterval: ctx.Duration(WriteRequestIntervalFlag.Name), DataSize: ctx.GlobalUint64(DataSizeFlag.Name), diff --git a/tools/traffic/workers/config.go b/tools/traffic/config/worker_config.go similarity index 98% rename from tools/traffic/workers/config.go rename to tools/traffic/config/worker_config.go index 3326029f79..84c00060d5 100644 --- a/tools/traffic/workers/config.go +++ b/tools/traffic/config/worker_config.go @@ -1,9 +1,9 @@ -package workers +package config import "time" // Config configures the traffic generator workers. -type Config struct { +type WorkerConfig struct { // The number of worker threads that generate write traffic. NumWriteInstances uint // The period of the submission rate of new blobs for each write worker thread. diff --git a/tools/traffic/test/blob_reader_test.go b/tools/traffic/test/blob_reader_test.go index 065453d363..9c389539c7 100644 --- a/tools/traffic/test/blob_reader_test.go +++ b/tools/traffic/test/blob_reader_test.go @@ -5,6 +5,7 @@ import ( "crypto/md5" "github.com/Layr-Labs/eigenda/common" tu "github.com/Layr-Labs/eigenda/common/testutils" + "github.com/Layr-Labs/eigenda/tools/traffic/config" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigenda/tools/traffic/workers" @@ -26,7 +27,7 @@ func TestBlobReaderNoOverflow(t *testing.T) { startTime := time.Unix(rand.Int63()%2_000_000_000, 0) ticker := newMockTicker(startTime) - config := &workers.Config{ + config := &config.WorkerConfig{ ReadOverflowTableSize: 0, } @@ -164,7 +165,7 @@ func TestBlobReaderWithOverflow(t *testing.T) { blobCount := 100 overflowTableSize := uint(rand.Intn(blobCount-1) + 1) - config := &workers.Config{ + config := &config.WorkerConfig{ ReadOverflowTableSize: overflowTableSize, } diff --git a/tools/traffic/test/blob_verifier_test.go b/tools/traffic/test/blob_verifier_test.go index ac9a35507b..a4c889e103 100644 --- a/tools/traffic/test/blob_verifier_test.go +++ b/tools/traffic/test/blob_verifier_test.go @@ -6,6 +6,7 @@ import ( disperser_rpc "github.com/Layr-Labs/eigenda/api/grpc/disperser" "github.com/Layr-Labs/eigenda/common" tu "github.com/Layr-Labs/eigenda/common/testutils" + "github.com/Layr-Labs/eigenda/tools/traffic/config" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigenda/tools/traffic/workers" @@ -64,7 +65,7 @@ func TestBlobVerifier(t *testing.T) { ticker := newMockTicker(startTime) requiredDownloads := rand.Intn(10) - config := &workers.Config{ + config := &config.WorkerConfig{ RequiredDownloads: float64(requiredDownloads), } diff --git a/tools/traffic/test/blob_writer_test.go b/tools/traffic/test/blob_writer_test.go index e67dfc7bdb..191aeb03d7 100644 --- a/tools/traffic/test/blob_writer_test.go +++ b/tools/traffic/test/blob_writer_test.go @@ -7,6 +7,7 @@ import ( "github.com/Layr-Labs/eigenda/common" tu "github.com/Layr-Labs/eigenda/common/testutils" "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/Layr-Labs/eigenda/tools/traffic/config" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/workers" "github.com/stretchr/testify/assert" @@ -42,7 +43,7 @@ func TestBlobWriter(t *testing.T) { customQuorum = []uint8{1, 2, 3} } - config := &workers.Config{ + config := &config.WorkerConfig{ DataSize: dataSize, SignerPrivateKey: signerPrivateKey, RandomizeBlobs: randomizeBlobs, diff --git a/tools/traffic/workers/blob_reader.go b/tools/traffic/workers/blob_reader.go index db8f2105d7..37d61ef84a 100644 --- a/tools/traffic/workers/blob_reader.go +++ b/tools/traffic/workers/blob_reader.go @@ -9,6 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/retriever/eth" + config2 "github.com/Layr-Labs/eigenda/tools/traffic/config" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigensdk-go/logging" @@ -32,7 +33,7 @@ type BlobReader struct { ticker InterceptableTicker // config contains the configuration for the generator. - config *Config + config *config2.WorkerConfig retriever clients.RetrievalClient chainClient eth.ChainClient @@ -66,7 +67,7 @@ func NewBlobReader( waitGroup *sync.WaitGroup, logger logging.Logger, ticker InterceptableTicker, - config *Config, + config *config2.WorkerConfig, retriever clients.RetrievalClient, chainClient eth.ChainClient, blobTable *table.BlobTable, diff --git a/tools/traffic/workers/blob_verifier.go b/tools/traffic/workers/blob_verifier.go index a39803e700..c34da4f65a 100644 --- a/tools/traffic/workers/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -4,6 +4,7 @@ import ( "context" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/api/grpc/disperser" + config2 "github.com/Layr-Labs/eigenda/tools/traffic/config" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" "github.com/Layr-Labs/eigensdk-go/logging" @@ -39,7 +40,7 @@ type BlobVerifier struct { logger logging.Logger // config contains the configuration for the generator. - config *Config + config *config2.WorkerConfig // A table of confirmed blobs. Blobs are added here when they are confirmed by the disperser service. table *table.BlobTable @@ -75,7 +76,7 @@ func NewBlobVerifier( waitGroup *sync.WaitGroup, logger logging.Logger, ticker InterceptableTicker, - config *Config, + config *config2.WorkerConfig, table *table.BlobTable, disperser clients.DisperserClient, generatorMetrics metrics.Metrics) BlobVerifier { diff --git a/tools/traffic/workers/blob_writer.go b/tools/traffic/workers/blob_writer.go index f71e2bb67e..c01d607005 100644 --- a/tools/traffic/workers/blob_writer.go +++ b/tools/traffic/workers/blob_writer.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/encoding/utils/codec" + config2 "github.com/Layr-Labs/eigenda/tools/traffic/config" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigensdk-go/logging" "sync" @@ -27,7 +28,7 @@ type BlobWriter struct { ticker InterceptableTicker // Config contains the configuration for the generator. - config *Config + config *config2.WorkerConfig // disperser is the client used to send blobs to the disperser. disperser clients.DisperserClient @@ -54,7 +55,7 @@ func NewBlobWriter( waitGroup *sync.WaitGroup, logger logging.Logger, ticker InterceptableTicker, - config *Config, + config *config2.WorkerConfig, disperser clients.DisperserClient, unconfirmedKeyHandler KeyHandler, generatorMetrics metrics.Metrics) BlobWriter { From 2e19a3d8230eb353e7431a641b65250ef50101da Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 31 Jul 2024 08:27:07 -0500 Subject: [PATCH 63/74] Use disperser client config. Signed-off-by: Cody Littley --- tools/traffic/config/config.go | 67 +++++++++++++++++----------------- tools/traffic/generator.go | 12 ++---- 2 files changed, 37 insertions(+), 42 deletions(-) diff --git a/tools/traffic/config/config.go b/tools/traffic/config/config.go index 071931e5db..f02e3a7142 100644 --- a/tools/traffic/config/config.go +++ b/tools/traffic/config/config.go @@ -2,6 +2,7 @@ package config import ( "errors" + "github.com/Layr-Labs/eigenda/api/clients" "time" "github.com/Layr-Labs/eigenda/common" @@ -10,16 +11,13 @@ import ( // Config configures a traffic generator. type Config struct { + // Logging configuration. LoggingConfig common.LoggerConfig - // The hostname of the disperser. - DisperserHostname string - // The port of the disperser. - DisperserPort string - // The timeout for the disperser. - DisperserTimeout time.Duration - // Whether to use a secure gRPC connection to the disperser. - DisperserUseSecureGrpcFlag bool + + // Configuration for the disperser client. + DisperserClientConfig *clients.Config + // The private key to use for signing requests. SignerPrivateKey string // Custom quorum numbers to use for the traffic generator. @@ -83,31 +81,34 @@ func NewConfig(ctx *cli.Context) (*Config, error) { } return &Config{ - LoggingConfig: *loggerConfig, - DisperserHostname: ctx.GlobalString(HostnameFlag.Name), - DisperserPort: ctx.GlobalString(GrpcPortFlag.Name), - DisperserTimeout: ctx.Duration(TimeoutFlag.Name), - DisperserUseSecureGrpcFlag: ctx.GlobalBool(UseSecureGrpcFlag.Name), - SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), - CustomQuorums: customQuorumsUint8, - DisableTlS: ctx.GlobalBool(DisableTLSFlag.Name), - MetricsHTTPPort: ctx.GlobalString(MetricsHTTPPortFlag.Name), - EthClientHostname: ctx.GlobalString(EthClientHostnameFlag.Name), - EthClientPort: ctx.GlobalString(EthClientPortFlag.Name), - BlsOperatorStateRetriever: ctx.String(BLSOperatorStateRetrieverFlag.Name), - EigenDAServiceManager: ctx.String(EigenDAServiceManagerFlag.Name), - EthClientRetries: ctx.Uint(EthClientRetriesFlag.Name), - TheGraphUrl: ctx.String(TheGraphUrlFlag.Name), - TheGraphPullInterval: ctx.Duration(TheGraphPullIntervalFlag.Name), - TheGraphRetries: ctx.Uint(TheGraphRetriesFlag.Name), - EncoderG1Path: ctx.String(EncoderG1PathFlag.Name), - EncoderG2Path: ctx.String(EncoderG2PathFlag.Name), - EncoderCacheDir: ctx.String(EncoderCacheDirFlag.Name), - EncoderSRSOrder: ctx.Uint64(EncoderSRSOrderFlag.Name), - EncoderSRSNumberToLoad: ctx.Uint64(EncoderSRSNumberToLoadFlag.Name), - EncoderNumWorkers: ctx.Uint64(EncoderNumWorkersFlag.Name), - RetrieverNumConnections: ctx.Uint(RetrieverNumConnectionsFlag.Name), - NodeClientTimeout: ctx.Duration(NodeClientTimeoutFlag.Name), + DisperserClientConfig: &clients.Config{ + Hostname: ctx.GlobalString(HostnameFlag.Name), + Port: ctx.GlobalString(GrpcPortFlag.Name), + Timeout: ctx.Duration(TimeoutFlag.Name), + UseSecureGrpcFlag: ctx.GlobalBool(UseSecureGrpcFlag.Name), + }, + + LoggingConfig: *loggerConfig, + SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), + CustomQuorums: customQuorumsUint8, + DisableTlS: ctx.GlobalBool(DisableTLSFlag.Name), + MetricsHTTPPort: ctx.GlobalString(MetricsHTTPPortFlag.Name), + EthClientHostname: ctx.GlobalString(EthClientHostnameFlag.Name), + EthClientPort: ctx.GlobalString(EthClientPortFlag.Name), + BlsOperatorStateRetriever: ctx.String(BLSOperatorStateRetrieverFlag.Name), + EigenDAServiceManager: ctx.String(EigenDAServiceManagerFlag.Name), + EthClientRetries: ctx.Uint(EthClientRetriesFlag.Name), + TheGraphUrl: ctx.String(TheGraphUrlFlag.Name), + TheGraphPullInterval: ctx.Duration(TheGraphPullIntervalFlag.Name), + TheGraphRetries: ctx.Uint(TheGraphRetriesFlag.Name), + EncoderG1Path: ctx.String(EncoderG1PathFlag.Name), + EncoderG2Path: ctx.String(EncoderG2PathFlag.Name), + EncoderCacheDir: ctx.String(EncoderCacheDirFlag.Name), + EncoderSRSOrder: ctx.Uint64(EncoderSRSOrderFlag.Name), + EncoderSRSNumberToLoad: ctx.Uint64(EncoderSRSNumberToLoadFlag.Name), + EncoderNumWorkers: ctx.Uint64(EncoderNumWorkersFlag.Name), + RetrieverNumConnections: ctx.Uint(RetrieverNumConnectionsFlag.Name), + NodeClientTimeout: ctx.Duration(NodeClientTimeoutFlag.Name), InstanceLaunchInterval: ctx.Duration(InstanceLaunchIntervalFlag.Name), diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 5234819547..b87d49f2be 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -64,7 +64,7 @@ func NewTrafficGenerator(config *config.Config, signer core.BlobRequestSigner) ( } clientConfig := clients.EigenDAClientConfig{ - RPC: config.DisperserHostname + ":" + config.DisperserPort, + RPC: config.DisperserClientConfig.Hostname + ":" + config.DisperserClientConfig.Port, DisableTLS: config.DisableTlS, SignerPrivateKeyHex: config.SignerPrivateKey, } @@ -86,13 +86,7 @@ func NewTrafficGenerator(config *config.Config, signer core.BlobRequestSigner) ( blobTable := table.NewBlobTable() - disperserConfig := clients.Config{ - Hostname: config.DisperserHostname, - Port: config.DisperserPort, - Timeout: config.DisperserTimeout, - UseSecureGrpcFlag: config.DisperserUseSecureGrpcFlag, - } - disperserClient := clients.NewDisperserClient(&disperserConfig, signer) + disperserClient := clients.NewDisperserClient(config.DisperserClientConfig, signer) statusVerifier := workers.NewBlobVerifier( &ctx, &waitGroup, @@ -140,7 +134,7 @@ func NewTrafficGenerator(config *config.Config, signer core.BlobRequestSigner) ( waitGroup: &waitGroup, generatorMetrics: generatorMetrics, logger: &logger, - disperserClient: clients.NewDisperserClient(&disperserConfig, signer), + disperserClient: clients.NewDisperserClient(config.DisperserClientConfig, signer), eigenDAClient: client, config: config, writers: writers, From 16f785cc76872e1abbf2ca1d0de9f111114d93a4 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 31 Jul 2024 08:44:43 -0500 Subject: [PATCH 64/74] Refactored more config. Signed-off-by: Cody Littley --- tools/traffic/config/config.go | 92 +++++++++++++++++++--------------- tools/traffic/generator.go | 19 ++----- 2 files changed, 55 insertions(+), 56 deletions(-) diff --git a/tools/traffic/config/config.go b/tools/traffic/config/config.go index f02e3a7142..fcf56976d8 100644 --- a/tools/traffic/config/config.go +++ b/tools/traffic/config/config.go @@ -2,7 +2,11 @@ package config import ( "errors" + "fmt" "github.com/Layr-Labs/eigenda/api/clients" + "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/encoding/kzg" + "github.com/Layr-Labs/eigenda/retriever" "time" "github.com/Layr-Labs/eigenda/common" @@ -18,6 +22,21 @@ type Config struct { // Configuration for the disperser client. DisperserClientConfig *clients.Config + // Configuration for the retriever client. + RetrievalClientConfig *retriever.Config + + //LoggerConfig common.LoggerConfig + //IndexerConfig indexer.Config + //MetricsConfig MetricsConfig + //ChainStateConfig thegraph.Config + // + //IndexerDataDir string + //Timeout time.Duration + //NumConnections int + //BLSOperatorStateRetrieverAddr string + //EigenDAServiceManagerAddr string + //UseGraph bool + // The private key to use for signing requests. SignerPrivateKey string // Custom quorum numbers to use for the traffic generator. @@ -26,34 +45,17 @@ type Config struct { DisableTlS bool // The port at which the metrics server listens for HTTP requests. MetricsHTTPPort string - // The hostname of the Ethereum client. - EthClientHostname string - // The port of the Ethereum client. - EthClientPort string + // The address of the BLS operator state retriever smart contract, in hex. BlsOperatorStateRetriever string - // The address of the EigenDA service manager smart contract, in hex. - EigenDAServiceManager string - // The number of times to retry an Ethereum client request. - EthClientRetries uint + // The URL of the subgraph instance. TheGraphUrl string // The interval at which to pull data from the subgraph. TheGraphPullInterval time.Duration // The number of times to retry a subgraph request. TheGraphRetries uint - // The path to the encoder G1 binary. - EncoderG1Path string - // The path to the encoder G2 binary. - EncoderG2Path string - // The path to the encoder cache directory. - EncoderCacheDir string - // The SRS order to use for the encoder. - EncoderSRSOrder uint64 - // The SRS number to load for the encoder. - EncoderSRSNumberToLoad uint64 - // The number of worker threads to use for the encoder. - EncoderNumWorkers uint64 + // The number of connections to use for the retriever. RetrieverNumConnections uint // The timeout for the node client. @@ -88,27 +90,37 @@ func NewConfig(ctx *cli.Context) (*Config, error) { UseSecureGrpcFlag: ctx.GlobalBool(UseSecureGrpcFlag.Name), }, - LoggingConfig: *loggerConfig, - SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), - CustomQuorums: customQuorumsUint8, - DisableTlS: ctx.GlobalBool(DisableTLSFlag.Name), - MetricsHTTPPort: ctx.GlobalString(MetricsHTTPPortFlag.Name), - EthClientHostname: ctx.GlobalString(EthClientHostnameFlag.Name), - EthClientPort: ctx.GlobalString(EthClientPortFlag.Name), + // TODO refactor flags + RetrievalClientConfig: &retriever.Config{ + EigenDAServiceManagerAddr: ctx.String(EigenDAServiceManagerFlag.Name), + EncoderConfig: kzg.KzgConfig{ + G1Path: ctx.String(EncoderG1PathFlag.Name), + G2Path: ctx.String(EncoderG2PathFlag.Name), + CacheDir: ctx.String(EncoderCacheDirFlag.Name), + SRSOrder: ctx.Uint64(EncoderSRSOrderFlag.Name), + SRSNumberToLoad: ctx.Uint64(EncoderSRSNumberToLoadFlag.Name), + NumWorker: ctx.Uint64(EncoderNumWorkersFlag.Name), + }, + EthClientConfig: geth.EthClientConfig{ + RPCURLs: []string{fmt.Sprintf("%s:%s", ctx.GlobalString(EthClientHostnameFlag.Name), ctx.GlobalString(EthClientPortFlag.Name))}, + NumRetries: ctx.Int(EthClientRetriesFlag.Name), + }, + }, + + LoggingConfig: *loggerConfig, + SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), + CustomQuorums: customQuorumsUint8, + DisableTlS: ctx.GlobalBool(DisableTLSFlag.Name), + MetricsHTTPPort: ctx.GlobalString(MetricsHTTPPortFlag.Name), + BlsOperatorStateRetriever: ctx.String(BLSOperatorStateRetrieverFlag.Name), - EigenDAServiceManager: ctx.String(EigenDAServiceManagerFlag.Name), - EthClientRetries: ctx.Uint(EthClientRetriesFlag.Name), - TheGraphUrl: ctx.String(TheGraphUrlFlag.Name), - TheGraphPullInterval: ctx.Duration(TheGraphPullIntervalFlag.Name), - TheGraphRetries: ctx.Uint(TheGraphRetriesFlag.Name), - EncoderG1Path: ctx.String(EncoderG1PathFlag.Name), - EncoderG2Path: ctx.String(EncoderG2PathFlag.Name), - EncoderCacheDir: ctx.String(EncoderCacheDirFlag.Name), - EncoderSRSOrder: ctx.Uint64(EncoderSRSOrderFlag.Name), - EncoderSRSNumberToLoad: ctx.Uint64(EncoderSRSNumberToLoadFlag.Name), - EncoderNumWorkers: ctx.Uint64(EncoderNumWorkersFlag.Name), - RetrieverNumConnections: ctx.Uint(RetrieverNumConnectionsFlag.Name), - NodeClientTimeout: ctx.Duration(NodeClientTimeoutFlag.Name), + + TheGraphUrl: ctx.String(TheGraphUrlFlag.Name), + TheGraphPullInterval: ctx.Duration(TheGraphPullIntervalFlag.Name), + TheGraphRetries: ctx.Uint(TheGraphRetriesFlag.Name), + + RetrieverNumConnections: ctx.Uint(RetrieverNumConnectionsFlag.Name), + NodeClientTimeout: ctx.Duration(NodeClientTimeoutFlag.Name), InstanceLaunchInterval: ctx.Duration(InstanceLaunchIntervalFlag.Name), diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index b87d49f2be..e8e8816352 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -6,7 +6,6 @@ import ( "github.com/Layr-Labs/eigenda/common/geth" "github.com/Layr-Labs/eigenda/core/eth" "github.com/Layr-Labs/eigenda/core/thegraph" - "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" "github.com/Layr-Labs/eigenda/tools/traffic/config" @@ -152,11 +151,7 @@ func buildRetriever(config *config.Config) (clients.RetrievalClient, retrivereth panic(fmt.Sprintf("Unable to instantiate logger: %s", err)) } - ethClientConfig := geth.EthClientConfig{ - RPCURLs: []string{config.EthClientHostname + ":" + config.EthClientPort}, - NumRetries: int(config.EthClientRetries), - } - gethClient, err := geth.NewMultiHomingClient(ethClientConfig, gethcommon.Address{}, logger) + gethClient, err := geth.NewMultiHomingClient(config.RetrievalClientConfig.EthClientConfig, gethcommon.Address{}, logger) if err != nil { panic(fmt.Sprintf("Unable to instantiate geth client: %s", err)) } @@ -165,7 +160,7 @@ func buildRetriever(config *config.Config) (clients.RetrievalClient, retrivereth logger, gethClient, config.BlsOperatorStateRetriever, - config.EigenDAServiceManager) + config.RetrievalClientConfig.EigenDAServiceManagerAddr) if err != nil { panic(fmt.Sprintf("Unable to instantiate transactor: %s", err)) } @@ -184,15 +179,7 @@ func buildRetriever(config *config.Config) (clients.RetrievalClient, retrivereth nodeClient := clients.NewNodeClient(config.NodeClientTimeout) - encoderConfig := kzg.KzgConfig{ - G1Path: config.EncoderG1Path, - G2Path: config.EncoderG2Path, - CacheDir: config.EncoderCacheDir, - SRSOrder: config.EncoderSRSOrder, - SRSNumberToLoad: config.EncoderSRSNumberToLoad, - NumWorker: config.EncoderNumWorkers, - } - v, err := verifier.NewVerifier(&encoderConfig, true) + v, err := verifier.NewVerifier(&config.RetrievalClientConfig.EncoderConfig, true) if err != nil { panic(fmt.Sprintf("Unable to build verifier: %s", err)) } From 6053d321bd04390bbeb00799562f85f23781af95 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 31 Jul 2024 09:01:39 -0500 Subject: [PATCH 65/74] Refactored more config. Need to refactor flags. Signed-off-by: Cody Littley --- tools/traffic/cmd/main.go | 10 +---- tools/traffic/config/config.go | 76 +++++++++++++++------------------- tools/traffic/generator.go | 24 ++++------- 3 files changed, 42 insertions(+), 68 deletions(-) diff --git a/tools/traffic/cmd/main.go b/tools/traffic/cmd/main.go index 02b8846a7a..8ad708339f 100644 --- a/tools/traffic/cmd/main.go +++ b/tools/traffic/cmd/main.go @@ -5,8 +5,6 @@ import ( "log" "os" - "github.com/Layr-Labs/eigenda/core" - "github.com/Layr-Labs/eigenda/core/auth" "github.com/Layr-Labs/eigenda/tools/traffic" "github.com/Layr-Labs/eigenda/tools/traffic/config" "github.com/urfave/cli" @@ -37,13 +35,7 @@ func trafficGeneratorMain(ctx *cli.Context) error { return err } - var signer core.BlobRequestSigner - if generatorConfig.SignerPrivateKey != "" { - log.Println("Using signer private key") - signer = auth.NewLocalBlobRequestSigner(generatorConfig.SignerPrivateKey) - } - - generator, err := traffic.NewTrafficGenerator(generatorConfig, signer) + generator, err := traffic.NewTrafficGenerator(generatorConfig) if err != nil { panic(fmt.Sprintf("failed to create new traffic generator\n%s", err)) } diff --git a/tools/traffic/config/config.go b/tools/traffic/config/config.go index fcf56976d8..7dac07b363 100644 --- a/tools/traffic/config/config.go +++ b/tools/traffic/config/config.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/thegraph" "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/retriever" "time" @@ -25,47 +26,25 @@ type Config struct { // Configuration for the retriever client. RetrievalClientConfig *retriever.Config - //LoggerConfig common.LoggerConfig - //IndexerConfig indexer.Config - //MetricsConfig MetricsConfig - //ChainStateConfig thegraph.Config - // - //IndexerDataDir string - //Timeout time.Duration - //NumConnections int - //BLSOperatorStateRetrieverAddr string - //EigenDAServiceManagerAddr string - //UseGraph bool - - // The private key to use for signing requests. - SignerPrivateKey string - // Custom quorum numbers to use for the traffic generator. - CustomQuorums []uint8 - // Whether to disable TLS for an insecure connection. - DisableTlS bool + // Configuration for the graph. + TheGraphConfig *thegraph.Config + + // Configuration for the EigenDA client. + EigenDaClientConfig *clients.EigenDAClientConfig + + // Configures the traffic generator workers. + WorkerConfig WorkerConfig + // The port at which the metrics server listens for HTTP requests. MetricsHTTPPort string - // The address of the BLS operator state retriever smart contract, in hex. BlsOperatorStateRetriever string - - // The URL of the subgraph instance. - TheGraphUrl string - // The interval at which to pull data from the subgraph. - TheGraphPullInterval time.Duration - // The number of times to retry a subgraph request. - TheGraphRetries uint - // The number of connections to use for the retriever. RetrieverNumConnections uint // The timeout for the node client. NodeClientTimeout time.Duration - // The amount of time to sleep after launching each worker thread. InstanceLaunchInterval time.Duration - - // Configures the traffic generator workers. - WorkerConfig WorkerConfig } func NewConfig(ctx *cli.Context) (*Config, error) { @@ -82,7 +61,7 @@ func NewConfig(ctx *cli.Context) (*Config, error) { customQuorumsUint8[i] = uint8(q) } - return &Config{ + config := &Config{ DisperserClientConfig: &clients.Config{ Hostname: ctx.GlobalString(HostnameFlag.Name), Port: ctx.GlobalString(GrpcPortFlag.Name), @@ -107,17 +86,23 @@ func NewConfig(ctx *cli.Context) (*Config, error) { }, }, - LoggingConfig: *loggerConfig, - SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), - CustomQuorums: customQuorumsUint8, - DisableTlS: ctx.GlobalBool(DisableTLSFlag.Name), - MetricsHTTPPort: ctx.GlobalString(MetricsHTTPPortFlag.Name), + TheGraphConfig: &thegraph.Config{ + Endpoint: ctx.String(TheGraphUrlFlag.Name), + PullInterval: ctx.Duration(TheGraphPullIntervalFlag.Name), + MaxRetries: ctx.Int(TheGraphRetriesFlag.Name), + }, - BlsOperatorStateRetriever: ctx.String(BLSOperatorStateRetrieverFlag.Name), + EigenDaClientConfig: &clients.EigenDAClientConfig{ + RPC: fmt.Sprintf("%s:%s", ctx.GlobalString(HostnameFlag.Name), ctx.GlobalString(GrpcPortFlag.Name)), + SignerPrivateKeyHex: ctx.String(SignerPrivateKeyFlag.Name), + DisableTLS: ctx.GlobalBool(DisableTLSFlag.Name), + }, - TheGraphUrl: ctx.String(TheGraphUrlFlag.Name), - TheGraphPullInterval: ctx.Duration(TheGraphPullIntervalFlag.Name), - TheGraphRetries: ctx.Uint(TheGraphRetriesFlag.Name), + LoggingConfig: *loggerConfig, + + MetricsHTTPPort: ctx.GlobalString(MetricsHTTPPortFlag.Name), + + BlsOperatorStateRetriever: ctx.String(BLSOperatorStateRetrieverFlag.Name), RetrieverNumConnections: ctx.Uint(RetrieverNumConnectionsFlag.Name), NodeClientTimeout: ctx.Duration(NodeClientTimeoutFlag.Name), @@ -146,5 +131,12 @@ func NewConfig(ctx *cli.Context) (*Config, error) { SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), CustomQuorums: customQuorumsUint8, }, - }, nil + } + + err = config.EigenDaClientConfig.CheckAndSetDefaults() + if err != nil { + return nil, err + } + + return config, nil } diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index e8e8816352..fa6835227f 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/auth" "github.com/Layr-Labs/eigenda/core/eth" "github.com/Layr-Labs/eigenda/core/thegraph" "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" @@ -56,24 +57,19 @@ type Generator struct { readers []*workers.BlobReader } -func NewTrafficGenerator(config *config.Config, signer core.BlobRequestSigner) (*Generator, error) { +func NewTrafficGenerator(config *config.Config) (*Generator, error) { logger, err := common.NewLogger(config.LoggingConfig) if err != nil { return nil, err } - clientConfig := clients.EigenDAClientConfig{ - RPC: config.DisperserClientConfig.Hostname + ":" + config.DisperserClientConfig.Port, - DisableTLS: config.DisableTlS, - SignerPrivateKeyHex: config.SignerPrivateKey, - } - err = clientConfig.CheckAndSetDefaults() - if err != nil { - return nil, err + var signer core.BlobRequestSigner + if config.EigenDaClientConfig.SignerPrivateKeyHex != "" { + signer = auth.NewLocalBlobRequestSigner(config.EigenDaClientConfig.SignerPrivateKeyHex) } logger2 := log.NewLogger(log.NewTerminalHandler(os.Stderr, true)) - client, err := clients.NewEigenDAClient(logger2, clientConfig) + client, err := clients.NewEigenDAClient(logger2, *config.EigenDaClientConfig) if err != nil { return nil, err } @@ -167,13 +163,7 @@ func buildRetriever(config *config.Config) (clients.RetrievalClient, retrivereth cs := eth.NewChainState(tx, gethClient) - // This is the indexer when config.UseGraph is true - chainStateConfig := thegraph.Config{ - Endpoint: config.TheGraphUrl, - PullInterval: config.TheGraphPullInterval, - MaxRetries: int(config.TheGraphRetries), - } - chainState := thegraph.MakeIndexedChainState(chainStateConfig, cs, logger) + chainState := thegraph.MakeIndexedChainState(*config.TheGraphConfig, cs, logger) var assignmentCoordinator core.AssignmentCoordinator = &core.StdAssignmentCoordinator{} From 9b87f4f8d435305235a8db52f83efc1c6f3efae5 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 31 Jul 2024 10:11:06 -0500 Subject: [PATCH 66/74] Incremental progress. Signed-off-by: Cody Littley --- retriever/config.go | 21 ++++++--- retriever/flags/flags.go | 27 ++++++----- tools/traffic/Makefile | 23 +++++++--- tools/traffic/config/config.go | 23 ++-------- tools/traffic/config/flags.go | 83 ++++------------------------------ 5 files changed, 58 insertions(+), 119 deletions(-) diff --git a/retriever/config.go b/retriever/config.go index 3f9a2b0d1c..c317980954 100644 --- a/retriever/config.go +++ b/retriever/config.go @@ -28,15 +28,10 @@ type Config struct { UseGraph bool } -func NewConfig(ctx *cli.Context) (*Config, error) { - loggerConfig, err := common.ReadLoggerCLIConfig(ctx, flags.FlagPrefix) - if err != nil { - return nil, err - } +func ReadRetrieverConfig(ctx *cli.Context) *Config { return &Config{ EncoderConfig: kzg.ReadCLIConfig(ctx), EthClientConfig: geth.ReadEthClientConfig(ctx), - LoggerConfig: *loggerConfig, IndexerConfig: indexer.ReadIndexerConfig(ctx), MetricsConfig: MetricsConfig{ HTTPPort: ctx.GlobalString(flags.MetricsHTTPPortFlag.Name), @@ -48,5 +43,17 @@ func NewConfig(ctx *cli.Context) (*Config, error) { BLSOperatorStateRetrieverAddr: ctx.GlobalString(flags.BlsOperatorStateRetrieverFlag.Name), EigenDAServiceManagerAddr: ctx.GlobalString(flags.EigenDAServiceManagerFlag.Name), UseGraph: ctx.GlobalBool(flags.UseGraphFlag.Name), - }, nil + } +} + +func NewConfig(ctx *cli.Context) (*Config, error) { + loggerConfig, err := common.ReadLoggerCLIConfig(ctx, flags.FlagPrefix) + if err != nil { + return nil, err + } + + config := ReadRetrieverConfig(ctx) + config.LoggerConfig = *loggerConfig + + return config, nil } diff --git a/retriever/flags/flags.go b/retriever/flags/flags.go index 66d6c9c222..c8f1276a7d 100644 --- a/retriever/flags/flags.go +++ b/retriever/flags/flags.go @@ -76,26 +76,25 @@ var ( } ) -var requiredFlags = []cli.Flag{ - HostnameFlag, - GrpcPortFlag, - TimeoutFlag, - BlsOperatorStateRetrieverFlag, - EigenDAServiceManagerFlag, -} - -var optionalFlags = []cli.Flag{ - NumConnectionsFlag, - IndexerDataDirFlag, - MetricsHTTPPortFlag, - UseGraphFlag, +func RetrieverFlags(envPrefix string) []cli.Flag { + return []cli.Flag{ + HostnameFlag, + GrpcPortFlag, + TimeoutFlag, + BlsOperatorStateRetrieverFlag, + EigenDAServiceManagerFlag, + NumConnectionsFlag, + IndexerDataDirFlag, + MetricsHTTPPortFlag, + UseGraphFlag, + } } // Flags contains the list of configuration options available to the binary. var Flags []cli.Flag func init() { - Flags = append(requiredFlags, optionalFlags...) + Flags = append(Flags, RetrieverFlags(envPrefix)...) Flags = append(Flags, kzg.CLIFlags(envPrefix)...) Flags = append(Flags, geth.EthClientFlags(envPrefix)...) Flags = append(Flags, common.LoggerCLIFlags(envPrefix, FlagPrefix)...) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 56f866255a..2d9cbc9544 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -7,23 +7,34 @@ build: clean go build -o ./bin/server ./cmd run: build + TRAFFIC_GENERATOR_LOG_FORMAT=text \ TRAFFIC_GENERATOR_DISPERSER_HOSTNAME=localhost \ TRAFFIC_GENERATOR_DISPERSER_PORT=32003 \ - TRAFFIC_GENERATOR_ETH_CLIENT_HOSTNAME=http://localhost \ - TRAFFIC_GENERATOR_ETH_CLIENT_PORT=8545 \ + RETRIEVER_HOSTNAME=localhost \ + RETRIEVER_GRPC_PORT=32003 \ + RETRIEVER_BLS_OPERATOR_STATE_RETRIVER=0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154 \ + RETRIEVER_TIMEOUT=10s \ TRAFFIC_GENERATOR_TIMEOUT=10s \ TRAFFIC_GENERATOR_NUM_WRITE_INSTANCES=1 \ TRAFFIC_GENERATOR_WRITE_REQUEST_INTERVAL=1s \ TRAFFIC_GENERATOR_DATA_SIZE=1000 \ TRAFFIC_GENERATOR_RANDOMIZE_BLOBS=true \ TRAFFIC_GENERATOR_SIGNER_PRIVATE_KEY_HEX=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \ + TRAFFIC_GENERATOR_PRIVATE_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \ TRAFFIC_GENERATOR_METRICS_HTTP_PORT=9101 \ TRAFFIC_GENERATOR_BLS_OPERATOR_STATE_RETRIEVER=0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154 \ - TRAFFIC_GENERATOR_EIGENDA_SERVICE_MANAGER=0x851356ae760d987E095750cCeb3bC6014560891C \ + TRAFFIC_GENERATOR_GRAPH_URL=http://localhost:8000/subgraphs/name/Layr-Labs/eigenda-operator-state \ TRAFFIC_GENERATOR_THE_GRAPH_URL=http://localhost:8000/subgraphs/name/Layr-Labs/eigenda-operator-state \ - TRAFFIC_GENERATOR_ENCODER_G1_PATH=../../inabox/resources/kzg/g1.point \ - TRAFFIC_GENERATOR_ENCODER_G2_PATH=../../inabox/resources/kzg/g2.point \ - TRAFFIC_GENERATOR_ENCODER_CACHE_DIR=../../inabox/resources/kzg/SRSTables \ + RETRIEVER_EIGENDA_SERVICE_MANAGER=0x851356ae760d987E095750cCeb3bC6014560891C \ + TRAFFIC_GENERATOR_G1_PATH=../../inabox/resources/kzg/g1.point \ + TRAFFIC_GENERATOR_G2_PATH=../../inabox/resources/kzg/g2.point \ + TRAFFIC_GENERATOR_CACHE_PATH=../../inabox/resources/kzg/SRSTables \ + TRAFFIC_GENERATOR_SRS_ORDER=3000 \ + TRAFFIC_GENERATOR_SRS_LOAD=3000 \ + TRAFFIC_GENERATOR_NUM_WORKERS=4 \ + TRAFFIC_GENERATOR_CHAIN_RPC=http://localhost:8545 \ + TRAFFIC_GENERATOR_NUM_RETRIES=2 \ ./bin/server + # TODO figure out what the appropriate values are for this prior to merging. These are the values that worked for me locally. \ No newline at end of file diff --git a/tools/traffic/config/config.go b/tools/traffic/config/config.go index 7dac07b363..f50ed9c681 100644 --- a/tools/traffic/config/config.go +++ b/tools/traffic/config/config.go @@ -4,9 +4,7 @@ import ( "errors" "fmt" "github.com/Layr-Labs/eigenda/api/clients" - "github.com/Layr-Labs/eigenda/common/geth" "github.com/Layr-Labs/eigenda/core/thegraph" - "github.com/Layr-Labs/eigenda/encoding/kzg" "github.com/Layr-Labs/eigenda/retriever" "time" @@ -61,6 +59,8 @@ func NewConfig(ctx *cli.Context) (*Config, error) { customQuorumsUint8[i] = uint8(q) } + retrieverConfig := retriever.ReadRetrieverConfig(ctx) + config := &Config{ DisperserClientConfig: &clients.Config{ Hostname: ctx.GlobalString(HostnameFlag.Name), @@ -69,22 +69,7 @@ func NewConfig(ctx *cli.Context) (*Config, error) { UseSecureGrpcFlag: ctx.GlobalBool(UseSecureGrpcFlag.Name), }, - // TODO refactor flags - RetrievalClientConfig: &retriever.Config{ - EigenDAServiceManagerAddr: ctx.String(EigenDAServiceManagerFlag.Name), - EncoderConfig: kzg.KzgConfig{ - G1Path: ctx.String(EncoderG1PathFlag.Name), - G2Path: ctx.String(EncoderG2PathFlag.Name), - CacheDir: ctx.String(EncoderCacheDirFlag.Name), - SRSOrder: ctx.Uint64(EncoderSRSOrderFlag.Name), - SRSNumberToLoad: ctx.Uint64(EncoderSRSNumberToLoadFlag.Name), - NumWorker: ctx.Uint64(EncoderNumWorkersFlag.Name), - }, - EthClientConfig: geth.EthClientConfig{ - RPCURLs: []string{fmt.Sprintf("%s:%s", ctx.GlobalString(EthClientHostnameFlag.Name), ctx.GlobalString(EthClientPortFlag.Name))}, - NumRetries: ctx.Int(EthClientRetriesFlag.Name), - }, - }, + RetrievalClientConfig: retrieverConfig, TheGraphConfig: &thegraph.Config{ Endpoint: ctx.String(TheGraphUrlFlag.Name), @@ -127,7 +112,7 @@ func NewConfig(ctx *cli.Context) (*Config, error) { RetrieveBlobChunksTimeout: ctx.Duration(RetrieveBlobChunksTimeoutFlag.Name), VerificationChannelCapacity: ctx.Uint(VerificationChannelCapacityFlag.Name), - EigenDAServiceManager: ctx.String(EigenDAServiceManagerFlag.Name), + EigenDAServiceManager: retrieverConfig.EigenDAServiceManagerAddr, SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), CustomQuorums: customQuorumsUint8, }, diff --git a/tools/traffic/config/flags.go b/tools/traffic/config/flags.go index e890f732a1..e85c3c6bf0 100644 --- a/tools/traffic/config/flags.go +++ b/tools/traffic/config/flags.go @@ -1,6 +1,11 @@ package config import ( + "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/thegraph" + "github.com/Layr-Labs/eigenda/encoding/kzg" + "github.com/Layr-Labs/eigenda/indexer" + "github.com/Layr-Labs/eigenda/retriever/flags" "time" "github.com/Layr-Labs/eigenda/common" @@ -64,37 +69,12 @@ var ( Required: false, EnvVar: common.PrefixEnvVar(envPrefix, "METRICS_HTTP_PORT"), } - EthClientHostnameFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "eth-client-hostname"), - Usage: "Hostname at which the Ethereum client is available.", - Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "ETH_CLIENT_HOSTNAME"), - } - EthClientPortFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "eth-client-port"), - Usage: "Port at which the Ethereum client is available.", - Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "ETH_CLIENT_PORT"), - } BLSOperatorStateRetrieverFlag = cli.StringFlag{ Name: common.PrefixFlag(FlagPrefix, "bls-operator-state-retriever"), Usage: "Hex address of the BLS operator state retriever contract.", Required: true, EnvVar: common.PrefixEnvVar(envPrefix, "BLS_OPERATOR_STATE_RETRIEVER"), } - EigenDAServiceManagerFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "eigenda-service-manager"), - Usage: "Hex address of the EigenDA service manager contract.", - Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "EIGENDA_SERVICE_MANAGER"), - } - EthClientRetriesFlag = cli.UintFlag{ - Name: common.PrefixFlag(FlagPrefix, "eth-client-retries"), - Usage: "Number of times to retry an Ethereum client request.", - Required: false, - Value: 2, - EnvVar: common.PrefixEnvVar(envPrefix, "ETH_CLIENT_RETRIES"), - } /* Common Configuration. */ @@ -162,44 +142,6 @@ var ( Value: 5, EnvVar: common.PrefixEnvVar(envPrefix, "THE_GRAPH_RETRIES"), } - EncoderG1PathFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "encoder-g1-path"), - Usage: "Path to the encoder G1 binary.", - Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "ENCODER_G1_PATH"), - } - EncoderG2PathFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "encoder-g2-path"), - Usage: "Path to the encoder G2 binary.", - Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "ENCODER_G2_PATH"), - } - EncoderCacheDirFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "encoder-cache-dir"), - Usage: "Path to the encoder cache directory.", - Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "ENCODER_CACHE_DIR"), - } - EncoderSRSOrderFlag = cli.UintFlag{ - Name: common.PrefixFlag(FlagPrefix, "encoder-srs-order"), - Usage: "The SRS order to use for the encoder.", - Required: false, - Value: 3000, - EnvVar: common.PrefixEnvVar(envPrefix, "ENCODER_SRS_ORDER"), - } - EncoderSRSNumberToLoadFlag = cli.UintFlag{ - Name: common.PrefixFlag(FlagPrefix, "encoder-srs-number-to-load"), - Usage: "The SRS number to load for the encoder.", - Required: false, - Value: 3000, - EnvVar: common.PrefixEnvVar(envPrefix, "ENCODER_SRS_NUMBER_TO_LOAD"), - } - EncoderNumWorkersFlag = cli.UintFlag{ - Name: common.PrefixFlag(FlagPrefix, "encoder-num-workers"), - Usage: "The number of worker threads to use for the encoder.", - Required: false, - Value: 4, - } RetrieverNumConnectionsFlag = cli.UintFlag{ Name: common.PrefixFlag(FlagPrefix, "retriever-num-connections"), Usage: "The number of connections to use for the retriever.", @@ -289,14 +231,8 @@ var ( var requiredFlags = []cli.Flag{ HostnameFlag, GrpcPortFlag, - EthClientHostnameFlag, - EthClientPortFlag, BLSOperatorStateRetrieverFlag, - EigenDAServiceManagerFlag, TheGraphUrlFlag, - EncoderG1PathFlag, - EncoderG2PathFlag, - EncoderCacheDirFlag, } var optionalFlags = []cli.Flag{ @@ -314,12 +250,8 @@ var optionalFlags = []cli.Flag{ RequiredDownloadsFlag, DisableTLSFlag, MetricsHTTPPortFlag, - EthClientRetriesFlag, TheGraphPullIntervalFlag, TheGraphRetriesFlag, - EncoderSRSOrderFlag, - EncoderSRSNumberToLoadFlag, - EncoderNumWorkersFlag, RetrieverNumConnectionsFlag, VerifierIntervalFlag, NodeClientTimeoutFlag, @@ -336,5 +268,10 @@ var Flags []cli.Flag func init() { Flags = append(requiredFlags, optionalFlags...) + Flags = append(Flags, flags.RetrieverFlags(envPrefix)...) + Flags = append(Flags, kzg.CLIFlags(envPrefix)...) Flags = append(Flags, common.LoggerCLIFlags(envPrefix, FlagPrefix)...) + Flags = append(Flags, geth.EthClientFlags(envPrefix)...) + Flags = append(Flags, indexer.CLIFlags(envPrefix)...) + Flags = append(Flags, thegraph.CLIFlags(envPrefix)...) } From 089b1a7d57c0a1bc2910f39f56b26b055f6b8829 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 31 Jul 2024 10:15:03 -0500 Subject: [PATCH 67/74] Finish changing config. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 3 +-- tools/traffic/config/config.go | 12 ++---------- tools/traffic/config/flags.go | 15 --------------- tools/traffic/generator.go | 4 ++-- 4 files changed, 5 insertions(+), 29 deletions(-) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 2d9cbc9544..251c3a32e6 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -13,6 +13,7 @@ run: build RETRIEVER_HOSTNAME=localhost \ RETRIEVER_GRPC_PORT=32003 \ RETRIEVER_BLS_OPERATOR_STATE_RETRIVER=0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154 \ + RETRIEVER_EIGENDA_SERVICE_MANAGER=0x851356ae760d987E095750cCeb3bC6014560891C \ RETRIEVER_TIMEOUT=10s \ TRAFFIC_GENERATOR_TIMEOUT=10s \ TRAFFIC_GENERATOR_NUM_WRITE_INSTANCES=1 \ @@ -22,10 +23,8 @@ run: build TRAFFIC_GENERATOR_SIGNER_PRIVATE_KEY_HEX=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \ TRAFFIC_GENERATOR_PRIVATE_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \ TRAFFIC_GENERATOR_METRICS_HTTP_PORT=9101 \ - TRAFFIC_GENERATOR_BLS_OPERATOR_STATE_RETRIEVER=0x5f3f1dBD7B74C6B46e8c44f98792A1dAf8d69154 \ TRAFFIC_GENERATOR_GRAPH_URL=http://localhost:8000/subgraphs/name/Layr-Labs/eigenda-operator-state \ TRAFFIC_GENERATOR_THE_GRAPH_URL=http://localhost:8000/subgraphs/name/Layr-Labs/eigenda-operator-state \ - RETRIEVER_EIGENDA_SERVICE_MANAGER=0x851356ae760d987E095750cCeb3bC6014560891C \ TRAFFIC_GENERATOR_G1_PATH=../../inabox/resources/kzg/g1.point \ TRAFFIC_GENERATOR_G2_PATH=../../inabox/resources/kzg/g2.point \ TRAFFIC_GENERATOR_CACHE_PATH=../../inabox/resources/kzg/SRSTables \ diff --git a/tools/traffic/config/config.go b/tools/traffic/config/config.go index f50ed9c681..6a7b1bd969 100644 --- a/tools/traffic/config/config.go +++ b/tools/traffic/config/config.go @@ -35,10 +35,6 @@ type Config struct { // The port at which the metrics server listens for HTTP requests. MetricsHTTPPort string - // The address of the BLS operator state retriever smart contract, in hex. - BlsOperatorStateRetriever string - // The number of connections to use for the retriever. - RetrieverNumConnections uint // The timeout for the node client. NodeClientTimeout time.Duration // The amount of time to sleep after launching each worker thread. @@ -85,12 +81,8 @@ func NewConfig(ctx *cli.Context) (*Config, error) { LoggingConfig: *loggerConfig, - MetricsHTTPPort: ctx.GlobalString(MetricsHTTPPortFlag.Name), - - BlsOperatorStateRetriever: ctx.String(BLSOperatorStateRetrieverFlag.Name), - - RetrieverNumConnections: ctx.Uint(RetrieverNumConnectionsFlag.Name), - NodeClientTimeout: ctx.Duration(NodeClientTimeoutFlag.Name), + MetricsHTTPPort: ctx.GlobalString(MetricsHTTPPortFlag.Name), + NodeClientTimeout: ctx.Duration(NodeClientTimeoutFlag.Name), InstanceLaunchInterval: ctx.Duration(InstanceLaunchIntervalFlag.Name), diff --git a/tools/traffic/config/flags.go b/tools/traffic/config/flags.go index e85c3c6bf0..951935db60 100644 --- a/tools/traffic/config/flags.go +++ b/tools/traffic/config/flags.go @@ -69,12 +69,6 @@ var ( Required: false, EnvVar: common.PrefixEnvVar(envPrefix, "METRICS_HTTP_PORT"), } - BLSOperatorStateRetrieverFlag = cli.StringFlag{ - Name: common.PrefixFlag(FlagPrefix, "bls-operator-state-retriever"), - Usage: "Hex address of the BLS operator state retriever contract.", - Required: true, - EnvVar: common.PrefixEnvVar(envPrefix, "BLS_OPERATOR_STATE_RETRIEVER"), - } /* Common Configuration. */ @@ -142,13 +136,6 @@ var ( Value: 5, EnvVar: common.PrefixEnvVar(envPrefix, "THE_GRAPH_RETRIES"), } - RetrieverNumConnectionsFlag = cli.UintFlag{ - Name: common.PrefixFlag(FlagPrefix, "retriever-num-connections"), - Usage: "The number of connections to use for the retriever.", - Required: false, - Value: 20, - EnvVar: common.PrefixEnvVar(envPrefix, "RETRIEVER_NUM_CONNECTIONS"), - } NodeClientTimeoutFlag = cli.DurationFlag{ Name: common.PrefixFlag(FlagPrefix, "node-client-timeout"), Usage: "The timeout for the node client.", @@ -231,7 +218,6 @@ var ( var requiredFlags = []cli.Flag{ HostnameFlag, GrpcPortFlag, - BLSOperatorStateRetrieverFlag, TheGraphUrlFlag, } @@ -252,7 +238,6 @@ var optionalFlags = []cli.Flag{ MetricsHTTPPortFlag, TheGraphPullIntervalFlag, TheGraphRetriesFlag, - RetrieverNumConnectionsFlag, VerifierIntervalFlag, NodeClientTimeoutFlag, FetchBatchHeaderTimeoutFlag, diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index fa6835227f..43bc4321c2 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -155,7 +155,7 @@ func buildRetriever(config *config.Config) (clients.RetrievalClient, retrivereth tx, err := eth.NewTransactor( logger, gethClient, - config.BlsOperatorStateRetriever, + config.RetrievalClientConfig.BLSOperatorStateRetrieverAddr, config.RetrievalClientConfig.EigenDAServiceManagerAddr) if err != nil { panic(fmt.Sprintf("Unable to instantiate transactor: %s", err)) @@ -180,7 +180,7 @@ func buildRetriever(config *config.Config) (clients.RetrievalClient, retrivereth assignmentCoordinator, nodeClient, v, - int(config.RetrieverNumConnections)) + config.RetrievalClientConfig.NumConnections) if err != nil { panic(fmt.Sprintf("Unable to build retriever: %s", err)) From 9d5c8b6ced8fc7acac645e66b840e1de3fed4184 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Mon, 19 Aug 2024 09:14:14 -0500 Subject: [PATCH 68/74] Refactored unit tests. Signed-off-by: Cody Littley --- tools/traffic/generator.go | 6 +- tools/traffic/test/mock_disperser.go | 96 ------------ tools/traffic/workers/blob_verifier.go | 54 ++----- .../{test => workers}/blob_verifier_test.go | 144 +++++++++--------- tools/traffic/workers/blob_writer.go | 14 +- tools/traffic/workers/blob_writer_test.go | 22 +-- tools/traffic/workers/key_handler.go | 7 - tools/traffic/workers/mock_key_handler.go | 24 --- tools/traffic/workers/unconfirmed_key.go | 15 ++ 9 files changed, 128 insertions(+), 254 deletions(-) delete mode 100644 tools/traffic/test/mock_disperser.go rename tools/traffic/{test => workers}/blob_verifier_test.go (52%) delete mode 100644 tools/traffic/workers/key_handler.go delete mode 100644 tools/traffic/workers/mock_key_handler.go create mode 100644 tools/traffic/workers/unconfirmed_key.go diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index 711d8048e6..8187487622 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -81,13 +81,15 @@ func NewTrafficGenerator(config *config.Config) (*Generator, error) { blobTable := table.NewBlobStore() + unconfirmedKeyChannel := make(chan *workers.UnconfirmedKey, 100) + disperserClient := clients.NewDisperserClient(config.DisperserClientConfig, signer) statusVerifier := workers.NewBlobVerifier( &ctx, &waitGroup, logger, - workers.NewTicker(config.WorkerConfig.VerifierInterval), &config.WorkerConfig, + unconfirmedKeyChannel, blobTable, disperserClient, generatorMetrics) @@ -100,7 +102,7 @@ func NewTrafficGenerator(config *config.Config) (*Generator, error) { logger, &config.WorkerConfig, disperserClient, - &statusVerifier, + unconfirmedKeyChannel, generatorMetrics) writers = append(writers, &writer) } diff --git a/tools/traffic/test/mock_disperser.go b/tools/traffic/test/mock_disperser.go deleted file mode 100644 index a03b3bb2ab..0000000000 --- a/tools/traffic/test/mock_disperser.go +++ /dev/null @@ -1,96 +0,0 @@ -package test - -import ( - "context" - disperser_rpc "github.com/Layr-Labs/eigenda/api/grpc/disperser" - "github.com/Layr-Labs/eigenda/disperser" - "github.com/stretchr/testify/assert" - "sync" - "testing" -) - -type mockDisperserClient struct { - t *testing.T - // if true, DisperseBlobAuthenticated is expected to be used, otherwise DisperseBlob is expected to be used - authenticated bool - - // The next status, key, and error to return from DisperseBlob or DisperseBlobAuthenticated - StatusToReturn disperser.BlobStatus - KeyToReturn []byte - DispenseErrorToReturn error - - // The previous values passed to DisperseBlob or DisperseBlobAuthenticated - ProvidedData []byte - ProvidedQuorum []uint8 - - // Incremented each time DisperseBlob or DisperseBlobAuthenticated is called. - DisperseCount uint - - // A map from key (in string form) to the status to return from GetBlobStatus. If nil, then an error is returned. - StatusMap map[string]disperser_rpc.BlobStatus - - // Incremented each time GetBlobStatus is called. - GetStatusCount uint - - lock *sync.Mutex -} - -func newMockDisperserClient(t *testing.T, lock *sync.Mutex, authenticated bool) *mockDisperserClient { - return &mockDisperserClient{ - t: t, - lock: lock, - authenticated: authenticated, - StatusMap: make(map[string]disperser_rpc.BlobStatus), - } -} - -func (m *mockDisperserClient) DisperseBlob( - ctx context.Context, - data []byte, - customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { - - m.lock.Lock() - defer m.lock.Unlock() - - assert.False(m.t, m.authenticated, "writer configured to use non-authenticated disperser method") - m.ProvidedData = data - m.ProvidedQuorum = customQuorums - m.DisperseCount++ - return &m.StatusToReturn, m.KeyToReturn, m.DispenseErrorToReturn -} - -func (m *mockDisperserClient) DisperseBlobAuthenticated( - ctx context.Context, - data []byte, - customQuorums []uint8) (*disperser.BlobStatus, []byte, error) { - - m.lock.Lock() - defer m.lock.Unlock() - - assert.True(m.t, m.authenticated, "writer configured to use authenticated disperser method") - m.ProvidedData = data - m.ProvidedQuorum = customQuorums - m.DisperseCount++ - return &m.StatusToReturn, m.KeyToReturn, m.DispenseErrorToReturn -} - -func (m *mockDisperserClient) GetBlobStatus(ctx context.Context, key []byte) (*disperser_rpc.BlobStatusReply, error) { - m.lock.Lock() - defer m.lock.Unlock() - - status := m.StatusMap[string(key)] - m.GetStatusCount++ - - return &disperser_rpc.BlobStatusReply{ - Status: status, - Info: &disperser_rpc.BlobInfo{ - BlobVerificationProof: &disperser_rpc.BlobVerificationProof{ - BatchMetadata: &disperser_rpc.BatchMetadata{}, - }, - }, - }, nil -} - -func (m *mockDisperserClient) RetrieveBlob(ctx context.Context, batchHeaderHash []byte, blobIndex uint32) ([]byte, error) { - panic("this method should not be called by the generator utility") -} diff --git a/tools/traffic/workers/blob_verifier.go b/tools/traffic/workers/blob_verifier.go index d8e3dddc6f..f3efa3f9e7 100644 --- a/tools/traffic/workers/blob_verifier.go +++ b/tools/traffic/workers/blob_verifier.go @@ -13,18 +13,6 @@ import ( "time" ) -// unconfirmedKey is a key that has not yet been confirmed by the disperser service. -type unconfirmedKey struct { - // The key of the blob. - key []byte - // The size of the blob in bytes. - size uint - // The checksum of the blob. - checksum [16]byte - // The time the blob was submitted to the disperser service. - submissionTime time.Time -} - // BlobVerifier periodically polls the disperser service to verify the status of blobs that were recently written. // When blobs become confirmed, the status verifier updates the blob blobsToRead accordingly. // This is a thread safe data structure. @@ -49,13 +37,10 @@ type BlobVerifier struct { dispenser clients.DisperserClient // The keys of blobs that have not yet been confirmed by the disperser service. - unconfirmedKeys []*unconfirmedKey + unconfirmedKeys []*UnconfirmedKey // Newly added keys that require verification. - keyChannel chan *unconfirmedKey - - // ticker is used to control the rate at which blobs queried for status. - ticker InterceptableTicker + keyChannel chan *UnconfirmedKey blobsInFlightMetric metrics.GaugeMetric getStatusLatencyMetric metrics.LatencyMetric @@ -75,8 +60,8 @@ func NewBlobVerifier( ctx *context.Context, waitGroup *sync.WaitGroup, logger logging.Logger, - ticker InterceptableTicker, config *config2.WorkerConfig, + keyChannel chan *UnconfirmedKey, table *table.BlobTable, disperser clients.DisperserClient, generatorMetrics metrics.Metrics) BlobVerifier { @@ -85,12 +70,11 @@ func NewBlobVerifier( ctx: ctx, waitGroup: waitGroup, logger: logger, - ticker: ticker, config: config, + keyChannel: keyChannel, table: table, dispenser: disperser, - unconfirmedKeys: make([]*unconfirmedKey, 0), - keyChannel: make(chan *unconfirmedKey, config.VerificationChannelCapacity), + unconfirmedKeys: make([]*UnconfirmedKey, 0), blobsInFlightMetric: generatorMetrics.NewGaugeMetric("blobs_in_flight"), getStatusLatencyMetric: generatorMetrics.NewLatencyMetric("get_status"), confirmationLatencyMetric: generatorMetrics.NewLatencyMetric("confirmation"), @@ -105,16 +89,6 @@ func NewBlobVerifier( } } -// AddUnconfirmedKey adds a key to the list of unconfirmed keys. -func (verifier *BlobVerifier) AddUnconfirmedKey(key []byte, checksum [16]byte, size uint) { - verifier.keyChannel <- &unconfirmedKey{ - key: key, - checksum: checksum, - size: size, - submissionTime: time.Now(), - } -} - // Start begins the status goroutine, which periodically polls // the disperser service to verify the status of blobs. func (verifier *BlobVerifier) Start() { @@ -124,7 +98,7 @@ func (verifier *BlobVerifier) Start() { // monitor periodically polls the disperser service to verify the status of blobs. func (verifier *BlobVerifier) monitor() { - ticker := verifier.ticker.GetTimeChannel() + ticker := time.NewTicker(verifier.config.VerifierInterval) for { select { case <-(*verifier.ctx).Done(): @@ -132,20 +106,20 @@ func (verifier *BlobVerifier) monitor() { return case key := <-verifier.keyChannel: verifier.unconfirmedKeys = append(verifier.unconfirmedKeys, key) - case <-ticker: + case <-ticker.C: verifier.poll() } } } // poll checks all unconfirmed keys to see if they have been confirmed by the disperser service. -// If a key is confirmed, it is added to the blob table and removed from the list of unconfirmed keys. +// If a Key is confirmed, it is added to the blob table and removed from the list of unconfirmed keys. func (verifier *BlobVerifier) poll() { // FUTURE WORK If the number of unconfirmed blobs is high and the time to confirm is high, this is not efficient. // Revisit this method if there are performance problems. - unconfirmedKeys := make([]*unconfirmedKey, 0) + unconfirmedKeys := make([]*UnconfirmedKey, 0) for _, key := range verifier.unconfirmedKeys { confirmed := verifier.checkStatusForBlob(key) if !confirmed { @@ -157,14 +131,14 @@ func (verifier *BlobVerifier) poll() { } // checkStatusForBlob checks the status of a blob. Returns true if the final blob status is known, false otherwise. -func (verifier *BlobVerifier) checkStatusForBlob(key *unconfirmedKey) bool { +func (verifier *BlobVerifier) checkStatusForBlob(key *UnconfirmedKey) bool { ctxTimeout, cancel := context.WithTimeout(*verifier.ctx, verifier.config.GetBlobStatusTimeout) defer cancel() status, err := metrics.InvokeAndReportLatency[*disperser.BlobStatusReply](verifier.getStatusLatencyMetric, func() (*disperser.BlobStatusReply, error) { - return verifier.dispenser.GetBlobStatus(ctxTimeout, key.key) + return verifier.dispenser.GetBlobStatus(ctxTimeout, key.Key) }) if err != nil { @@ -208,12 +182,12 @@ func (verifier *BlobVerifier) checkStatusForBlob(key *unconfirmedKey) bool { } // forwardToReader forwards a blob to the reader. Only called once the blob is ready to be read. -func (verifier *BlobVerifier) forwardToReader(key *unconfirmedKey, status *disperser.BlobStatusReply) { +func (verifier *BlobVerifier) forwardToReader(key *UnconfirmedKey, status *disperser.BlobStatusReply) { batchHeaderHash := status.GetInfo().BlobVerificationProof.BatchMetadata.BatchHeaderHash blobIndex := status.GetInfo().BlobVerificationProof.GetBlobIndex() confirmationTime := time.Now() - confirmationLatency := confirmationTime.Sub(key.submissionTime) + confirmationLatency := confirmationTime.Sub(key.SubmissionTime) verifier.confirmationLatencyMetric.ReportLatency(confirmationLatency) requiredDownloads := verifier.config.RequiredDownloads @@ -238,7 +212,7 @@ func (verifier *BlobVerifier) forwardToReader(key *unconfirmedKey, status *dispe downloadCount = int32(requiredDownloads) } - blobMetadata, err := table.NewBlobMetadata(key.key, key.checksum, key.size, uint(blobIndex), batchHeaderHash, int(downloadCount)) + blobMetadata, err := table.NewBlobMetadata(key.Key, key.Checksum, key.Size, uint(blobIndex), batchHeaderHash, int(downloadCount)) if err != nil { verifier.logger.Error("failed to create blob metadata", "err:", err) return diff --git a/tools/traffic/test/blob_verifier_test.go b/tools/traffic/workers/blob_verifier_test.go similarity index 52% rename from tools/traffic/test/blob_verifier_test.go rename to tools/traffic/workers/blob_verifier_test.go index b795005cb9..b85ad623ed 100644 --- a/tools/traffic/test/blob_verifier_test.go +++ b/tools/traffic/workers/blob_verifier_test.go @@ -1,4 +1,4 @@ -package test +package workers import ( "context" @@ -9,8 +9,8 @@ import ( "github.com/Layr-Labs/eigenda/tools/traffic/config" "github.com/Layr-Labs/eigenda/tools/traffic/metrics" "github.com/Layr-Labs/eigenda/tools/traffic/table" - "github.com/Layr-Labs/eigenda/tools/traffic/workers" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "golang.org/x/exp/rand" "sync" "testing" @@ -61,8 +61,6 @@ func TestBlobVerifier(t *testing.T) { waitGroup := sync.WaitGroup{} logger, err := common.NewLogger(common.DefaultLoggerConfig()) assert.Nil(t, err) - startTime := time.Unix(rand.Int63()%2_000_000_000, 0) - ticker := newMockTicker(startTime) requiredDownloads := rand.Intn(10) config := &config.WorkerConfig{ @@ -73,27 +71,25 @@ func TestBlobVerifier(t *testing.T) { verifierMetrics := metrics.NewMockMetrics() - lock := sync.Mutex{} + disperserClient := &MockDisperserClient{} - disperserClient := newMockDisperserClient(t, &lock, true) - - verifier := workers.NewBlobVerifier( + verifier := NewBlobVerifier( &ctx, &waitGroup, logger, - ticker, config, + make(chan *UnconfirmedKey), blobTable, disperserClient, verifierMetrics) - verifier.Start() - expectedGetStatusCount := 0 statusCounts := make(map[disperser_rpc.BlobStatus]int) checksums := make(map[string][16]byte) sizes := make(map[string]uint) + statusMap := make(map[string]disperser_rpc.BlobStatus) + for i := 0; i < 100; i++ { // Add some new keys to track. @@ -112,90 +108,96 @@ func TestBlobVerifier(t *testing.T) { sizes[string(key)] = uint(size) stringifiedKey := string(key) - disperserClient.StatusMap[stringifiedKey] = disperser_rpc.BlobStatus_UNKNOWN + statusMap[stringifiedKey] = disperser_rpc.BlobStatus_UNKNOWN + + unconfirmedKey := &UnconfirmedKey{ + Key: key, + Checksum: checksum, + Size: uint(size), + SubmissionTime: time.Now(), + } - verifier.AddUnconfirmedKey(key, checksum, uint(size)) + verifier.unconfirmedKeys = append(verifier.unconfirmedKeys, unconfirmedKey) } + // Reset the mock disperser client. + disperserClient.mock = mock.Mock{} + expectedGetStatusCount = 0 + // Choose some new statuses to be returned. // Count the number of status queries we expect to see in this iteration. - for key, status := range disperserClient.StatusMap { - if !isStatusTerminal(status) { + for key, status := range statusMap { + var newStatus disperser_rpc.BlobStatus + if isStatusTerminal(status) { + newStatus = status + } else { // Blobs in a non-terminal status will be queried again. expectedGetStatusCount += 1 // Set the next status to be returned. - newStatus := getRandomStatus() - disperserClient.StatusMap[key] = newStatus + newStatus = getRandomStatus() + statusMap[key] = newStatus + statusCounts[newStatus] += 1 } + disperserClient.mock.On("GetBlobStatus", []byte(key)).Return( + &disperser_rpc.BlobStatusReply{ + Status: newStatus, + Info: &disperser_rpc.BlobInfo{ + BlobVerificationProof: &disperser_rpc.BlobVerificationProof{ + BatchMetadata: &disperser_rpc.BatchMetadata{ + BatchHeaderHash: make([]byte, 0), + }, + }, + }, + }, nil) } - // Advance to the next cycle, allowing the verifier to process the new keys. - ticker.Tick(time.Second) + // Simulate advancement of time, allowing the verifier to process the new keys. + verifier.poll() - // Data is inserted asynchronously, so we may need to wait for it to be processed. - tu.AssertEventuallyTrue(t, func() bool { - lock.Lock() - defer lock.Unlock() + // Validate the number of calls made to the disperser client. + disperserClient.mock.AssertNumberOfCalls(t, "GetBlobStatus", expectedGetStatusCount) - // Validate the number of calls made to the disperser client. - if int(disperserClient.GetStatusCount) < expectedGetStatusCount { - return false - } + // Read the data in the table into a map for quick lookup. + tableData := make(map[string]*table.BlobMetadata) + for _, metadata := range blobTable.GetAll() { + tableData[string(metadata.Key)] = metadata + } - // Read the data in the table into a map for quick lookup. - tableData := make(map[string]*table.BlobMetadata) + blobsInFlight := 0 + for key, status := range statusMap { + metadata, present := tableData[key] - for _, metadata := range blobTable.GetAll() { - tableData[string(metadata.Key)] = metadata + if !isStatusTerminal(status) { + blobsInFlight++ } - blobsInFlight := 0 - for key, status := range disperserClient.StatusMap { - metadata, present := tableData[key] - - if !isStatusTerminal(status) { - blobsInFlight++ - } - - if isStatusSuccess(status) { - // Successful blobs should be in the table. - if !present { - // Blob might not yet be in table due to timing. - return false - } - } else { - // Non-successful blobs should not be in the table. - assert.False(t, present) - } - - // Verify metadata. - if present { - assert.Equal(t, checksums[key], metadata.Checksum) - assert.Equal(t, sizes[key], metadata.Size) - assert.Equal(t, requiredDownloads, metadata.RemainingReadPermits) - } + if isStatusSuccess(status) { + // Successful blobs should be in the table. + assert.True(t, present) + } else { + // Non-successful blobs should not be in the table. + assert.False(t, present) } - // Verify metrics. - for status, count := range statusCounts { - metricName := fmt.Sprintf("get_status_%s", status.String()) - if float64(count) != verifierMetrics.GetCount(metricName) { - return false - } - } - if float64(blobsInFlight) != verifierMetrics.GetGaugeValue("blobs_in_flight") { - fmt.Printf("expected blobs_in_flight to be %d, got %f\n", blobsInFlight, verifierMetrics.GetCount("blobs_in_flight")) - return false + // Verify metadata. + if present { + assert.Equal(t, checksums[key], metadata.Checksum) + assert.Equal(t, sizes[key], metadata.Size) + assert.Equal(t, requiredDownloads, metadata.RemainingReadPermits) } + } - return true - }, time.Second) + // Verify metrics. + for status, count := range statusCounts { + metricName := fmt.Sprintf("get_status_%s", status.String()) + assert.Equal(t, float64(count), verifierMetrics.GetCount(metricName)) + } + if float64(blobsInFlight) != verifierMetrics.GetGaugeValue("blobs_in_flight") { + assert.Equal(t, float64(blobsInFlight), verifierMetrics.GetGaugeValue("blobs_in_flight")) + } } - assert.Equal(t, expectedGetStatusCount, int(disperserClient.GetStatusCount)) - assert.Equal(t, 0, int(disperserClient.DisperseCount)) - cancel() tu.ExecuteWithTimeout(func() { waitGroup.Wait() diff --git a/tools/traffic/workers/blob_writer.go b/tools/traffic/workers/blob_writer.go index a30a7e5bdd..81b6c8a9ef 100644 --- a/tools/traffic/workers/blob_writer.go +++ b/tools/traffic/workers/blob_writer.go @@ -32,7 +32,7 @@ type BlobWriter struct { disperser clients.DisperserClient // Unconfirmed keys are sent here. - unconfirmedKeyHandler KeyHandler + unconfirmedKeyChannel chan *UnconfirmedKey // fixedRandomData contains random data for blobs if RandomizeBlobs is false, and nil otherwise. fixedRandomData []byte @@ -54,7 +54,7 @@ func NewBlobWriter( logger logging.Logger, config *config.WorkerConfig, disperser clients.DisperserClient, - unconfirmedKeyHandler KeyHandler, + unconfirmedKeyChannel chan *UnconfirmedKey, generatorMetrics metrics.Metrics) BlobWriter { var fixedRandomData []byte @@ -77,7 +77,7 @@ func NewBlobWriter( logger: logger, config: config, disperser: disperser, - unconfirmedKeyHandler: unconfirmedKeyHandler, + unconfirmedKeyChannel: unconfirmedKeyChannel, fixedRandomData: fixedRandomData, writeLatencyMetric: generatorMetrics.NewLatencyMetric("write"), writeSuccessMetric: generatorMetrics.NewCountMetric("write_success"), @@ -123,7 +123,13 @@ func (writer *BlobWriter) writeNextBlob() { writer.writeSuccessMetric.Increment() checksum := md5.Sum(data) - writer.unconfirmedKeyHandler.AddUnconfirmedKey(key, checksum, uint(len(data))) + + writer.unconfirmedKeyChannel <- &UnconfirmedKey{ + Key: key, + Checksum: checksum, + Size: uint(len(data)), + SubmissionTime: time.Now(), + } } // getRandomData returns a slice of random data to be used for a blob. diff --git a/tools/traffic/workers/blob_writer_test.go b/tools/traffic/workers/blob_writer_test.go index 723894490a..dcd70841c6 100644 --- a/tools/traffic/workers/blob_writer_test.go +++ b/tools/traffic/workers/blob_writer_test.go @@ -26,6 +26,7 @@ func TestBlobWriter(t *testing.T) { assert.Nil(t, err) dataSize := rand.Uint64()%1024 + 64 + encodedDataSize := len(codec.ConvertByPaddingEmptyByte(make([]byte, dataSize))) authenticated := rand.Intn(2) == 0 var signerPrivateKey string @@ -55,9 +56,7 @@ func TestBlobWriter(t *testing.T) { } disperserClient := &MockDisperserClient{} - unconfirmedKeyHandler := &MockKeyHandler{} - unconfirmedKeyHandler.mock.On( - "AddUnconfirmedKey", mock.Anything, mock.Anything, mock.Anything).Return(nil) + unconfirmedKeyChannel := make(chan *UnconfirmedKey, 100) generatorMetrics := metrics.NewMockMetrics() @@ -67,7 +66,7 @@ func TestBlobWriter(t *testing.T) { logger, config, disperserClient, - unconfirmedKeyHandler, + unconfirmedKeyChannel, generatorMetrics) errorCount := 0 @@ -83,7 +82,7 @@ func TestBlobWriter(t *testing.T) { errorToReturn = nil } - // This is the key that will be assigned to the next blob. + // This is the Key that will be assigned to the next blob. keyToReturn := make([]byte, 32) _, err = rand.Read(keyToReturn) assert.Nil(t, err) @@ -96,21 +95,24 @@ func TestBlobWriter(t *testing.T) { writer.writeNextBlob() disperserClient.mock.AssertNumberOfCalls(t, functionName, 1) - unconfirmedKeyHandler.mock.AssertNumberOfCalls(t, "AddUnconfirmedKey", i+1-errorCount) if errorToReturn == nil { - dataSentToDisperser := disperserClient.mock.Calls[0].Arguments.Get(0).([]byte) assert.NotNil(t, dataSentToDisperser) - // Strip away the extra encoding bytes. We should have data of the expected size. + // Strip away the extra encoding bytes. We should have data of the expected Size. decodedData := codec.RemoveEmptyByteFromPaddedBytes(dataSentToDisperser) assert.Equal(t, dataSize, uint64(len(decodedData))) - // Verify that the proper data was sent to the unconfirmed key handler. + // Verify that the proper data was sent to the unconfirmed Key handler. checksum := md5.Sum(dataSentToDisperser) - unconfirmedKeyHandler.mock.AssertCalled(t, "AddUnconfirmedKey", keyToReturn, checksum, uint(len(dataSentToDisperser))) + unconfirmedKey, ok := <-unconfirmedKeyChannel + + assert.True(t, ok) + assert.Equal(t, keyToReturn, unconfirmedKey.Key) + assert.Equal(t, uint(encodedDataSize), unconfirmedKey.Size) + assert.Equal(t, checksum, unconfirmedKey.Checksum) // Verify that data has the proper amount of randomness. if previousData != nil { diff --git a/tools/traffic/workers/key_handler.go b/tools/traffic/workers/key_handler.go deleted file mode 100644 index 30c8b5ed9c..0000000000 --- a/tools/traffic/workers/key_handler.go +++ /dev/null @@ -1,7 +0,0 @@ -package workers - -// KeyHandler is an interface describing an object that can accept unconfirmed keys. -type KeyHandler interface { - // AddUnconfirmedKey accepts an unconfirmed blob key, the checksum of the blob, and the size of the blob in bytes. - AddUnconfirmedKey(key []byte, checksum [16]byte, size uint) -} diff --git a/tools/traffic/workers/mock_key_handler.go b/tools/traffic/workers/mock_key_handler.go deleted file mode 100644 index 2c48de995b..0000000000 --- a/tools/traffic/workers/mock_key_handler.go +++ /dev/null @@ -1,24 +0,0 @@ -package workers - -import ( - "github.com/stretchr/testify/mock" -) - -var _ KeyHandler = (*MockKeyHandler)(nil) - -// MockKeyHandler is a stand-in for the blob verifier's UnconfirmedKeyHandler. -type MockKeyHandler struct { - mock mock.Mock - - ProvidedKey []byte - ProvidedChecksum [16]byte - ProvidedSize uint -} - -func (m *MockKeyHandler) AddUnconfirmedKey(key []byte, checksum [16]byte, size uint) { - m.mock.Called(key, checksum, size) - - m.ProvidedKey = key - m.ProvidedChecksum = checksum - m.ProvidedSize = size -} diff --git a/tools/traffic/workers/unconfirmed_key.go b/tools/traffic/workers/unconfirmed_key.go new file mode 100644 index 0000000000..c86b8f1dcd --- /dev/null +++ b/tools/traffic/workers/unconfirmed_key.go @@ -0,0 +1,15 @@ +package workers + +import "time" + +// UnconfirmedKey is a Key that has not yet been confirmed by the disperser service. +type UnconfirmedKey struct { + // The Key of the blob. + Key []byte + // The Size of the blob in bytes. + Size uint + // The Checksum of the blob. + Checksum [16]byte + // The time the blob was submitted to the disperser service. + SubmissionTime time.Time +} From 1dfe628613ce1316dd426adf67df3aaae2f5c8d7 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 4 Sep 2024 09:40:38 -0500 Subject: [PATCH 69/74] Debugging stuff. Signed-off-by: Cody Littley --- inabox/deploy/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inabox/deploy/utils.go b/inabox/deploy/utils.go index 6f989fd393..200a7b640d 100644 --- a/inabox/deploy/utils.go +++ b/inabox/deploy/utils.go @@ -162,8 +162,8 @@ func execForgeScript(script, privateKey string, deployer *ContractDeployer, extr err := cmd.Run() if err != nil { - log.Print(fmt.Sprint(err) + ": " + stderr.String()) - log.Panicf("Failed to execute forge script. Err: %s", err) + log.Panicf("Failed to execute forge script: %s\n Err: %s\n--- std out ---\n%s\n--- std err ---\n%s\n", + cmd, err, out.String(), stderr.String()) } else { log.Print(out.String()) } From 75de0828e6e1b910e66081b1e363cb938d0d2680 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 3 Oct 2024 14:30:32 -0500 Subject: [PATCH 70/74] Cleanup. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 3 --- tools/traffic/test/blob_reader_test.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 251c3a32e6..9c80dcce03 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -34,6 +34,3 @@ run: build TRAFFIC_GENERATOR_CHAIN_RPC=http://localhost:8545 \ TRAFFIC_GENERATOR_NUM_RETRIES=2 \ ./bin/server - - -# TODO figure out what the appropriate values are for this prior to merging. These are the values that worked for me locally. \ No newline at end of file diff --git a/tools/traffic/test/blob_reader_test.go b/tools/traffic/test/blob_reader_test.go index 1fffad2510..3582c19cf3 100644 --- a/tools/traffic/test/blob_reader_test.go +++ b/tools/traffic/test/blob_reader_test.go @@ -16,7 +16,7 @@ import ( "time" ) -// TestBlobReaderNoOptionalReads tests the BlobReader's basic functionality' +// TestBlobReaderNoOptionalReads tests the BlobReader's basic functionality func TestBlobReader(t *testing.T) { tu.InitializeRandom() From 8df796c8c5332422d8e8efd7976cbd95d2e43d8c Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 3 Oct 2024 14:53:56 -0500 Subject: [PATCH 71/74] Revert unintentional changes. Signed-off-by: Cody Littley --- common/ratelimit/limiter_cli.go | 2 +- disperser/apiserver/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/ratelimit/limiter_cli.go b/common/ratelimit/limiter_cli.go index 48f84fc75e..33ff407a17 100644 --- a/common/ratelimit/limiter_cli.go +++ b/common/ratelimit/limiter_cli.go @@ -42,7 +42,7 @@ func RatelimiterCLIFlags(envPrefix string, flagPrefix string) []cli.Flag { }, cli.BoolFlag{ Name: common.PrefixFlag(flagPrefix, CountFailedFlagName), - Usage: "DisperseCount failed requests", + Usage: "Count failed requests", EnvVar: common.PrefixEnvVar(envPrefix, "COUNT_FAILED"), }, cli.IntFlag{ diff --git a/disperser/apiserver/server.go b/disperser/apiserver/server.go index 6fc357e1b4..440586605f 100644 --- a/disperser/apiserver/server.go +++ b/disperser/apiserver/server.go @@ -162,7 +162,7 @@ func (s *DispersalServer) DisperseBlobAuthenticated(stream pb.Disperser_Disperse resultCh := make(chan *pb.AuthenticatedRequest) errCh := make(chan error) - // Start stream.Recv() in a goroutine + // Run stream.Recv() in a goroutine go func() { in, err := stream.Recv() if err != nil { From 8d4caf6e60cacfec93fb32fa3d4d5b3a8f80594f Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 3 Oct 2024 15:01:56 -0500 Subject: [PATCH 72/74] Don't remove old generator yet. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 17 +- tools/traffic/cmd/main.go | 20 ++- tools/traffic/cmd2/main.go | 44 ++++++ tools/traffic/config.go | 56 +++++++ tools/traffic/flags/flags.go | 112 ++++++++++++++ tools/traffic/generator.go | 265 ++++++++++---------------------- tools/traffic/generator_test.go | 71 +++++++++ tools/traffic/generator_v2.go | 217 ++++++++++++++++++++++++++ 8 files changed, 614 insertions(+), 188 deletions(-) create mode 100644 tools/traffic/cmd2/main.go create mode 100644 tools/traffic/config.go create mode 100644 tools/traffic/flags/flags.go create mode 100644 tools/traffic/generator_test.go create mode 100644 tools/traffic/generator_v2.go diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 9c80dcce03..1a1e0efab3 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -7,6 +7,21 @@ build: clean go build -o ./bin/server ./cmd run: build + TRAFFIC_GENERATOR_HOSTNAME=localhost \ + TRAFFIC_GENERATOR_GRPC_PORT=32001 \ + TRAFFIC_GENERATOR_TIMEOUT=10s \ + TRAFFIC_GENERATOR_NUM_INSTANCES=1 \ + TRAFFIC_GENERATOR_REQUEST_INTERVAL=1s \ + TRAFFIC_GENERATOR_DATA_SIZE=1000 \ + TRAFFIC_GENERATOR_RANDOMIZE_BLOBS=true \ + ./bin/server + +build2: clean + # cd ../.. && make protoc + go mod tidy + go build -o ./bin/server2 ./cmd2 + +run2: build2 TRAFFIC_GENERATOR_LOG_FORMAT=text \ TRAFFIC_GENERATOR_DISPERSER_HOSTNAME=localhost \ TRAFFIC_GENERATOR_DISPERSER_PORT=32003 \ @@ -33,4 +48,4 @@ run: build TRAFFIC_GENERATOR_NUM_WORKERS=4 \ TRAFFIC_GENERATOR_CHAIN_RPC=http://localhost:8545 \ TRAFFIC_GENERATOR_NUM_RETRIES=2 \ - ./bin/server + ./bin/server2 diff --git a/tools/traffic/cmd/main.go b/tools/traffic/cmd/main.go index 8ad708339f..62a508629c 100644 --- a/tools/traffic/cmd/main.go +++ b/tools/traffic/cmd/main.go @@ -5,8 +5,10 @@ import ( "log" "os" + "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/core/auth" "github.com/Layr-Labs/eigenda/tools/traffic" - "github.com/Layr-Labs/eigenda/tools/traffic/config" + "github.com/Layr-Labs/eigenda/tools/traffic/flags" "github.com/urfave/cli" ) @@ -22,7 +24,7 @@ func main() { app.Name = "da-traffic-generator" app.Usage = "EigenDA Traffic Generator" app.Description = "Service for generating traffic to EigenDA disperser" - app.Flags = config.Flags + app.Flags = flags.Flags app.Action = trafficGeneratorMain if err := app.Run(os.Args); err != nil { log.Fatalf("application failed: %v", err) @@ -30,15 +32,21 @@ func main() { } func trafficGeneratorMain(ctx *cli.Context) error { - generatorConfig, err := config.NewConfig(ctx) + config, err := traffic.NewConfig(ctx) if err != nil { return err } - generator, err := traffic.NewTrafficGenerator(generatorConfig) + var signer core.BlobRequestSigner + if config.SignerPrivateKey != "" { + log.Println("Using signer private key") + signer = auth.NewLocalBlobRequestSigner(config.SignerPrivateKey) + } + + generator, err := traffic.NewTrafficGenerator(config, signer) if err != nil { - panic(fmt.Sprintf("failed to create new traffic generator\n%s", err)) + panic("failed to create new traffic generator") } - return generator.Start() + return generator.Run() } diff --git a/tools/traffic/cmd2/main.go b/tools/traffic/cmd2/main.go new file mode 100644 index 0000000000..8ad708339f --- /dev/null +++ b/tools/traffic/cmd2/main.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "log" + "os" + + "github.com/Layr-Labs/eigenda/tools/traffic" + "github.com/Layr-Labs/eigenda/tools/traffic/config" + "github.com/urfave/cli" +) + +var ( + version = "" + gitCommit = "" + gitDate = "" +) + +func main() { + app := cli.NewApp() + app.Version = fmt.Sprintf("%s-%s-%s", version, gitCommit, gitDate) + app.Name = "da-traffic-generator" + app.Usage = "EigenDA Traffic Generator" + app.Description = "Service for generating traffic to EigenDA disperser" + app.Flags = config.Flags + app.Action = trafficGeneratorMain + if err := app.Run(os.Args); err != nil { + log.Fatalf("application failed: %v", err) + } +} + +func trafficGeneratorMain(ctx *cli.Context) error { + generatorConfig, err := config.NewConfig(ctx) + if err != nil { + return err + } + + generator, err := traffic.NewTrafficGenerator(generatorConfig) + if err != nil { + panic(fmt.Sprintf("failed to create new traffic generator\n%s", err)) + } + + return generator.Start() +} diff --git a/tools/traffic/config.go b/tools/traffic/config.go new file mode 100644 index 0000000000..590c607590 --- /dev/null +++ b/tools/traffic/config.go @@ -0,0 +1,56 @@ +package traffic + +import ( + "errors" + "time" + + "github.com/Layr-Labs/eigenda/api/clients" + "github.com/Layr-Labs/eigenda/common" + "github.com/Layr-Labs/eigenda/tools/traffic/flags" + "github.com/urfave/cli" +) + +type Config struct { + clients.Config + + NumInstances uint + RequestInterval time.Duration + DataSize uint64 + LoggingConfig common.LoggerConfig + RandomizeBlobs bool + InstanceLaunchInterval time.Duration + + SignerPrivateKey string + CustomQuorums []uint8 +} + +func NewConfig(ctx *cli.Context) (*Config, error) { + loggerConfig, err := common.ReadLoggerCLIConfig(ctx, flags.FlagPrefix) + if err != nil { + return nil, err + } + customQuorums := ctx.GlobalIntSlice(flags.CustomQuorumNumbersFlag.Name) + customQuorumsUint8 := make([]uint8, len(customQuorums)) + for i, q := range customQuorums { + if q < 0 || q > 255 { + return nil, errors.New("invalid custom quorum number") + } + customQuorumsUint8[i] = uint8(q) + } + return &Config{ + Config: *clients.NewConfig( + ctx.GlobalString(flags.HostnameFlag.Name), + ctx.GlobalString(flags.GrpcPortFlag.Name), + ctx.Duration(flags.TimeoutFlag.Name), + ctx.GlobalBool(flags.UseSecureGrpcFlag.Name), + ), + NumInstances: ctx.GlobalUint(flags.NumInstancesFlag.Name), + RequestInterval: ctx.Duration(flags.RequestIntervalFlag.Name), + DataSize: ctx.GlobalUint64(flags.DataSizeFlag.Name), + LoggingConfig: *loggerConfig, + RandomizeBlobs: ctx.GlobalBool(flags.RandomizeBlobsFlag.Name), + InstanceLaunchInterval: ctx.Duration(flags.InstanceLaunchIntervalFlag.Name), + SignerPrivateKey: ctx.String(flags.SignerPrivateKeyFlag.Name), + CustomQuorums: customQuorumsUint8, + }, nil +} diff --git a/tools/traffic/flags/flags.go b/tools/traffic/flags/flags.go new file mode 100644 index 0000000000..aca2eb11a3 --- /dev/null +++ b/tools/traffic/flags/flags.go @@ -0,0 +1,112 @@ +package flags + +import ( + "time" + + "github.com/Layr-Labs/eigenda/common" + "github.com/urfave/cli" +) + +const ( + FlagPrefix = "traffic-generator" + envPrefix = "TRAFFIC_GENERATOR" +) + +var ( + /* Required Flags */ + + HostnameFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "disperser-hostname"), + Usage: "Hostname at which disperser service is available", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "HOSTNAME"), + } + GrpcPortFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "disperser-port"), + Usage: "Port at which a disperser listens for grpc calls", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "GRPC_PORT"), + } + TimeoutFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "timeout"), + Usage: "Amount of time to wait for GPRC", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "TIMEOUT"), + Value: 10 * time.Second, + } + NumInstancesFlag = cli.UintFlag{ + Name: common.PrefixFlag(FlagPrefix, "num-instances"), + Usage: "Number of generator instances to run in parallel", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "NUM_INSTANCES"), + } + RequestIntervalFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "request-interval"), + Usage: "Duration between requests", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "REQUEST_INTERVAL"), + Value: 30 * time.Second, + } + DataSizeFlag = cli.Uint64Flag{ + Name: common.PrefixFlag(FlagPrefix, "data-size"), + Usage: "Size of the data blob", + Required: true, + EnvVar: common.PrefixEnvVar(envPrefix, "DATA_SIZE"), + } + RandomizeBlobsFlag = cli.BoolFlag{ + Name: common.PrefixFlag(FlagPrefix, "randomize-blobs"), + Usage: "Whether to randomzie blob data", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "RANDOMIZE_BLOBS"), + } + InstanceLaunchIntervalFlag = cli.DurationFlag{ + Name: common.PrefixFlag(FlagPrefix, "instance-launch-interva"), + Usage: "Duration between generator instance launches", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "INSTANCE_LAUNCH_INTERVAL"), + Value: 1 * time.Second, + } + UseSecureGrpcFlag = cli.BoolFlag{ + Name: common.PrefixFlag(FlagPrefix, "use-secure-grpc"), + Usage: "Whether to use secure grpc", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "USE_SECURE_GRPC"), + } + SignerPrivateKeyFlag = cli.StringFlag{ + Name: common.PrefixFlag(FlagPrefix, "signer-private-key-hex"), + Usage: "Private key to use for signing requests", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "SIGNER_PRIVATE_KEY_HEX"), + } + CustomQuorumNumbersFlag = cli.IntSliceFlag{ + Name: common.PrefixFlag(FlagPrefix, "custom-quorum-numbers"), + Usage: "Custom quorum numbers to use for the traffic generator", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "CUSTOM_QUORUM_NUMBERS"), + } +) + +var requiredFlags = []cli.Flag{ + HostnameFlag, + GrpcPortFlag, + NumInstancesFlag, + RequestIntervalFlag, + DataSizeFlag, +} + +var optionalFlags = []cli.Flag{ + TimeoutFlag, + RandomizeBlobsFlag, + InstanceLaunchIntervalFlag, + UseSecureGrpcFlag, + SignerPrivateKeyFlag, + CustomQuorumNumbersFlag, +} + +// Flags contains the list of configuration options available to the binary. +var Flags []cli.Flag + +func init() { + Flags = append(requiredFlags, optionalFlags...) + Flags = append(Flags, common.LoggerCLIFlags(envPrefix, FlagPrefix)...) +} diff --git a/tools/traffic/generator.go b/tools/traffic/generator.go index c903b15af8..2f8731ffd5 100644 --- a/tools/traffic/generator.go +++ b/tools/traffic/generator.go @@ -2,20 +2,8 @@ package traffic import ( "context" - "fmt" - "github.com/Layr-Labs/eigenda/common/geth" - "github.com/Layr-Labs/eigenda/core/auth" - "github.com/Layr-Labs/eigenda/core/eth" - "github.com/Layr-Labs/eigenda/core/thegraph" - "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" - retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" - "github.com/Layr-Labs/eigenda/tools/traffic/config" - "github.com/Layr-Labs/eigenda/tools/traffic/metrics" - "github.com/Layr-Labs/eigenda/tools/traffic/table" - "github.com/Layr-Labs/eigenda/tools/traffic/workers" - "github.com/Layr-Labs/eigensdk-go/logging" - gethcommon "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" + "crypto/rand" + "encoding/hex" "os" "os/signal" "sync" @@ -25,193 +13,108 @@ import ( "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/Layr-Labs/eigensdk-go/logging" ) -// Generator simulates read/write traffic to the DA service. -// -// ┌------------┐ ┌------------┐ -// | writer |-┐ ┌----------------┐ | reader |-┐ -// └------------┘ |-┐ -------> | status tracker | -------> └------------┘ |-┐ -// └------------┘ | └----------------┘ └------------┘ | -// └------------┘ └------------┘ -// -// The traffic generator is built from three principal components: one or more writers -// that write blobs, a statusTracker that polls the dispenser service until blobs are confirmed, -// and one or more readers that read blobs. -// -// When a writer finishes writing a blob, it sends information about that blob to the statusTracker. -// When the statusTracker observes that a blob has been confirmed, it sends information about the blob -// to the readers. The readers only attempt to read blobs that have been confirmed by the statusTracker. -type Generator struct { - ctx *context.Context - cancel *context.CancelFunc - waitGroup *sync.WaitGroup - generatorMetrics metrics.Metrics - logger *logging.Logger - disperserClient clients.DisperserClient - eigenDAClient *clients.EigenDAClient - config *config.Config - - writers []*workers.BlobWriter - statusTracker *workers.BlobStatusTracker - readers []*workers.BlobReader +type TrafficGenerator struct { + Logger logging.Logger + DisperserClient clients.DisperserClient + Config *Config } -func NewTrafficGenerator(config *config.Config) (*Generator, error) { - logger, err := common.NewLogger(config.LoggingConfig) - if err != nil { - return nil, err - } - - var signer core.BlobRequestSigner - if config.EigenDAClientConfig.SignerPrivateKeyHex != "" { - signer = auth.NewLocalBlobRequestSigner(config.EigenDAClientConfig.SignerPrivateKeyHex) - } - - logger2 := log.NewLogger(log.NewTerminalHandler(os.Stderr, true)) - client, err := clients.NewEigenDAClient(logger2, *config.EigenDAClientConfig) +func NewTrafficGenerator(config *Config, signer core.BlobRequestSigner) (*TrafficGenerator, error) { + loggerConfig := common.DefaultLoggerConfig() + logger, err := common.NewLogger(loggerConfig) if err != nil { return nil, err } - ctx, cancel := context.WithCancel(context.Background()) - waitGroup := sync.WaitGroup{} - - generatorMetrics := metrics.NewMetrics(config.MetricsHTTPPort, logger) - - blobTable := table.NewBlobStore() - - unconfirmedKeyChannel := make(chan *workers.UnconfirmedKey, 100) - - disperserClient := clients.NewDisperserClient(config.DisperserClientConfig, signer) - statusVerifier := workers.NewBlobStatusTracker( - &ctx, - &waitGroup, - logger, - &config.WorkerConfig, - unconfirmedKeyChannel, - blobTable, - disperserClient, - generatorMetrics) - - writers := make([]*workers.BlobWriter, 0) - for i := 0; i < int(config.WorkerConfig.NumWriteInstances); i++ { - writer := workers.NewBlobWriter( - &ctx, - &waitGroup, - logger, - &config.WorkerConfig, - disperserClient, - unconfirmedKeyChannel, - generatorMetrics) - writers = append(writers, &writer) - } - - retriever, chainClient := buildRetriever(config) - - readers := make([]*workers.BlobReader, 0) - for i := 0; i < int(config.WorkerConfig.NumReadInstances); i++ { - reader := workers.NewBlobReader( - &ctx, - &waitGroup, - logger, - &config.WorkerConfig, - retriever, - chainClient, - blobTable, - generatorMetrics) - readers = append(readers, &reader) - } - - return &Generator{ - ctx: &ctx, - cancel: &cancel, - waitGroup: &waitGroup, - generatorMetrics: generatorMetrics, - logger: &logger, - disperserClient: clients.NewDisperserClient(config.DisperserClientConfig, signer), - eigenDAClient: client, - config: config, - writers: writers, - statusTracker: &statusVerifier, - readers: readers, + return &TrafficGenerator{ + Logger: logger, + DisperserClient: clients.NewDisperserClient(&config.Config, signer), + Config: config, }, nil } -// buildRetriever creates a retriever client for the traffic generator. -func buildRetriever(config *config.Config) (clients.RetrievalClient, retrivereth.ChainClient) { - loggerConfig := common.DefaultLoggerConfig() - - logger, err := common.NewLogger(loggerConfig) - if err != nil { - panic(fmt.Sprintf("Unable to instantiate logger: %s", err)) - } - - gethClient, err := geth.NewMultiHomingClient(config.RetrievalClientConfig.EthClientConfig, gethcommon.Address{}, logger) - if err != nil { - panic(fmt.Sprintf("Unable to instantiate geth client: %s", err)) - } - - tx, err := eth.NewTransactor( - logger, - gethClient, - config.RetrievalClientConfig.BLSOperatorStateRetrieverAddr, - config.RetrievalClientConfig.EigenDAServiceManagerAddr) - if err != nil { - panic(fmt.Sprintf("Unable to instantiate transactor: %s", err)) +func (g *TrafficGenerator) Run() error { + ctx, cancel := context.WithCancel(context.Background()) + var wg sync.WaitGroup + for i := 0; i < int(g.Config.NumInstances); i++ { + wg.Add(1) + go func() { + defer wg.Done() + _ = g.StartTraffic(ctx) + }() + time.Sleep(g.Config.InstanceLaunchInterval) } + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, syscall.SIGTERM) + <-signals - cs := eth.NewChainState(tx, gethClient) - - chainState := thegraph.MakeIndexedChainState(*config.TheGraphConfig, cs, logger) - - var assignmentCoordinator core.AssignmentCoordinator = &core.StdAssignmentCoordinator{} - - nodeClient := clients.NewNodeClient(config.NodeClientTimeout) + cancel() + wg.Wait() + return nil +} - v, err := verifier.NewVerifier(&config.RetrievalClientConfig.EncoderConfig, true) +func (g *TrafficGenerator) StartTraffic(ctx context.Context) error { + data := make([]byte, g.Config.DataSize) + _, err := rand.Read(data) if err != nil { - panic(fmt.Sprintf("Unable to build statusTracker: %s", err)) + return err } - retriever, err := clients.NewRetrievalClient( - logger, - chainState, - assignmentCoordinator, - nodeClient, - v, - config.RetrievalClientConfig.NumConnections) - - if err != nil { - panic(fmt.Sprintf("Unable to build retriever: %s", err)) + paddedData := codec.ConvertByPaddingEmptyByte(data) + + ticker := time.NewTicker(g.Config.RequestInterval) + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if g.Config.RandomizeBlobs { + _, err := rand.Read(data) + if err != nil { + return err + } + paddedData = codec.ConvertByPaddingEmptyByte(data) + + err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) + if err != nil { + g.Logger.Error("failed to send blob request", "err:", err) + } + paddedData = nil + } else { + err = g.sendRequest(ctx, paddedData[:g.Config.DataSize]) + if err != nil { + g.Logger.Error("failed to send blob request", "err:", err) + } + } + + } } - - chainClient := retrivereth.NewChainClient(gethClient, logger) - - return retriever, chainClient } -// Start instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. -func (generator *Generator) Start() error { - - generator.generatorMetrics.Start() - generator.statusTracker.Start() - - for _, writer := range generator.writers { - writer.Start() - time.Sleep(generator.config.InstanceLaunchInterval) - } - - for _, reader := range generator.readers { - reader.Start() - time.Sleep(generator.config.InstanceLaunchInterval) +func (g *TrafficGenerator) sendRequest(ctx context.Context, data []byte) error { + ctxTimeout, cancel := context.WithTimeout(ctx, g.Config.Timeout) + defer cancel() + + if g.Config.SignerPrivateKey != "" { + blobStatus, key, err := g.DisperserClient.DisperseBlobAuthenticated(ctxTimeout, data, g.Config.CustomQuorums) + if err != nil { + return err + } + + g.Logger.Info("successfully dispersed new blob", "authenticated", true, "key", hex.EncodeToString(key), "status", blobStatus.String()) + return nil + } else { + blobStatus, key, err := g.DisperserClient.DisperseBlob(ctxTimeout, data, g.Config.CustomQuorums) + if err != nil { + return err + } + + g.Logger.Info("successfully dispersed new blob", "authenticated", false, "key", hex.EncodeToString(key), "status", blobStatus.String()) + return nil } - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt, syscall.SIGTERM) - <-signals - - (*generator.cancel)() - generator.waitGroup.Wait() - return nil } diff --git a/tools/traffic/generator_test.go b/tools/traffic/generator_test.go new file mode 100644 index 0000000000..b530ef5bd2 --- /dev/null +++ b/tools/traffic/generator_test.go @@ -0,0 +1,71 @@ +package traffic_test + +import ( + "context" + "testing" + "time" + + "github.com/Layr-Labs/eigenda/api/clients" + clientsmock "github.com/Layr-Labs/eigenda/api/clients/mock" + "github.com/Layr-Labs/eigenda/disperser" + "github.com/Layr-Labs/eigenda/tools/traffic" + "github.com/Layr-Labs/eigensdk-go/logging" + + "github.com/stretchr/testify/mock" +) + +func TestTrafficGenerator(t *testing.T) { + disperserClient := clientsmock.NewMockDisperserClient() + logger := logging.NewNoopLogger() + trafficGenerator := &traffic.TrafficGenerator{ + Logger: logger, + Config: &traffic.Config{ + Config: clients.Config{ + Timeout: 1 * time.Second, + }, + DataSize: 1000_000, + RequestInterval: 2 * time.Second, + }, + DisperserClient: disperserClient, + } + + processing := disperser.Processing + disperserClient.On("DisperseBlob", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&processing, []byte{1}, nil) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + _ = trafficGenerator.StartTraffic(ctx) + }() + time.Sleep(5 * time.Second) + cancel() + disperserClient.AssertNumberOfCalls(t, "DisperseBlob", 2) +} + +func TestTrafficGeneratorAuthenticated(t *testing.T) { + disperserClient := clientsmock.NewMockDisperserClient() + logger := logging.NewNoopLogger() + + trafficGenerator := &traffic.TrafficGenerator{ + Logger: logger, + Config: &traffic.Config{ + Config: clients.Config{ + Timeout: 1 * time.Second, + }, + DataSize: 1000_000, + RequestInterval: 2 * time.Second, + SignerPrivateKey: "Hi", + }, + DisperserClient: disperserClient, + } + + processing := disperser.Processing + disperserClient.On("DisperseBlobAuthenticated", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&processing, []byte{1}, nil) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + _ = trafficGenerator.StartTraffic(ctx) + }() + time.Sleep(5 * time.Second) + cancel() + disperserClient.AssertNumberOfCalls(t, "DisperseBlobAuthenticated", 2) +} diff --git a/tools/traffic/generator_v2.go b/tools/traffic/generator_v2.go new file mode 100644 index 0000000000..cb9165d9ca --- /dev/null +++ b/tools/traffic/generator_v2.go @@ -0,0 +1,217 @@ +package traffic + +import ( + "context" + "fmt" + "github.com/Layr-Labs/eigenda/common/geth" + "github.com/Layr-Labs/eigenda/core/auth" + "github.com/Layr-Labs/eigenda/core/eth" + "github.com/Layr-Labs/eigenda/core/thegraph" + "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" + retrivereth "github.com/Layr-Labs/eigenda/retriever/eth" + "github.com/Layr-Labs/eigenda/tools/traffic/config" + "github.com/Layr-Labs/eigenda/tools/traffic/metrics" + "github.com/Layr-Labs/eigenda/tools/traffic/table" + "github.com/Layr-Labs/eigenda/tools/traffic/workers" + "github.com/Layr-Labs/eigensdk-go/logging" + gethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/Layr-Labs/eigenda/api/clients" + "github.com/Layr-Labs/eigenda/common" + "github.com/Layr-Labs/eigenda/core" +) + +// Generator simulates read/write traffic to the DA service. +// +// ┌------------┐ ┌------------┐ +// | writer |-┐ ┌----------------┐ | reader |-┐ +// └------------┘ |-┐ -------> | status tracker | -------> └------------┘ |-┐ +// └------------┘ | └----------------┘ └------------┘ | +// └------------┘ └------------┘ +// +// The traffic generator is built from three principal components: one or more writers +// that write blobs, a statusTracker that polls the dispenser service until blobs are confirmed, +// and one or more readers that read blobs. +// +// When a writer finishes writing a blob, it sends information about that blob to the statusTracker. +// When the statusTracker observes that a blob has been confirmed, it sends information about the blob +// to the readers. The readers only attempt to read blobs that have been confirmed by the statusTracker. +type Generator struct { + ctx *context.Context + cancel *context.CancelFunc + waitGroup *sync.WaitGroup + generatorMetrics metrics.Metrics + logger *logging.Logger + disperserClient clients.DisperserClient + eigenDAClient *clients.EigenDAClient + config *config.Config + + writers []*workers.BlobWriter + statusTracker *workers.BlobStatusTracker + readers []*workers.BlobReader +} + +func NewTrafficGeneratorV2(config *config.Config) (*Generator, error) { + logger, err := common.NewLogger(config.LoggingConfig) + if err != nil { + return nil, err + } + + var signer core.BlobRequestSigner + if config.EigenDAClientConfig.SignerPrivateKeyHex != "" { + signer = auth.NewLocalBlobRequestSigner(config.EigenDAClientConfig.SignerPrivateKeyHex) + } + + logger2 := log.NewLogger(log.NewTerminalHandler(os.Stderr, true)) + client, err := clients.NewEigenDAClient(logger2, *config.EigenDAClientConfig) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithCancel(context.Background()) + waitGroup := sync.WaitGroup{} + + generatorMetrics := metrics.NewMetrics(config.MetricsHTTPPort, logger) + + blobTable := table.NewBlobStore() + + unconfirmedKeyChannel := make(chan *workers.UnconfirmedKey, 100) + + disperserClient := clients.NewDisperserClient(config.DisperserClientConfig, signer) + statusVerifier := workers.NewBlobStatusTracker( + &ctx, + &waitGroup, + logger, + &config.WorkerConfig, + unconfirmedKeyChannel, + blobTable, + disperserClient, + generatorMetrics) + + writers := make([]*workers.BlobWriter, 0) + for i := 0; i < int(config.WorkerConfig.NumWriteInstances); i++ { + writer := workers.NewBlobWriter( + &ctx, + &waitGroup, + logger, + &config.WorkerConfig, + disperserClient, + unconfirmedKeyChannel, + generatorMetrics) + writers = append(writers, &writer) + } + + retriever, chainClient := buildRetriever(config) + + readers := make([]*workers.BlobReader, 0) + for i := 0; i < int(config.WorkerConfig.NumReadInstances); i++ { + reader := workers.NewBlobReader( + &ctx, + &waitGroup, + logger, + &config.WorkerConfig, + retriever, + chainClient, + blobTable, + generatorMetrics) + readers = append(readers, &reader) + } + + return &Generator{ + ctx: &ctx, + cancel: &cancel, + waitGroup: &waitGroup, + generatorMetrics: generatorMetrics, + logger: &logger, + disperserClient: clients.NewDisperserClient(config.DisperserClientConfig, signer), + eigenDAClient: client, + config: config, + writers: writers, + statusTracker: &statusVerifier, + readers: readers, + }, nil +} + +// buildRetriever creates a retriever client for the traffic generator. +func buildRetriever(config *config.Config) (clients.RetrievalClient, retrivereth.ChainClient) { + loggerConfig := common.DefaultLoggerConfig() + + logger, err := common.NewLogger(loggerConfig) + if err != nil { + panic(fmt.Sprintf("Unable to instantiate logger: %s", err)) + } + + gethClient, err := geth.NewMultiHomingClient(config.RetrievalClientConfig.EthClientConfig, gethcommon.Address{}, logger) + if err != nil { + panic(fmt.Sprintf("Unable to instantiate geth client: %s", err)) + } + + tx, err := eth.NewTransactor( + logger, + gethClient, + config.RetrievalClientConfig.BLSOperatorStateRetrieverAddr, + config.RetrievalClientConfig.EigenDAServiceManagerAddr) + if err != nil { + panic(fmt.Sprintf("Unable to instantiate transactor: %s", err)) + } + + cs := eth.NewChainState(tx, gethClient) + + chainState := thegraph.MakeIndexedChainState(*config.TheGraphConfig, cs, logger) + + var assignmentCoordinator core.AssignmentCoordinator = &core.StdAssignmentCoordinator{} + + nodeClient := clients.NewNodeClient(config.NodeClientTimeout) + + v, err := verifier.NewVerifier(&config.RetrievalClientConfig.EncoderConfig, true) + if err != nil { + panic(fmt.Sprintf("Unable to build statusTracker: %s", err)) + } + + retriever, err := clients.NewRetrievalClient( + logger, + chainState, + assignmentCoordinator, + nodeClient, + v, + config.RetrievalClientConfig.NumConnections) + + if err != nil { + panic(fmt.Sprintf("Unable to build retriever: %s", err)) + } + + chainClient := retrivereth.NewChainClient(gethClient, logger) + + return retriever, chainClient +} + +// Start instantiates goroutines that generate read/write traffic, continues until a SIGTERM is observed. +func (generator *Generator) Start() error { + + generator.generatorMetrics.Start() + generator.statusTracker.Start() + + for _, writer := range generator.writers { + writer.Start() + time.Sleep(generator.config.InstanceLaunchInterval) + } + + for _, reader := range generator.readers { + reader.Start() + time.Sleep(generator.config.InstanceLaunchInterval) + } + + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, syscall.SIGTERM) + <-signals + + (*generator.cancel)() + generator.waitGroup.Wait() + return nil +} From aac8c64bc9587c1e7b5a54ac57621c18c0cb98b3 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 3 Oct 2024 15:03:04 -0500 Subject: [PATCH 73/74] Cleanup. Signed-off-by: Cody Littley --- tools/traffic/table/blob_store_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/traffic/table/blob_store_test.go b/tools/traffic/table/blob_store_test.go index 2d1114a9ec..99c9c91a02 100644 --- a/tools/traffic/table/blob_store_test.go +++ b/tools/traffic/table/blob_store_test.go @@ -10,12 +10,10 @@ import ( // randomMetadata generates a random BlobMetadata instance. func randomMetadata(t *testing.T, permits int) *BlobMetadata { key := make([]byte, 32) - batchHeaderHash := [32]byte{} checksum := [16]byte{} _, _ = rand.Read(key) _, _ = rand.Read(checksum[:]) - _, _ = rand.Read(batchHeaderHash[:]) - + batchHeaderHash := [32]byte{} metadata, err := NewBlobMetadata(key, checksum, 1024, 0, batchHeaderHash, permits) assert.Nil(t, err) From b2b4e63d03736736e6babbf86e623507b91e1a81 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 3 Oct 2024 15:09:04 -0500 Subject: [PATCH 74/74] Revert unecessary change. Signed-off-by: Cody Littley --- tools/traffic/cmd2/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/traffic/cmd2/main.go b/tools/traffic/cmd2/main.go index 8ad708339f..51b41f56bc 100644 --- a/tools/traffic/cmd2/main.go +++ b/tools/traffic/cmd2/main.go @@ -35,7 +35,7 @@ func trafficGeneratorMain(ctx *cli.Context) error { return err } - generator, err := traffic.NewTrafficGenerator(generatorConfig) + generator, err := traffic.NewTrafficGeneratorV2(generatorConfig) if err != nil { panic(fmt.Sprintf("failed to create new traffic generator\n%s", err)) }