From ca5f699467a8ef958167c56907ea60372bff0e8e Mon Sep 17 00:00:00 2001 From: Cody Littley <56973212+cody-littley@users.noreply.github.com> Date: Thu, 19 Dec 2024 08:47:36 -0600 Subject: [PATCH] Sign get chunks (#1022) Signed-off-by: Cody Littley --- api/clients/v2/relay_client.go | 66 +++++++++++-- node/node.go | 4 + node/node_v2_test.go | 9 +- relay/mock/ics.go | 68 +++++++++++++ relay/server_test.go | 173 +++++++++++++++++++++++++++------ 5 files changed, 283 insertions(+), 37 deletions(-) create mode 100644 relay/mock/ics.go diff --git a/api/clients/v2/relay_client.go b/api/clients/v2/relay_client.go index f43d747ab8..d6be360643 100644 --- a/api/clients/v2/relay_client.go +++ b/api/clients/v2/relay_client.go @@ -2,7 +2,10 @@ package clients import ( "context" + "errors" "fmt" + "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/relay/auth" "sync" relaygrpc "github.com/Layr-Labs/eigenda/api/grpc/relay" @@ -12,9 +15,14 @@ import ( "google.golang.org/grpc" ) +// MessageSigner is a function that signs a message with a private BLS key. +type MessageSigner func(ctx context.Context, data [32]byte) (*core.Signature, error) + type RelayClientConfig struct { Sockets map[corev2.RelayKey]string UseSecureGrpcFlag bool + OperatorID *core.OperatorID + MessageSigner MessageSigner } type ChunkRequestByRange struct { @@ -62,7 +70,7 @@ var _ RelayClient = (*relayClient)(nil) // NewRelayClient creates a new RelayClient that connects to the relays specified in the config. // It keeps a connection to each relay and reuses it for subsequent requests, and the connection is lazily instantiated. -func NewRelayClient(config *RelayClientConfig, logger logging.Logger) (*relayClient, error) { +func NewRelayClient(config *RelayClientConfig, logger logging.Logger) (RelayClient, error) { if config == nil || len(config.Sockets) <= 0 { return nil, fmt.Errorf("invalid config: %v", config) } @@ -97,7 +105,32 @@ func (c *relayClient) GetBlob(ctx context.Context, relayKey corev2.RelayKey, blo return res.GetBlob(), nil } -func (c *relayClient) GetChunksByRange(ctx context.Context, relayKey corev2.RelayKey, requests []*ChunkRequestByRange) ([][]byte, error) { +// signGetChunksRequest signs the GetChunksRequest with the operator's private key +// and sets the signature in the request. +func (c *relayClient) signGetChunksRequest(ctx context.Context, request *relaygrpc.GetChunksRequest) error { + if c.config.OperatorID == nil { + return errors.New("no operator ID provided in config, cannot sign get chunks request") + } + if c.config.MessageSigner == nil { + return errors.New("no message signer provided in config, cannot sign get chunks request") + } + + hash := auth.HashGetChunksRequest(request) + hashArray := [32]byte{} + copy(hashArray[:], hash) + signature, err := c.config.MessageSigner(ctx, hashArray) + if err != nil { + return fmt.Errorf("failed to sign get chunks request: %v", err) + } + request.OperatorSignature = signature.Serialize() + return nil +} + +func (c *relayClient) GetChunksByRange( + ctx context.Context, + relayKey corev2.RelayKey, + requests []*ChunkRequestByRange) ([][]byte, error) { + if len(requests) == 0 { return nil, fmt.Errorf("no requests") } @@ -118,10 +151,17 @@ func (c *relayClient) GetChunksByRange(ctx context.Context, relayKey corev2.Rela }, } } - res, err := client.GetChunks(ctx, &relaygrpc.GetChunksRequest{ + + request := &relaygrpc.GetChunksRequest{ ChunkRequests: grpcRequests, - }) + OperatorId: c.config.OperatorID[:], + } + err = c.signGetChunksRequest(ctx, request) + if err != nil { + return nil, err + } + res, err := client.GetChunks(ctx, request) if err != nil { return nil, err } @@ -129,7 +169,11 @@ func (c *relayClient) GetChunksByRange(ctx context.Context, relayKey corev2.Rela return res.GetData(), nil } -func (c *relayClient) GetChunksByIndex(ctx context.Context, relayKey corev2.RelayKey, requests []*ChunkRequestByIndex) ([][]byte, error) { +func (c *relayClient) GetChunksByIndex( + ctx context.Context, + relayKey corev2.RelayKey, + requests []*ChunkRequestByIndex) ([][]byte, error) { + if len(requests) == 0 { return nil, fmt.Errorf("no requests") } @@ -150,9 +194,17 @@ func (c *relayClient) GetChunksByIndex(ctx context.Context, relayKey corev2.Rela }, } } - res, err := client.GetChunks(ctx, &relaygrpc.GetChunksRequest{ + + request := &relaygrpc.GetChunksRequest{ ChunkRequests: grpcRequests, - }) + OperatorId: c.config.OperatorID[:], + } + err = c.signGetChunksRequest(ctx, request) + if err != nil { + return nil, err + } + + res, err := client.GetChunks(ctx, request) if err != nil { return nil, err diff --git a/node/node.go b/node/node.go index e4652fccfc..6ae337e4b0 100644 --- a/node/node.go +++ b/node/node.go @@ -280,6 +280,8 @@ func NewNode( relayClient, err = clients.NewRelayClient(&clients.RelayClientConfig{ Sockets: relayURLs, UseSecureGrpcFlag: config.UseSecureGrpc, + OperatorID: &config.ID, + MessageSigner: n.SignMessage, }, logger) if err != nil { @@ -460,6 +462,8 @@ func (n *Node) RefreshOnchainState(ctx context.Context) error { relayClient, err := clients.NewRelayClient(&clients.RelayClientConfig{ Sockets: relayURLs, UseSecureGrpcFlag: n.Config.UseSecureGrpc, + OperatorID: &n.Config.ID, + MessageSigner: n.SignMessage, }, n.Logger) if err != nil { n.Logger.Error("error creating relay client", "err", err) diff --git a/node/node_v2_test.go b/node/node_v2_test.go index bb26694138..7bace8e57c 100644 --- a/node/node_v2_test.go +++ b/node/node_v2_test.go @@ -181,8 +181,15 @@ func TestRefreshOnchainStateSuccess(t *testing.T) { relayURLs := map[v2.RelayKey]string{ 0: "http://localhost:8080", } + + messageSigner := func(ctx context.Context, data [32]byte) (*core.Signature, error) { + return nil, nil + } + relayClient, err := clients.NewRelayClient(&clients.RelayClientConfig{ - Sockets: relayURLs, + Sockets: relayURLs, + OperatorID: &c.node.Config.ID, + MessageSigner: messageSigner, }, c.node.Logger) require.NoError(t, err) // set up non-mock client diff --git a/relay/mock/ics.go b/relay/mock/ics.go new file mode 100644 index 0000000000..d72bb5c5e9 --- /dev/null +++ b/relay/mock/ics.go @@ -0,0 +1,68 @@ +package mock + +import ( + "context" + "github.com/Layr-Labs/eigenda/core" + "github.com/stretchr/testify/mock" +) + +var _ core.IndexedChainState = (*IndexedChainState)(nil) + +// IndexedChainState is a mock implementation of core.IndexedChainState. +type IndexedChainState struct { + Mock mock.Mock +} + +func (m *IndexedChainState) GetCurrentBlockNumber() (uint, error) { + args := m.Mock.Called() + return args.Get(0).(uint), args.Error(1) +} + +func (m *IndexedChainState) GetOperatorState( + ctx context.Context, + blockNumber uint, + quorums []core.QuorumID) (*core.OperatorState, error) { + + args := m.Mock.Called(blockNumber, quorums) + return args.Get(0).(*core.OperatorState), args.Error(1) +} + +func (m *IndexedChainState) GetOperatorStateByOperator( + ctx context.Context, + blockNumber uint, + operator core.OperatorID) (*core.OperatorState, error) { + + args := m.Mock.Called(blockNumber, operator) + return args.Get(0).(*core.OperatorState), args.Error(1) +} + +func (m *IndexedChainState) GetOperatorSocket( + ctx context.Context, + blockNumber uint, + operator core.OperatorID) (string, error) { + + args := m.Mock.Called(blockNumber, operator) + return args.Get(0).(string), args.Error(1) +} + +func (m *IndexedChainState) GetIndexedOperatorState( + ctx context.Context, + blockNumber uint, + quorums []core.QuorumID) (*core.IndexedOperatorState, error) { + + args := m.Mock.Called(blockNumber, quorums) + return args.Get(0).(*core.IndexedOperatorState), args.Error(1) +} + +func (m *IndexedChainState) GetIndexedOperators( + ctx context.Context, + blockNumber uint) (map[core.OperatorID]*core.IndexedOperatorInfo, error) { + + args := m.Mock.Called(blockNumber) + return args.Get(0).(map[core.OperatorID]*core.IndexedOperatorInfo), args.Error(1) +} + +func (m *IndexedChainState) Start(context context.Context) error { + args := m.Mock.Called() + return args.Error(0) +} diff --git a/relay/server_test.go b/relay/server_test.go index 801ebeef87..94adcba01d 100644 --- a/relay/server_test.go +++ b/relay/server_test.go @@ -2,7 +2,10 @@ package relay import ( "context" - "math/rand" + "encoding/binary" + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/Layr-Labs/eigenda/relay/auth" + "github.com/Layr-Labs/eigenda/relay/mock" "testing" "time" @@ -30,6 +33,8 @@ func defaultConfig() *Config { ChunkCacheSize: 1024 * 1024, ChunkMaxConcurrency: 32, MaxKeysPerGetChunksRequest: 1024, + AuthenticationKeyCacheSize: 1024, + AuthenticationDisabled: false, RateLimits: limiter.Config{ MaxGetBlobOpsPerSecond: 1024, GetBlobOpsBurstiness: 1024, @@ -47,7 +52,6 @@ func defaultConfig() *Config { GetChunkBytesBurstinessClient: 2 * 1024 * 1024, MaxConcurrentGetChunkOpsClient: 1, }, - AuthenticationDisabled: true, Timeouts: TimeoutConfig{ GetBlobTimeout: 10 * time.Second, GetChunksTimeout: 10 * time.Second, @@ -76,7 +80,20 @@ func getBlob(t *testing.T, request *pb.GetBlobRequest) (*pb.GetBlobReply, error) return response, err } -func getChunks(t *testing.T, request *pb.GetChunksRequest) (*pb.GetChunksReply, error) { +func getChunks( + t *testing.T, + random *random.TestRandom, + operatorKeys map[uint32]*core.KeyPair, + request *pb.GetChunksRequest) (*pb.GetChunksReply, error) { + + // Choose a random operator to send this request as. Operator IDs are expected to be sequential starting at 0. + operatorID := random.Uint32() % uint32(len(operatorKeys)) + operatorIDBytes := make([]byte, 32) + binary.BigEndian.PutUint32(operatorIDBytes[24:], operatorID) + request.OperatorId = operatorIDBytes + signature := auth.SignGetChunksRequest(operatorKeys[operatorID], request) + request.OperatorSignature = signature + var opts []grpc.DialOption opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) @@ -93,7 +110,7 @@ func getChunks(t *testing.T, request *pb.GetChunksRequest) (*pb.GetChunksReply, } func TestReadWriteBlobs(t *testing.T) { - tu.InitializeRandom() + rand := random.NewTestRandom(t) logger, err := common.NewLogger(common.DefaultLoggerConfig()) require.NoError(t, err) @@ -106,6 +123,12 @@ func TestReadWriteBlobs(t *testing.T) { blobStore := buildBlobStore(t, logger) chainReader := newMockChainReader() + ics := &mock.IndexedChainState{} + blockNumber := uint(rand.Uint32()) + ics.Mock.On("GetCurrentBlockNumber").Return(blockNumber, nil) + operatorInfo := make(map[core.OperatorID]*core.IndexedOperatorInfo) + ics.Mock.On("GetIndexedOperators", blockNumber).Return(operatorInfo, nil) + // This is the server used to read it back config := defaultConfig() server, err := NewServer( @@ -116,7 +139,7 @@ func TestReadWriteBlobs(t *testing.T) { blobStore, nil, /* not used in this test*/ chainReader, - nil /* not used in this test*/) + ics) require.NoError(t, err) go func() { @@ -176,7 +199,7 @@ func TestReadWriteBlobs(t *testing.T) { } func TestReadNonExistentBlob(t *testing.T) { - tu.InitializeRandom() + rand := random.NewTestRandom(t) logger, err := common.NewLogger(common.DefaultLoggerConfig()) require.NoError(t, err) @@ -188,6 +211,12 @@ func TestReadNonExistentBlob(t *testing.T) { metadataStore := buildMetadataStore(t) blobStore := buildBlobStore(t, logger) + ics := &mock.IndexedChainState{} + blockNumber := uint(rand.Uint32()) + ics.Mock.On("GetCurrentBlockNumber").Return(blockNumber, nil) + operatorInfo := make(map[core.OperatorID]*core.IndexedOperatorInfo) + ics.Mock.On("GetIndexedOperators", blockNumber).Return(operatorInfo, nil) + // This is the server used to read it back config := defaultConfig() chainReader := newMockChainReader() @@ -199,7 +228,7 @@ func TestReadNonExistentBlob(t *testing.T) { blobStore, nil, /* not used in this test */ chainReader, - nil /* not used in this test*/) + ics) require.NoError(t, err) go func() { @@ -223,7 +252,7 @@ func TestReadNonExistentBlob(t *testing.T) { } func TestReadWriteBlobsWithSharding(t *testing.T) { - tu.InitializeRandom() + rand := random.NewTestRandom(t) logger, err := common.NewLogger(common.DefaultLoggerConfig()) require.NoError(t, err) @@ -245,6 +274,12 @@ func TestReadWriteBlobsWithSharding(t *testing.T) { } } + ics := &mock.IndexedChainState{} + blockNumber := uint(rand.Uint32()) + ics.Mock.On("GetCurrentBlockNumber").Return(blockNumber, nil) + operatorInfo := make(map[core.OperatorID]*core.IndexedOperatorInfo) + ics.Mock.On("GetIndexedOperators", blockNumber).Return(operatorInfo, nil) + // This is the server used to read it back config := defaultConfig() config.RelayIDs = shardList @@ -257,7 +292,7 @@ func TestReadWriteBlobsWithSharding(t *testing.T) { blobStore, nil, /* not used in this test*/ chainReader, - nil /* not used in this test*/) + ics) require.NoError(t, err) go func() { @@ -353,7 +388,7 @@ func TestReadWriteBlobsWithSharding(t *testing.T) { } func TestReadWriteChunks(t *testing.T) { - tu.InitializeRandom() + rand := random.NewTestRandom(t) logger, err := common.NewLogger(common.DefaultLoggerConfig()) require.NoError(t, err) @@ -365,6 +400,26 @@ func TestReadWriteChunks(t *testing.T) { metadataStore := buildMetadataStore(t) chunkReader, chunkWriter := buildChunkStore(t, logger) + operatorCount := rand.Intn(3) + 1 + operatorKeys := make(map[uint32]*core.KeyPair) + operatorInfo := make(map[core.OperatorID]*core.IndexedOperatorInfo) + for i := 0; i < operatorCount; i++ { + keypair := rand.BLS() + operatorKeys[uint32(i)] = keypair + + var operatorID core.OperatorID + binary.BigEndian.PutUint32(operatorID[24:], uint32(i)) + operatorInfo[operatorID] = &core.IndexedOperatorInfo{ + PubkeyG1: keypair.GetPubKeyG1(), + PubkeyG2: keypair.GetPubKeyG2(), + } + } + + ics := &mock.IndexedChainState{} + blockNumber := uint(rand.Uint32()) + ics.Mock.On("GetCurrentBlockNumber").Return(blockNumber, nil) + ics.Mock.On("GetIndexedOperators", blockNumber).Return(operatorInfo, nil) + // This is the server used to read it back config := defaultConfig() config.RateLimits.MaxGetChunkOpsPerSecond = 1000 @@ -380,7 +435,7 @@ func TestReadWriteChunks(t *testing.T) { nil, /* not used in this test*/ chunkReader, chainReader, - nil /* not used in this test*/) + ics) require.NoError(t, err) go func() { @@ -438,7 +493,7 @@ func TestReadWriteChunks(t *testing.T) { ChunkRequests: requestedChunks, } - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) require.NoError(t, err) require.Equal(t, 1, len(response.Data)) @@ -472,7 +527,7 @@ func TestReadWriteChunks(t *testing.T) { ChunkRequests: requestedChunks, } - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) require.NoError(t, err) require.Equal(t, 1, len(response.Data)) @@ -505,7 +560,7 @@ func TestReadWriteChunks(t *testing.T) { ChunkRequests: requestedChunks, } - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) require.NoError(t, err) require.Equal(t, 1, len(response.Data)) @@ -541,7 +596,7 @@ func TestReadWriteChunks(t *testing.T) { ChunkRequests: requestedChunks, } - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) require.NoError(t, err) require.Equal(t, 1, len(response.Data)) @@ -558,7 +613,7 @@ func TestReadWriteChunks(t *testing.T) { } func TestBatchedReadWriteChunks(t *testing.T) { - tu.InitializeRandom() + rand := random.NewTestRandom(t) logger, err := common.NewLogger(common.DefaultLoggerConfig()) require.NoError(t, err) @@ -570,6 +625,26 @@ func TestBatchedReadWriteChunks(t *testing.T) { metadataStore := buildMetadataStore(t) chunkReader, chunkWriter := buildChunkStore(t, logger) + operatorCount := rand.Intn(3) + 1 + operatorKeys := make(map[uint32]*core.KeyPair) + operatorInfo := make(map[core.OperatorID]*core.IndexedOperatorInfo) + for i := 0; i < operatorCount; i++ { + keypair := rand.BLS() + operatorKeys[uint32(i)] = keypair + + var operatorID core.OperatorID + binary.BigEndian.PutUint32(operatorID[24:], uint32(i)) + operatorInfo[operatorID] = &core.IndexedOperatorInfo{ + PubkeyG1: keypair.GetPubKeyG1(), + PubkeyG2: keypair.GetPubKeyG2(), + } + } + + ics := &mock.IndexedChainState{} + blockNumber := uint(rand.Uint32()) + ics.Mock.On("GetCurrentBlockNumber").Return(blockNumber, nil) + ics.Mock.On("GetIndexedOperators", blockNumber).Return(operatorInfo, nil) + // This is the server used to read it back config := defaultConfig() chainReader := newMockChainReader() @@ -581,7 +656,7 @@ func TestBatchedReadWriteChunks(t *testing.T) { nil, /* not used in this test */ chunkReader, chainReader, - nil /* not used in this test*/) + ics) require.NoError(t, err) go func() { @@ -654,7 +729,7 @@ func TestBatchedReadWriteChunks(t *testing.T) { ChunkRequests: requestedChunks, } - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) require.NoError(t, err) require.Equal(t, keyCount, len(response.Data)) @@ -673,7 +748,7 @@ func TestBatchedReadWriteChunks(t *testing.T) { } func TestReadWriteChunksWithSharding(t *testing.T) { - tu.InitializeRandom() + rand := random.NewTestRandom(t) logger, err := common.NewLogger(common.DefaultLoggerConfig()) require.NoError(t, err) @@ -696,6 +771,26 @@ func TestReadWriteChunksWithSharding(t *testing.T) { } shardMap := make(map[v2.BlobKey][]v2.RelayKey) + operatorCount := rand.Intn(3) + 1 + operatorKeys := make(map[uint32]*core.KeyPair) + operatorInfo := make(map[core.OperatorID]*core.IndexedOperatorInfo) + for i := 0; i < operatorCount; i++ { + keypair := rand.BLS() + operatorKeys[uint32(i)] = keypair + + var operatorID core.OperatorID + binary.BigEndian.PutUint32(operatorID[24:], uint32(i)) + operatorInfo[operatorID] = &core.IndexedOperatorInfo{ + PubkeyG1: keypair.GetPubKeyG1(), + PubkeyG2: keypair.GetPubKeyG2(), + } + } + + ics := &mock.IndexedChainState{} + blockNumber := uint(rand.Uint32()) + ics.Mock.On("GetCurrentBlockNumber").Return(blockNumber, nil) + ics.Mock.On("GetIndexedOperators", blockNumber).Return(operatorInfo, nil) + // This is the server used to read it back config := defaultConfig() config.RelayIDs = shardList @@ -712,7 +807,7 @@ func TestReadWriteChunksWithSharding(t *testing.T) { nil, /* not used in this test*/ chunkReader, chainReader, - nil /* not used in this test*/) + ics) require.NoError(t, err) go func() { @@ -786,7 +881,7 @@ func TestReadWriteChunksWithSharding(t *testing.T) { } } - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) if isBlobInCorrectShard { require.NoError(t, err) @@ -836,7 +931,7 @@ func TestReadWriteChunksWithSharding(t *testing.T) { } if isBlobInCorrectShard { - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) require.NoError(t, err) require.Equal(t, 1, len(response.Data)) @@ -848,7 +943,7 @@ func TestReadWriteChunksWithSharding(t *testing.T) { require.Equal(t, data[i], frame) } } else { - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) require.Error(t, err) require.Nil(t, response) } @@ -884,7 +979,7 @@ func TestReadWriteChunksWithSharding(t *testing.T) { } if isBlobInCorrectShard { - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) require.NoError(t, err) require.Equal(t, 1, len(response.Data)) @@ -931,7 +1026,7 @@ func TestReadWriteChunksWithSharding(t *testing.T) { } if isBlobInCorrectShard { - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) require.NoError(t, err) require.Equal(t, 1, len(response.Data)) @@ -945,7 +1040,7 @@ func TestReadWriteChunksWithSharding(t *testing.T) { } } } else { - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) require.Error(t, err) require.Nil(t, response) } @@ -953,7 +1048,7 @@ func TestReadWriteChunksWithSharding(t *testing.T) { } func TestBatchedReadWriteChunksWithSharding(t *testing.T) { - tu.InitializeRandom() + rand := random.NewTestRandom(t) logger, err := common.NewLogger(common.DefaultLoggerConfig()) require.NoError(t, err) @@ -976,6 +1071,26 @@ func TestBatchedReadWriteChunksWithSharding(t *testing.T) { } shardMap := make(map[v2.BlobKey][]v2.RelayKey) + operatorCount := rand.Intn(3) + 1 + operatorKeys := make(map[uint32]*core.KeyPair) + operatorInfo := make(map[core.OperatorID]*core.IndexedOperatorInfo) + for i := 0; i < operatorCount; i++ { + keypair := rand.BLS() + operatorKeys[uint32(i)] = keypair + + var operatorID core.OperatorID + binary.BigEndian.PutUint32(operatorID[24:], uint32(i)) + operatorInfo[operatorID] = &core.IndexedOperatorInfo{ + PubkeyG1: keypair.GetPubKeyG1(), + PubkeyG2: keypair.GetPubKeyG2(), + } + } + + ics := &mock.IndexedChainState{} + blockNumber := uint(rand.Uint32()) + ics.Mock.On("GetCurrentBlockNumber").Return(blockNumber, nil) + ics.Mock.On("GetIndexedOperators", blockNumber).Return(operatorInfo, nil) + // This is the server used to read it back config := defaultConfig() config.RelayIDs = shardList @@ -992,7 +1107,7 @@ func TestBatchedReadWriteChunksWithSharding(t *testing.T) { nil, /* not used in this test */ chunkReader, chainReader, - nil /* not used in this test*/) + ics) require.NoError(t, err) go func() { @@ -1099,7 +1214,7 @@ func TestBatchedReadWriteChunksWithSharding(t *testing.T) { } } - response, err := getChunks(t, request) + response, err := getChunks(t, rand, operatorKeys, request) if allInCorrectShard { require.NoError(t, err)