DAS protobuffs

api/proto/disperser/disperser.proto

@@ -31,7 +31,19 @@ service Disperser {
 	// api/proto/retriever/retriever.proto).
 	// The blob should have been initially dispersed via this Disperser service
 	// for this API to work.
-	rpc RetrieveBlob(RetrieveBlobRequest) returns (RetrieveBlobReply) {}
+	rpc GetBlob(GetBlobRequest) returns (GetBlobReply) {}
+
+	// Retrieves the requested chunk from the Disperser's backend.
+	rpc GetChunk(GetChunkRequest) returns (GetChunkReply) {}
+
+	/////////////////////////////////
+	// Deprecated RPCs, do not use //
+	/////////////////////////////////
+
+	// Deprecated: use GetBlob instead.
+	rpc RetrieveBlob(RetrieveBlobRequest) returns (RetrieveBlobReply) {
+		option deprecated = true;
+	}
 }
 
 // Requests and Responses
@@ -115,17 +127,39 @@ message BlobStatusReply {
 	BlobInfo info = 2;
 }
 
-// RetrieveBlobRequest contains parameters to retrieve the blob.
-message RetrieveBlobRequest {
+// GetBlobRequest contains parameters to retrieve the blob.
+message GetBlobRequest {
 	bytes batch_header_hash = 1;
 	uint32 blob_index = 2;
 }
 
-// RetrieveBlobReply contains the retrieved blob data
-message RetrieveBlobReply {
+// GetBlobReply contains the retrieved blob data
+message GetBlobReply {
 	bytes data = 1;
 }
 
+// TODO this is identical to the node.proto definition. Should we share this definition?
+// A request to retrieve a specific chunk.
+message GetChunkRequest {
+	// The hash of the blob's ReducedBatchHeader defined on-chain.
+	bytes batch_header_hash = 1;
+	// Which blob in the batch to retrieve for (note: a batch is logically an ordered
+	// list of blobs).
+	uint32 blob_index = 2;
+	// Which quorum of the blob to retrieve for (note: a blob can have multiple
+	// quorums and the chunks for different quorums at a Node can be different).
+	// The ID must be in range [0, 254].
+	uint32 quorum_id = 3;
+	// Which chunk in the blob to retrieve.
+	uint32 chunk_index = 4;
+}
+
+// A reply to GetChunkRequest.
+message GetChunkReply {
+	// The chunk requested.
+	bytes chunk = 1;
+}
+
 // Data Types
 
 // BlobStatus represents the status of a blob.
@@ -250,3 +284,20 @@ message BatchHeader {
 	// (e.g. operator stakes) at this block number.
 	uint32 reference_block_number = 4;
 }
+
+/////////////////////////////////////
+// Deprecated messages, do not use //
+/////////////////////////////////////
+
+// Deprecated: use GetBlobRequest instead.
+message RetrieveBlobRequest {
+	option deprecated = true;
+	bytes batch_header_hash = 1;
+	uint32 blob_index = 2;
+}
+
+// Deprecated: use GetBlobReply instead.
+message RetrieveBlobReply {
+	option deprecated = true;
+	bytes data = 1;
+}

api/proto/lightnode/lightnode.proto

@@ -0,0 +1,65 @@
+syntax = "proto3";
+package lightnode;
+option go_package = "github.com/Layr-Labs/eigenda/api/grpc/lightnode";
+
+service LightNode {
+	// GetChunk retrieves a specific chunk held by the light node.
+	rpc GetChunk(GetChunkRequest) returns (GetChunkReply) {}
+
+	// StreamAvailabilityStatus streams the availability status of all chunks assigned to the light node.
+	// For use by a DA node for monitoring the availability of chunks through its constellation of agent light nodes.
+	rpc StreamAvailabilityStatus(StreamAvailabilityStatusRequest) returns (stream SampledChunk) {}
+}
+
+// TODO these message types are duplicated from node.proto, perhaps they should share a common definition
+
+// A request to retrieve a specific chunk.
+message GetChunkRequest {
+	// The hash of the blob's ReducedBatchHeader defined on-chain.
+	bytes batch_header_hash = 1;
+	// Which blob in the batch to retrieve for (note: a batch is logically an ordered
+	// list of blobs).
+	uint32 blob_index = 2;
+	// Which quorum of the blob to retrieve for (note: a blob can have multiple
+	// quorums and the chunks for different quorums at a Node can be different).
+	// The ID must be in range [0, 254].
+	uint32 quorum_id = 3;
+	// Which chunk in the blob to retrieve.
+	uint32 chunk_index = 4;
+}
+
+// This describes how the chunks returned in GetChunkReply is encoded.
+enum ChunkEncodingFormat {
+	UNKNOWN = 0;
+	GNARK = 1;
+	GOB = 2;
+}
+
+// A reply to RetrieveChunkRequest.
+message GetChunkReply {
+	// The chunk requested per RetrieveChunkRequest.
+	bytes chunk = 1;
+	// The encoding format of the chunk.
+	ChunkEncodingFormat chunk_encoding_format = 2;
+}
+
+// A request from a DA node to an agent light node to stream the availability status of all chunks
+// assigned to the light node.
+message StreamAvailabilityStatusRequest {
+	// TODO describe protocol for authentication
+	bytes authentication_token = 1;
+}
+
+// For every chunk verified by an agent light node, the agent light node sends a message to its parent DA node
+// to indicate that the chunk is available.
+message SampledChunk {
+	// The hash of the blob's ReducedBatchHeader defined on-chain.
+	bytes batch_header_hash = 1;
+	// The index of a blob within its batch (note: a batch is logically an ordered
+	// list of blobs).
+	uint32 blob_index = 2;
+	// The quorum ID of the blob.
+	uint32 quorum_id = 3;
+	// Which chunk in the blob this light node is responsible for sampling.
+	uint32 chunk_index = 4;
+}

api/proto/node/node.proto

@@ -27,12 +27,32 @@ service Dispersal {
 }
 
 service Retrieval {
-	// RetrieveChunks retrieves the chunks for a blob custodied at the Node.
-	rpc RetrieveChunks(RetrieveChunksRequest) returns (RetrieveChunksReply) {}
+	// GetChunks retrieves all of the chunks for a blob held by the node.
+	rpc GetChunks(GetChunksRequest) returns (GetChunksReply) {}
+	// GetChunk retrieves a specific chunk for a blob custodied at the Node.
+	rpc GetChunk(GetChunkRequest) returns (GetChunkReply) {}
+	// StreamChunks requests a stream of chunks with a specific index.
+	rpc StreamChunks(stream StreamChunksRequest) returns (stream GetChunkReply) {}
 	// GetBlobHeader is similar to RetrieveChunks, this just returns the header of the blob.
 	rpc GetBlobHeader(GetBlobHeaderRequest) returns (GetBlobHeaderReply) {}
-	// Retrieve node info metadata
-	rpc NodeInfo(NodeInfoRequest) returns (NodeInfoReply) {}
+	// StreamBlobHeaders streams the blob headers to the client.
+	rpc StreamBlobHeaders(StreamBlobHeadersRequest) returns (stream GetBlobHeaderReply) {}
+	// Retrieve node info metadata.
+	rpc GetNodeInfo(GetNodeInfoRequest) returns (GetNodeInfoReply) {}
+
+	/////////////////////////////////
+	// Deprecated RPCs, do not use //
+	/////////////////////////////////
+
+	// Deprecated: Use GetChunks instead.
+	rpc RetrieveChunks(RetrieveChunksRequest) returns (RetrieveChunksReply) {
+		option deprecated = true;
+	}
+
+	// Deprecated: Use GetNodeInfo instead.
+	rpc NodeInfo(NodeInfoRequest) returns (NodeInfoReply) {
+		option deprecated = true;
+	}
 }
 
 // Requests and replies
@@ -74,8 +94,17 @@ message AttestBatchReply {
 	bytes signature = 1;
 }
 
-message RetrieveChunksRequest {
-	// The hash of the ReducedBatchHeader defined onchain, see:
+// This describes how the chunks returned in GetChunksReply are encoded.
+// Used to facilitate the decoding of chunks.
+enum ChunkEncodingFormat {
+	UNKNOWN = 0;
+	GNARK = 1;
+	GOB = 2;
+}
+
+// Request all of the chunks for a blob held by the node.
+message GetChunksRequest {
+	// The hash of the ReducedBatchHeader defined on-chain, see:
 	// https://github.com/Layr-Labs/eigenda/blob/master/contracts/src/interfaces/IEigenDAServiceManager.sol#L43
 	// This identifies which batch to retrieve for.
 	bytes batch_header_hash = 1;
@@ -88,21 +117,45 @@ message RetrieveChunksRequest {
 	uint32 quorum_id = 3;
 }
 
-// This describes how the chunks returned in RetrieveChunksReply are encoded.
-// Used to facilitate the decoding of chunks.
-enum ChunkEncodingFormat {
-	UNKNOWN = 0;
-	GNARK = 1;
-	GOB = 2;
-}
-
-message RetrieveChunksReply {
+// Reply to GetChunksRequest.
+message GetChunksReply {
 	// All chunks the Node is storing for the requested blob per RetrieveChunksRequest.
 	repeated bytes chunks = 1;
 	// How the above chunks are encoded.
 	ChunkEncodingFormat chunk_encoding_format = 2;
 }
 
+// A request to retrieve a specific chunk.
+message GetChunkRequest {
+	// The hash of the blob's ReducedBatchHeader defined on-chain.
+	bytes batch_header_hash = 1;
+	// Which blob in the batch to retrieve for (note: a batch is logically an ordered
+	// list of blobs).
+	uint32 blob_index = 2;
+	// Which quorum of the blob to retrieve for (note: a blob can have multiple
+	// quorums and the chunks for different quorums at a Node can be different).
+	// The ID must be in range [0, 254].
+	uint32 quorum_id = 3;
+	// Which chunk in the blob to retrieve.
+	uint32 chunk_index = 4;
+}
+
+// A reply to RetrieveChunkRequest.
+message GetChunkReply {
+	// The chunk requested per RetrieveChunkRequest.
+	bytes chunk = 1;
+	// The encoding format of the chunk.
+	ChunkEncodingFormat chunk_encoding_format = 2;
+}
+
+// Request that all new chunks with a particular index be sent to the client.
+message StreamChunksRequest {
+	// The quorum ID of the chunks to stream.
+	uint32 quorum_id = 1;
+	// The chunk index to start streaming from.
+	uint32 chunk_index = 2;
+}
+
 // See RetrieveChunksRequest for documentation of each parameter of GetBlobHeaderRequest.
 message GetBlobHeaderRequest {
 	bytes batch_header_hash = 1;
@@ -119,6 +172,12 @@ message GetBlobHeaderReply {
 	MerkleProof proof = 2;
 }
 
+// Request that all new blob headers be sent to the client.
+message StreamBlobHeadersRequest {
+	// The quorum ID of the headers to stream.
+	uint32 quorum_id = 1;
+}
+
 message MerkleProof {
 	// The proof itself.
 	repeated bytes hashes = 1;
@@ -204,15 +263,61 @@ message BatchHeader {
 	uint32 reference_block_number = 3;
 }
 
-// Node info request
+// Node info request.
+message GetNodeInfoRequest {
+}
+
+// Node info reply.
+message GetNodeInfoReply {
+	string semver = 1;
+	string arch = 2;
+	string os = 3;
+	uint32 num_cpu = 4;
+	uint64 mem_bytes = 5;
+}
+
+/////////////////////////////////////
+// Deprecated messages, do not use //
+/////////////////////////////////////
+
+// Deprecated: Use RetrieveChunkRequest instead.
+message RetrieveChunksRequest {
+	// Use GetChunksRequest instead.
+	option deprecated = true;
+	// The hash of the ReducedBatchHeader defined on-chain, see:
+	// https://github.com/Layr-Labs/eigenda/blob/master/contracts/src/interfaces/IEigenDAServiceManager.sol#L43
+	// This identifies which batch to retrieve for.
+	bytes batch_header_hash = 1;
+	// Which blob in the batch to retrieve for (note: a batch is logically an ordered
+	// list of blobs).
+	uint32 blob_index = 2;
+	// Which quorum of the blob to retrieve for (note: a blob can have multiple
+	// quorums and the chunks for different quorums at a Node can be different).
+	// The ID must be in range [0, 254].
+	uint32 quorum_id = 3;
+}
+
+// Deprecated: Use RetrieveChunkReply instead.
+message RetrieveChunksReply {
+	// Use GetChunksReply instead.
+	option deprecated = true;
+	// All chunks the Node is storing for the requested blob per RetrieveChunksRequest.
+	repeated bytes chunks = 1;
+	// How the above chunks are encoded.
+	ChunkEncodingFormat chunk_encoding_format = 2;
+}
+
+// Deprecated: Use GetNodeInfoRequest instead.
 message NodeInfoRequest {
+	option deprecated = true;
 }
 
-// Node info reply
+// Deprecated: Use NodeInfoReply instead.
 message NodeInfoReply {
+	option deprecated = true;
 	string semver = 1;
 	string arch = 2;
 	string os = 3;
 	uint32 num_cpu = 4;
 	uint64 mem_bytes = 5;
-}
+}