Merge branch 'master' of https://github.com/Layr-Labs/eigenda into ch…

…unkencodingnode

README.md

@@ -10,11 +10,11 @@
 
 EigenDA is a secure, high-throughput, and decentralized data availability (DA) service built on top of Ethereum using the [EigenLayer](https://github.com/Layr-Labs/eigenlayer-contracts) restaking primitives.
 
-To understand more how EigenDA works and how it transforms the modern landscape of data availability, continue reading [EigenDA introduction](https://www.blog.eigenlayer.xyz/intro-to-eigenda-hyperscale-data-availability-for-rollups/).
+To understand more about how EigenDA works and how it transforms the modern landscape of data availability, continue reading [EigenDA introduction](https://www.blog.eigenlayer.xyz/intro-to-eigenda-hyperscale-data-availability-for-rollups/).
 
 To dive deep into the technical details, continue reading [EigenDA protocol spec](https://github.com/Layr-Labs/eigenda/blob/master/docs/spec/overview.md).
 
-If you're interested in integrating your rollup with EigenDA, please fill out the [EigenDA questionnaire](https://docs.google.com/forms/d/e/1FAIpQLSez6PG-BL6C6Mc4QY1M--vbV219OGL_0Euv2zhJ1HmcUiU7cw/viewform).
+If you're interested in integrating your rollup with EigenDA, please fill out the [EigenDA Partner Registration](https://docs.google.com/forms/d/e/1FAIpQLSdXvfxgRfIHWYu90FqN-2yyhgrYm9oExr0jSy7ERzbMUimJew/viewform).
 
 ## API Documentation
 
@@ -38,4 +38,4 @@ We welcome all contributions! There are many ways to contribute to the project,
 - [Open an Issue](https://github.com/Layr-Labs/eigenda/issues/new/choose)
 - [EigenLayer/EigenDA forum](https://forum.eigenlayer.xyz/c/eigenda/9)
 - [Email](mailto:[email protected])
-- [Follow us on Twitter](https://twitter.com/eigenlayer)
+- [Follow us on Twitter](https://twitter.com/eigen_da)

api/clients/node_client.go

@@ -2,6 +2,7 @@ package clients
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"github.com/Layr-Labs/eigenda/api/grpc/node"
@@ -120,7 +121,23 @@ func (c client) GetChunks(
 
 	chunks := make([]*encoding.Frame, len(reply.GetChunks()))
 	for i, data := range reply.GetChunks() {
-		chunk, err := new(encoding.Frame).Deserialize(data)
+		var chunk *encoding.Frame
+		switch reply.GetEncoding() {
+		case node.ChunkEncoding_GNARK:
+			chunk, err = new(encoding.Frame).DeserializeGnark(data)
+		case node.ChunkEncoding_GOB:
+			chunk, err = new(encoding.Frame).Deserialize(data)
+		case node.ChunkEncoding_UNKNOWN:
+			// For backward compatibility, we fallback the UNKNOWN to GNARK
+			chunk, err = new(encoding.Frame).DeserializeGnark(data)
+			if err != nil {
+				chunksChan <- RetrievedChunks{
+					OperatorID: opID,
+					Err:        errors.New("UNKNOWN chunk encoding format"),
+					Chunks:     nil,
+				}
+			}
+		}
 		if err != nil {
 			chunksChan <- RetrievedChunks{
 				OperatorID: opID,

api/docs/disperser.md

@@ -154,7 +154,7 @@ replay attacks in the event that a signature is leaked.
 | Field | Type | Label | Description |
 | ----- | ---- | ----- | ----------- |
 | commitment | [common.G1Commitment](#common-G1Commitment) |  | KZG commitment of the blob. |
-| data_length | [uint32](#uint32) |  | The length of the blob in symbols (each symbol is 31 bytes). |
+| data_length | [uint32](#uint32) |  | The length of the blob in symbols (each symbol is 32 bytes). |
 | blob_quorum_params | [BlobQuorumParam](#disperser-BlobQuorumParam) | repeated | The params of the quorums that this blob participates in. |
 
 
@@ -270,7 +270,7 @@ BlobStatusRequest is used to query the status of a blob.
 
 | Field | Type | Label | Description |
 | ----- | ---- | ----- | ----------- |
-| data | [bytes](#bytes) |  | The data to be dispersed. The size of data must be <= 2MiB. |
+| data | [bytes](#bytes) |  | The data to be dispersed. The size of data must be <= 2MiB. Every 32 bytes of data chunk is interpreted as an integer in big endian format where the lower address has more significant bits. The integer must stay in the valid range to be interpreted as a field element on the bn254 curve. The valid range is 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 containing slightly less than 254 bits and more than 253 bits. If any one of the 32 bytes chunk is outside the range, the whole request is deemed as invalid, and rejected. |
 | custom_quorum_numbers | [uint32](#uint32) | repeated | The quorums to which the blob will be sent, in addition to the required quorums which are configured on the EigenDA smart contract. If required quorums are included here, an error will be returned. The disperser will ensure that the encoded blobs for each quorum are all processed within the same batch. |
 | account_id | [string](#string) |  | The account ID of the client. This should be a hex-encoded string of the ECSDA public key corresponding to the key used by the client to sign the BlobAuthHeader. |
 
@@ -315,7 +315,17 @@ RetrieveBlobRequest contains parameters to retrieve the blob.
 <a name="disperser-BlobStatus"></a>
 
 ### BlobStatus
-
+BlobStatus represents the status of a blob.
+The status of a blob is updated as the blob is processed by the disperser.
+The status of a blob can be queried by the client using the GetBlobStatus API.
+Intermediate states are states that the blob can be in while being processed, and it can be updated to a differet state:
+- PROCESSING
+- DISPERSING
+- CONFIRMED
+Terminal states are states that will not be updated to a different state:
+- FAILED
+- FINALIZED
+- INSUFFICIENT_SIGNATURES
 
 | Name | Number | Description |
 | ---- | ------ | ----------- |
@@ -325,6 +335,7 @@ RetrieveBlobRequest contains parameters to retrieve the blob.
 | FAILED | 3 | FAILED means that the blob has failed permanently (for reasons other than insufficient signatures, which is a separate state) |
 | FINALIZED | 4 | FINALIZED means that the block containing the blob&#39;s confirmation transaction has been finalized on Ethereum |
 | INSUFFICIENT_SIGNATURES | 5 | INSUFFICIENT_SIGNATURES means that the confirmation threshold for the blob was not met for at least one quorum. |
+| DISPERSING | 6 | DISPERSING means that the blob is currently being dispersed to DA Nodes and being confirmed onchain |
 
 
 
@@ -339,7 +350,7 @@ Disperser defines the public APIs for dispersing blobs.
 
 | Method Name | Request Type | Response Type | Description |
 | ----------- | ------------ | ------------- | ------------|
-| DisperseBlob | [DisperseBlobRequest](#disperser-DisperseBlobRequest) | [DisperseBlobReply](#disperser-DisperseBlobReply) | This API accepts blob to disperse from clients. This executes the dispersal async, i.e. it returns once the request is accepted. The client could use GetBlobStatus() API to poll the processing status of the blob. |
+| DisperseBlob | [DisperseBlobRequest](#disperser-DisperseBlobRequest) | [DisperseBlobReply](#disperser-DisperseBlobReply) | This API accepts blob to disperse from clients. This executes the dispersal async, i.e. it returns once the request is accepted. The client could use GetBlobStatus() API to poll the the processing status of the blob. |
 | DisperseBlobAuthenticated | [AuthenticatedRequest](#disperser-AuthenticatedRequest) stream | [AuthenticatedReply](#disperser-AuthenticatedReply) stream | DisperseBlobAuthenticated is similar to DisperseBlob, except that it requires the client to authenticate itself via the AuthenticationData message. The protoco is as follows: 1. The client sends a DisperseBlobAuthenticated request with the DisperseBlobRequest message 2. The Disperser sends back a BlobAuthHeader message containing information for the client to verify and sign. 3. The client verifies the BlobAuthHeader and sends back the signed BlobAuthHeader in an 	 AuthenticationData message. 4. The Disperser verifies the signature and returns a DisperseBlobReply message. |
 | GetBlobStatus | [BlobStatusRequest](#disperser-BlobStatusRequest) | [BlobStatusReply](#disperser-BlobStatusReply) | This API is meant to be polled for the blob status. |
 | RetrieveBlob | [RetrieveBlobRequest](#disperser-RetrieveBlobRequest) | [RetrieveBlobReply](#disperser-RetrieveBlobReply) | This retrieves the requested blob from the Disperser&#39;s backend. This is a more efficient way to retrieve blobs than directly retrieving from the DA Nodes (see detail about this approach in api/proto/retriever/retriever.proto). The blob should have been initially dispersed via this Disperser service for this API to work. |

api/docs/node.md

@@ -4,6 +4,8 @@
 ## Table of Contents
 
 - [node/node.proto](#node_node-proto)
+    - [AttestBatchReply](#node-AttestBatchReply)
+    - [AttestBatchRequest](#node-AttestBatchRequest)
     - [BatchHeader](#node-BatchHeader)
     - [Blob](#node-Blob)
     - [BlobHeader](#node-BlobHeader)
@@ -13,11 +15,17 @@
     - [GetBlobHeaderReply](#node-GetBlobHeaderReply)
     - [GetBlobHeaderRequest](#node-GetBlobHeaderRequest)
     - [MerkleProof](#node-MerkleProof)
+    - [NodeInfoReply](#node-NodeInfoReply)
+    - [NodeInfoRequest](#node-NodeInfoRequest)
     - [RetrieveChunksReply](#node-RetrieveChunksReply)
     - [RetrieveChunksRequest](#node-RetrieveChunksRequest)
+    - [StoreBlobsReply](#node-StoreBlobsReply)
+    - [StoreBlobsRequest](#node-StoreBlobsRequest)
     - [StoreChunksReply](#node-StoreChunksReply)
     - [StoreChunksRequest](#node-StoreChunksRequest)
 
+    - [ChunkEncoding](#node-ChunkEncoding)
+
     - [Dispersal](#node-Dispersal)
     - [Retrieval](#node-Retrieval)
 
@@ -35,6 +43,37 @@
 
 
 
+<a name="node-AttestBatchReply"></a>
+
+### AttestBatchReply
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| signature | [bytes](#bytes) |  |  |
+
+
+
+
+
+
+<a name="node-AttestBatchRequest"></a>
+
+### AttestBatchRequest
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| batch_header | [BatchHeader](#node-BatchHeader) |  | header of the batch |
+| blob_header_hashes | [bytes](#bytes) | repeated | the header hashes of all blobs in the batch |
+
+
+
+
+
+
 <a name="node-BatchHeader"></a>
 
 ### BatchHeader
@@ -86,6 +125,7 @@ single operator node.
 | length | [uint32](#uint32) |  | The length of the original blob in number of symbols (in the field where the polynomial is defined). |
 | quorum_headers | [BlobQuorumInfo](#node-BlobQuorumInfo) | repeated | The params of the quorums that this blob participates in. |
 | account_id | [string](#string) |  | The ID of the user who is dispersing this blob to EigenDA. |
+| reference_block_number | [uint32](#uint32) |  | The reference block number whose state is used to encode the blob |
 
 
 
@@ -122,6 +162,7 @@ operator and a single quorum.
 | Field | Type | Label | Description |
 | ----- | ---- | ----- | ----------- |
 | chunks | [bytes](#bytes) | repeated | Each chunk corresponds to a collection of points on the polynomial. Each chunk has same number of points. |
+| bundle | [bytes](#bytes) |  | All chunks of the bundle encoded in a byte array. |
 
 
 
@@ -195,6 +236,35 @@ See RetrieveChunksRequest for documentation of each parameter of GetBlobHeaderRe
 
 
 
+<a name="node-NodeInfoReply"></a>
+
+### NodeInfoReply
+Node info reply
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| semver | [string](#string) |  |  |
+| arch | [string](#string) |  |  |
+| os | [string](#string) |  |  |
+| num_cpu | [uint32](#uint32) |  |  |
+| mem_bytes | [uint64](#uint64) |  |  |
+
+
+
+
+
+
+<a name="node-NodeInfoRequest"></a>
+
+### NodeInfoRequest
+Node info request
+
+
+
+
+
+
 <a name="node-RetrieveChunksReply"></a>
 
 ### RetrieveChunksReply
@@ -204,6 +274,7 @@ See RetrieveChunksRequest for documentation of each parameter of GetBlobHeaderRe
 | Field | Type | Label | Description |
 | ----- | ---- | ----- | ----------- |
 | chunks | [bytes](#bytes) | repeated | All chunks the Node is storing for the requested blob per RetrieveChunksRequest. |
+| encoding | [ChunkEncoding](#node-ChunkEncoding) |  | How the above chunks encoded. |
 
 
 
@@ -227,6 +298,37 @@ See RetrieveChunksRequest for documentation of each parameter of GetBlobHeaderRe
 
 
 
+<a name="node-StoreBlobsReply"></a>
+
+### StoreBlobsReply
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| signatures | [google.protobuf.BytesValue](#google-protobuf-BytesValue) | repeated | The operator&#39;s BLS sgnature signed on the blob header hashes. The ordering of the signatures must match the ordering of the blobs sent in the request, with empty signatures in the places for discarded blobs. |
+
+
+
+
+
+
+<a name="node-StoreBlobsRequest"></a>
+
+### StoreBlobsRequest
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| blobs | [Blob](#node-Blob) | repeated | Blobs to store |
+| reference_block_number | [uint32](#uint32) |  | The reference block number whose state is used to encode the blobs |
+
+
+
+
+
+
 <a name="node-StoreChunksReply"></a>
 
 ### StoreChunksReply
@@ -259,6 +361,20 @@ See RetrieveChunksRequest for documentation of each parameter of GetBlobHeaderRe
 
 
 
+
+<a name="node-ChunkEncoding"></a>
+
+### ChunkEncoding
+This describes how the chunks returned in RetrieveChunksReply are encoded.
+Used to facilitate the decoding of chunks.
+
+| Name | Number | Description |
+| ---- | ------ | ----------- |
+| UNKNOWN | 0 |  |
+| GNARK | 1 |  |
+| GOB | 2 |  |
+
+
 
 
 
@@ -272,6 +388,9 @@ See RetrieveChunksRequest for documentation of each parameter of GetBlobHeaderRe
 | Method Name | Request Type | Response Type | Description |
 | ----------- | ------------ | ------------- | ------------|
 | StoreChunks | [StoreChunksRequest](#node-StoreChunksRequest) | [StoreChunksReply](#node-StoreChunksReply) | StoreChunks validates that the chunks match what the Node is supposed to receive ( different Nodes are responsible for different chunks, as EigenDA is horizontally sharded) and is correctly coded (e.g. each chunk must be a valid KZG multiproof) according to the EigenDA protocol. It also stores the chunks along with metadata for the protocol-defined length of custody. It will return a signature at the end to attest to the data in this request it has processed. |
+| StoreBlobs | [StoreBlobsRequest](#node-StoreBlobsRequest) | [StoreBlobsReply](#node-StoreBlobsReply) | StoreBlobs is simiar to StoreChunks, but it stores the blobs using a different storage schema so that the stored blobs can later be aggregated by AttestBatch method to a bigger batch. StoreBlobs &#43; AttestBatch will eventually replace and deprecate StoreChunks method. |
+| AttestBatch | [AttestBatchRequest](#node-AttestBatchRequest) | [AttestBatchReply](#node-AttestBatchReply) | AttestBatch is used to aggregate the batches stored by StoreBlobs method to a bigger batch. It will return a signature at the end to attest to the aggregated batch. |
+| NodeInfo | [NodeInfoRequest](#node-NodeInfoRequest) | [NodeInfoReply](#node-NodeInfoReply) | Retrieve node info metadata |
 
 
 <a name="node-Retrieval"></a>
@@ -282,7 +401,8 @@ See RetrieveChunksRequest for documentation of each parameter of GetBlobHeaderRe
 | Method Name | Request Type | Response Type | Description |
 | ----------- | ------------ | ------------- | ------------|
 | RetrieveChunks | [RetrieveChunksRequest](#node-RetrieveChunksRequest) | [RetrieveChunksReply](#node-RetrieveChunksReply) | RetrieveChunks retrieves the chunks for a blob custodied at the Node. |
-| GetBlobHeader | [GetBlobHeaderRequest](#node-GetBlobHeaderRequest) | [GetBlobHeaderReply](#node-GetBlobHeaderReply) | Similar to RetrieveChunks, this just returns the header of the blob. |
+| GetBlobHeader | [GetBlobHeaderRequest](#node-GetBlobHeaderRequest) | [GetBlobHeaderReply](#node-GetBlobHeaderReply) | GetBlobHeader is similar to RetrieveChunks, this just returns the header of the blob. |
+| NodeInfo | [NodeInfoRequest](#node-NodeInfoRequest) | [NodeInfoReply](#node-NodeInfoReply) | Retrieve node info metadata |
 
 
 

api/grpc/mock/node_disperser_client.go

@@ -0,0 +1,39 @@
+package mock
+
+import (
+	"context"
+
+	"github.com/Layr-Labs/eigenda/api/grpc/node"
+	"github.com/stretchr/testify/mock"
+	"google.golang.org/grpc"
+)
+
+type MockNodeDispersalClient struct {
+	mock.Mock
+}
+
+var _ node.DispersalClient = (*MockNodeDispersalClient)(nil)
+
+func NewMockDispersalClient() *MockNodeDispersalClient {
+	return &MockNodeDispersalClient{}
+}
+
+func (m *MockNodeDispersalClient) StoreChunks(ctx context.Context, in *node.StoreChunksRequest, opts ...grpc.CallOption) (*node.StoreChunksReply, error) {
+	args := m.Called()
+	return args.Get(0).(*node.StoreChunksReply), args.Error(1)
+}
+
+func (m *MockNodeDispersalClient) StoreBlobs(ctx context.Context, in *node.StoreBlobsRequest, opts ...grpc.CallOption) (*node.StoreBlobsReply, error) {
+	args := m.Called()
+	return args.Get(0).(*node.StoreBlobsReply), args.Error(1)
+}
+
+func (m *MockNodeDispersalClient) AttestBatch(ctx context.Context, in *node.AttestBatchRequest, opts ...grpc.CallOption) (*node.AttestBatchReply, error) {
+	args := m.Called()
+	return args.Get(0).(*node.AttestBatchReply), args.Error(1)
+}
+
+func (m *MockNodeDispersalClient) NodeInfo(ctx context.Context, in *node.NodeInfoRequest, opts ...grpc.CallOption) (*node.NodeInfoReply, error) {
+	args := m.Called()
+	return args.Get(0).(*node.NodeInfoReply), args.Error(1)
+}

common/aws/dynamodb/client.go

@@ -105,6 +105,18 @@ func (c *Client) PutItem(ctx context.Context, tableName string, item Item) (err
 	return nil
 }
 
+func (c *Client) PutItemWithCondition(ctx context.Context, tableName string, item Item, condition string) (err error) {
+	_, err = c.dynamoClient.PutItem(ctx, &dynamodb.PutItemInput{
+		TableName: aws.String(tableName), Item: item,
+		ConditionExpression: aws.String(condition),
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // PutItems puts items in batches of 25 items (which is a limit DynamoDB imposes)
 // It returns the items that failed to be put.
 func (c *Client) PutItems(ctx context.Context, tableName string, items []Item) ([]Item, error) {
@@ -166,6 +178,20 @@ func (c *Client) QueryIndex(ctx context.Context, tableName string, indexName str
 	return response.Items, nil
 }
 
+// Query returns all items in the primary index that match the given expression
+func (c *Client) Query(ctx context.Context, tableName string, keyCondition string, expAttributeValues ExpresseionValues) ([]Item, error) {
+	response, err := c.dynamoClient.Query(ctx, &dynamodb.QueryInput{
+		TableName:                 aws.String(tableName),
+		KeyConditionExpression:    aws.String(keyCondition),
+		ExpressionAttributeValues: expAttributeValues,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return response.Items, nil
+}
+
 // QueryIndexCount returns the count of the items in the index that match the given key
 func (c *Client) QueryIndexCount(ctx context.Context, tableName string, indexName string, keyCondition string, expAttributeValues ExpresseionValues) (int32, error) {
 	response, err := c.dynamoClient.Query(ctx, &dynamodb.QueryInput{