Merge branch 'master' into client-v2-get

.gitattributes

@@ -0,0 +1,6 @@
+# Auto-generated files should not be rendered in diffs.
+api/docs/*.md             linguist-generated=true
+api/docs/*.html           linguist-generated=true
+*.pb.go                   linguist-generated=true
+inabox/deploy/env_vars.go linguist-generated=true
+# contracts/bindings/*.go   linguist-generated=true      Enable once bindings are checked in CI

.github/workflows/codeQL-scanning.yaml → .github/workflows/codeql-scanning.yaml

@@ -1,4 +1,4 @@
-name: "CodeQL scanning"
+name: "codeql-scanning"
 
 on:
   push:
@@ -19,8 +19,6 @@ on:
       - 'api/**'
       - '.github/codeql/**'
       - '.github/workflows/codeql-analysis.yml'
-   paths-ignore:
-      - 'contracts/bindings/**'
   schedule:
     - cron: '0 9 * * *'
 

Dockerfile

@@ -76,6 +76,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \
 
 # Controller build stage
 FROM common-builder AS controller-builder
+COPY node/auth /app/node/auth
 WORKDIR /app/disperser
 RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \

Makefile

@@ -59,6 +59,9 @@ dataapi-build:
 unit-tests:
 	./test.sh
 
+fuzz-tests:
+	go test --fuzz=FuzzParseSignatureKMS -fuzztime=5m ./common
+
 integration-tests-churner:
 	go test -v ./churner/tests
 

api/builder/generate-docs.sh

@@ -26,61 +26,92 @@ done
 # Sort the proto files alphabetically. Required for deterministic output.
 IFS=$'\n' PROTO_FILES=($(sort <<<"${PROTO_FILES[*]}")); unset IFS
 
-# Generate unified HTML doc
-echo "Generating unified HTML documentation..."
-docker run --rm \
-  -v "${DOCS_DIR}":/out \
-  -v "${PROTO_DIR}":/protos \
-  pseudomuto/protoc-gen-doc \
-  "${PROTO_FILES[@]}" \
-  --doc_opt=html,eigenda-protos.html 2>/dev/null
-
-if [ $? -ne 0 ]; then
-  echo "Failed to generate unified HTML documentation."
-  exit 1
-fi
+PID_LIST=()
 
-# Generate unified markdown doc
-echo "Generating unified markdown documentation..."
-docker run --rm \
-  -v "${DOCS_DIR}":/out \
-  -v "${PROTO_DIR}":/protos \
-  pseudomuto/protoc-gen-doc \
-  "${PROTO_FILES[@]}" \
-  --doc_opt=markdown,eigenda-protos.md 2>/dev/null
-
-if [ $? -ne 0 ]; then
-  echo "Failed to generate unified markdown documentation."
-  exit 1
-fi
+# Generate unified HTML doc
+generateUnifiedHTML() {
+  echo "Generating unified HTML documentation..."
+  docker run --rm \
+    -v "${DOCS_DIR}":/out \
+    -v "${PROTO_DIR}":/protos \
+    pseudomuto/protoc-gen-doc \
+    "${PROTO_FILES[@]}" \
+    --doc_opt=html,eigenda-protos.html 2>/dev/null &
 
-# Generate individual markdown/HTML docs
-for PROTO_FILE in "${PROTO_FILES[@]}"; do
-  PROTO_NAME=$(basename "${PROTO_FILE}" .proto)
+  if [ $? -ne 0 ]; then
+    echo "Failed to generate unified HTML documentation."
+    exit 1
+  fi
+}
+generateUnifiedHTML &
+PID_LIST+=($!)
 
-  echo "Generating markdown documentation for ${PROTO_NAME}..."
+# Generate unified markdown doc
+generateUnifiedMD() {
+  echo "Generating unified markdown documentation..."
   docker run --rm \
     -v "${DOCS_DIR}":/out \
     -v "${PROTO_DIR}":/protos \
     pseudomuto/protoc-gen-doc \
-    "${PROTO_FILE}" \
-    --doc_opt=markdown,"${PROTO_NAME}.md" 2>/dev/null
+    "${PROTO_FILES[@]}" \
+    --doc_opt=markdown,eigenda-protos.md 2>/dev/null
 
   if [ $? -ne 0 ]; then
-    echo "Failed to generate documentation for ${PROTO_NAME}."
+    echo "Failed to generate unified markdown documentation."
     exit 1
   fi
+}
+generateUnifiedMD &
+PID_LIST+=($!)
 
-  echo "Generating HTML documentation for ${PROTO_NAME}..."
+# Generate markdown docs for a proto file. First argument is the name of the proto file, second is the path.
+generateMD() {
+  echo "Generating markdown documentation for ${1}..."
   docker run --rm \
     -v "${DOCS_DIR}":/out \
     -v "${PROTO_DIR}":/protos \
     pseudomuto/protoc-gen-doc \
-    "${PROTO_FILE}" \
-    --doc_opt=html,"${PROTO_NAME}.html" 2>/dev/null
+    "${2}" \
+    --doc_opt=markdown,"${1}.md" 2>/dev/null
 
   if [ $? -ne 0 ]; then
-    echo "Failed to generate documentation for ${PROTO_NAME}."
+    echo "Failed to generate documentation for ${1}."
     exit 1
   fi
-done
+}
+
+# Generate HTML docs for a proto file. First argument is the name of the proto file, second is the path.
+generateHTML() {
+    echo "Generating HTML documentation for ${1}..."
+    docker run --rm \
+      -v "${DOCS_DIR}":/out \
+      -v "${PROTO_DIR}":/protos \
+      pseudomuto/protoc-gen-doc \
+      "${2}" \
+      --doc_opt=html,"${1}.html" 2>/dev/null
+
+    if [ $? -ne 0 ]; then
+      echo "Failed to generate documentation for ${1}."
+      exit 1
+    fi
+}
+
+# Generate individual markdown/HTML docs
+for PROTO_FILE in "${PROTO_FILES[@]}"; do
+  PROTO_NAME=$(basename "${PROTO_FILE}" .proto)
+
+  generateMD "${PROTO_NAME}" "${PROTO_FILE}" &
+  PID_LIST+=($!)
+
+  generateHTML "${PROTO_NAME}" "${PROTO_FILE}" &
+  PID_LIST+=($!)
+done
+
+# Wait for all processes to finish
+for PID in "${PID_LIST[@]}"; do
+  wait $PID
+  if [ $? -ne 0 ]; then
+    echo "Failed to wait for process $PID."
+    exit 1
+  fi
+done

api/clients/mock/static_request_signer.go

@@ -0,0 +1,30 @@
+package mock
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"github.com/Layr-Labs/eigenda/api/clients/v2"
+	v2 "github.com/Layr-Labs/eigenda/api/grpc/node/v2"
+	"github.com/Layr-Labs/eigenda/node/auth"
+)
+
+var _ clients.DispersalRequestSigner = &staticRequestSigner{}
+
+// StaticRequestSigner is a DispersalRequestSigner that signs requests with a static key (i.e. it doesn't use AWS KMS).
+// Useful for testing.
+type staticRequestSigner struct {
+	key *ecdsa.PrivateKey
+}
+
+func NewStaticRequestSigner(key *ecdsa.PrivateKey) clients.DispersalRequestSigner {
+	return &staticRequestSigner{
+		key: key,
+	}
+}
+
+func (s *staticRequestSigner) SignStoreChunksRequest(
+	ctx context.Context,
+	request *v2.StoreChunksRequest) ([]byte, error) {
+
+	return auth.SignStoreChunksRequest(s.key, request)
+}

api/clients/v2/accountant.go

@@ -26,26 +26,23 @@ type Accountant struct {
 
 	// local accounting
 	// contains 3 bins; circular wrapping of indices
-	binRecords        []BinRecord
+	periodRecords     []PeriodRecord
 	usageLock         sync.Mutex
 	cumulativePayment *big.Int
 
 	// number of bins in the circular accounting, restricted by minNumBins which is 3
 	numBins uint32
 }
 
-type BinRecord struct {
+type PeriodRecord struct {
 	Index uint32
 	Usage uint64
 }
 
 func NewAccountant(accountID string, reservation *core.ReservedPayment, onDemand *core.OnDemandPayment, reservationWindow uint32, pricePerSymbol uint32, minNumSymbols uint32, numBins uint32) *Accountant {
-	//TODO: client storage; currently every instance starts fresh but on-chain or a small store makes more sense
-	// Also client is currently responsible for supplying network params, we need to add RPC in order to be automatic
-	// There's a subsequent PR that handles populating the accountant with on-chain state from the disperser
-	binRecords := make([]BinRecord, numBins)
-	for i := range binRecords {
-		binRecords[i] = BinRecord{Index: uint32(i), Usage: 0}
+	periodRecords := make([]PeriodRecord, numBins)
+	for i := range periodRecords {
+		periodRecords[i] = PeriodRecord{Index: uint32(i), Usage: 0}
 	}
 	a := Accountant{
 		accountID:         accountID,
@@ -54,7 +51,7 @@ func NewAccountant(accountID string, reservation *core.ReservedPayment, onDemand
 		reservationWindow: reservationWindow,
 		pricePerSymbol:    pricePerSymbol,
 		minNumSymbols:     minNumSymbols,
-		binRecords:        binRecords,
+		periodRecords:     periodRecords,
 		cumulativePayment: big.NewInt(0),
 		numBins:           max(numBins, uint32(meterer.MinNumBins)),
 	}
@@ -74,22 +71,22 @@ func (a *Accountant) BlobPaymentInfo(ctx context.Context, numSymbols uint32, quo
 
 	a.usageLock.Lock()
 	defer a.usageLock.Unlock()
-	relativeBinRecord := a.GetRelativeBinRecord(currentReservationPeriod)
-	relativeBinRecord.Usage += symbolUsage
+	relativePeriodRecord := a.GetRelativePeriodRecord(currentReservationPeriod)
+	relativePeriodRecord.Usage += symbolUsage
 
 	// first attempt to use the active reservation
 	binLimit := a.reservation.SymbolsPerSecond * uint64(a.reservationWindow)
-	if relativeBinRecord.Usage <= binLimit {
+	if relativePeriodRecord.Usage <= binLimit {
 		if err := QuorumCheck(quorumNumbers, a.reservation.QuorumNumbers); err != nil {
 			return 0, big.NewInt(0), err
 		}
 		return currentReservationPeriod, big.NewInt(0), nil
 	}
 
-	overflowBinRecord := a.GetRelativeBinRecord(currentReservationPeriod + 2)
+	overflowPeriodRecord := a.GetRelativePeriodRecord(currentReservationPeriod + 2)
 	// Allow one overflow when the overflow bin is empty, the current usage and new length are both less than the limit
-	if overflowBinRecord.Usage == 0 && relativeBinRecord.Usage-symbolUsage < binLimit && symbolUsage <= binLimit {
-		overflowBinRecord.Usage += relativeBinRecord.Usage - binLimit
+	if overflowPeriodRecord.Usage == 0 && relativePeriodRecord.Usage-symbolUsage < binLimit && symbolUsage <= binLimit {
+		overflowPeriodRecord.Usage += relativePeriodRecord.Usage - binLimit
 		if err := QuorumCheck(quorumNumbers, a.reservation.QuorumNumbers); err != nil {
 			return 0, big.NewInt(0), err
 		}
@@ -98,7 +95,7 @@ func (a *Accountant) BlobPaymentInfo(ctx context.Context, numSymbols uint32, quo
 
 	// reservation not available, rollback reservation records, attempt on-demand
 	//todo: rollback on-demand if disperser respond with some type of rejection?
-	relativeBinRecord.Usage -= symbolUsage
+	relativePeriodRecord.Usage -= symbolUsage
 	incrementRequired := big.NewInt(int64(a.PaymentCharged(numSymbols)))
 	a.cumulativePayment.Add(a.cumulativePayment, incrementRequired)
 	if a.cumulativePayment.Cmp(a.onDemand.CumulativePayment) <= 0 {
@@ -143,16 +140,16 @@ func (a *Accountant) SymbolsCharged(numSymbols uint32) uint32 {
 	return uint32(core.RoundUpDivide(uint(numSymbols), uint(a.minNumSymbols))) * a.minNumSymbols
 }
 
-func (a *Accountant) GetRelativeBinRecord(index uint32) *BinRecord {
+func (a *Accountant) GetRelativePeriodRecord(index uint32) *PeriodRecord {
 	relativeIndex := index % a.numBins
-	if a.binRecords[relativeIndex].Index != uint32(index) {
-		a.binRecords[relativeIndex] = BinRecord{
+	if a.periodRecords[relativeIndex].Index != uint32(index) {
+		a.periodRecords[relativeIndex] = PeriodRecord{
 			Index: uint32(index),
 			Usage: 0,
 		}
 	}
 
-	return &a.binRecords[relativeIndex]
+	return &a.periodRecords[relativeIndex]
 }
 
 // SetPaymentState sets the accountant's state from the disperser's response
@@ -214,18 +211,18 @@ func (a *Accountant) SetPaymentState(paymentState *disperser_rpc.GetPaymentState
 		}
 	}
 
-	binRecords := make([]BinRecord, len(paymentState.GetBinRecords()))
-	for i, record := range paymentState.GetBinRecords() {
+	periodRecords := make([]PeriodRecord, len(paymentState.GetPeriodRecords()))
+	for i, record := range paymentState.GetPeriodRecords() {
 		if record == nil {
-			binRecords[i] = BinRecord{Index: 0, Usage: 0}
+			periodRecords[i] = PeriodRecord{Index: 0, Usage: 0}
 		} else {
-			binRecords[i] = BinRecord{
+			periodRecords[i] = PeriodRecord{
 				Index: record.Index,
 				Usage: record.Usage,
 			}
 		}
 	}
-	a.binRecords = binRecords
+	a.periodRecords = periodRecords
 	return nil
 }