diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..6133218a0f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Auto-generated files should not be rendered in diffs. +api/docs/*.md linguist-generated=true +api/docs/*.html linguist-generated=true +*.pb.go linguist-generated=true +inabox/deploy/env_vars.go linguist-generated=true +# contracts/bindings/*.go linguist-generated=true Enable once bindings are checked in CI \ No newline at end of file diff --git a/.github/workflows/codeQL-scanning.yaml b/.github/workflows/codeql-scanning.yaml similarity index 94% rename from .github/workflows/codeQL-scanning.yaml rename to .github/workflows/codeql-scanning.yaml index e3511339e9..46bca7add3 100644 --- a/.github/workflows/codeQL-scanning.yaml +++ b/.github/workflows/codeql-scanning.yaml @@ -1,4 +1,4 @@ -name: "CodeQL scanning" +name: "codeql-scanning" on: push: @@ -19,8 +19,6 @@ on: - 'api/**' - '.github/codeql/**' - '.github/workflows/codeql-analysis.yml' - paths-ignore: - - 'contracts/bindings/**' schedule: - cron: '0 9 * * *' diff --git a/Dockerfile b/Dockerfile index 0140473fc3..64f146dd8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,6 +76,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ # Controller build stage FROM common-builder AS controller-builder +COPY node/auth /app/node/auth WORKDIR /app/disperser RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ diff --git a/Makefile b/Makefile index 033741a38d..f092560872 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,9 @@ dataapi-build: unit-tests: ./test.sh +fuzz-tests: + go test --fuzz=FuzzParseSignatureKMS -fuzztime=5m ./common + integration-tests-churner: go test -v ./churner/tests diff --git a/api/builder/generate-docs.sh b/api/builder/generate-docs.sh index 011e0da37b..85badf0234 100755 --- a/api/builder/generate-docs.sh +++ b/api/builder/generate-docs.sh @@ -26,61 +26,92 @@ done # Sort the proto files alphabetically. Required for deterministic output. IFS=$'\n' PROTO_FILES=($(sort <<<"${PROTO_FILES[*]}")); unset IFS -# Generate unified HTML doc -echo "Generating unified HTML documentation..." -docker run --rm \ - -v "${DOCS_DIR}":/out \ - -v "${PROTO_DIR}":/protos \ - pseudomuto/protoc-gen-doc \ - "${PROTO_FILES[@]}" \ - --doc_opt=html,eigenda-protos.html 2>/dev/null - -if [ $? -ne 0 ]; then - echo "Failed to generate unified HTML documentation." - exit 1 -fi +PID_LIST=() -# Generate unified markdown doc -echo "Generating unified markdown documentation..." -docker run --rm \ - -v "${DOCS_DIR}":/out \ - -v "${PROTO_DIR}":/protos \ - pseudomuto/protoc-gen-doc \ - "${PROTO_FILES[@]}" \ - --doc_opt=markdown,eigenda-protos.md 2>/dev/null - -if [ $? -ne 0 ]; then - echo "Failed to generate unified markdown documentation." - exit 1 -fi +# Generate unified HTML doc +generateUnifiedHTML() { + echo "Generating unified HTML documentation..." + docker run --rm \ + -v "${DOCS_DIR}":/out \ + -v "${PROTO_DIR}":/protos \ + pseudomuto/protoc-gen-doc \ + "${PROTO_FILES[@]}" \ + --doc_opt=html,eigenda-protos.html 2>/dev/null & -# Generate individual markdown/HTML docs -for PROTO_FILE in "${PROTO_FILES[@]}"; do - PROTO_NAME=$(basename "${PROTO_FILE}" .proto) + if [ $? -ne 0 ]; then + echo "Failed to generate unified HTML documentation." + exit 1 + fi +} +generateUnifiedHTML & +PID_LIST+=($!) - echo "Generating markdown documentation for ${PROTO_NAME}..." +# Generate unified markdown doc +generateUnifiedMD() { + echo "Generating unified markdown documentation..." docker run --rm \ -v "${DOCS_DIR}":/out \ -v "${PROTO_DIR}":/protos \ pseudomuto/protoc-gen-doc \ - "${PROTO_FILE}" \ - --doc_opt=markdown,"${PROTO_NAME}.md" 2>/dev/null + "${PROTO_FILES[@]}" \ + --doc_opt=markdown,eigenda-protos.md 2>/dev/null if [ $? -ne 0 ]; then - echo "Failed to generate documentation for ${PROTO_NAME}." + echo "Failed to generate unified markdown documentation." exit 1 fi +} +generateUnifiedMD & +PID_LIST+=($!) - echo "Generating HTML documentation for ${PROTO_NAME}..." +# Generate markdown docs for a proto file. First argument is the name of the proto file, second is the path. +generateMD() { + echo "Generating markdown documentation for ${1}..." docker run --rm \ -v "${DOCS_DIR}":/out \ -v "${PROTO_DIR}":/protos \ pseudomuto/protoc-gen-doc \ - "${PROTO_FILE}" \ - --doc_opt=html,"${PROTO_NAME}.html" 2>/dev/null + "${2}" \ + --doc_opt=markdown,"${1}.md" 2>/dev/null if [ $? -ne 0 ]; then - echo "Failed to generate documentation for ${PROTO_NAME}." + echo "Failed to generate documentation for ${1}." exit 1 fi -done \ No newline at end of file +} + +# Generate HTML docs for a proto file. First argument is the name of the proto file, second is the path. +generateHTML() { + echo "Generating HTML documentation for ${1}..." + docker run --rm \ + -v "${DOCS_DIR}":/out \ + -v "${PROTO_DIR}":/protos \ + pseudomuto/protoc-gen-doc \ + "${2}" \ + --doc_opt=html,"${1}.html" 2>/dev/null + + if [ $? -ne 0 ]; then + echo "Failed to generate documentation for ${1}." + exit 1 + fi +} + +# Generate individual markdown/HTML docs +for PROTO_FILE in "${PROTO_FILES[@]}"; do + PROTO_NAME=$(basename "${PROTO_FILE}" .proto) + + generateMD "${PROTO_NAME}" "${PROTO_FILE}" & + PID_LIST+=($!) + + generateHTML "${PROTO_NAME}" "${PROTO_FILE}" & + PID_LIST+=($!) +done + +# Wait for all processes to finish +for PID in "${PID_LIST[@]}"; do + wait $PID + if [ $? -ne 0 ]; then + echo "Failed to wait for process $PID." + exit 1 + fi +done diff --git a/api/clients/mock/static_request_signer.go b/api/clients/mock/static_request_signer.go new file mode 100644 index 0000000000..cb05368bcc --- /dev/null +++ b/api/clients/mock/static_request_signer.go @@ -0,0 +1,30 @@ +package mock + +import ( + "context" + "crypto/ecdsa" + "github.com/Layr-Labs/eigenda/api/clients/v2" + v2 "github.com/Layr-Labs/eigenda/api/grpc/node/v2" + "github.com/Layr-Labs/eigenda/node/auth" +) + +var _ clients.DispersalRequestSigner = &staticRequestSigner{} + +// StaticRequestSigner is a DispersalRequestSigner that signs requests with a static key (i.e. it doesn't use AWS KMS). +// Useful for testing. +type staticRequestSigner struct { + key *ecdsa.PrivateKey +} + +func NewStaticRequestSigner(key *ecdsa.PrivateKey) clients.DispersalRequestSigner { + return &staticRequestSigner{ + key: key, + } +} + +func (s *staticRequestSigner) SignStoreChunksRequest( + ctx context.Context, + request *v2.StoreChunksRequest) ([]byte, error) { + + return auth.SignStoreChunksRequest(s.key, request) +} diff --git a/api/clients/v2/accountant.go b/api/clients/v2/accountant.go index 02dd58d9f6..be0030d3f8 100644 --- a/api/clients/v2/accountant.go +++ b/api/clients/v2/accountant.go @@ -26,7 +26,7 @@ type Accountant struct { // local accounting // contains 3 bins; circular wrapping of indices - binRecords []BinRecord + periodRecords []PeriodRecord usageLock sync.Mutex cumulativePayment *big.Int @@ -34,18 +34,15 @@ type Accountant struct { numBins uint32 } -type BinRecord struct { +type PeriodRecord struct { Index uint32 Usage uint64 } func NewAccountant(accountID string, reservation *core.ReservedPayment, onDemand *core.OnDemandPayment, reservationWindow uint32, pricePerSymbol uint32, minNumSymbols uint32, numBins uint32) *Accountant { - //TODO: client storage; currently every instance starts fresh but on-chain or a small store makes more sense - // Also client is currently responsible for supplying network params, we need to add RPC in order to be automatic - // There's a subsequent PR that handles populating the accountant with on-chain state from the disperser - binRecords := make([]BinRecord, numBins) - for i := range binRecords { - binRecords[i] = BinRecord{Index: uint32(i), Usage: 0} + periodRecords := make([]PeriodRecord, numBins) + for i := range periodRecords { + periodRecords[i] = PeriodRecord{Index: uint32(i), Usage: 0} } a := Accountant{ accountID: accountID, @@ -54,7 +51,7 @@ func NewAccountant(accountID string, reservation *core.ReservedPayment, onDemand reservationWindow: reservationWindow, pricePerSymbol: pricePerSymbol, minNumSymbols: minNumSymbols, - binRecords: binRecords, + periodRecords: periodRecords, cumulativePayment: big.NewInt(0), numBins: max(numBins, uint32(meterer.MinNumBins)), } @@ -74,22 +71,22 @@ func (a *Accountant) BlobPaymentInfo(ctx context.Context, numSymbols uint32, quo a.usageLock.Lock() defer a.usageLock.Unlock() - relativeBinRecord := a.GetRelativeBinRecord(currentReservationPeriod) - relativeBinRecord.Usage += symbolUsage + relativePeriodRecord := a.GetRelativePeriodRecord(currentReservationPeriod) + relativePeriodRecord.Usage += symbolUsage // first attempt to use the active reservation binLimit := a.reservation.SymbolsPerSecond * uint64(a.reservationWindow) - if relativeBinRecord.Usage <= binLimit { + if relativePeriodRecord.Usage <= binLimit { if err := QuorumCheck(quorumNumbers, a.reservation.QuorumNumbers); err != nil { return 0, big.NewInt(0), err } return currentReservationPeriod, big.NewInt(0), nil } - overflowBinRecord := a.GetRelativeBinRecord(currentReservationPeriod + 2) + overflowPeriodRecord := a.GetRelativePeriodRecord(currentReservationPeriod + 2) // Allow one overflow when the overflow bin is empty, the current usage and new length are both less than the limit - if overflowBinRecord.Usage == 0 && relativeBinRecord.Usage-symbolUsage < binLimit && symbolUsage <= binLimit { - overflowBinRecord.Usage += relativeBinRecord.Usage - binLimit + if overflowPeriodRecord.Usage == 0 && relativePeriodRecord.Usage-symbolUsage < binLimit && symbolUsage <= binLimit { + overflowPeriodRecord.Usage += relativePeriodRecord.Usage - binLimit if err := QuorumCheck(quorumNumbers, a.reservation.QuorumNumbers); err != nil { return 0, big.NewInt(0), err } @@ -98,7 +95,7 @@ func (a *Accountant) BlobPaymentInfo(ctx context.Context, numSymbols uint32, quo // reservation not available, rollback reservation records, attempt on-demand //todo: rollback on-demand if disperser respond with some type of rejection? - relativeBinRecord.Usage -= symbolUsage + relativePeriodRecord.Usage -= symbolUsage incrementRequired := big.NewInt(int64(a.PaymentCharged(numSymbols))) a.cumulativePayment.Add(a.cumulativePayment, incrementRequired) if a.cumulativePayment.Cmp(a.onDemand.CumulativePayment) <= 0 { @@ -143,16 +140,16 @@ func (a *Accountant) SymbolsCharged(numSymbols uint32) uint32 { return uint32(core.RoundUpDivide(uint(numSymbols), uint(a.minNumSymbols))) * a.minNumSymbols } -func (a *Accountant) GetRelativeBinRecord(index uint32) *BinRecord { +func (a *Accountant) GetRelativePeriodRecord(index uint32) *PeriodRecord { relativeIndex := index % a.numBins - if a.binRecords[relativeIndex].Index != uint32(index) { - a.binRecords[relativeIndex] = BinRecord{ + if a.periodRecords[relativeIndex].Index != uint32(index) { + a.periodRecords[relativeIndex] = PeriodRecord{ Index: uint32(index), Usage: 0, } } - return &a.binRecords[relativeIndex] + return &a.periodRecords[relativeIndex] } // SetPaymentState sets the accountant's state from the disperser's response @@ -214,18 +211,18 @@ func (a *Accountant) SetPaymentState(paymentState *disperser_rpc.GetPaymentState } } - binRecords := make([]BinRecord, len(paymentState.GetBinRecords())) - for i, record := range paymentState.GetBinRecords() { + periodRecords := make([]PeriodRecord, len(paymentState.GetPeriodRecords())) + for i, record := range paymentState.GetPeriodRecords() { if record == nil { - binRecords[i] = BinRecord{Index: 0, Usage: 0} + periodRecords[i] = PeriodRecord{Index: 0, Usage: 0} } else { - binRecords[i] = BinRecord{ + periodRecords[i] = PeriodRecord{ Index: record.Index, Usage: record.Usage, } } } - a.binRecords = binRecords + a.periodRecords = periodRecords return nil } diff --git a/api/clients/v2/accountant_test.go b/api/clients/v2/accountant_test.go index ac8672ff5f..ba71c7a5b3 100644 --- a/api/clients/v2/accountant_test.go +++ b/api/clients/v2/accountant_test.go @@ -43,7 +43,7 @@ func TestNewAccountant(t *testing.T) { assert.Equal(t, reservationWindow, accountant.reservationWindow) assert.Equal(t, pricePerSymbol, accountant.pricePerSymbol) assert.Equal(t, minNumSymbols, accountant.minNumSymbols) - assert.Equal(t, []BinRecord{{Index: 0, Usage: 0}, {Index: 1, Usage: 0}, {Index: 2, Usage: 0}}, accountant.binRecords) + assert.Equal(t, []PeriodRecord{{Index: 0, Usage: 0}, {Index: 1, Usage: 0}, {Index: 2, Usage: 0}}, accountant.periodRecords) assert.Equal(t, big.NewInt(0), accountant.cumulativePayment) } @@ -76,7 +76,7 @@ func TestAccountBlob_Reservation(t *testing.T) { assert.NoError(t, err) assert.Equal(t, meterer.GetReservationPeriod(uint64(time.Now().Unix()), reservationWindow), header.ReservationPeriod) assert.Equal(t, big.NewInt(0), header.CumulativePayment) - assert.Equal(t, isRotation([]uint64{500, 0, 0}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{500, 0, 0}, mapRecordUsage(accountant.periodRecords)), true) symbolLength = uint32(700) @@ -85,7 +85,7 @@ func TestAccountBlob_Reservation(t *testing.T) { assert.NoError(t, err) assert.NotEqual(t, 0, header.ReservationPeriod) assert.Equal(t, big.NewInt(0), header.CumulativePayment) - assert.Equal(t, isRotation([]uint64{1200, 0, 200}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{1200, 0, 200}, mapRecordUsage(accountant.periodRecords)), true) // Second call should use on-demand payment header, err = accountant.AccountBlob(ctx, 300, quorums, salt) @@ -125,7 +125,7 @@ func TestAccountBlob_OnDemand(t *testing.T) { expectedPayment := big.NewInt(int64(numSymbols * pricePerSymbol)) assert.Equal(t, uint32(0), header.ReservationPeriod) assert.Equal(t, expectedPayment, header.CumulativePayment) - assert.Equal(t, isRotation([]uint64{0, 0, 0}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{0, 0, 0}, mapRecordUsage(accountant.periodRecords)), true) assert.Equal(t, expectedPayment, accountant.cumulativePayment) } @@ -225,7 +225,7 @@ func TestAccountBlob_BinRotation(t *testing.T) { // First call _, err = accountant.AccountBlob(ctx, 800, quorums, salt) assert.NoError(t, err) - assert.Equal(t, isRotation([]uint64{800, 0, 0}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{800, 0, 0}, mapRecordUsage(accountant.periodRecords)), true) // next reservation duration time.Sleep(1000 * time.Millisecond) @@ -233,12 +233,12 @@ func TestAccountBlob_BinRotation(t *testing.T) { // Second call _, err = accountant.AccountBlob(ctx, 300, quorums, salt) assert.NoError(t, err) - assert.Equal(t, isRotation([]uint64{800, 300, 0}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{800, 300, 0}, mapRecordUsage(accountant.periodRecords)), true) // Third call _, err = accountant.AccountBlob(ctx, 500, quorums, salt) assert.NoError(t, err) - assert.Equal(t, isRotation([]uint64{800, 800, 0}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{800, 800, 0}, mapRecordUsage(accountant.periodRecords)), true) } func TestConcurrentBinRotationAndAccountBlob(t *testing.T) { @@ -279,7 +279,7 @@ func TestConcurrentBinRotationAndAccountBlob(t *testing.T) { wg.Wait() // Check final state - usages := mapRecordUsage(accountant.binRecords) + usages := mapRecordUsage(accountant.periodRecords) assert.Equal(t, uint64(1000), usages[0]+usages[1]+usages[2]) } @@ -313,14 +313,14 @@ func TestAccountBlob_ReservationWithOneOverflow(t *testing.T) { assert.Equal(t, salt, header.Salt) assert.Equal(t, meterer.GetReservationPeriod(uint64(now), reservationWindow), header.ReservationPeriod) assert.Equal(t, big.NewInt(0), header.CumulativePayment) - assert.Equal(t, isRotation([]uint64{800, 0, 0}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{800, 0, 0}, mapRecordUsage(accountant.periodRecords)), true) // Second call: Allow one overflow header, err = accountant.AccountBlob(ctx, 500, quorums, salt+1) assert.NoError(t, err) assert.Equal(t, salt+1, header.Salt) assert.Equal(t, big.NewInt(0), header.CumulativePayment) - assert.Equal(t, isRotation([]uint64{1300, 0, 300}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{1300, 0, 300}, mapRecordUsage(accountant.periodRecords)), true) // Third call: Should use on-demand payment header, err = accountant.AccountBlob(ctx, 200, quorums, salt+2) @@ -328,7 +328,7 @@ func TestAccountBlob_ReservationWithOneOverflow(t *testing.T) { assert.Equal(t, salt+2, header.Salt) assert.Equal(t, uint32(0), header.ReservationPeriod) assert.Equal(t, big.NewInt(200), header.CumulativePayment) - assert.Equal(t, isRotation([]uint64{1300, 0, 300}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{1300, 0, 300}, mapRecordUsage(accountant.periodRecords)), true) } func TestAccountBlob_ReservationOverflowReset(t *testing.T) { @@ -357,12 +357,12 @@ func TestAccountBlob_ReservationOverflowReset(t *testing.T) { // full reservation _, err = accountant.AccountBlob(ctx, 1000, quorums, salt) assert.NoError(t, err) - assert.Equal(t, isRotation([]uint64{1000, 0, 0}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{1000, 0, 0}, mapRecordUsage(accountant.periodRecords)), true) // no overflow header, err := accountant.AccountBlob(ctx, 500, quorums, salt) assert.NoError(t, err) - assert.Equal(t, isRotation([]uint64{1000, 0, 0}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{1000, 0, 0}, mapRecordUsage(accountant.periodRecords)), true) assert.Equal(t, big.NewInt(500), header.CumulativePayment) // Wait for next reservation duration @@ -371,7 +371,7 @@ func TestAccountBlob_ReservationOverflowReset(t *testing.T) { // Third call: Should use new bin and allow overflow again _, err = accountant.AccountBlob(ctx, 500, quorums, salt) assert.NoError(t, err) - assert.Equal(t, isRotation([]uint64{1000, 500, 0}, mapRecordUsage(accountant.binRecords)), true) + assert.Equal(t, isRotation([]uint64{1000, 500, 0}, mapRecordUsage(accountant.periodRecords)), true) } func TestQuorumCheck(t *testing.T) { @@ -431,7 +431,7 @@ func TestQuorumCheck(t *testing.T) { } } -func mapRecordUsage(records []BinRecord) []uint64 { +func mapRecordUsage(records []PeriodRecord) []uint64 { return []uint64{records[0].Usage, records[1].Usage, records[2].Usage} } diff --git a/api/clients/v2/dispersal_request_signer.go b/api/clients/v2/dispersal_request_signer.go new file mode 100644 index 0000000000..4ab4a9e091 --- /dev/null +++ b/api/clients/v2/dispersal_request_signer.go @@ -0,0 +1,62 @@ +package clients + +import ( + "context" + "crypto/ecdsa" + "fmt" + grpc "github.com/Layr-Labs/eigenda/api/grpc/node/v2" + "github.com/Layr-Labs/eigenda/api/hashing" + aws2 "github.com/Layr-Labs/eigenda/common/aws" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/kms" +) + +// DispersalRequestSigner encapsulates the logic for signing GetChunks requests. +type DispersalRequestSigner interface { + // SignStoreChunksRequest signs a StoreChunksRequest. Does not modify the request + // (i.e. it does not insert the signature). + SignStoreChunksRequest(ctx context.Context, request *grpc.StoreChunksRequest) ([]byte, error) +} + +var _ DispersalRequestSigner = &requestSigner{} + +type requestSigner struct { + keyID string + publicKey *ecdsa.PublicKey + keyManager *kms.Client +} + +// NewDispersalRequestSigner creates a new DispersalRequestSigner. +func NewDispersalRequestSigner( + ctx context.Context, + region string, + endpoint string, + keyID string) (DispersalRequestSigner, error) { + + keyManager := kms.New(kms.Options{ + Region: region, + BaseEndpoint: aws.String(endpoint), + }) + + key, err := aws2.LoadPublicKeyKMS(ctx, keyManager, keyID) + if err != nil { + return nil, fmt.Errorf("failed to get ecdsa public key: %w", err) + } + + return &requestSigner{ + keyID: keyID, + publicKey: key, + keyManager: keyManager, + }, nil +} + +func (s *requestSigner) SignStoreChunksRequest(ctx context.Context, request *grpc.StoreChunksRequest) ([]byte, error) { + hash := hashing.HashStoreChunksRequest(request) + + signature, err := aws2.SignKMS(ctx, s.keyManager, s.keyID, s.publicKey, hash) + if err != nil { + return nil, fmt.Errorf("failed to sign request: %w", err) + } + + return signature, nil +} diff --git a/api/clients/v2/dispersal_request_signer_test.go b/api/clients/v2/dispersal_request_signer_test.go new file mode 100644 index 0000000000..8ef094fe93 --- /dev/null +++ b/api/clients/v2/dispersal_request_signer_test.go @@ -0,0 +1,129 @@ +package clients + +import ( + "context" + aws2 "github.com/Layr-Labs/eigenda/common/aws" + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/Layr-Labs/eigenda/inabox/deploy" + "github.com/Layr-Labs/eigenda/node/auth" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/kms/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ory/dockertest/v3" + "github.com/stretchr/testify/require" + "log" + "os" + "path/filepath" + "runtime" + "testing" +) + +var ( + dockertestPool *dockertest.Pool + dockertestResource *dockertest.Resource +) + +const ( + localstackPort = "4570" + localstackHost = "http://0.0.0.0:4570" + region = "us-east-1" +) + +func setup(t *testing.T) { + deployLocalStack := !(os.Getenv("DEPLOY_LOCALSTACK") == "false") + + _, b, _, _ := runtime.Caller(0) + rootPath := filepath.Join(filepath.Dir(b), "../../..") + changeDirectory(filepath.Join(rootPath, "inabox")) + + if deployLocalStack { + var err error + dockertestPool, dockertestResource, err = deploy.StartDockertestWithLocalstackContainer(localstackPort) + require.NoError(t, err) + } +} + +func changeDirectory(path string) { + err := os.Chdir(path) + if err != nil { + + currentDirectory, err := os.Getwd() + if err != nil { + log.Printf("Failed to get current directory. Error: %s", err) + } + + log.Panicf("Failed to change directories. CWD: %s, Error: %s", currentDirectory, err) + } + + newDir, err := os.Getwd() + if err != nil { + log.Panicf("Failed to get working directory. Error: %s", err) + } + log.Printf("Current Working Directory: %s\n", newDir) +} + +func teardown() { + deployLocalStack := !(os.Getenv("DEPLOY_LOCALSTACK") == "false") + + if deployLocalStack { + deploy.PurgeDockertestResources(dockertestPool, dockertestResource) + } +} + +func TestRequestSigning(t *testing.T) { + rand := random.NewTestRandom(t) + setup(t) + defer teardown() + + keyManager := kms.New(kms.Options{ + Region: region, + BaseEndpoint: aws.String(localstackHost), + }) + + for i := 0; i < 10; i++ { + createKeyOutput, err := keyManager.CreateKey(context.Background(), &kms.CreateKeyInput{ + KeySpec: types.KeySpecEccSecgP256k1, + KeyUsage: types.KeyUsageTypeSignVerify, + }) + require.NoError(t, err) + + keyID := *createKeyOutput.KeyMetadata.KeyId + + key, err := aws2.LoadPublicKeyKMS(context.Background(), keyManager, keyID) + require.NoError(t, err) + + publicAddress := crypto.PubkeyToAddress(*key) + + for j := 0; j < 10; j++ { + request := auth.RandomStoreChunksRequest(rand) + request.Signature = nil + + signer, err := NewDispersalRequestSigner(context.Background(), region, localstackHost, keyID) + require.NoError(t, err) + + // Test a valid signature. + signature, err := signer.SignStoreChunksRequest(context.Background(), request) + require.NoError(t, err) + + require.Nil(t, request.Signature) + request.Signature = signature + err = auth.VerifyStoreChunksRequest(publicAddress, request) + require.NoError(t, err) + + // Changing a byte in the middle of the signature should make the verification fail + badSignature := make([]byte, len(signature)) + copy(badSignature, signature) + badSignature[10] = badSignature[10] + 1 + request.Signature = badSignature + err = auth.VerifyStoreChunksRequest(publicAddress, request) + require.Error(t, err) + + // Changing a byte in the middle of the request should make the verification fail + request.DisperserID = request.DisperserID + 1 + request.Signature = signature + err = auth.VerifyStoreChunksRequest(publicAddress, request) + require.Error(t, err) + } + } +} diff --git a/api/clients/v2/node_client.go b/api/clients/v2/node_client.go index 0d31f699dc..0a379b7bfe 100644 --- a/api/clients/v2/node_client.go +++ b/api/clients/v2/node_client.go @@ -3,6 +3,7 @@ package clients import ( "context" "fmt" + "github.com/Layr-Labs/eigenda/api" "sync" commonpb "github.com/Layr-Labs/eigenda/api/grpc/common/v2" @@ -24,21 +25,23 @@ type NodeClient interface { } type nodeClient struct { - config *NodeClientConfig - initOnce sync.Once - conn *grpc.ClientConn + config *NodeClientConfig + initOnce sync.Once + conn *grpc.ClientConn + requestSigner DispersalRequestSigner dispersalClient nodegrpc.DispersalClient } var _ NodeClient = (*nodeClient)(nil) -func NewNodeClient(config *NodeClientConfig) (*nodeClient, error) { +func NewNodeClient(config *NodeClientConfig, requestSigner DispersalRequestSigner) (NodeClient, error) { if config == nil || config.Hostname == "" || config.Port == "" { return nil, fmt.Errorf("invalid config: %v", config) } return &nodeClient{ - config: config, + config: config, + requestSigner: requestSigner, }, nil } @@ -60,8 +63,7 @@ func (c *nodeClient) StoreChunks(ctx context.Context, batch *corev2.Batch) (*cor } } - // Call the gRPC method to store chunks - response, err := c.dispersalClient.StoreChunks(ctx, &nodegrpc.StoreChunksRequest{ + request := &nodegrpc.StoreChunksRequest{ Batch: &commonpb.Batch{ Header: &commonpb.BatchHeader{ BatchRoot: batch.BatchHeader.BatchRoot[:], @@ -69,7 +71,20 @@ func (c *nodeClient) StoreChunks(ctx context.Context, batch *corev2.Batch) (*cor }, BlobCertificates: blobCerts, }, - }) + DisperserID: api.EigenLabsDisperserID, // this will need to be updated when dispersers are decentralized + } + + if c.requestSigner != nil { + // Sign the request to store chunks + signature, err := c.requestSigner.SignStoreChunksRequest(ctx, request) + if err != nil { + return nil, fmt.Errorf("failed to sign store chunks request: %v", err) + } + request.Signature = signature + } + + // Call the gRPC method to store chunks + response, err := c.dispersalClient.StoreChunks(ctx, request) if err != nil { return nil, err } diff --git a/api/clients/v2/relay_client.go b/api/clients/v2/relay_client.go index 00ad656392..a808ac8ec7 100644 --- a/api/clients/v2/relay_client.go +++ b/api/clients/v2/relay_client.go @@ -6,10 +6,9 @@ import ( "fmt" "sync" - "github.com/Layr-Labs/eigenda/core" - "github.com/Layr-Labs/eigenda/relay/auth" - relaygrpc "github.com/Layr-Labs/eigenda/api/grpc/relay" + "github.com/Layr-Labs/eigenda/api/hashing" + "github.com/Layr-Labs/eigenda/core" corev2 "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/hashicorp/go-multierror" @@ -76,7 +75,7 @@ func NewRelayClient(config *RelayClientConfig, logger logging.Logger) (RelayClie return nil, fmt.Errorf("invalid config: %v", config) } - logger.Info("creating relay client", "config", config) + logger.Info("creating relay client", "urls", config.Sockets) initOnce := sync.Map{} for key := range config.Sockets { @@ -116,7 +115,7 @@ func (c *relayClient) signGetChunksRequest(ctx context.Context, request *relaygr return errors.New("no message signer provided in config, cannot sign get chunks request") } - hash := auth.HashGetChunksRequest(request) + hash := hashing.HashGetChunksRequest(request) hashArray := [32]byte{} copy(hashArray[:], hash) signature, err := c.config.MessageSigner(ctx, hashArray) diff --git a/api/clients/v2/retrieval_client.go b/api/clients/v2/retrieval_client.go index a253bfd79a..4ffc0c5f4b 100644 --- a/api/clients/v2/retrieval_client.go +++ b/api/clients/v2/retrieval_client.go @@ -136,6 +136,10 @@ func (r *retrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHe indices = append(indices, assignmentIndices...) } + if len(chunks) == 0 { + return nil, errors.New("failed to retrieve any chunks") + } + return r.verifier.Decode( chunks, indices, diff --git a/api/clients/v2/verification/blob_verifier.go b/api/clients/v2/verification/blob_verifier.go new file mode 100644 index 0000000000..d6a11d350a --- /dev/null +++ b/api/clients/v2/verification/blob_verifier.go @@ -0,0 +1,76 @@ +package verification + +import ( + "context" + "fmt" + + "github.com/Layr-Labs/eigenda/common" + + disperser "github.com/Layr-Labs/eigenda/api/grpc/disperser/v2" + verifierBindings "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDABlobVerifier" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + gethcommon "github.com/ethereum/go-ethereum/common" +) + +// BlobVerifier is responsible for making eth calls against the BlobVerifier contract to ensure cryptographic and +// structural integrity of V2 certificates +// +// The blob verifier contract is located at https://github.com/Layr-Labs/eigenda/blob/master/contracts/src/core/EigenDABlobVerifier.sol +type BlobVerifier struct { + // go binding around the EigenDABlobVerifier ethereum contract + blobVerifierCaller *verifierBindings.ContractEigenDABlobVerifierCaller +} + +// NewBlobVerifier constructs a BlobVerifier +func NewBlobVerifier( + ethClient *common.EthClient, // the eth client, which should already be set up + blobVerifierAddress string, // the hex address of the EigenDABlobVerifier contract +) (*BlobVerifier, error) { + + verifierCaller, err := verifierBindings.NewContractEigenDABlobVerifierCaller( + gethcommon.HexToAddress(blobVerifierAddress), + *ethClient) + + if err != nil { + return nil, fmt.Errorf("bind to verifier contract at %s: %s", blobVerifierAddress, err) + } + + return &BlobVerifier{ + blobVerifierCaller: verifierCaller, + }, nil +} + +// VerifyBlobV2FromSignedBatch calls the verifyBlobV2FromSignedBatch view function on the EigenDABlobVerifier contract +// +// This method returns nil if the blob is successfully verified. Otherwise, it returns an error. +// +// It is the responsibility of the caller to configure a timeout on the ctx, if a timeout is required. +func (v *BlobVerifier) VerifyBlobV2FromSignedBatch( + ctx context.Context, + // The signed batch that contains the blob being verified. This is obtained from the disperser, and is used + // to verify that the described blob actually exists in a valid batch. + signedBatch *disperser.SignedBatch, + // Contains all necessary information about the blob, so that it can be verified. + blobVerificationProof *disperser.BlobVerificationInfo, +) error { + convertedSignedBatch, err := verifierBindings.ConvertSignedBatch(signedBatch) + if err != nil { + return fmt.Errorf("convert signed batch: %s", err) + } + + convertedBlobVerificationProof, err := verifierBindings.ConvertVerificationProof(blobVerificationProof) + if err != nil { + return fmt.Errorf("convert blob verification proof: %s", err) + } + + err = v.blobVerifierCaller.VerifyBlobV2FromSignedBatch( + &bind.CallOpts{Context: ctx}, + *convertedSignedBatch, + *convertedBlobVerificationProof) + + if err != nil { + return fmt.Errorf("verify blob v2 from signed batch: %s", err) + } + + return nil +} diff --git a/api/constants.go b/api/constants.go new file mode 100644 index 0000000000..0f3772cce7 --- /dev/null +++ b/api/constants.go @@ -0,0 +1,4 @@ +package api + +// EigenLabsDisperserID is the ID of the disperser that is managed by Eigen Labs. +const EigenLabsDisperserID = uint32(0) diff --git a/api/docs/common.html b/api/docs/common.html index 1c47ca8d1b..522008e9f9 100644 --- a/api/docs/common.html +++ b/api/docs/common.html @@ -175,23 +175,19 @@

Table of Contents

  • - common/v2/common.proto + common/common.proto