From 44608b7ae59aa0832390c1fb6a71bf9d24d5891b Mon Sep 17 00:00:00 2001 From: rajivpoc Date: Sun, 16 Apr 2023 17:46:53 -0400 Subject: [PATCH 1/4] feat: bidtraces v3 with eligible timestamp ms --- cmd/tool/export-data-api-payloads-bids.go | 4 +- common/types.go | 43 ++++++ database/typesconv.go | 28 ++++ services/api/service.go | 4 +- services/api/service_test.go | 157 ++++++++++++++++++++++ 5 files changed, 232 insertions(+), 4 deletions(-) diff --git a/cmd/tool/export-data-api-payloads-bids.go b/cmd/tool/export-data-api-payloads-bids.go index 3a8de66c..a3f561a3 100644 --- a/cmd/tool/export-data-api-payloads-bids.go +++ b/cmd/tool/export-data-api-payloads-bids.go @@ -60,9 +60,9 @@ var DataAPIExportBids = &cobra.Command{ } log.Infof("got %d bids", len(bids)) - entries := make([]common.BidTraceV2WithTimestampJSON, len(bids)) + entries := make([]common.BidTraceV3WithEligibleTimestampMsJSON, len(bids)) for i, bid := range bids { - entries[i] = database.BuilderSubmissionEntryToBidTraceV2WithTimestampJSON(bid) + entries[i] = database.BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(bid) } if len(entries) == 0 { diff --git a/common/types.go b/common/types.go index 2dde1096..89a7f50c 100644 --- a/common/types.go +++ b/common/types.go @@ -299,6 +299,49 @@ func (b *BidTraceV2WithTimestampJSON) ToCSVRecord() []string { } } +type BidTraceV3WithEligibleTimestampMsJSON struct { + BidTraceV2WithTimestampJSON + EligibleTimestampMs int64 `json:"eligible_timestamp_ms,string,omitempty"` +} + +func (b *BidTraceV3WithEligibleTimestampMsJSON) CSVHeader() []string { + return []string{ + "slot", + "parent_hash", + "block_hash", + "builder_pubkey", + "proposer_pubkey", + "proposer_fee_recipient", + "gas_limit", + "gas_used", + "value", + "num_tx", + "block_number", + "timestamp", + "timestamp_ms", + "eligible_timestamp_ms", + } +} + +func (b *BidTraceV3WithEligibleTimestampMsJSON) ToCSVRecord() []string { + return []string{ + fmt.Sprint(b.Slot), + b.ParentHash, + b.BlockHash, + b.BuilderPubkey, + b.ProposerPubkey, + b.ProposerFeeRecipient, + fmt.Sprint(b.GasLimit), + fmt.Sprint(b.GasUsed), + b.Value, + fmt.Sprint(b.NumTx), + fmt.Sprint(b.BlockNumber), + fmt.Sprint(b.Timestamp), + fmt.Sprint(b.TimestampMs), + fmt.Sprint(b.EligibleTimestampMs), + } +} + type SignedBlindedBeaconBlock struct { Bellatrix *boostTypes.SignedBlindedBeaconBlock Capella *apiv1capella.SignedBlindedBeaconBlock diff --git a/database/typesconv.go b/database/typesconv.go index 2a5fba41..f7eee276 100644 --- a/database/typesconv.go +++ b/database/typesconv.go @@ -74,3 +74,31 @@ func BuilderSubmissionEntryToBidTraceV2WithTimestampJSON(payload *BuilderBlockSu }, } } + +func BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(payload *BuilderBlockSubmissionEntry) common.BidTraceV3WithEligibleTimestampMsJSON { + timestamp := payload.InsertedAt + if payload.ReceivedAt.Valid { + timestamp = payload.ReceivedAt.Time + } + + return common.BidTraceV3WithEligibleTimestampMsJSON{ + EligibleTimestampMs: payload.EligibleAt.Time.UnixMilli(), + BidTraceV2WithTimestampJSON: common.BidTraceV2WithTimestampJSON{ + Timestamp: timestamp.Unix(), + TimestampMs: timestamp.UnixMilli(), + BidTraceV2JSON: common.BidTraceV2JSON{ + Slot: payload.Slot, + ParentHash: payload.ParentHash, + BlockHash: payload.BlockHash, + BuilderPubkey: payload.BuilderPubkey, + ProposerPubkey: payload.ProposerPubkey, + ProposerFeeRecipient: payload.ProposerFeeRecipient, + GasLimit: payload.GasLimit, + GasUsed: payload.GasUsed, + Value: payload.Value, + NumTx: payload.NumTx, + BlockNumber: payload.BlockNumber, + }, + }, + } +} diff --git a/services/api/service.go b/services/api/service.go index b5a36c08..c2bab245 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -1710,9 +1710,9 @@ func (api *RelayAPI) handleDataBuilderBidsReceived(w http.ResponseWriter, req *h return } - response := make([]common.BidTraceV2WithTimestampJSON, len(blockSubmissions)) + response := make([]common.BidTraceV3WithEligibleTimestampMsJSON, len(blockSubmissions)) for i, payload := range blockSubmissions { - response[i] = database.BuilderSubmissionEntryToBidTraceV2WithTimestampJSON(payload) + response[i] = database.BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(payload) } api.RespondOK(w, response) diff --git a/services/api/service_test.go b/services/api/service_test.go index e0c1854d..c185b7e9 100644 --- a/services/api/service_test.go +++ b/services/api/service_test.go @@ -329,3 +329,160 @@ func TestDataApiGetDataProposerPayloadDelivered(t *testing.T) { } }) } + +func TestDataApiGetBuilderBlocksReceived(t *testing.T) { + path := "/relay/v1/data/bidtraces/builder_blocks_received" + + t.Run("Reject requests with cursor", func(t *testing.T) { + backend := newTestBackend(t, 1) + rr := backend.request(http.MethodGet, path+"?cursor=1", nil) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "cursor argument not supported") + }) + + t.Run("Accept valid slot", func(t *testing.T) { + backend := newTestBackend(t, 1) + + validSlot := uint64(2) + validSlotPath := fmt.Sprintf("%s?slot=%d", path, validSlot) + rr := backend.request(http.MethodGet, validSlotPath, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Accept valid slot", func(t *testing.T) { + backend := newTestBackend(t, 1) + + validSlot := uint64(2) + validSlotPath := fmt.Sprintf("%s?slot=%d", path, validSlot) + rr := backend.request(http.MethodGet, validSlotPath, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Reject invalid slot", func(t *testing.T) { + backend := newTestBackend(t, 1) + + invalidSlots := []string{ + "-1", + "1.1", + } + + for _, invalidSlot := range invalidSlots { + invalidSlotPath := fmt.Sprintf("%s?slot=%s", path, invalidSlot) + rr := backend.request(http.MethodGet, invalidSlotPath, nil) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "invalid slot argument") + } + }) + + t.Run("Accept valid block_hash", func(t *testing.T) { + backend := newTestBackend(t, 1) + + validBlockHash := "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rr := backend.request(http.MethodGet, path+"?block_hash="+validBlockHash, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Reject invalid block_hash", func(t *testing.T) { + backend := newTestBackend(t, 1) + + invalidBlockHashes := []string{ + // One character too long. + "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", + // One character too short. + "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + // Missing the 0x prefix. + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + // Has an invalid hex character ('z' at the end). + "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaz", + } + + for _, invalidBlockHash := range invalidBlockHashes { + rr := backend.request(http.MethodGet, path+"?block_hash="+invalidBlockHash, nil) + t.Log(invalidBlockHash) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "invalid block_hash argument") + } + }) + + t.Run("Accept valid block_number", func(t *testing.T) { + backend := newTestBackend(t, 1) + + validBlockNumber := uint64(2) + validBlockNumberPath := fmt.Sprintf("%s?block_number=%d", path, validBlockNumber) + rr := backend.request(http.MethodGet, validBlockNumberPath, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Reject invalid block_number", func(t *testing.T) { + backend := newTestBackend(t, 1) + + invalidBlockNumbers := []string{ + "-1", + "1.1", + } + + for _, invalidBlockNumber := range invalidBlockNumbers { + invalidBlockNumberPath := fmt.Sprintf("%s?block_number=%s", path, invalidBlockNumber) + rr := backend.request(http.MethodGet, invalidBlockNumberPath, nil) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "invalid block_number argument") + } + }) + + t.Run("Accept valid builder_pubkey", func(t *testing.T) { + backend := newTestBackend(t, 1) + + validBuilderPubkey := "0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b90890792" + rr := backend.request(http.MethodGet, path+"?builder_pubkey="+validBuilderPubkey, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Reject invalid builder_pubkey", func(t *testing.T) { + backend := newTestBackend(t, 1) + + invalidBuilderPubkeys := []string{ + // One character too long. + "0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b908907921", + // One character too short. + "0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b9089079", + // Missing the 0x prefix. + "6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b90890792", + // Has an invalid hex character ('z' at the end). + "0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b9089079z", + } + + for _, invalidBuilderPubkey := range invalidBuilderPubkeys { + rr := backend.request(http.MethodGet, path+"?builder_pubkey="+invalidBuilderPubkey, nil) + t.Log(invalidBuilderPubkey) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "invalid builder_pubkey argument") + } + }) + + t.Run("Reject no slot or block_hash or block_number or builder_pubkey", func(t *testing.T) { + backend := newTestBackend(t, 1) + rr := backend.request(http.MethodGet, path, nil) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "need to query for specific slot or block_hash or block_number or builder_pubkey") + }) + + t.Run("Accept valid limit", func(t *testing.T) { + backend := newTestBackend(t, 1) + blockNumber := uint64(1) + limit := uint64(1) + limitPath := fmt.Sprintf("%s?block_number=%d&limit=%d", path, blockNumber, limit) + rr := backend.request(http.MethodGet, limitPath, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Reject above max limit", func(t *testing.T) { + backend := newTestBackend(t, 1) + blockNumber := uint64(1) + maximumLimit := uint64(500) + oneAboveMaxLimit := maximumLimit + 1 + limitPath := fmt.Sprintf("%s?block_number=%d&limit=%d", path, blockNumber, oneAboveMaxLimit) + rr := backend.request(http.MethodGet, limitPath, nil) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), fmt.Sprintf("maximum limit is %d", maximumLimit)) + }) +} From c484aa14c0a4f4fba7c48c675c82e2822a7adde7 Mon Sep 17 00:00:00 2001 From: rajivpoc Date: Sun, 16 Apr 2023 20:41:53 -0400 Subject: [PATCH 2/4] chore: delivered payloads use bidtracev3 --- cmd/tool/export-data-api-payloads-bids.go | 4 +- .../export-data-api-payloads-delivered.go | 4 +- common/types.go | 33 ++++++--- database/typesconv.go | 69 +++++++++++++------ services/api/service.go | 8 +-- 5 files changed, 81 insertions(+), 37 deletions(-) diff --git a/cmd/tool/export-data-api-payloads-bids.go b/cmd/tool/export-data-api-payloads-bids.go index a3f561a3..242cbd28 100644 --- a/cmd/tool/export-data-api-payloads-bids.go +++ b/cmd/tool/export-data-api-payloads-bids.go @@ -60,9 +60,9 @@ var DataAPIExportBids = &cobra.Command{ } log.Infof("got %d bids", len(bids)) - entries := make([]common.BidTraceV3WithEligibleTimestampMsJSON, len(bids)) + entries := make([]common.BidTraceV3JSON, len(bids)) for i, bid := range bids { - entries[i] = database.BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(bid) + entries[i] = database.BuilderSubmissionEntryToBidTraceV3JSON(bid) } if len(entries) == 0 { diff --git a/cmd/tool/export-data-api-payloads-delivered.go b/cmd/tool/export-data-api-payloads-delivered.go index 3060d0b8..c896ddd8 100644 --- a/cmd/tool/export-data-api-payloads-delivered.go +++ b/cmd/tool/export-data-api-payloads-delivered.go @@ -72,9 +72,9 @@ var DataAPIExportPayloads = &cobra.Command{ } log.Infof("got %d payloads", len(deliveredPayloads)) - entries := make([]common.BidTraceV2JSON, len(deliveredPayloads)) + entries := make([]common.BidTraceV3JSON, len(deliveredPayloads)) for i, payload := range deliveredPayloads { - entries[i] = database.DeliveredPayloadEntryToBidTraceV2JSON(payload) + entries[i] = database.DeliveredPayloadEntryToBidTraceV3JSON(payload) } if len(entries) == 0 { diff --git a/common/types.go b/common/types.go index 89a7f50c..eaf50f63 100644 --- a/common/types.go +++ b/common/types.go @@ -299,12 +299,25 @@ func (b *BidTraceV2WithTimestampJSON) ToCSVRecord() []string { } } -type BidTraceV3WithEligibleTimestampMsJSON struct { - BidTraceV2WithTimestampJSON - EligibleTimestampMs int64 `json:"eligible_timestamp_ms,string,omitempty"` -} - -func (b *BidTraceV3WithEligibleTimestampMsJSON) CSVHeader() []string { +type BidTraceV3JSON struct { + Slot uint64 `json:"slot,string"` + ParentHash string `json:"parent_hash"` + BlockHash string `json:"block_hash"` + BuilderPubkey string `json:"builder_pubkey"` + ProposerPubkey string `json:"proposer_pubkey"` + ProposerFeeRecipient string `json:"proposer_fee_recipient"` + GasLimit uint64 `json:"gas_limit,string"` + GasUsed uint64 `json:"gas_used,string"` + Value string `json:"value"` + NumTx uint64 `json:"num_tx,string"` + BlockNumber uint64 `json:"block_number,string"` + Timestamp int64 `json:"timestamp,string,omitempty"` + TimestampMs int64 `json:"timestamp_ms,string,omitempty"` + EligibleAtTimestampMs int64 `json:"eligible_at_timestamp_ms,string,omitempty"` + SignedAtTimestampMs int64 `json:"signed_at_timestamp_ms,string,omitempty"` +} + +func (b *BidTraceV3JSON) CSVHeader() []string { return []string{ "slot", "parent_hash", @@ -319,11 +332,12 @@ func (b *BidTraceV3WithEligibleTimestampMsJSON) CSVHeader() []string { "block_number", "timestamp", "timestamp_ms", - "eligible_timestamp_ms", + "eligible_at_timestamp_ms", + "signed_at_timestamp_ms", } } -func (b *BidTraceV3WithEligibleTimestampMsJSON) ToCSVRecord() []string { +func (b *BidTraceV3JSON) ToCSVRecord() []string { return []string{ fmt.Sprint(b.Slot), b.ParentHash, @@ -338,7 +352,8 @@ func (b *BidTraceV3WithEligibleTimestampMsJSON) ToCSVRecord() []string { fmt.Sprint(b.BlockNumber), fmt.Sprint(b.Timestamp), fmt.Sprint(b.TimestampMs), - fmt.Sprint(b.EligibleTimestampMs), + fmt.Sprint(b.EligibleAtTimestampMs), + fmt.Sprint(b.SignedAtTimestampMs), } } diff --git a/database/typesconv.go b/database/typesconv.go index f7eee276..b75525e9 100644 --- a/database/typesconv.go +++ b/database/typesconv.go @@ -75,30 +75,59 @@ func BuilderSubmissionEntryToBidTraceV2WithTimestampJSON(payload *BuilderBlockSu } } -func BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(payload *BuilderBlockSubmissionEntry) common.BidTraceV3WithEligibleTimestampMsJSON { +func DeliveredPayloadEntryToBidTraceV3JSON(payload *DeliveredPayloadEntry) common.BidTraceV3JSON { + bidTrace := common.BidTraceV3JSON{ + Slot: payload.Slot, + ParentHash: payload.ParentHash, + BlockHash: payload.BlockHash, + BuilderPubkey: payload.BuilderPubkey, + ProposerPubkey: payload.ProposerPubkey, + ProposerFeeRecipient: payload.ProposerFeeRecipient, + GasLimit: payload.GasLimit, + GasUsed: payload.GasUsed, + Value: payload.Value, + NumTx: payload.NumTx, + BlockNumber: payload.BlockNumber, + Timestamp: int64(0), + TimestampMs: int64(0), + SignedAtTimestampMs: int64(0), + EligibleAtTimestampMs: int64(0), + } + + if payload.SignedAt.Valid { + bidTrace.SignedAtTimestampMs = payload.SignedAt.Time.UnixMilli() + } + + return bidTrace +} + +func BuilderSubmissionEntryToBidTraceV3JSON(payload *BuilderBlockSubmissionEntry) common.BidTraceV3JSON { timestamp := payload.InsertedAt if payload.ReceivedAt.Valid { timestamp = payload.ReceivedAt.Time } - return common.BidTraceV3WithEligibleTimestampMsJSON{ - EligibleTimestampMs: payload.EligibleAt.Time.UnixMilli(), - BidTraceV2WithTimestampJSON: common.BidTraceV2WithTimestampJSON{ - Timestamp: timestamp.Unix(), - TimestampMs: timestamp.UnixMilli(), - BidTraceV2JSON: common.BidTraceV2JSON{ - Slot: payload.Slot, - ParentHash: payload.ParentHash, - BlockHash: payload.BlockHash, - BuilderPubkey: payload.BuilderPubkey, - ProposerPubkey: payload.ProposerPubkey, - ProposerFeeRecipient: payload.ProposerFeeRecipient, - GasLimit: payload.GasLimit, - GasUsed: payload.GasUsed, - Value: payload.Value, - NumTx: payload.NumTx, - BlockNumber: payload.BlockNumber, - }, - }, + bidtrace := common.BidTraceV3JSON{ + Timestamp: timestamp.Unix(), + TimestampMs: timestamp.UnixMilli(), + Slot: payload.Slot, + ParentHash: payload.ParentHash, + BlockHash: payload.BlockHash, + BuilderPubkey: payload.BuilderPubkey, + ProposerPubkey: payload.ProposerPubkey, + ProposerFeeRecipient: payload.ProposerFeeRecipient, + GasLimit: payload.GasLimit, + GasUsed: payload.GasUsed, + Value: payload.Value, + NumTx: payload.NumTx, + BlockNumber: payload.BlockNumber, + EligibleAtTimestampMs: int64(0), + SignedAtTimestampMs: int64(0), } + + if payload.EligibleAt.Valid { + bidtrace.EligibleAtTimestampMs = payload.EligibleAt.Time.UnixMilli() + } + + return bidtrace } diff --git a/services/api/service.go b/services/api/service.go index c2bab245..543eb187 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -1625,9 +1625,9 @@ func (api *RelayAPI) handleDataProposerPayloadDelivered(w http.ResponseWriter, r return } - response := make([]common.BidTraceV2JSON, len(deliveredPayloads)) + response := make([]common.BidTraceV3JSON, len(deliveredPayloads)) for i, payload := range deliveredPayloads { - response[i] = database.DeliveredPayloadEntryToBidTraceV2JSON(payload) + response[i] = database.DeliveredPayloadEntryToBidTraceV3JSON(payload) } api.RespondOK(w, response) @@ -1710,9 +1710,9 @@ func (api *RelayAPI) handleDataBuilderBidsReceived(w http.ResponseWriter, req *h return } - response := make([]common.BidTraceV3WithEligibleTimestampMsJSON, len(blockSubmissions)) + response := make([]common.BidTraceV3JSON, len(blockSubmissions)) for i, payload := range blockSubmissions { - response[i] = database.BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(payload) + response[i] = database.BuilderSubmissionEntryToBidTraceV3JSON(payload) } api.RespondOK(w, response) From 22acbc7bd07621f8494aef5dcc0d8586ac4ae42c Mon Sep 17 00:00:00 2001 From: rajivpoc Date: Sun, 16 Apr 2023 17:46:53 -0400 Subject: [PATCH 3/4] feat: bidtraces v3 with eligible timestamp ms --- cmd/tool/export-data-api-payloads-bids.go | 4 +- common/types.go | 43 ++++++ database/typesconv.go | 27 ++++ services/api/service.go | 4 +- services/api/service_test.go | 158 +++++++++++++++++++++- 5 files changed, 231 insertions(+), 5 deletions(-) diff --git a/cmd/tool/export-data-api-payloads-bids.go b/cmd/tool/export-data-api-payloads-bids.go index 3a8de66c..a3f561a3 100644 --- a/cmd/tool/export-data-api-payloads-bids.go +++ b/cmd/tool/export-data-api-payloads-bids.go @@ -60,9 +60,9 @@ var DataAPIExportBids = &cobra.Command{ } log.Infof("got %d bids", len(bids)) - entries := make([]common.BidTraceV2WithTimestampJSON, len(bids)) + entries := make([]common.BidTraceV3WithEligibleTimestampMsJSON, len(bids)) for i, bid := range bids { - entries[i] = database.BuilderSubmissionEntryToBidTraceV2WithTimestampJSON(bid) + entries[i] = database.BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(bid) } if len(entries) == 0 { diff --git a/common/types.go b/common/types.go index 2e54782f..ece1b2b3 100644 --- a/common/types.go +++ b/common/types.go @@ -274,6 +274,49 @@ func (b *BidTraceV2WithTimestampJSON) ToCSVRecord() []string { } } +type BidTraceV3WithEligibleTimestampMsJSON struct { + BidTraceV2WithTimestampJSON + EligibleTimestampMs int64 `json:"eligible_timestamp_ms,string,omitempty"` +} + +func (b *BidTraceV3WithEligibleTimestampMsJSON) CSVHeader() []string { + return []string{ + "slot", + "parent_hash", + "block_hash", + "builder_pubkey", + "proposer_pubkey", + "proposer_fee_recipient", + "gas_limit", + "gas_used", + "value", + "num_tx", + "block_number", + "timestamp", + "timestamp_ms", + "eligible_timestamp_ms", + } +} + +func (b *BidTraceV3WithEligibleTimestampMsJSON) ToCSVRecord() []string { + return []string{ + fmt.Sprint(b.Slot), + b.ParentHash, + b.BlockHash, + b.BuilderPubkey, + b.ProposerPubkey, + b.ProposerFeeRecipient, + fmt.Sprint(b.GasLimit), + fmt.Sprint(b.GasUsed), + b.Value, + fmt.Sprint(b.NumTx), + fmt.Sprint(b.BlockNumber), + fmt.Sprint(b.Timestamp), + fmt.Sprint(b.TimestampMs), + fmt.Sprint(b.EligibleTimestampMs), + } +} + type SignedBlindedBeaconBlock struct { Bellatrix *boostTypes.SignedBlindedBeaconBlock Capella *apiv1capella.SignedBlindedBeaconBlock diff --git a/database/typesconv.go b/database/typesconv.go index bf33ddea..41db8c37 100644 --- a/database/typesconv.go +++ b/database/typesconv.go @@ -120,3 +120,30 @@ func ExecutionPayloadEntryToExecutionPayload(executionPayloadEntry *ExecutionPay return nil, ErrUnsupportedExecutionPayload } } +func BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(payload *BuilderBlockSubmissionEntry) common.BidTraceV3WithEligibleTimestampMsJSON { + timestamp := payload.InsertedAt + if payload.ReceivedAt.Valid { + timestamp = payload.ReceivedAt.Time + } + + return common.BidTraceV3WithEligibleTimestampMsJSON{ + EligibleTimestampMs: payload.EligibleAt.Time.UnixMilli(), + BidTraceV2WithTimestampJSON: common.BidTraceV2WithTimestampJSON{ + Timestamp: timestamp.Unix(), + TimestampMs: timestamp.UnixMilli(), + BidTraceV2JSON: common.BidTraceV2JSON{ + Slot: payload.Slot, + ParentHash: payload.ParentHash, + BlockHash: payload.BlockHash, + BuilderPubkey: payload.BuilderPubkey, + ProposerPubkey: payload.ProposerPubkey, + ProposerFeeRecipient: payload.ProposerFeeRecipient, + GasLimit: payload.GasLimit, + GasUsed: payload.GasUsed, + Value: payload.Value, + NumTx: payload.NumTx, + BlockNumber: payload.BlockNumber, + }, + }, + } +} diff --git a/services/api/service.go b/services/api/service.go index dcbf54ce..672c626a 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -2187,9 +2187,9 @@ func (api *RelayAPI) handleDataBuilderBidsReceived(w http.ResponseWriter, req *h return } - response := make([]common.BidTraceV2WithTimestampJSON, len(blockSubmissions)) + response := make([]common.BidTraceV3WithEligibleTimestampMsJSON, len(blockSubmissions)) for i, payload := range blockSubmissions { - response[i] = database.BuilderSubmissionEntryToBidTraceV2WithTimestampJSON(payload) + response[i] = database.BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(payload) } api.RespondOK(w, response) diff --git a/services/api/service_test.go b/services/api/service_test.go index 70ab067c..6a7b678f 100644 --- a/services/api/service_test.go +++ b/services/api/service_test.go @@ -369,7 +369,6 @@ func TestDataApiGetDataProposerPayloadDelivered(t *testing.T) { } }) } - func TestBuilderSubmitBlockSSZ(t *testing.T) { requestPayloadJSONBytes := common.LoadGzippedBytes(t, "../../testdata/submitBlockPayloadCapella_Goerli.json.gz") @@ -489,3 +488,160 @@ func gzipBytes(t *testing.T, b []byte) []byte { require.NoError(t, zw.Close()) return buf.Bytes() } + +func TestDataApiGetBuilderBlocksReceived(t *testing.T) { + path := "/relay/v1/data/bidtraces/builder_blocks_received" + + t.Run("Reject requests with cursor", func(t *testing.T) { + backend := newTestBackend(t, 1) + rr := backend.request(http.MethodGet, path+"?cursor=1", nil) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "cursor argument not supported") + }) + + t.Run("Accept valid slot", func(t *testing.T) { + backend := newTestBackend(t, 1) + + validSlot := uint64(2) + validSlotPath := fmt.Sprintf("%s?slot=%d", path, validSlot) + rr := backend.request(http.MethodGet, validSlotPath, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Accept valid slot", func(t *testing.T) { + backend := newTestBackend(t, 1) + + validSlot := uint64(2) + validSlotPath := fmt.Sprintf("%s?slot=%d", path, validSlot) + rr := backend.request(http.MethodGet, validSlotPath, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Reject invalid slot", func(t *testing.T) { + backend := newTestBackend(t, 1) + + invalidSlots := []string{ + "-1", + "1.1", + } + + for _, invalidSlot := range invalidSlots { + invalidSlotPath := fmt.Sprintf("%s?slot=%s", path, invalidSlot) + rr := backend.request(http.MethodGet, invalidSlotPath, nil) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "invalid slot argument") + } + }) + + t.Run("Accept valid block_hash", func(t *testing.T) { + backend := newTestBackend(t, 1) + + validBlockHash := "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + rr := backend.request(http.MethodGet, path+"?block_hash="+validBlockHash, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Reject invalid block_hash", func(t *testing.T) { + backend := newTestBackend(t, 1) + + invalidBlockHashes := []string{ + // One character too long. + "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", + // One character too short. + "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + // Missing the 0x prefix. + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + // Has an invalid hex character ('z' at the end). + "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaz", + } + + for _, invalidBlockHash := range invalidBlockHashes { + rr := backend.request(http.MethodGet, path+"?block_hash="+invalidBlockHash, nil) + t.Log(invalidBlockHash) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "invalid block_hash argument") + } + }) + + t.Run("Accept valid block_number", func(t *testing.T) { + backend := newTestBackend(t, 1) + + validBlockNumber := uint64(2) + validBlockNumberPath := fmt.Sprintf("%s?block_number=%d", path, validBlockNumber) + rr := backend.request(http.MethodGet, validBlockNumberPath, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Reject invalid block_number", func(t *testing.T) { + backend := newTestBackend(t, 1) + + invalidBlockNumbers := []string{ + "-1", + "1.1", + } + + for _, invalidBlockNumber := range invalidBlockNumbers { + invalidBlockNumberPath := fmt.Sprintf("%s?block_number=%s", path, invalidBlockNumber) + rr := backend.request(http.MethodGet, invalidBlockNumberPath, nil) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "invalid block_number argument") + } + }) + + t.Run("Accept valid builder_pubkey", func(t *testing.T) { + backend := newTestBackend(t, 1) + + validBuilderPubkey := "0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b90890792" + rr := backend.request(http.MethodGet, path+"?builder_pubkey="+validBuilderPubkey, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Reject invalid builder_pubkey", func(t *testing.T) { + backend := newTestBackend(t, 1) + + invalidBuilderPubkeys := []string{ + // One character too long. + "0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b908907921", + // One character too short. + "0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b9089079", + // Missing the 0x prefix. + "6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b90890792", + // Has an invalid hex character ('z' at the end). + "0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b9089079z", + } + + for _, invalidBuilderPubkey := range invalidBuilderPubkeys { + rr := backend.request(http.MethodGet, path+"?builder_pubkey="+invalidBuilderPubkey, nil) + t.Log(invalidBuilderPubkey) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "invalid builder_pubkey argument") + } + }) + + t.Run("Reject no slot or block_hash or block_number or builder_pubkey", func(t *testing.T) { + backend := newTestBackend(t, 1) + rr := backend.request(http.MethodGet, path, nil) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), "need to query for specific slot or block_hash or block_number or builder_pubkey") + }) + + t.Run("Accept valid limit", func(t *testing.T) { + backend := newTestBackend(t, 1) + blockNumber := uint64(1) + limit := uint64(1) + limitPath := fmt.Sprintf("%s?block_number=%d&limit=%d", path, blockNumber, limit) + rr := backend.request(http.MethodGet, limitPath, nil) + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("Reject above max limit", func(t *testing.T) { + backend := newTestBackend(t, 1) + blockNumber := uint64(1) + maximumLimit := uint64(500) + oneAboveMaxLimit := maximumLimit + 1 + limitPath := fmt.Sprintf("%s?block_number=%d&limit=%d", path, blockNumber, oneAboveMaxLimit) + rr := backend.request(http.MethodGet, limitPath, nil) + require.Equal(t, http.StatusBadRequest, rr.Code) + require.Contains(t, rr.Body.String(), fmt.Sprintf("maximum limit is %d", maximumLimit)) + }) +} From 155cce2b059979ce6bc471425ab58d9c34b50d42 Mon Sep 17 00:00:00 2001 From: rajivpoc Date: Sun, 16 Apr 2023 20:41:53 -0400 Subject: [PATCH 4/4] chore: delivered payloads use bidtracev3 --- cmd/tool/export-data-api-payloads-bids.go | 4 +- .../export-data-api-payloads-delivered.go | 4 +- common/types.go | 33 ++++++--- database/typesconv.go | 70 +++++++++++++------ services/api/service.go | 8 +-- 5 files changed, 82 insertions(+), 37 deletions(-) diff --git a/cmd/tool/export-data-api-payloads-bids.go b/cmd/tool/export-data-api-payloads-bids.go index a3f561a3..242cbd28 100644 --- a/cmd/tool/export-data-api-payloads-bids.go +++ b/cmd/tool/export-data-api-payloads-bids.go @@ -60,9 +60,9 @@ var DataAPIExportBids = &cobra.Command{ } log.Infof("got %d bids", len(bids)) - entries := make([]common.BidTraceV3WithEligibleTimestampMsJSON, len(bids)) + entries := make([]common.BidTraceV3JSON, len(bids)) for i, bid := range bids { - entries[i] = database.BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(bid) + entries[i] = database.BuilderSubmissionEntryToBidTraceV3JSON(bid) } if len(entries) == 0 { diff --git a/cmd/tool/export-data-api-payloads-delivered.go b/cmd/tool/export-data-api-payloads-delivered.go index 3060d0b8..c896ddd8 100644 --- a/cmd/tool/export-data-api-payloads-delivered.go +++ b/cmd/tool/export-data-api-payloads-delivered.go @@ -72,9 +72,9 @@ var DataAPIExportPayloads = &cobra.Command{ } log.Infof("got %d payloads", len(deliveredPayloads)) - entries := make([]common.BidTraceV2JSON, len(deliveredPayloads)) + entries := make([]common.BidTraceV3JSON, len(deliveredPayloads)) for i, payload := range deliveredPayloads { - entries[i] = database.DeliveredPayloadEntryToBidTraceV2JSON(payload) + entries[i] = database.DeliveredPayloadEntryToBidTraceV3JSON(payload) } if len(entries) == 0 { diff --git a/common/types.go b/common/types.go index ece1b2b3..b25d8cdb 100644 --- a/common/types.go +++ b/common/types.go @@ -274,12 +274,25 @@ func (b *BidTraceV2WithTimestampJSON) ToCSVRecord() []string { } } -type BidTraceV3WithEligibleTimestampMsJSON struct { - BidTraceV2WithTimestampJSON - EligibleTimestampMs int64 `json:"eligible_timestamp_ms,string,omitempty"` -} - -func (b *BidTraceV3WithEligibleTimestampMsJSON) CSVHeader() []string { +type BidTraceV3JSON struct { + Slot uint64 `json:"slot,string"` + ParentHash string `json:"parent_hash"` + BlockHash string `json:"block_hash"` + BuilderPubkey string `json:"builder_pubkey"` + ProposerPubkey string `json:"proposer_pubkey"` + ProposerFeeRecipient string `json:"proposer_fee_recipient"` + GasLimit uint64 `json:"gas_limit,string"` + GasUsed uint64 `json:"gas_used,string"` + Value string `json:"value"` + NumTx uint64 `json:"num_tx,string"` + BlockNumber uint64 `json:"block_number,string"` + Timestamp int64 `json:"timestamp,string,omitempty"` + TimestampMs int64 `json:"timestamp_ms,string,omitempty"` + EligibleAtTimestampMs int64 `json:"eligible_at_timestamp_ms,string,omitempty"` + SignedAtTimestampMs int64 `json:"signed_at_timestamp_ms,string,omitempty"` +} + +func (b *BidTraceV3JSON) CSVHeader() []string { return []string{ "slot", "parent_hash", @@ -294,11 +307,12 @@ func (b *BidTraceV3WithEligibleTimestampMsJSON) CSVHeader() []string { "block_number", "timestamp", "timestamp_ms", - "eligible_timestamp_ms", + "eligible_at_timestamp_ms", + "signed_at_timestamp_ms", } } -func (b *BidTraceV3WithEligibleTimestampMsJSON) ToCSVRecord() []string { +func (b *BidTraceV3JSON) ToCSVRecord() []string { return []string{ fmt.Sprint(b.Slot), b.ParentHash, @@ -313,7 +327,8 @@ func (b *BidTraceV3WithEligibleTimestampMsJSON) ToCSVRecord() []string { fmt.Sprint(b.BlockNumber), fmt.Sprint(b.Timestamp), fmt.Sprint(b.TimestampMs), - fmt.Sprint(b.EligibleTimestampMs), + fmt.Sprint(b.EligibleAtTimestampMs), + fmt.Sprint(b.SignedAtTimestampMs), } } diff --git a/database/typesconv.go b/database/typesconv.go index 41db8c37..5c444483 100644 --- a/database/typesconv.go +++ b/database/typesconv.go @@ -120,30 +120,60 @@ func ExecutionPayloadEntryToExecutionPayload(executionPayloadEntry *ExecutionPay return nil, ErrUnsupportedExecutionPayload } } -func BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(payload *BuilderBlockSubmissionEntry) common.BidTraceV3WithEligibleTimestampMsJSON { + +func DeliveredPayloadEntryToBidTraceV3JSON(payload *DeliveredPayloadEntry) common.BidTraceV3JSON { + bidTrace := common.BidTraceV3JSON{ + Slot: payload.Slot, + ParentHash: payload.ParentHash, + BlockHash: payload.BlockHash, + BuilderPubkey: payload.BuilderPubkey, + ProposerPubkey: payload.ProposerPubkey, + ProposerFeeRecipient: payload.ProposerFeeRecipient, + GasLimit: payload.GasLimit, + GasUsed: payload.GasUsed, + Value: payload.Value, + NumTx: payload.NumTx, + BlockNumber: payload.BlockNumber, + Timestamp: int64(0), + TimestampMs: int64(0), + SignedAtTimestampMs: int64(0), + EligibleAtTimestampMs: int64(0), + } + + if payload.SignedAt.Valid { + bidTrace.SignedAtTimestampMs = payload.SignedAt.Time.UnixMilli() + } + + return bidTrace +} + +func BuilderSubmissionEntryToBidTraceV3JSON(payload *BuilderBlockSubmissionEntry) common.BidTraceV3JSON { timestamp := payload.InsertedAt if payload.ReceivedAt.Valid { timestamp = payload.ReceivedAt.Time } - return common.BidTraceV3WithEligibleTimestampMsJSON{ - EligibleTimestampMs: payload.EligibleAt.Time.UnixMilli(), - BidTraceV2WithTimestampJSON: common.BidTraceV2WithTimestampJSON{ - Timestamp: timestamp.Unix(), - TimestampMs: timestamp.UnixMilli(), - BidTraceV2JSON: common.BidTraceV2JSON{ - Slot: payload.Slot, - ParentHash: payload.ParentHash, - BlockHash: payload.BlockHash, - BuilderPubkey: payload.BuilderPubkey, - ProposerPubkey: payload.ProposerPubkey, - ProposerFeeRecipient: payload.ProposerFeeRecipient, - GasLimit: payload.GasLimit, - GasUsed: payload.GasUsed, - Value: payload.Value, - NumTx: payload.NumTx, - BlockNumber: payload.BlockNumber, - }, - }, + bidtrace := common.BidTraceV3JSON{ + Timestamp: timestamp.Unix(), + TimestampMs: timestamp.UnixMilli(), + Slot: payload.Slot, + ParentHash: payload.ParentHash, + BlockHash: payload.BlockHash, + BuilderPubkey: payload.BuilderPubkey, + ProposerPubkey: payload.ProposerPubkey, + ProposerFeeRecipient: payload.ProposerFeeRecipient, + GasLimit: payload.GasLimit, + GasUsed: payload.GasUsed, + Value: payload.Value, + NumTx: payload.NumTx, + BlockNumber: payload.BlockNumber, + EligibleAtTimestampMs: int64(0), + SignedAtTimestampMs: int64(0), } + + if payload.EligibleAt.Valid { + bidtrace.EligibleAtTimestampMs = payload.EligibleAt.Time.UnixMilli() + } + + return bidtrace } diff --git a/services/api/service.go b/services/api/service.go index 672c626a..af750cf2 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -2102,9 +2102,9 @@ func (api *RelayAPI) handleDataProposerPayloadDelivered(w http.ResponseWriter, r return } - response := make([]common.BidTraceV2JSON, len(deliveredPayloads)) + response := make([]common.BidTraceV3JSON, len(deliveredPayloads)) for i, payload := range deliveredPayloads { - response[i] = database.DeliveredPayloadEntryToBidTraceV2JSON(payload) + response[i] = database.DeliveredPayloadEntryToBidTraceV3JSON(payload) } api.RespondOK(w, response) @@ -2187,9 +2187,9 @@ func (api *RelayAPI) handleDataBuilderBidsReceived(w http.ResponseWriter, req *h return } - response := make([]common.BidTraceV3WithEligibleTimestampMsJSON, len(blockSubmissions)) + response := make([]common.BidTraceV3JSON, len(blockSubmissions)) for i, payload := range blockSubmissions { - response[i] = database.BuilderSubmissionEntryToBidTraceV3WithEligibleTimestampMsJSON(payload) + response[i] = database.BuilderSubmissionEntryToBidTraceV3JSON(payload) } api.RespondOK(w, response)