build(deps): bump golang.org/x/net from 0.10.0 to 0.17.0

.github/dependabot.yml

@@ -6,6 +6,6 @@ updates:
   - package-ecosystem: gomod
     directory: /
     schedule:
-      interval: daily
+      interval: weekly
     reviewers:
-      - "metachris"
+      - alextes

.github/workflows/build.yml

@@ -0,0 +1,37 @@
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches:
+      - "*"
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: alextes
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract short SHA
+        run: echo "SHORT_SHA=$(echo ${{ github.sha }} | cut -c 1-7)" >> $GITHUB_ENV
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: |
+            ultrasoundorg/mev-boost-relay:${{ env.SHORT_SHA }}
+            ultrasoundorg/mev-boost-relay:latest

cmd/api.go

@@ -55,6 +55,7 @@ func init() {
 	apiCmd.Flags().StringSliceVar(&beaconNodeURIs, "beacon-uris", defaultBeaconURIs, "beacon endpoints")
 	apiCmd.Flags().StringVar(&redisURI, "redis-uri", defaultRedisURI, "redis uri")
 	apiCmd.Flags().StringVar(&redisReadonlyURI, "redis-readonly-uri", defaultRedisReadonlyURI, "redis readonly uri")
+	apiCmd.Flags().StringVar(&redisArchiveURI, "redis-archive-uri", defaultRedisArchiveURI, "redis block submission archive uri")
 	apiCmd.Flags().StringVar(&postgresDSN, "db", defaultPostgresDSN, "PostgreSQL DSN")
 	apiCmd.Flags().StringSliceVar(&memcachedURIs, "memcached-uris", defaultMemcachedURIs,
 		"Enable memcached, typically used as secondary backup to Redis for redundancy")
@@ -107,12 +108,14 @@ var apiCmd = &cobra.Command{
 		beaconClient := beaconclient.NewMultiBeaconClient(log, beaconInstances)
 
 		// Connect to Redis
-		if redisReadonlyURI == "" {
-			log.Infof("Connecting to Redis at %s ...", redisURI)
-		} else {
-			log.Infof("Connecting to Redis at %s / readonly: %s ...", redisURI, redisReadonlyURI)
+		log.Infof("Connecting to Redis at %s ...", redisURI)
+		if redisReadonlyURI != "" {
+			log.Infof("Connecting to readonly Redis at %s ...", redisReadonlyURI)
+		}
+		if redisArchiveURI != "" {
+			log.Infof("Connecting to block submission archive Redis at %s ...", redisArchiveURI)
 		}
-		redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, redisReadonlyURI)
+		redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, redisReadonlyURI, redisArchiveURI)
 		if err != nil {
 			log.WithError(err).Fatalf("Failed to connect to Redis at %s", redisURI)
 		}

cmd/housekeeper.go

@@ -68,7 +68,7 @@ var housekeeperCmd = &cobra.Command{
 		beaconClient := beaconclient.NewMultiBeaconClient(log, beaconInstances)
 
 		// Connect to Redis and setup the datastore
-		redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, "")
+		redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, "", "")
 		if err != nil {
 			log.WithError(err).Fatalf("Failed to connect to Redis at %s", redisURI)
 		}

cmd/variables.go

@@ -11,6 +11,7 @@ var (
 	defaultBeaconURIs       = common.GetSliceEnv("BEACON_URIS", []string{"http://localhost:3500"})
 	defaultRedisURI         = common.GetEnv("REDIS_URI", "localhost:6379")
 	defaultRedisReadonlyURI = common.GetEnv("REDIS_READONLY_URI", "")
+	defaultRedisArchiveURI  = common.GetEnv("REDIS_ARCHIVE_URI", "")
 	defaultPostgresDSN      = common.GetEnv("POSTGRES_DSN", "")
 	defaultMemcachedURIs    = common.GetSliceEnv("MEMCACHED_URIS", nil)
 	defaultLogJSON          = os.Getenv("LOG_JSON") != ""
@@ -19,6 +20,7 @@ var (
 	beaconNodeURIs   []string
 	redisURI         string
 	redisReadonlyURI string
+	redisArchiveURI  string
 	postgresDSN      string
 	memcachedURIs    []string
 

cmd/website.go

@@ -67,12 +67,14 @@ var websiteCmd = &cobra.Command{
 		log.Debug(networkInfo.String())
 
 		// Connect to Redis
-		if redisReadonlyURI == "" {
-			log.Infof("Connecting to Redis at %s ...", redisURI)
-		} else {
-			log.Infof("Connecting to Redis at %s / readonly: %s ...", redisURI, redisReadonlyURI)
+		log.Infof("Connecting to Redis at %s ...", redisURI)
+		if redisReadonlyURI != "" {
+			log.Infof("Connecting to readonly Redis at %s ...", redisReadonlyURI)
+		}
+		if redisArchiveURI != "" {
+			log.Infof("Connecting to block submission archive Redis at %s ...", redisArchiveURI)
 		}
-		redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, redisReadonlyURI)
+		redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, redisReadonlyURI, redisArchiveURI)
 		if err != nil {
 			log.WithError(err).Fatalf("Failed to connect to Redis at %s", redisURI)
 		}

common/logging.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"os"
+	"time"
 
 	"github.com/sirupsen/logrus"
 )
@@ -11,10 +12,13 @@ func LogSetup(json bool, logLevel string) *logrus.Entry {
 	log.Logger.SetOutput(os.Stdout)
 
 	if json {
-		log.Logger.SetFormatter(&logrus.JSONFormatter{})
+		log.Logger.SetFormatter(&logrus.JSONFormatter{
+			TimestampFormat: time.RFC3339Nano,
+		})
 	} else {
 		log.Logger.SetFormatter(&logrus.TextFormatter{
-			FullTimestamp: true,
+			TimestampFormat: time.RFC3339Nano,
+			FullTimestamp:   true,
 		})
 	}
 

datastore/datastore.go

@@ -219,7 +219,20 @@ func (ds *Datastore) GetGetPayloadResponse(log *logrus.Entry, slot uint64, propo
 		}
 	}
 
-	// 3. try to get from database (should not happen, it's just a backup)
+	// 3. try to get it from redis archive
+	if ds.redis.archiveClient != nil {
+		resp, err = ds.redis.GetArchivedExecutionPayload(slot, _proposerPubkey, _blockHash)
+		if errors.Is(err, redis.Nil) {
+			log.WithError(err).Info("execution payload not found in redis archive")
+		} else if err != nil {
+			log.WithError(err).Error("error getting execution payload from redis archive")
+		} else if resp != nil {
+			log.Info("getPayload response from redis archive")
+			return resp, nil
+		}
+	}
+
+	// 4. try to get from database. This may happen when a proposer asks for a bid which was offered by another relay
 	executionPayloadEntry, err := ds.db.GetExecutionPayloadEntryBySlotPkHash(slot, proposerPubkey, blockHash)
 	if errors.Is(err, sql.ErrNoRows) {
 		log.WithError(err).Warn("execution payload not found in database")

datastore/datastore_test.go

@@ -15,7 +15,7 @@ func setupTestDatastore(t *testing.T, mockDB *database.MockDB) *Datastore {
 	redisTestServer, err := miniredis.Run()
 	require.NoError(t, err)
 
-	redisDs, err := NewRedisCache("", redisTestServer.Addr(), "")
+	redisDs, err := NewRedisCache("", redisTestServer.Addr(), "", "")
 	require.NoError(t, err)
 
 	ds, err := NewDatastore(redisDs, nil, mockDB)

datastore/redis.go

@@ -80,12 +80,14 @@
 }
 
 type RedisCache struct {
+	archiveClient  *redis.Client
 	client         *redis.Client
 	readonlyClient *redis.Client
 
 	// prefixes (keys generated with a function)
 	prefixGetHeaderResponse           string
 	prefixExecPayloadCapella          string
+	prefixExecPayloadCapellaJSON      string
 	prefixBidTrace                    string
 	prefixBlockBuilderLatestBids      string // latest bid for a given slot
 	prefixBlockBuilderLatestBidsValue string // value of latest bid for a given slot
@@ -105,7 +107,7 @@
 	keyLastHashDelivered  string
 }
 
-func NewRedisCache(prefix, redisURI, readonlyURI string) (*RedisCache, error) {
+func NewRedisCache(prefix, redisURI, readonlyURI, archiveURI string) (*RedisCache, error) {
 	client, err := connectRedis(redisURI)
 	if err != nil {
 		return nil, err
@@ -119,13 +121,26 @@
 		}
 	}
 
+	// By default we use the same client for block submission archiving, unless
+	// a different URI is provided. If latency is a concern, running a separate
+	// client connected to a separate instance is advisable.
+	archiveClient := client
+	if archiveURI != "" {
+		archiveClient, err = connectRedis(archiveURI)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return &RedisCache{
+		archiveClient:  archiveClient,
 		client:         client,
 		readonlyClient: roClient,
 
-		prefixGetHeaderResponse:  fmt.Sprintf("%s/%s:cache-gethead-response", redisPrefix, prefix),
-		prefixExecPayloadCapella: fmt.Sprintf("%s/%s:cache-execpayload-capella", redisPrefix, prefix),
-		prefixBidTrace:           fmt.Sprintf("%s/%s:cache-bid-trace", redisPrefix, prefix),
+		prefixGetHeaderResponse:      fmt.Sprintf("%s/%s:cache-gethead-response", redisPrefix, prefix),
+		prefixExecPayloadCapella:     fmt.Sprintf("%s/%s:cache-execpayload-capella", redisPrefix, prefix),
+		prefixExecPayloadCapellaJSON: fmt.Sprintf("%s/%s:cache-execpayload-capella-json", redisPrefix, prefix),
+		prefixBidTrace:               fmt.Sprintf("%s/%s:cache-bid-trace", redisPrefix, prefix),
 
 		prefixBlockBuilderLatestBids:      fmt.Sprintf("%s/%s:block-builder-latest-bid", redisPrefix, prefix),       // hashmap for slot+parentHash+proposerPubkey with builderPubkey as field
 		prefixBlockBuilderLatestBidsValue: fmt.Sprintf("%s/%s:block-builder-latest-bid-value", redisPrefix, prefix), // hashmap for slot+parentHash+proposerPubkey with builderPubkey as field
@@ -153,6 +168,10 @@
 	return fmt.Sprintf("%s:%d_%s_%s", r.prefixExecPayloadCapella, slot, proposerPubkey, blockHash)
 }
 
+func (r *RedisCache) keyExecPayloadCapellaJSON(slot uint64, proposerPubkey, blockHash string) string {
+	return fmt.Sprintf("%s:%d_%s_%s", r.prefixExecPayloadCapellaJSON, slot, proposerPubkey, blockHash)
+}
+
 func (r *RedisCache) keyCacheBidTrace(slot uint64, proposerPubkey, blockHash string) string {
 	return fmt.Sprintf("%s:%d_%s_%s", r.prefixBidTrace, slot, proposerPubkey, blockHash)
 }
@@ -742,3 +761,40 @@
 func (r *RedisCache) NewTxPipeline() redis.Pipeliner { //nolint:ireturn
 	return r.client.TxPipeline()
 }
+
+func (r *RedisCache) ArchivePayloadRequest(payload []interface{}) error {
+	return r.archiveClient.XAdd(context.Background(), &redis.XAddArgs{
+		// We expect payload request logs to be at most 10 KiB in size. This
+		// means we can expect the stream to eventually take up
+		// 10_000 * 10 KiB = 100 MiB.
+		MaxLen: 10_000,
+		Approx: true,
+		Stream: "payload-request-archive",
+		Values: payload,
+	}).Err()
+}
+
+func (r *RedisCache) GetArchivedExecutionPayload(slot uint64, proposerPubkey, blockHash string) (*common.VersionedExecutionPayload, error) {
+	if r.archiveClient == nil {
+		return nil, errors.New("archive client not configured")
+	}
+
+	resp := new(common.VersionedExecutionPayload)
+	capellaPayload := new(capella.ExecutionPayload)
+
+	key := r.keyExecPayloadCapellaJSON(slot, proposerPubkey, blockHash)
+	val, err := r.archiveClient.Get(context.Background(), key).Result()
+	if err != nil {
+		return nil, err
+	}
+
+	err = capellaPayload.UnmarshalJSON([]byte(val))
+	if err != nil {
+		return nil, err
+	}
+
+	resp.Capella = new(api.VersionedExecutionPayload)
+	resp.Capella.Capella = capellaPayload
+	resp.Capella.Version = consensusspec.DataVersionCapella
+	return resp, nil
+}

datastore/redis_test.go

@@ -26,7 +26,7 @@ func setupTestRedis(t *testing.T) *RedisCache {
 
 	redisTestServer, err := miniredis.Run()
 	require.NoError(t, err)
-	redisService, err := NewRedisCache("", redisTestServer.Addr(), "")
+	redisService, err := NewRedisCache("", redisTestServer.Addr(), "", "")
 	// redisService, err := NewRedisCache("", "localhost:6379", "")
 	require.NoError(t, err)
 
@@ -325,25 +325,25 @@ func TestRedisURIs(t *testing.T) {
 	require.NoError(t, err)
 
 	// test connection with and without protocol
-	_, err = NewRedisCache("", redisTestServer.Addr(), "")
+	_, err = NewRedisCache("", redisTestServer.Addr(), "", "")
 	require.NoError(t, err)
-	_, err = NewRedisCache("", "redis://"+redisTestServer.Addr(), "")
+	_, err = NewRedisCache("", "redis://"+redisTestServer.Addr(), "", "")
 	require.NoError(t, err)
 
 	// test connection w/ credentials
 	username := "user"
 	password := "pass"
 	redisTestServer.RequireUserAuth(username, password)
 	fullURL := "redis://" + username + ":" + password + "@" + redisTestServer.Addr()
-	_, err = NewRedisCache("", fullURL, "")
+	_, err = NewRedisCache("", fullURL, "", "")
 	require.NoError(t, err)
 
 	// ensure malformed URL throws error
 	malformURL := "http://" + username + ":" + password + "@" + redisTestServer.Addr()
-	_, err = NewRedisCache("", malformURL, "")
+	_, err = NewRedisCache("", malformURL, "", "")
 	require.Error(t, err)
 	malformURL = "redis://" + username + ":" + "wrongpass" + "@" + redisTestServer.Addr()
-	_, err = NewRedisCache("", malformURL, "")
+	_, err = NewRedisCache("", malformURL, "", "")
 	require.Error(t, err)
 }
 

go.mod

@@ -98,9 +98,9 @@ require (
 	github.com/yuin/gopher-lua v1.1.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
 	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -435,8 +435,8 @@ golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU=
 golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
@@ -469,8 +469,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -515,10 +515,10 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

services/api/optimistic_test.go

@@ -116,7 +116,7 @@ func startTestBackend(t *testing.T) (*phase0.BLSPubKey, *bls.SecretKey, *testBac
 	}
 	redisTestServer, err := miniredis.Run()
 	require.NoError(t, err)
-	mockRedis, err := datastore.NewRedisCache("", redisTestServer.Addr(), "")
+	mockRedis, err := datastore.NewRedisCache("", redisTestServer.Addr(), "", "")
 	require.NoError(t, err)
 	mockDS, err := datastore.NewDatastore(mockRedis, nil, mockDB)
 	require.NoError(t, err)