diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 83f5768c..f66c0cf3 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -6,6 +6,6 @@ updates:
- package-ecosystem: gomod
directory: /
schedule:
- interval: daily
+ interval: weekly
reviewers:
- - "metachris"
\ No newline at end of file
+ - alextes
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 00000000..63c1d9ab
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,37 @@
+name: Build and Push Docker Image
+
+on:
+ push:
+ branches:
+ - "*"
+
+jobs:
+ docker:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+
+ - name: Login to Docker Hub
+ uses: docker/login-action@v2
+ with:
+ username: alextes
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Extract short SHA
+ run: echo "SHORT_SHA=$(echo ${{ github.sha }} | cut -c 1-7)" >> $GITHUB_ENV
+
+ - name: Build and push
+ uses: docker/build-push-action@v4
+ with:
+ context: .
+ push: true
+ tags: |
+ ultrasoundorg/mev-boost-relay:${{ env.SHORT_SHA }}
+ ultrasoundorg/mev-boost-relay:latest
diff --git a/README.md b/README.md
index 2b16580b..53c83672 100644
--- a/README.md
+++ b/README.md
@@ -1,391 +1,3 @@
-# MEV-Boost Relay
+# Ultra Sound MEV Boost Relay
-[](https://goreportcard.com/report/github.com/flashbots/mev-boost-relay)
-[](https://github.com/flashbots/mev-boost-relay/actions?query=workflow%3A%22Checks%22)
-[](https://hub.docker.com/r/flashbots/mev-boost-relay/tags)
-
-MEV-Boost Relay for Ethereum proposer/builder separation (PBS).
-
-Currently live at:
-
-* [boost-relay.flashbots.net](https://boost-relay.flashbots.net) (also on [Goerli](https://boost-relay-goerli.flashbots.net) and [Sepolia](https://boost-relay-sepolia.flashbots.net))
-* [relay.ultrasound.money](https://relay.ultrasound.money), [agnostic-relay.net](https://agnostic-relay.net), bloXroute relays ([light fork](https://github.com/bloXroute-Labs/mev-relay))
-* [mainnet.aestus.live](https://mainnet.aestus.live), [relay.edennetwork.io/info](https://relay.edennetwork.io/info), [mainnet-relay.securerpc.com](https://mainnet-relay.securerpc.com)
-
-Alternatives (not audited or endorsed): [blocknative/dreamboat](https://github.com/blocknative/dreamboat), [manifold/mev-freelay](https://github.com/manifoldfinance/mev-freelay)
-
-### See also
-
-* [Docker images](https://hub.docker.com/r/flashbots/mev-boost-relay)
-* [mev-boost](https://github.com/flashbots/mev-boost)
-* [Relay API specs](https://flashbots.github.io/relay-specs)
-* [Guide for running mev-boost-relay at scale](https://flashbots.notion.site/Running-mev-boost-relay-at-scale-draft-4040ccd5186c425d9a860cbb29bbfe09)
-* [Running relay and builders in custom devnets](https://gist.github.com/metachris/66df812f2920e6b0047afb9fdaf7df91#using-unnamed-devnets)
-* [More docs](/docs/docs/)
-
-### Components
-
-The relay consists of three main components, which are designed to run and scale independently, and to be as simple as possible:
-
-1. [API](https://github.com/flashbots/mev-boost-relay/tree/main/services/api): Services that provide APIs for (a) proposers, (b) block builders, (c) data.
-1. [Website](https://github.com/flashbots/mev-boost-relay/tree/main/services/website): Serving the [website requests](https://boost-relay.flashbots.net/) (information is pulled from Redis and database).
-1. [Housekeeper](https://github.com/flashbots/mev-boost-relay/tree/main/services/housekeeper): Updates known validators, proposer duties, and more in the background. Only a single instance of this should run.
-
-### Dependencies
-
-1. Redis
-1. PostgreSQL
-1. one or more beacon nodes
-1. block submission validation nodes
-1. [optional] Memcached
-
-### Beacon nodes / CL clients
-
-- The relay services need access to one or more beacon node for event subscriptions (in particular the `head` and `payload_attributes` topics).
-- You can specify multiple beacon nodes by providing a comma separated list of beacon node URIs.
-- The beacon nodes need to support the [`payload_attributes` SSE event](https://github.com/ethereum/beacon-APIs/pull/305).
-- Support the [v2 CL publish block endpoint](https://github.com/ethereum/beacon-APIs/pull/317) in the current main branch, since August 2. This is still
- experimental and may or may not fully work. It requires at least one of these CL clients
- - **Lighthouse+** [v4.3.0](https://github.com/sigp/lighthouse/releases) or later. Here's a [quick guide](https://gist.github.com/metachris/bcae9ae42e2fc834804241f991351c4e) for setting up Lighthouse.
- - **Prysm** [v4.0.6](https://github.com/prysmaticlabs/prysm/releases) or later.
-- The latest release (v0.26) still uses the old V1 broadcast endpoint using CL clients with custom validate-before-broadcast patches (see [README of the release for more details](https://github.com/flashbots/mev-boost-relay/tree/v0.26#beacon-nodes--cl-clients))
-
-**Relays are strongly advised to run multiple beacon nodes!**
-* The reason is that on getPayload, the block has to be validated and broadcast by a local beacon node before it is returned to the proposer.
-* If the local beacon nodes don't accept it (i.e. because it's down), the block won't be returned to the proposer, which leads to the proposer missing the slot.
-* The relay makes the validate+broadcast request to all beacon nodes concurrently, and returns as soon as the first request is successful.
-
-### Security
-
-A security assessment for the relay was conducted on 2022-08-22 by [lotusbumi](https://github.com/lotusbumi). Additional information can be found in the [Security](#security) section of this repository.
-
-If you find a security vulnerability on this project or any other initiative related to Flashbots, please let us know sending an email to security@flashbots.net.
-
----
-
-# Background
-
-MEV is a centralizing force on Ethereum. Unattended, the competition for MEV opportunities leads to consensus security instability and permissioned communication infrastructure between traders and block producers. This erodes neutrality, transparency, decentralization, and permissionlessness.
-
-Flashbots is a research and development organization working on mitigating the negative externalities of MEV. Flashbots started as a builder specializing in MEV extraction in proof-of-work Ethereum to democratize access to MEV and make the most profitable blocks available to all miners. >90% of miners are outsourcing some of their block construction to Flashbots today.
-
-The mev-boost relay is a trusted mediator between block producers and block builders. It enables all Ethereum proof-of-stake validators to offer their blockspace to not just Flashbots but other builders as well. This opens up the market to more builders and creates competition between them, leading to more revenue and choice for validators, and better censorship-resistance for Ethereum.
-
-In the future, [proposer/builder separation](https://ethresear.ch/t/two-slot-proposer-builder-separation/10980) will be enshrined in the Ethereum protocol itself to further harden its trust model.
-
-Read more in [Why run mev-boost?](https://writings.flashbots.net/writings/why-run-mevboost/) and in the [Frequently Asked Questions](https://github.com/flashbots/mev-boost/wiki/Frequently-Asked-Questions).
-
----
-
-# Usage
-
-## Running Postgres, Redis and Memcached
-```bash
-# Start PostgreSQL & Redis individually:
-docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres postgres
-docker run -d -p 6379:6379 redis
-
-# [optional] Start Memcached
-docker run -d -p 11211:11211 memcached
-
-# Or with docker-compose:
-docker-compose up
-```
-
-Note: docker-compose also runs an Adminer (a web frontend for Postgres) on http://localhost:8093/?username=postgres (db: `postgres`, username: `postgres`, password: `postgres`)
-
-Now start the services:
-
-```bash
-# The housekeeper sets up the validators, and does various housekeeping
-go run . housekeeper --network sepolia --db postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable
-
-# Run APIs for sepolia (using a dummy BLS secret key)
-go run . api --network sepolia --secret-key 0x607a11b45a7219cc61a3d9c5fd08c7eebd602a6a19a977f8d3771d5711a550f2 --db postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable
-
-# Run Website for sepolia
-go run . website --network sepolia --db postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable
-
-# Query status
-curl localhost:9062/eth/v1/builder/status
-
-# Send test validator registrations
-curl -X POST -H'Content-Encoding: gzip' localhost:9062/eth/v1/builder/validators --data-binary @testdata/valreg2.json.gz
-
-# Delete previous registrations
-redis-cli DEL boost-relay/sepolia:validators-registration boost-relay/sepolia:validators-registration-timestamp
-```
-
-
-## Environment variables
-
-#### General
-
-* `ACTIVE_VALIDATOR_HOURS` - number of hours to track active proposers in redis (default: `3`)
-* `API_MAX_HEADER_BYTES` - http maximum header bytes (default: `60_000`)
-* `API_TIMEOUT_READ_MS` - http read timeout in milliseconds (default: `1_500`)
-* `API_TIMEOUT_READHEADER_MS` - http read header timeout in milliseconds (default: `600`)
-* `API_TIMEOUT_WRITE_MS` - http write timeout in milliseconds (default: `10_000`)
-* `API_TIMEOUT_IDLE_MS` - http idle timeout in milliseconds (default: `3_000`)
-* `API_SHUTDOWN_WAIT_SEC` - how long to wait on shutdown before stopping server, to allow draining of requests (default: `30`)
-* `API_SHUTDOWN_STOP_SENDING_BIDS` - whether API should stop sending bids during shutdown (nly useful in single-instance/testnet setups, default: `false`)
-* `BLOCKSIM_MAX_CONCURRENT` - maximum number of concurrent block-sim requests (0 for no maximum, default: `4`)
-* `BLOCKSIM_TIMEOUT_MS` - builder block submission validation request timeout (default: `3000`)
-* `BROADCAST_MODE` - which broadcast mode to use for block publishing (default: `consensus_and_equivocation`)
-* `DB_DONT_APPLY_SCHEMA` - disable applying DB schema on startup (useful for connecting data API to read-only replica)
-* `DB_TABLE_PREFIX` - prefix to use for db tables (default uses `dev`)
-* `GETPAYLOAD_RETRY_TIMEOUT_MS` - getPayload retry getting a payload if first try failed (default: `100`)
-* `MEMCACHED_URIS` - optional comma separated list of memcached endpoints, typically used as secondary storage alongside Redis
-* `MEMCACHED_EXPIRY_SECONDS` - item expiry timeout when using memcache (default: `45`)
-* `MEMCACHED_CLIENT_TIMEOUT_MS` - client timeout in milliseconds (default: `250`)
-* `MEMCACHED_MAX_IDLE_CONNS` - client max idle conns (default: `10`)
-* `NUM_ACTIVE_VALIDATOR_PROCESSORS` - proposer API - number of goroutines to listen to the active validators channel
-* `NUM_VALIDATOR_REG_PROCESSORS` - proposer API - number of goroutines to listen to the validator registration channel
-* `NO_HEADER_USERAGENTS` - proposer API - comma separated list of user agents for which no bids should be returned
-* `ENABLE_BUILDER_CANCELLATIONS` - whether to enable block builder cancellations
-* `REDIS_URI` - main redis URI (default: `localhost:6379`)
-* `REDIS_READONLY_URI` - optional, a secondary redis instance for heavy read operations
-
-#### Feature Flags
-
-* `DISABLE_PAYLOAD_DATABASE_STORAGE` - builder API - disable storing execution payloads in the database (i.e. when using memcached as data availability redundancy)
-* `DISABLE_LOWPRIO_BUILDERS` - reject block submissions by low-prio builders
-* `FORCE_GET_HEADER_204` - force 204 as getHeader response
-* `ENABLE_IGNORABLE_VALIDATION_ERRORS` - enable ignorable validation errors
-* `USE_V2_PUBLISH_BLOCK_ENDPOINT` - uses the v2 publish block endpoint on the beacon node
-
-#### Development Environment Variables
-
-* `RUN_DB_TESTS` - when set to "1" enables integration tests with Postgres using endpoint specified by environment variable `TEST_DB_DSN`
-* `RUN_INTEGRATION_TESTS` - when set to "1" enables integration tests, currently used for testing Memcached using comma separated list of endpoints specified by `MEMCACHED_URIS`
-* `TEST_DB_DSN` - specifies connection string using Data Source Name (DSN) for Postgres (default: postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable)
-
-#### Redis tuning
-
-* `REDIS_CONNECTION_POOL_SIZE`, `REDIS_MIN_IDLE_CONNECTIONS`, `REDIS_READ_TIMEOUT_SEC`, `REDIS_POOL_TIMEOUT_SEC`, `REDIS_WRITE_TIMEOUT_SEC` (see also [the code here](https://github.com/flashbots/mev-boost-relay/blob/e39cd38010de26bf9a51d1a3e77fc235ea87b12f/datastore/redis.go#L35-L41))
-
-## Updating the website
-
-* Edit the HTML in `services/website/website.html`
-* Edit template values in `testdata/website-htmldata.json`
-* Generate a static version of the website with `go run scripts/website-staticgen/main.go`
-
-This builds a local copy of the template and saves it in `website-index.html`
-
-The website is using:
-* [PureCSS](https://purecss.io/)
-* [HeroIcons](https://heroicons.com/)
-
----
-
-# Technical Notes
-
-See [ARCHITECTURE.md](ARCHITECTURE.md) and [Running MEV-Boost-Relay at scale](https://flashbots.notion.site/Draft-Running-a-relay-4040ccd5186c425d9a860cbb29bbfe09) for more technical details!
-
-## Storing execution payloads and redundant data availability
-
-By default, the execution payloads for all block submission are stored in Redis and also in the Postgres database,
-to provide redundant data availability for getPayload responses. But the database table is not pruned automatically,
-because it takes a lot of resources to rebuild the indexes (and a better option is using `TRUNCATE`).
-
-Storing all the payloads in the database can lead to terabytes of data in this particular table. Now it's also possible
-to use memcached as a second data availability layer. Using memcached is optional and disabled by default.
-
-To enable memcached, you just need to supply the memcached URIs either via environment variable (i.e.
-`MEMCACHED_URIS=localhost:11211`) or through command line flag (`--memcached-uris`).
-
-You can disable storing the execution payloads in the database with this environment variable:
-`DISABLE_PAYLOAD_DATABASE_STORAGE=1`.
-
-## Builder submission validation nodes
-
-You can use the [builder project](https://github.com/flashbots/builder) to validate block builder submissions: https://github.com/flashbots/builder
-
-Here's an example systemd config:
-
-
-/etc/systemd/system/geth.service
-
-```ini
-[Unit]
-Description=mev-boost
-Wants=network-online.target
-After=network-online.target
-
-[Service]
-User=ubuntu
-Group=ubuntu
-Environment=HOME=/home/ubuntu
-Type=simple
-KillMode=mixed
-KillSignal=SIGINT
-TimeoutStopSec=90
-Restart=on-failure
-RestartSec=10s
-ExecStart=/home/ubuntu/builder/build/bin/geth \
- --syncmode=snap \
- --datadir /var/lib/goethereum \
- --metrics \
- --metrics.expensive \
- --http \
- --http.api="engine,eth,web3,net,debug,flashbots" \
- --http.corsdomain "*" \
- --http.addr "0.0.0.0" \
- --http.port 8545 \
- --http.vhosts '*' \
- --ws \
- --ws.api="engine,eth,web3,net,debug" \
- --ws.addr 0.0.0.0 \
- --ws.port 8546 \
- --ws.api engine,eth,net,web3 \
- --ws.origins '*' \
- --graphql \
- --graphql.corsdomain '*' \
- --graphql.vhosts '*' \
- --authrpc.addr="0.0.0.0" \
- --authrpc.jwtsecret=/var/lib/goethereum/jwtsecret \
- --authrpc.vhosts '*' \
- --cache=8192
-
-[Install]
-WantedBy=multi-user.target
-```
-
-
-Sending blocks to the validation node:
-
-- The built-in [blocksim-ratelimiter](services/api/blocksim_ratelimiter.go) is a simple example queue implementation.
-- By default, `BLOCKSIM_MAX_CONCURRENT` is set to 4, which allows 4 concurrent block simulations per API node
-- For production use, use the [prio-load-balancer](https://github.com/flashbots/prio-load-balancer) project for a single priority queue,
- and disable the internal concurrency limit (set `BLOCKSIM_MAX_CONCURRENT` to `0`).
-
-## Beacon node setup
-
-### Lighthouse
-
-- Lighthouse with validation and equivocaation check before broadcast: https://github.com/sigp/lighthouse/pull/4168
-- with `--always-prepare-payload` and `--prepare-payload-lookahead 12000` flags, and some junk feeRecipeint
-
-Here's a [quick guide](https://gist.github.com/metachris/bcae9ae42e2fc834804241f991351c4e) for setting up Lighthouse.
-
-Here's an example Lighthouse systemd config:
-
-
-/etc/systemd/system/lighthouse.service
-
-```ini
-[Unit]
-Description=Lighthouse
-After=network.target
-Wants=network.target
-
-[Service]
-User=ubuntu
-Group=ubuntu
-Type=simple
-Restart=always
-RestartSec=5
-TimeoutStopSec=180
-ExecStart=/home/ubuntu/.cargo/bin/lighthouse bn \
- --network mainnet \
- --checkpoint-sync-url=https://mainnet-checkpoint-sync.attestant.io \
- --eth1 \
- --http \
- --http-address "0.0.0.0" \
- --http-port 3500 \
- --datadir=/mnt/data/lighthouse \
- --http-allow-sync-stalled \
- --execution-endpoints=http://localhost:8551 \
- --jwt-secrets=/var/lib/goethereum/jwtsecret \
- --disable-deposit-contract-sync \
- --always-prepare-payload \
- --prepare-payload-lookahead 12000
-
-[Install]
-WantedBy=default.target
-```
-
-
-
-
-### Prysm
-
-- Prysm with validation and equivocaation check before broadcast: https://github.com/prysmaticlabs/prysm/pull/12335
-- use `--grpc-max-msg-size 104857600`, because by default the getAllValidators response is too big and fails
-
-Here's an example Prysm systemd config:
-
-
-/etc/systemd/system/prysm.service
-
-```ini
-[Unit]
-Description=Prysm
-After=network.target
-Wants=network.target
-
-[Service]
-User=ubuntu
-Group=ubuntu
-Type=simple
-Restart=always
-RestartSec=5
-TimeoutStopSec=180
-ExecStart=/home/ubuntu/prysm/bazel-bin/cmd/beacon-chain/beacon-chain_/beacon-chain \
- --accept-terms-of-use \
- --enable-debug-rpc-endpoints \
- --checkpoint-sync-url=https://mainnet-checkpoint-sync.attestant.io \
- --genesis-beacon-api-url=https://mainnet-checkpoint-sync.attestant.io \
- --grpc-gateway-host "0.0.0.0" \
- --datadir=/mnt/data/prysm \
- --p2p-max-peers 100 \
- --execution-endpoint=http://localhost:8551 \
- --jwt-secret=/var/lib/goethereum/jwtsecret \
- --min-sync-peers=1 \
- --grpc-max-msg-size 104857600 \
- --prepare-all-payloads \
- --disable-reorg-late-blocks
-
-[Install]
-WantedBy=default.target
-```
-
-
-
-## Bid Cancellations
-
-Block builders can opt into cancellations by submitting blocks to `/relay/v1/builder/blocks?cancellations=1`. This may incur a performance penalty (i.e. validation of submissions taking significantly longer). See also https://github.com/flashbots/mev-boost-relay/issues/348
-
----
-
-# Maintainers
-
-- [@metachris](https://twitter.com/metachris)
-- [@Ruteri](https://twitter.com/mmrosum)
-- [@avalonche](https://github.com/avalonche)
-
-# Contributing
-
-[Flashbots](https://flashbots.net) is a research and development collective working on mitigating the negative externalities of decentralized economies. We contribute with the larger free software community to illuminate the dark forest.
-
-You are welcome here <3.
-
-- If you have a question, feedback or a bug report for this project, please [open a new Issue](https://github.com/flashbots/mev-boost/issues).
-- If you would like to contribute with code, check the [CONTRIBUTING file](CONTRIBUTING.md) for further info about the development environment.
-- We just ask you to be nice. Read our [code of conduct](CODE_OF_CONDUCT.md).
-
-# Security
-
-If you find a security vulnerability on this project or any other initiative related to Flashbots, please let us know sending an email to security@flashbots.net.
-
-## Audits
-
-- [20220822](docs/audit-20220822.md), by [lotusbumi](https://github.com/lotusbumi).
-
-# License
-
-The code in this project is free software under the [AGPL License version 3 or later](LICENSE).
-
----
-
-Made with ☀️ by the ⚡🤖 collective.
+Lightly modified fork of [flashbots/mev-boost-relay](https://github.com/flashbots/mev-boost-relay). Major parts are unused or rewritten and live in other ultra sound repos. Some public, some private.
diff --git a/cmd/api.go b/cmd/api.go
index 89abbf85..bac28b7a 100644
--- a/cmd/api.go
+++ b/cmd/api.go
@@ -55,6 +55,7 @@ func init() {
apiCmd.Flags().StringSliceVar(&beaconNodeURIs, "beacon-uris", defaultBeaconURIs, "beacon endpoints")
apiCmd.Flags().StringVar(&redisURI, "redis-uri", defaultRedisURI, "redis uri")
apiCmd.Flags().StringVar(&redisReadonlyURI, "redis-readonly-uri", defaultRedisReadonlyURI, "redis readonly uri")
+ apiCmd.Flags().StringVar(&redisArchiveURI, "redis-archive-uri", defaultRedisArchiveURI, "redis block submission archive uri")
apiCmd.Flags().StringVar(&postgresDSN, "db", defaultPostgresDSN, "PostgreSQL DSN")
apiCmd.Flags().StringSliceVar(&memcachedURIs, "memcached-uris", defaultMemcachedURIs,
"Enable memcached, typically used as secondary backup to Redis for redundancy")
@@ -107,12 +108,14 @@ var apiCmd = &cobra.Command{
beaconClient := beaconclient.NewMultiBeaconClient(log, beaconInstances)
// Connect to Redis
- if redisReadonlyURI == "" {
- log.Infof("Connecting to Redis at %s ...", redisURI)
- } else {
- log.Infof("Connecting to Redis at %s / readonly: %s ...", redisURI, redisReadonlyURI)
+ log.Infof("Connecting to Redis at %s ...", redisURI)
+ if redisReadonlyURI != "" {
+ log.Infof("Connecting to readonly Redis at %s ...", redisReadonlyURI)
+ }
+ if redisArchiveURI != "" {
+ log.Infof("Connecting to block submission archive Redis at %s ...", redisArchiveURI)
}
- redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, redisReadonlyURI)
+ redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, redisReadonlyURI, redisArchiveURI)
if err != nil {
log.WithError(err).Fatalf("Failed to connect to Redis at %s", redisURI)
}
diff --git a/cmd/housekeeper.go b/cmd/housekeeper.go
index 9b03d8be..02e6477f 100644
--- a/cmd/housekeeper.go
+++ b/cmd/housekeeper.go
@@ -68,7 +68,7 @@ var housekeeperCmd = &cobra.Command{
beaconClient := beaconclient.NewMultiBeaconClient(log, beaconInstances)
// Connect to Redis and setup the datastore
- redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, "")
+ redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, "", "")
if err != nil {
log.WithError(err).Fatalf("Failed to connect to Redis at %s", redisURI)
}
diff --git a/cmd/variables.go b/cmd/variables.go
index 5f49e7e3..5428feca 100644
--- a/cmd/variables.go
+++ b/cmd/variables.go
@@ -11,6 +11,7 @@ var (
defaultBeaconURIs = common.GetSliceEnv("BEACON_URIS", []string{"http://localhost:3500"})
defaultRedisURI = common.GetEnv("REDIS_URI", "localhost:6379")
defaultRedisReadonlyURI = common.GetEnv("REDIS_READONLY_URI", "")
+ defaultRedisArchiveURI = common.GetEnv("REDIS_ARCHIVE_URI", "")
defaultPostgresDSN = common.GetEnv("POSTGRES_DSN", "")
defaultMemcachedURIs = common.GetSliceEnv("MEMCACHED_URIS", nil)
defaultLogJSON = os.Getenv("LOG_JSON") != ""
@@ -19,6 +20,7 @@ var (
beaconNodeURIs []string
redisURI string
redisReadonlyURI string
+ redisArchiveURI string
postgresDSN string
memcachedURIs []string
diff --git a/cmd/website.go b/cmd/website.go
index e075f43a..7e032db5 100644
--- a/cmd/website.go
+++ b/cmd/website.go
@@ -67,12 +67,14 @@ var websiteCmd = &cobra.Command{
log.Debug(networkInfo.String())
// Connect to Redis
- if redisReadonlyURI == "" {
- log.Infof("Connecting to Redis at %s ...", redisURI)
- } else {
- log.Infof("Connecting to Redis at %s / readonly: %s ...", redisURI, redisReadonlyURI)
+ log.Infof("Connecting to Redis at %s ...", redisURI)
+ if redisReadonlyURI != "" {
+ log.Infof("Connecting to readonly Redis at %s ...", redisReadonlyURI)
+ }
+ if redisArchiveURI != "" {
+ log.Infof("Connecting to block submission archive Redis at %s ...", redisArchiveURI)
}
- redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, redisReadonlyURI)
+ redis, err := datastore.NewRedisCache(networkInfo.Name, redisURI, redisReadonlyURI, redisArchiveURI)
if err != nil {
log.WithError(err).Fatalf("Failed to connect to Redis at %s", redisURI)
}
diff --git a/common/logging.go b/common/logging.go
index 117f0a89..ba9326ec 100644
--- a/common/logging.go
+++ b/common/logging.go
@@ -2,6 +2,7 @@ package common
import (
"os"
+ "time"
"github.com/sirupsen/logrus"
)
@@ -11,10 +12,13 @@ func LogSetup(json bool, logLevel string) *logrus.Entry {
log.Logger.SetOutput(os.Stdout)
if json {
- log.Logger.SetFormatter(&logrus.JSONFormatter{})
+ log.Logger.SetFormatter(&logrus.JSONFormatter{
+ TimestampFormat: time.RFC3339Nano,
+ })
} else {
log.Logger.SetFormatter(&logrus.TextFormatter{
- FullTimestamp: true,
+ TimestampFormat: time.RFC3339Nano,
+ FullTimestamp: true,
})
}
diff --git a/datastore/datastore.go b/datastore/datastore.go
index 23044072..cbe41a04 100644
--- a/datastore/datastore.go
+++ b/datastore/datastore.go
@@ -219,7 +219,20 @@ func (ds *Datastore) GetGetPayloadResponse(log *logrus.Entry, slot uint64, propo
}
}
- // 3. try to get from database (should not happen, it's just a backup)
+ // 3. try to get it from redis archive
+ if ds.redis.archiveClient != nil {
+ resp, err = ds.redis.GetArchivedExecutionPayload(slot, _proposerPubkey, _blockHash)
+ if errors.Is(err, redis.Nil) {
+ log.WithError(err).Info("execution payload not found in redis archive")
+ } else if err != nil {
+ log.WithError(err).Error("error getting execution payload from redis archive")
+ } else if resp != nil {
+ log.Info("getPayload response from redis archive")
+ return resp, nil
+ }
+ }
+
+ // 4. try to get from database. This may happen when a proposer asks for a bid which was offered by another relay
executionPayloadEntry, err := ds.db.GetExecutionPayloadEntryBySlotPkHash(slot, proposerPubkey, blockHash)
if errors.Is(err, sql.ErrNoRows) {
log.WithError(err).Warn("execution payload not found in database")
diff --git a/datastore/datastore_test.go b/datastore/datastore_test.go
index f6a4b157..b622389c 100644
--- a/datastore/datastore_test.go
+++ b/datastore/datastore_test.go
@@ -15,7 +15,7 @@ func setupTestDatastore(t *testing.T, mockDB *database.MockDB) *Datastore {
redisTestServer, err := miniredis.Run()
require.NoError(t, err)
- redisDs, err := NewRedisCache("", redisTestServer.Addr(), "")
+ redisDs, err := NewRedisCache("", redisTestServer.Addr(), "", "")
require.NoError(t, err)
ds, err := NewDatastore(redisDs, nil, mockDB)
diff --git a/datastore/redis.go b/datastore/redis.go
index 54cd7af6..594fe6eb 100644
--- a/datastore/redis.go
+++ b/datastore/redis.go
@@ -80,12 +80,14 @@ func connectRedis(redisURI string) (*redis.Client, error) {
}
type RedisCache struct {
+ archiveClient *redis.Client
client *redis.Client
readonlyClient *redis.Client
// prefixes (keys generated with a function)
prefixGetHeaderResponse string
prefixExecPayloadCapella string
+ prefixExecPayloadCapellaJSON string
prefixBidTrace string
prefixBlockBuilderLatestBids string // latest bid for a given slot
prefixBlockBuilderLatestBidsValue string // value of latest bid for a given slot
@@ -105,7 +107,7 @@ type RedisCache struct {
keyLastHashDelivered string
}
-func NewRedisCache(prefix, redisURI, readonlyURI string) (*RedisCache, error) {
+func NewRedisCache(prefix, redisURI, readonlyURI, archiveURI string) (*RedisCache, error) {
client, err := connectRedis(redisURI)
if err != nil {
return nil, err
@@ -119,13 +121,26 @@ func NewRedisCache(prefix, redisURI, readonlyURI string) (*RedisCache, error) {
}
}
+ // By default we use the same client for block submission archiving, unless
+ // a different URI is provided. If latency is a concern, running a separate
+ // client connected to a separate instance is advisable.
+ archiveClient := client
+ if archiveURI != "" {
+ archiveClient, err = connectRedis(archiveURI)
+ if err != nil {
+ return nil, err
+ }
+ }
+
return &RedisCache{
+ archiveClient: archiveClient,
client: client,
readonlyClient: roClient,
- prefixGetHeaderResponse: fmt.Sprintf("%s/%s:cache-gethead-response", redisPrefix, prefix),
- prefixExecPayloadCapella: fmt.Sprintf("%s/%s:cache-execpayload-capella", redisPrefix, prefix),
- prefixBidTrace: fmt.Sprintf("%s/%s:cache-bid-trace", redisPrefix, prefix),
+ prefixGetHeaderResponse: fmt.Sprintf("%s/%s:cache-gethead-response", redisPrefix, prefix),
+ prefixExecPayloadCapella: fmt.Sprintf("%s/%s:cache-execpayload-capella", redisPrefix, prefix),
+ prefixExecPayloadCapellaJSON: fmt.Sprintf("%s/%s:cache-execpayload-capella-json", redisPrefix, prefix),
+ prefixBidTrace: fmt.Sprintf("%s/%s:cache-bid-trace", redisPrefix, prefix),
prefixBlockBuilderLatestBids: fmt.Sprintf("%s/%s:block-builder-latest-bid", redisPrefix, prefix), // hashmap for slot+parentHash+proposerPubkey with builderPubkey as field
prefixBlockBuilderLatestBidsValue: fmt.Sprintf("%s/%s:block-builder-latest-bid-value", redisPrefix, prefix), // hashmap for slot+parentHash+proposerPubkey with builderPubkey as field
@@ -153,6 +168,10 @@ func (r *RedisCache) keyExecPayloadCapella(slot uint64, proposerPubkey, blockHas
return fmt.Sprintf("%s:%d_%s_%s", r.prefixExecPayloadCapella, slot, proposerPubkey, blockHash)
}
+func (r *RedisCache) keyExecPayloadCapellaJSON(slot uint64, proposerPubkey, blockHash string) string {
+ return fmt.Sprintf("%s:%d_%s_%s", r.prefixExecPayloadCapellaJSON, slot, proposerPubkey, blockHash)
+}
+
func (r *RedisCache) keyCacheBidTrace(slot uint64, proposerPubkey, blockHash string) string {
return fmt.Sprintf("%s:%d_%s_%s", r.prefixBidTrace, slot, proposerPubkey, blockHash)
}
@@ -742,3 +761,40 @@ func (r *RedisCache) NewPipeline() redis.Pipeliner { //nolint:ireturn,nolintlint
func (r *RedisCache) NewTxPipeline() redis.Pipeliner { //nolint:ireturn
return r.client.TxPipeline()
}
+
+func (r *RedisCache) ArchivePayloadRequest(payload []interface{}) error {
+ return r.archiveClient.XAdd(context.Background(), &redis.XAddArgs{
+ // We expect payload request logs to be at most 10 KiB in size. This
+ // means we can expect the stream to eventually take up
+ // 10_000 * 10 KiB = 100 MiB.
+ MaxLen: 10_000,
+ Approx: true,
+ Stream: "payload-request-archive",
+ Values: payload,
+ }).Err()
+}
+
+func (r *RedisCache) GetArchivedExecutionPayload(slot uint64, proposerPubkey, blockHash string) (*common.VersionedExecutionPayload, error) {
+ if r.archiveClient == nil {
+ return nil, errors.New("archive client not configured")
+ }
+
+ resp := new(common.VersionedExecutionPayload)
+ capellaPayload := new(capella.ExecutionPayload)
+
+ key := r.keyExecPayloadCapellaJSON(slot, proposerPubkey, blockHash)
+ val, err := r.archiveClient.Get(context.Background(), key).Result()
+ if err != nil {
+ return nil, err
+ }
+
+ err = capellaPayload.UnmarshalJSON([]byte(val))
+ if err != nil {
+ return nil, err
+ }
+
+ resp.Capella = new(api.VersionedExecutionPayload)
+ resp.Capella.Capella = capellaPayload
+ resp.Capella.Version = consensusspec.DataVersionCapella
+ return resp, nil
+}
diff --git a/datastore/redis_test.go b/datastore/redis_test.go
index 679085fe..d16b2f9d 100644
--- a/datastore/redis_test.go
+++ b/datastore/redis_test.go
@@ -26,7 +26,7 @@ func setupTestRedis(t *testing.T) *RedisCache {
redisTestServer, err := miniredis.Run()
require.NoError(t, err)
- redisService, err := NewRedisCache("", redisTestServer.Addr(), "")
+ redisService, err := NewRedisCache("", redisTestServer.Addr(), "", "")
// redisService, err := NewRedisCache("", "localhost:6379", "")
require.NoError(t, err)
@@ -325,9 +325,9 @@ func TestRedisURIs(t *testing.T) {
require.NoError(t, err)
// test connection with and without protocol
- _, err = NewRedisCache("", redisTestServer.Addr(), "")
+ _, err = NewRedisCache("", redisTestServer.Addr(), "", "")
require.NoError(t, err)
- _, err = NewRedisCache("", "redis://"+redisTestServer.Addr(), "")
+ _, err = NewRedisCache("", "redis://"+redisTestServer.Addr(), "", "")
require.NoError(t, err)
// test connection w/ credentials
@@ -335,15 +335,15 @@ func TestRedisURIs(t *testing.T) {
password := "pass"
redisTestServer.RequireUserAuth(username, password)
fullURL := "redis://" + username + ":" + password + "@" + redisTestServer.Addr()
- _, err = NewRedisCache("", fullURL, "")
+ _, err = NewRedisCache("", fullURL, "", "")
require.NoError(t, err)
// ensure malformed URL throws error
malformURL := "http://" + username + ":" + password + "@" + redisTestServer.Addr()
- _, err = NewRedisCache("", malformURL, "")
+ _, err = NewRedisCache("", malformURL, "", "")
require.Error(t, err)
malformURL = "redis://" + username + ":" + "wrongpass" + "@" + redisTestServer.Addr()
- _, err = NewRedisCache("", malformURL, "")
+ _, err = NewRedisCache("", malformURL, "", "")
require.Error(t, err)
}
diff --git a/go.mod b/go.mod
index 443b6bdd..87bf6dcb 100644
--- a/go.mod
+++ b/go.mod
@@ -98,9 +98,9 @@ require (
github.com/yuin/gopher-lua v1.1.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.25.0 // indirect
- golang.org/x/crypto v0.13.0 // indirect
- golang.org/x/net v0.10.0 // indirect
- golang.org/x/sys v0.12.0 // indirect
+ golang.org/x/crypto v0.14.0 // indirect
+ golang.org/x/net v0.17.0 // indirect
+ golang.org/x/sys v0.13.0 // indirect
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/go.sum b/go.sum
index 7bc728d6..760d5253 100644
--- a/go.sum
+++ b/go.sum
@@ -435,8 +435,8 @@ golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU=
golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
@@ -469,8 +469,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -515,10 +515,10 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
diff --git a/services/api/optimistic_test.go b/services/api/optimistic_test.go
index b0f6e491..1e39c125 100644
--- a/services/api/optimistic_test.go
+++ b/services/api/optimistic_test.go
@@ -116,7 +116,7 @@ func startTestBackend(t *testing.T) (*phase0.BLSPubKey, *bls.SecretKey, *testBac
}
redisTestServer, err := miniredis.Run()
require.NoError(t, err)
- mockRedis, err := datastore.NewRedisCache("", redisTestServer.Addr(), "")
+ mockRedis, err := datastore.NewRedisCache("", redisTestServer.Addr(), "", "")
require.NoError(t, err)
mockDS, err := datastore.NewDatastore(mockRedis, nil, mockDB)
require.NoError(t, err)
diff --git a/services/api/service.go b/services/api/service.go
index 084497fb..9e6436c6 100644
--- a/services/api/service.go
+++ b/services/api/service.go
@@ -57,10 +57,11 @@ var (
var (
// Proposer API (builder-specs)
- pathStatus = "/eth/v1/builder/status"
- pathRegisterValidator = "/eth/v1/builder/validators"
- pathGetHeader = "/eth/v1/builder/header/{slot:[0-9]+}/{parent_hash:0x[a-fA-F0-9]+}/{pubkey:0x[a-fA-F0-9]+}"
- pathGetPayload = "/eth/v1/builder/blinded_blocks"
+ pathStatus = "/eth/v1/builder/status"
+ pathRegisterValidator = "/eth/v1/builder/validators"
+ pathGetHeader = "/eth/v1/builder/header/{slot:[0-9]+}/{parent_hash:0x[a-fA-F0-9]+}/{pubkey:0x[a-fA-F0-9]+}"
+ pathGetPayload = "/eth/v1/builder/blinded_blocks"
+ pathGetRegistrationMeta = "/ultrasound/v1/registration_meta"
// Block builder API
pathBuilderGetValidators = "/relay/v1/builder/validators"
@@ -106,6 +107,11 @@ var (
})
)
+var (
+ pubkeyMetadata = make(map[string]registrationMetadata)
+ pubkeyMetadataMutex = &sync.Mutex{}
+)
+
// RelayAPIOpts contains the options for a relay
type RelayAPIOpts struct {
Log *logrus.Entry
@@ -218,6 +224,15 @@ type RelayAPI struct {
blockBuildersCache map[string]*blockBuilderCacheEntry
}
+type registrationMetadata struct {
+ FeeRecipient string `json:"fee_recipient"`
+ GasLimit uint64 `json:"gas_limit"`
+ IPAddress string `json:"ip_address"`
+ ReceivedAt int64 `json:"received_at"`
+ Timestamp uint64 `json:"timestamp"`
+ UserAgent string `json:"user_agent"`
+}
+
// NewRelayAPI creates a new service. if builders is nil, allow any builder
func NewRelayAPI(opts RelayAPIOpts) (api *RelayAPI, err error) {
if opts.Log == nil {
@@ -335,6 +350,7 @@ func (api *RelayAPI) getRouter() http.Handler {
r.HandleFunc(pathRegisterValidator, api.handleRegisterValidator).Methods(http.MethodPost)
r.HandleFunc(pathGetHeader, api.handleGetHeader).Methods(http.MethodGet)
r.HandleFunc(pathGetPayload, api.handleGetPayload).Methods(http.MethodPost)
+ r.HandleFunc(pathGetRegistrationMeta, api.handleGetRegistrationMeta).Methods(http.MethodGet)
}
// Builder API
@@ -838,7 +854,7 @@ func (api *RelayAPI) Respond(w http.ResponseWriter, code int, response any) {
// write the json response
if err := json.NewEncoder(w).Encode(response); err != nil {
- api.log.WithField("response", response).WithError(err).Error("Couldn't write response")
+ api.log.WithError(err).Error("Couldn't write response")
http.Error(w, "", http.StatusInternalServerError)
}
}
@@ -1037,6 +1053,20 @@ func (api *RelayAPI) handleRegisterValidator(w http.ResponseWriter, req *http.Re
}
}
+ // Store the metadata in memory
+ metadata := registrationMetadata{
+ FeeRecipient: signedValidatorRegistration.Message.FeeRecipient.String(),
+ GasLimit: signedValidatorRegistration.Message.GasLimit,
+ // Until we have a way to record latency directly, we ping IPs.
+ IPAddress: req.Header.Get("X-Real-IP"),
+ ReceivedAt: start.UnixMilli(),
+ Timestamp: signedValidatorRegistration.Message.Timestamp,
+ UserAgent: ua,
+ }
+ pubkeyMetadataMutex.Lock()
+ pubkeyMetadata[pkHex.String()] = metadata
+ pubkeyMetadataMutex.Unlock()
+
// Now we have a new registration to process
numRegNew += 1
@@ -1354,34 +1384,127 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request)
}
}()
+ // Export metadata at end of request
+ abortReason := ""
+ timeBeforePublish := int64(0)
+ timeAfterPublish := int64(0)
+
+ defer func() {
+ archivePayloadLog := []interface{}{
+ "content_length", strconv.FormatInt(req.ContentLength, 10),
+ "decoded_at", decodeTime.UnixMilli(),
+ "finished_at", fmt.Sprint(time.Now().UTC().UnixMilli()),
+ "head_slot", strconv.FormatUint(headSlot, 10),
+ "proposer_pubkey", proposerPubkey.String(),
+ "received_at", receivedAt.UnixMilli(),
+ }
+
+ if ua != "" {
+ archivePayloadLog = append(archivePayloadLog, "user_agent", ua)
+ }
+
+ ip := req.Header.Get("X-Real-IP")
+
+ if ip != "" {
+ archivePayloadLog = append(archivePayloadLog, "ip", ip)
+ }
+
+ jsonPayload, err := json.Marshal(payload)
+ if err != nil {
+ log.WithError(err).Error("could not marshal payload")
+ } else {
+ archivePayloadLog = append(archivePayloadLog, "payload", string(jsonPayload))
+ }
+
+ if abortReason != "" {
+ archivePayloadLog = append(archivePayloadLog, "abort_reason", abortReason)
+ }
+
+ reqIDParam := req.URL.Query().Get("id")
+ if reqIDParam != "" {
+ archivePayloadLog = append(archivePayloadLog, "req_id_param", reqIDParam)
+ }
+
+ if timeBeforePublish != 0 {
+ archivePayloadLog = append(archivePayloadLog, "time_before_publish", timeBeforePublish)
+ }
+
+ if timeAfterPublish != 0 {
+ archivePayloadLog = append(archivePayloadLog, "time_after_publish", timeAfterPublish)
+ }
+
+ // Until we have a way to record latency directly, we ping IPs.
+ ipAddress := req.Header.Get("X-Real-IP")
+ if ipAddress != "" {
+ archivePayloadLog = append(archivePayloadLog, "ip_address", ipAddress)
+ }
+
+ err = api.redis.ArchivePayloadRequest(archivePayloadLog)
+ if err != nil {
+ log.WithError(err).Error("failed to archive payload request")
+ }
+
+ log.Debug(fmt.Sprintf("successfully archived payload request, block_hash: %s", payload.BlockHash()))
+ }()
+
// Get the response - from Redis, Memcache or DB
- // note that recent mev-boost versions only send getPayload to relays that provided the bid
+ // note that recent mev-boost versions only send getPayload to relays that provided the bid, older versions send getPayload to all relays.
+ // Additionally, proposers may feel it's safer to ask for a bid from all relays and fork.
getPayloadResp, err = api.datastore.GetGetPayloadResponse(log, payload.Slot(), proposerPubkey.String(), payload.BlockHash())
if err != nil || getPayloadResp == nil {
- log.WithError(err).Warn("failed getting execution payload (1/2)")
- time.Sleep(time.Duration(timeoutGetPayloadRetryMs) * time.Millisecond)
+ log.WithError(err).Warn("failed first attempt to get execution payload")
- // Try again
+ // Wait, then try again.
+ time.Sleep(time.Duration(timeoutGetPayloadRetryMs) * time.Millisecond)
getPayloadResp, err = api.datastore.GetGetPayloadResponse(log, payload.Slot(), proposerPubkey.String(), payload.BlockHash())
+
if err != nil || getPayloadResp == nil {
// Still not found! Error out now.
if errors.Is(err, datastore.ErrExecutionPayloadNotFound) {
- // Couldn't find the execution payload, maybe it never was submitted to our relay! Check that now
- _, err := api.db.GetBlockSubmissionEntry(payload.Slot(), proposerPubkey.String(), payload.BlockHash())
- if errors.Is(err, sql.ErrNoRows) {
- log.Warn("failed getting execution payload (2/2) - payload not found, block was never submitted to this relay")
- api.RespondError(w, http.StatusBadRequest, "no execution payload for this request - block was never seen by this relay")
- } else if err != nil {
- log.WithError(err).Error("failed getting execution payload (2/2) - payload not found, and error on checking bids")
- } else {
- log.Error("failed getting execution payload (2/2) - payload not found, but found bid in database")
+ // Couldn't find the execution payload, three options:
+ // 1. We're storing all payloads in postgres, we never received
+ // or served the bid, but someone still asked us for it. We can
+ // check this.
+ // 2. We do not store all payloads in postgres. The bid was
+ // never the top bid, so we didn't store it in Redis or
+ // Memcached either. We received, but never served the bid, but
+ // someone still asked us for it.
+ // 3. The bid was accepted but the payload was lost in all
+ // active stores. This is a critical error! If this ever
+ // happens, we have work to do.
+ // Annoyingly, we can't currently distinguish between 2 and 3!
+
+ // Check for case 1 if possible.
+ if !api.ffDisablePayloadDBStorage {
+ _, err := api.db.GetBlockSubmissionEntry(payload.Slot(), proposerPubkey.String(), payload.BlockHash())
+ if errors.Is(err, sql.ErrNoRows) {
+ abortReason = "execution-payload-not-found"
+ log.Info("failed second attempt to get execution payload, discovered block was never submitted to this relay")
+ api.RespondError(w, http.StatusBadRequest, "no execution payload for this request, block was never seen by this relay")
+ return
+ }
+ if err != nil {
+ abortReason = "execution-payload-retrieval-error"
+ log.WithError(err).Error("failed second attempt to get execution payload, hit an error while checking if block was submitted to this relay")
+ api.RespondError(w, http.StatusInternalServerError, "no execution payload for this request, hit an error while checking if block was submitted to this relay")
+ return
+ }
}
- } else { // some other error
- log.WithError(err).Error("failed getting execution payload (2/2) - error")
+
+ // Case 2 or 3, we don't know which.
+ abortReason = "execution-payload-not-found"
+ log.Warn("failed second attempt to get execution payload, not found case, block was never submitted to this relay or bid was accepted but payload was lost")
+ api.RespondError(w, http.StatusBadRequest, "no execution payload for this request, block was never seen by this relay or bid was accepted but payload was lost, if you got this bid from us, please contact the relay")
+ return
+ } else {
+ abortReason = "execution-payload-retrieval-error"
+ log.WithError(err).Error("failed second attempt to get execution payload, error case")
+ api.RespondError(w, http.StatusInternalServerError, "no execution payload for this request")
+ return
}
- api.RespondError(w, http.StatusBadRequest, "no execution payload for this request")
- return
}
+
+ // The second attempt succeeded. We may continue.
}
// Now we know this relay also has the payload
@@ -1443,7 +1566,7 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request)
}
// Publish the signed beacon block via beacon-node
- timeBeforePublish := time.Now().UTC().UnixMilli()
+ timeBeforePublish = time.Now().UTC().UnixMilli()
log = log.WithField("timestampBeforePublishing", timeBeforePublish)
signedBeaconBlock := common.SignedBlindedBeaconBlockToBeaconBlock(payload, getPayloadResp)
code, err := api.beaconClient.PublishBlock(signedBeaconBlock) // errors are logged inside
@@ -1452,7 +1575,7 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request)
api.RespondError(w, http.StatusBadRequest, "failed to publish block")
return
}
- timeAfterPublish := time.Now().UTC().UnixMilli()
+ timeAfterPublish = time.Now().UTC().UnixMilli()
msNeededForPublishing = uint64(timeAfterPublish - timeBeforePublish)
log = log.WithField("timestampAfterPublishing", timeAfterPublish)
log.WithField("msNeededForPublishing", msNeededForPublishing).Info("block published through beacon node")
@@ -1469,6 +1592,24 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request)
log.Info("execution payload delivered")
}
+func (api *RelayAPI) handleGetRegistrationMeta(w http.ResponseWriter, req *http.Request) {
+ log := api.log.WithFields(logrus.Fields{
+ "method": "getRegistrationMeta",
+ "ua": req.UserAgent(),
+ "headSlot": api.headSlot.Load(),
+ })
+
+ w.Header().Set("Content-Type", "application/json")
+
+ if err := json.NewEncoder(w).Encode(pubkeyMetadata); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ log.WithError(err).Error("failed to encode pubkeyMetadata")
+ return
+ }
+
+ log.Debug("getRegistrationMeta request processed")
+}
+
// --------------------
//
// BLOCK BUILDER APIS
diff --git a/services/api/service_test.go b/services/api/service_test.go
index a993ec69..19c362a9 100644
--- a/services/api/service_test.go
+++ b/services/api/service_test.go
@@ -56,7 +56,7 @@ func newTestBackend(t require.TestingT, numBeaconNodes int) *testBackend {
redisClient, err := miniredis.Run()
require.NoError(t, err)
- redisCache, err := datastore.NewRedisCache("", redisClient.Addr(), "")
+ redisCache, err := datastore.NewRedisCache("", redisClient.Addr(), "", "")
require.NoError(t, err)
db := database.MockDB{}