DIFFS: Merge 'merged_develop' to gwyneth

.github/workflows/checks.yaml

@@ -55,7 +55,7 @@ jobs:
           version: nightly
 
       - name: Install native dependencies
-        run: sudo apt-get install -y libsqlite3-dev
+        run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
 
       - name: Lint
         run: make lint

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
     "crates/rbuilder",
+    "crates/reth-rbuilder",
     "crates/rbuilder/src/test_utils",
     "crates/rbuilder/src/telemetry/metrics_macros"
 ]
@@ -23,12 +24,16 @@ edition = "2021"
 
 [workspace.dependencies]
 reth = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
+reth-cli-util = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-db = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
+reth-db-api = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-db-common = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-errors = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-libmdbx = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-node-api = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
+reth-node-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
+reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-trie = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-basic-payload-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
@@ -38,6 +43,13 @@ reth-provider = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6", f
 reth-chainspec = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-evm = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
 reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", tag = "v1.0.6" }
+clap = { version = "4.4.3" }
+eyre = { version = "0.6.8" }
+libc = { version = "0.2.161" }
+tikv-jemallocator = { version = "0.5.4" }
+tokio = "1.38.0"
+tokio-util = "0.7.11"
+tracing = "0.1.37"
 
 # version is copied from reth  "v1.0.6" dependencies
 revm = { version = "14.0.0", features = [
@@ -86,12 +98,18 @@ revm-inspectors = { git = "https://github.com/taikoxyz/revm-inspectors.git", bra
 
 [patch."https://github.com/paradigmxyz/reth"]
 reth = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
+reth-cli-util = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth"}
 reth-db = {git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
+reth-db-api = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
+reth-execution-errors = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
+reth-trie-db = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
 reth-db-common = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
 reth-errors = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
 reth-libmdbx = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
 reth-payload-builder = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
 reth-node-api = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
+reth-node-builder = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
+reth-node-ethereum = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
 reth-trie = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
 reth-trie-parallel = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
 reth-basic-payload-builder = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
@@ -101,6 +119,3 @@ reth-provider = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth
 reth-chainspec = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
 reth-evm = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
 reth-evm-ethereum = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
-reth-db-api = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
-reth-execution-errors = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
-reth-trie-db = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }

Dockerfile

@@ -1,19 +1,71 @@
-# Dockerfile for building rbuilder executable
-FROM lukemathwalker/cargo-chef:latest-rust-1 AS builder
+#
+# Base container (with sccache and cargo-chef)
+#
+# - https://github.com/mozilla/sccache
+# - https://github.com/LukeMathWalker/cargo-chef
+#
+# Based on https://depot.dev/blog/rust-dockerfile-best-practices
+#
+FROM rust:1.81 as base
+
+ARG FEATURES
+
+RUN cargo install sccache --version ^0.8
+RUN cargo install cargo-chef --version ^0.1
+
+RUN apt-get update \
+    && apt-get install -y clang libclang-dev
+
+ENV CARGO_HOME=/usr/local/cargo
+ENV RUSTC_WRAPPER=sccache
+ENV SCCACHE_DIR=/sccache
+
+#
+# Planner container (running "cargo chef prepare")
+#
+FROM base AS planner
 WORKDIR /app
 
-# Install system dependencies
-RUN apt-get update && apt-get -y upgrade && apt-get install -y libclang-dev pkg-config git
+COPY ./Cargo.lock ./Cargo.lock
+COPY ./Cargo.toml ./Cargo.toml
+COPY ./.git ./.git
+COPY ./crates/ ./crates/
 
-# Copy the entire rbuilder repository
-COPY . .
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
+    cargo chef prepare --recipe-path recipe.json
 
-# Build the executable
-RUN cargo build --release
+#
+# Builder container (running "cargo chef cook" and "cargo build --release")
+#
+FROM base as builder
+WORKDIR /app
+# Default binary filename rbuilder
+# Alternatively can be set to "reth-rbuilder" - to have reth included in the binary
+ARG RBUILDER_BIN="rbuilder"
+COPY --from=planner /app/recipe.json recipe.json
+
+RUN --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
+    cargo chef cook --release --recipe-path recipe.json
+
+COPY ./Cargo.lock ./Cargo.lock
+COPY ./Cargo.toml ./Cargo.toml
+COPY ./.git ./.git
+COPY ./crates/ ./crates/
+
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
+    cargo build --release --features="$FEATURES" --bin=${RBUILDER_BIN}
+
+#
+# Runtime container
+#
+FROM gcr.io/distroless/cc-debian12
+WORKDIR /app
 
-# Stage for the executable
-FROM scratch
-COPY --from=builder /app/target/release/rbuilder /usr/local/bin/rbuilder
+ARG RBUILDER_BIN="rbuilder"
+COPY --from=builder /app/target/release/${RBUILDER_BIN} /app/rbuilder
 
-# Mark the executable as the entrypoint
-ENTRYPOINT ["/usr/local/bin/rbuilder"]
+ENTRYPOINT ["/app/rbuilder"]

README.md

@@ -84,15 +84,22 @@ You can use [builder-playground](https://github.com/flashbots/builder-playground
 
 First, start [builder-playground](https://github.com/flashbots/builder-playground):
 
-```
+```bash
 git clone [email protected]:flashbots/builder-playground.git
 cd builder-playground
 go run main.go
 ```
 
-Then, run `rbuilder` using the `config-playground.toml` config file:
+Next, update `config-playground.toml` with fully-qualified paths for entries containing `$HOME`:
 
+```bash
+# replaces '$HOME' with the actual value of "$HOME"
+sed -i "s|\$HOME|$HOME|g" config-playground.toml
 ```
+
+Then run `rbuilder` using the `config-playground.toml` config file:
+
+```bash
 cargo run --bin rbuilder run config-playground.toml
 ```
 

config-backtest-example.toml

@@ -28,3 +28,8 @@ sorting = "max-profit"
 failed_order_retries = 1
 drop_failed_orders = true
 
+[[builders]]
+name = "parallel"
+algo = "parallel-builder"
+discard_txs = true
+num_threads = 25

config-live-example.toml

@@ -32,8 +32,6 @@ max_concurrent_seals = 4
 # genesis_fork_version = "0x00112233"
 
 sbundle_mergeabe_signers = []
-# slot_delta_to_start_submits_ms is usually negative since we start bidding BEFORE the slot start
-# slot_delta_to_start_submits_ms = -5000
 live_builders = ["mp-ordering", "mgp-ordering", "merging"]
 
 [[relays]]
@@ -58,3 +56,9 @@ discard_txs = true
 sorting = "max-profit"
 failed_order_retries = 1
 drop_failed_orders = true
+
+[[builders]]
+name = "parallel"
+algo = "parallel-builder"
+discard_txs = true
+num_threads = 25

config-playground.toml

@@ -13,7 +13,7 @@ coinbase_secret_key = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf
 cl_node_url = ["http://localhost:3500"]
 jsonrpc_server_port = 8645
 jsonrpc_server_ip = "0.0.0.0"
-el_node_ipc_path = "/tmp/reth.ipc"
+el_node_ipc_path = "$HOME/.playground/devnet/reth.ipc"
 extra_data = "⚡🤖"
 
 dry_run = false
@@ -25,8 +25,6 @@ dry_run_validation_url = "http://localhost:8545"
 ignore_cancellable_orders = true
 
 sbundle_mergeabe_signers = []
-# slot_delta_to_start_submits_ms is usually negative since we start bidding BEFORE the slot start
-# slot_delta_to_start_submits_ms = -5000
 live_builders = ["mp-ordering"]
 
 [[relays]]