Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(mono): relative path + cli change for ports and datadirs #16

Open
wants to merge 8 commits into
base: gwyneth
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2,175 changes: 1,130 additions & 1,045 deletions Cargo.lock

Large diffs are not rendered by default.

48 changes: 24 additions & 24 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -78,29 +78,29 @@ alloy-signer-local = { version = "0.3.0" }
alloy-sol-types = { version = "0.8.2", default-features = false }

[patch.crates-io]
revm = { git = "https://github.com/taikoxyz/revm.git", branch = "v43-gwyneth" }
revm-primitives = { git = "https://github.com/taikoxyz/revm.git", branch = "v43-gwyneth" }
revm-interpreter = { git = "https://github.com/taikoxyz/revm.git", branch = "v43-gwyneth" }
revm-precompile = { git = "https://github.com/taikoxyz/revm.git", branch = "v43-gwyneth" }
revm-inspectors = { git = "https://github.com/taikoxyz/revm-inspectors.git", branch = "main-rbuilder" }
revm = { path = "../revm/crates/revm" }
revm-primitives = { path = "../revm/crates/primitives" }
revm-interpreter = { path = "../revm/crates/interpreter" }
revm-precompile = { path = "../revm/crates/precompile" }
revm-inspectors = { path = "../revm-inspectors" }

[patch."https://github.com/paradigmxyz/reth"]
reth = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-db = {git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-db-common = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-errors = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-libmdbx = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-payload-builder = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-node-api = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-trie = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-trie-parallel = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-basic-payload-builder = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-node-core = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-primitives = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-provider = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-chainspec = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-evm = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-evm-ethereum = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-db-api = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-execution-errors = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth-trie-db = { git = "https://github.com/taikoxyz/gwyneth", branch = "gwyneth" }
reth = { path = "../reth/bin/reth" }
reth-db = {path = "../reth/crates/storage/db"}
reth-db-common = { path = "../reth/crates/storage/db-common" }
reth-errors = { path = "../reth/crates/errors" }
reth-libmdbx = { path = "../reth/crates/storage/libmdbx-rs" }
reth-payload-builder = { path = "../reth/crates/payload/builder" }
reth-node-api = { path = "../reth/crates/node/api" }
reth-trie = { path = "../reth/crates/trie/trie" }
reth-trie-parallel = { path = "../reth/crates/trie/parallel" }
reth-basic-payload-builder = { path = "../reth/crates/payload/basic" }
reth-node-core = { path = "../reth/crates/node/core" }
reth-primitives = { path = "../reth/crates/primitives" }
reth-provider = { path = "../reth/crates/storage/provider" }
reth-chainspec = { path = "../reth/crates/chainspec" }
reth-evm = { path = "../reth/crates/evm" }
reth-evm-ethereum = { path = "../reth/crates/ethereum/evm" }
reth-db-api = { path = "../reth/crates/storage/db-api" }
reth-execution-errors = { path = "../reth/crates/evm/execution-errors" }
reth-trie-db = { path = "../reth/crates/trie/db" }
97 changes: 25 additions & 72 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,74 +1,27 @@
#
# Base container (with sccache and cargo-chef)
#
# - https://github.com/mozilla/sccache
# - https://github.com/LukeMathWalker/cargo-chef
#
# Based on https://depot.dev/blog/rust-dockerfile-best-practices
#
FROM rust:1.81 as base

ARG FEATURES

RUN cargo install sccache --version ^0.8
RUN cargo install cargo-chef --version ^0.1

RUN apt-get update \
&& apt-get install -y clang libclang-dev

ENV CARGO_HOME=/usr/local/cargo
ENV RUSTC_WRAPPER=sccache
ENV SCCACHE_DIR=/sccache

#
# Planner container (running "cargo chef prepare")
#
FROM base AS planner
WORKDIR /app

COPY ./Cargo.lock ./Cargo.lock
COPY ./Cargo.toml ./Cargo.toml
COPY ./.git ./.git
COPY ./crates/ ./crates/

RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
cargo chef prepare --recipe-path recipe.json

#
# Builder container (running "cargo chef cook" and "cargo build --release")
#
FROM base as builder
FROM lukemathwalker/cargo-chef:latest-rust-1 AS builder
RUN apt-get update && apt-get -y upgrade && apt-get install -y libclang-dev

COPY ./rbuilder/Cargo.lock /rbuilder/Cargo.lock
COPY ./rbuilder/Cargo.toml /rbuilder/Cargo.toml
COPY ./rbuilder/crates /rbuilder/crates
COPY ./reth/Cargo.lock ./reth/Cargo.lock
COPY ./reth/Cargo.toml ./reth/Cargo.toml
COPY ./reth/crates ./reth/crates
COPY ./reth/bin ./reth/bin
COPY ./reth/examples ./reth/examples
COPY ./reth/testing ./reth/testing
COPY ./revm ./revm
COPY ./revm-inspectors ./revm-inspectors
RUN pwd && ls

WORKDIR /rbuilder
RUN cargo build --release

FROM ubuntu:22.04 AS runtime
COPY --from=builder /rbuilder/target/release/rbuilder /usr/local/bin
COPY ./reth/crates/ethereum/node/tests/assets /network-configs
RUN cat /network-configs/genesis.json
WORKDIR /app
# RUN rbuilder

COPY --from=planner /app/recipe.json recipe.json

RUN --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
cargo chef cook --release --recipe-path recipe.json

COPY ./Cargo.lock ./Cargo.lock
COPY ./Cargo.toml ./Cargo.toml
COPY ./.git ./.git
COPY ./crates/ ./crates/

RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
cargo build --release --features="$FEATURES"

#
# Runtime container
#
FROM gcr.io/distroless/cc-debian12

WORKDIR /app

# RUN apk add libssl3 ca-certificates
# RUN apt-get update \
# && apt-get install -y libssl3 ca-certificates \
# && rm -rf /var/lib/apt/lists/*

COPY --from=builder /app/target/release/rbuilder /app/rbuilder

ENTRYPOINT ["/app/rbuilder"]
ENTRYPOINT ["/usr/local/bin/rbuilder"]
43 changes: 30 additions & 13 deletions config-gwyneth-reth.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# This is auto-generated from kurtosis template:
# gwyneth-mono/ethereum-package/static_files/gwyneth/rbuilder_config.toml.tmpl
log_json = false
log_level = "info,rbuilder=debug"
redacted_telemetry_server_port = 6061
Expand All @@ -6,28 +8,43 @@ full_telemetry_server_port = 6060
full_telemetry_server_ip = "0.0.0.0"

chain = "/network-configs/genesis.json"
reth_datadir = "/data/reth/execution-data/"
# We can have multiple, separated by comma
l2_reth_datadirs = ["/data/reth/gwyneth-167010"]
reth_datadir = "/data/reth/execution-data"
el_node_ipc_path = "/tmp/ipc/l1.ipc"

gwyneth_chain_ids = [

160010,
160011,
]
l2_reth_datadirs = [

"/data/reth/gwyneth-160010",
"/data/reth/gwyneth-160011",
]
l2_ipc_paths = [

"/tmp/ipc/l2.ipc-160010",
"/tmp/ipc/l2.ipc-160011",
]
l2_server_ports = [

9647,
9648,
]


coinbase_secret_key = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
relay_secret_key = "5eae315483f028b5cdd5d1090ff0c7618b18737ea9bf3c35047189db22835c48"
optimistic_relay_secret_key = "env:OPTIMISTIC_RELAY_SECRET_KEY"

# This shall be something like: "http://172.16.152.12:4000". To gather the correct IP, in (host machine) terminal we can query by:
# docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' CL_CONTAINER_HASH
# but it is auto-filled from now by the taiko reth's 'make install' process
cl_node_url = [""]
cl_node_url = "http://172.16.32.12:4000"
jsonrpc_server_port = 9646
jsonrpc_server_ip = "0.0.0.0"
# Not exactly the same because the random postfix is generated every time, but something like: "/tmp/reth.ipc-3ZTH1DJ4"
el_node_ipc_path = "/tmp/reth.ipc"
# We can have multiple, separated by comma
l2_el_node_ipc_paths = ["/tmp/reth.ipc-167010"]

extra_data = "⚡🤖"

dry_run = false
dry_run_validation_url = "http://localhost:8545"
dry_run_validation_url = "http://172.16.32.10:8545"

# blocks_processor_url can be an API service to record bids and transactions. It is not required.
# blocks_processor_url = "http://block_processor.internal"
Expand All @@ -46,7 +63,7 @@ url = "http://0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e
priority = 0
use_ssz_for_submit = false
use_gzip_for_submit = false
l1_rpc_url = "http://localhost:8545"
l1_rpc_url = "http://172.16.32.10:8545"
l1_proposer_pk = "39725efee3fb28614de3bacaffe4cc4bd8c436257e2c8bb887c4b5c4be45e76d"
l1_smart_contract_address = "0x9fCF7D13d10dEdF17d0f24C62f0cf4ED462f65b7"

Expand Down
6 changes: 4 additions & 2 deletions crates/rbuilder/src/backtest/execute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use crate::{
primitives::SimulatedOrder,
utils::clean_extradata,
};
use ahash::HashSet;
use ahash::{HashMap, HashSet};
use alloy_primitives::{Address, U256};
use reth::providers::ProviderFactory;
use reth_chainspec::ChainSpec;
Expand Down Expand Up @@ -86,8 +86,10 @@ pub fn backtest_prepare_ctx_for_block<DB: Database + Clone>(
block_data.winning_bid_trace.proposer_fee_recipient,
Some(builder_signer),
);
let mut provider_factories = HashMap::default();
provider_factories.insert(chain_spec.chain.id(), provider_factory.clone());
let (sim_orders, sim_errors) =
simulate_all_orders_with_sim_tree(provider_factory.clone(), &ctx, &orders, false)?;
simulate_all_orders_with_sim_tree(provider_factories, &ctx, &orders, false)?;
Ok(BacktestBlockInput {
ctx,
sim_orders,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ pub fn sim_historical_block(
let txs = extract_onchain_block_txs(&onchain_block)?;

let suggested_fee_recipient = find_suggested_fee_recipient(&onchain_block, &txs);
let coinbase = onchain_block.header.miner;
let coinbase = ChainAddress(chain_spec.chain.id(), onchain_block.header.miner);

let ctx = BlockBuildingContext::from_onchain_block(
onchain_block,
Expand All @@ -50,7 +50,7 @@ pub fn sim_historical_block(
None,
);

let state_provider = provider_factory.history_by_block_hash(ctx.attributes.parent)?;
let state_provider = provider_factory.history_by_block_hash(ctx.chains[&chain_spec.chain().id()].attributes.parent)?;
let mut partial_block = PartialBlock::new(true, None);
let mut state = BlockState::new(state_provider, chain_spec.chain().id());

Expand All @@ -62,8 +62,6 @@ pub fn sim_historical_block(
let mut cumulative_blob_gas_used = 0;
let mut written_slots: HashMap<SlotKey, Vec<B256>> = HashMap::default();

let coinbase = ChainAddress(chain_spec.chain.id(), coinbase);

for (idx, tx) in txs.into_iter().enumerate() {
let coinbase_balance_before = state.balance(coinbase)?;
let mut accumulator_tracer = AccumulatorSimulationTracer::default();
Expand Down
3 changes: 2 additions & 1 deletion crates/rbuilder/src/bin/debug-bench-machine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use rbuilder::{
use reth::providers::BlockNumReader;
use reth_payload_builder::database::SyncCachedReads as CachedReads;
use reth_provider::StateProvider;
use revm_primitives::ChainAddress;
use std::{path::PathBuf, sync::Arc, time::Instant};
use tracing::{debug, info};

Expand Down Expand Up @@ -62,7 +63,7 @@ async fn main() -> eyre::Result<()> {
txs.len()
);

let coinbase = onchain_block.header.miner;
let coinbase = ChainAddress(chain_spec.chain.id(), onchain_block.header.miner);

let ctx = BlockBuildingContext::from_onchain_block(
onchain_block,
Expand Down
8 changes: 5 additions & 3 deletions crates/rbuilder/src/bin/dummy-builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use rbuilder::{
base_config::{
DEFAULT_EL_NODE_IPC_PATH, DEFAULT_INCOMING_BUNDLES_PORT, DEFAULT_IP,
DEFAULT_RETH_DB_PATH,
}, config::create_provider_factory, layer2_info::Layer2Info, order_input::{
}, config::create_provider_factory, layer2_info::{self, Layer2Info}, order_input::{
OrderInputConfig, DEFAULT_INPUT_CHANNEL_BUFFER_SIZE, DEFAULT_RESULTS_CHANNEL_TIMEOUT,
DEFAULT_SERVE_MAX_CONNECTIONS,
}, payload_events::{MevBoostSlotData, MevBoostSlotDataGenerator}, simulation::SimulatedOrderCommand, LiveBuilder
Expand Down Expand Up @@ -66,6 +66,8 @@ async fn main() -> eyre::Result<()> {
cancel.clone(),
);

let layer2_info = Layer2Info::new(vec![], &vec![], &vec![], &vec![]).await?;

let builder = LiveBuilder::<Arc<DatabaseEnv>, MevBoostSlotDataGenerator> {
watchdog_timeout: Duration::from_secs(10000),
error_storage_path: None,
Expand Down Expand Up @@ -95,7 +97,7 @@ async fn main() -> eyre::Result<()> {
extra_rpc: RpcModule::new(()),
sink_factory: Box::new(TraceBlockSinkFactory {}),
builders: vec![Arc::new(DummyBuildingAlgorithm::new(10))],
layer2_info: Layer2Info::new(vec![], HashMap::default()).await?,
layer2_info,
};

let ctrlc = tokio::spawn(async move {
Expand Down Expand Up @@ -192,7 +194,7 @@ impl DummyBuildingAlgorithm {
&self,
orders: Vec<SimulatedOrder>,
provider_factory: HashMap<u64, ProviderFactory<DB>>,
ctx: &HashMap<u64, BlockBuildingContext>,
ctx: &BlockBuildingContext,
) -> eyre::Result<Box<dyn BlockBuildingHelper>> {
let mut block_building_helper = BlockBuildingHelperFromDB::new(
provider_factory.clone(),
Expand Down
Loading