diff --git a/.github/workflows/base_benchmarks.yaml b/.github/workflows/base_benchmarks.yaml index e279ec487..23e6da1ea 100644 --- a/.github/workflows/base_benchmarks.yaml +++ b/.github/workflows/base_benchmarks.yaml @@ -18,7 +18,7 @@ jobs: - name: Install Rust run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && echo "$HOME/.cargo/bin" >> $GITHUB_PATH - name: Install dependencies - run: sudo add-apt-repository ppa:ethereum/ethereum && sudo apt update && sudo apt install -y solc build-essential pkg-config libssl-dev cmake protobuf-compiler + run: sudo add-apt-repository ppa:ethereum/ethereum && sudo apt update && sudo apt install -y solc build-essential pkg-config libssl-dev cmake protobuf-compiler libsystemd-dev - name: Track base branch benchmarks run: | bencher run \ diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 79f6d030c..4e53691f2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,7 +49,7 @@ jobs: - name: Install nightly run: rustup toolchain install nightly - name: Install dependencies - run: sudo add-apt-repository ppa:ethereum/ethereum && sudo apt update && sudo apt install -y solc build-essential pkg-config libssl-dev cmake protobuf-compiler + run: sudo add-apt-repository ppa:ethereum/ethereum && sudo apt update && sudo apt install -y solc build-essential pkg-config libssl-dev cmake protobuf-compiler libsystemd-dev - uses: actions/checkout@v4 with: submodules: recursive diff --git a/.github/workflows/pr_benchmarks.yaml b/.github/workflows/pr_benchmarks.yaml index 9e6d44e7a..bfd346c18 100644 --- a/.github/workflows/pr_benchmarks.yaml +++ b/.github/workflows/pr_benchmarks.yaml @@ -20,7 +20,7 @@ jobs: - name: Install Rust run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && echo "$HOME/.cargo/bin" >> $GITHUB_PATH - name: Install dependencies - run: sudo add-apt-repository ppa:ethereum/ethereum && sudo apt update && sudo apt install -y solc build-essential pkg-config libssl-dev cmake protobuf-compiler + run: sudo add-apt-repository ppa:ethereum/ethereum && sudo apt update && sudo apt install -y solc build-essential pkg-config libssl-dev cmake protobuf-compiler libsystemd-dev - name: Track PR Benchmarks run: | bencher run \ diff --git a/Cargo.lock b/Cargo.lock index 0dd0f2a47..956ab0dbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1441,6 +1441,12 @@ dependencies = [ "serde", ] +[[package]] +name = "build-env" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1522ac6ee801a11bf9ef3f80403f4ede6eb41291fac3dde3de09989679305f25" + [[package]] name = "build_const" version = "0.2.2" @@ -2075,6 +2081,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "cstr-argument" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bd9c8e659a473bce955ae5c35b116af38af11a7acb0b480e01f3ed348aeb40" +dependencies = [ + "cfg-if", + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -3106,7 +3122,28 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "foreign-types-shared", + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] @@ -3115,6 +3152,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -5405,6 +5448,17 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libsystemd-sys" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed080163caa59cc29b34bce2209b737149a4bac148cd9a8b04e4c12822798119" +dependencies = [ + "build-env", + "libc", + "pkg-config", +] + [[package]] name = "libz-sys" version = "1.1.20" @@ -6048,7 +6102,7 @@ checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ "bitflags 2.6.0", "cfg-if", - "foreign-types", + "foreign-types 0.3.2", "libc", "once_cell", "openssl-macros", @@ -8758,6 +8812,21 @@ dependencies = [ "libc", ] +[[package]] +name = "systemd" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afec0101d9ae8ab26aedf0840109df689938ea7e538aa03df4369f1854f11562" +dependencies = [ + "cstr-argument", + "foreign-types 0.5.0", + "libc", + "libsystemd-sys", + "log", + "memchr", + "utf8-cstr", +] + [[package]] name = "tap" version = "1.0.1" @@ -9564,6 +9633,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" +[[package]] +name = "utf8-cstr" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55bcbb425141152b10d5693095950b51c3745d019363fc2929ffd8f61449b628" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -10485,6 +10560,7 @@ dependencies = [ "sha2", "sha3", "sled", + "systemd", "tempfile", "thiserror 2.0.6", "time", @@ -10496,6 +10572,7 @@ dependencies = [ "tracing", "tracing-subscriber", "ureq", + "url", "vergen", "zilliqa", "zilliqa-macros", diff --git a/Dockerfile b/Dockerfile index f482dd23c..6e81452ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM rust:1.83.0-slim-bullseye as builder ARG is_release=false RUN apt update -y && \ apt upgrade -y && \ - apt install -y protobuf-compiler + apt install -y protobuf-compiler libsystemd-dev pkg-config RUN apt autoremove diff --git a/z2/resources/config.tera.toml b/z2/resources/config.tera.toml index d8bcca590..db714b1e3 100644 --- a/z2/resources/config.tera.toml +++ b/z2/resources/config.tera.toml @@ -11,6 +11,9 @@ data_dir = "/data" consensus.genesis_accounts = [ ["{{ genesis_address }}", "900_000_000_000_000_000_000_000_000" ] ] consensus.genesis_deposits = [ ["{{ bootstrap_bls_public_key }}", "{{ bootstrap_peer_id }}", "20_000_000_000_000_000_000_000_000", "0x0000000000000000000000000000000000000000", "{{ genesis_address }}"] ] +# API gateway +remote_api_url = "{{ remote_api_url }}" + # Reward parameters consensus.rewards_per_hour = "51_000_000_000_000_000_000_000" consensus.blocks_per_hour = 3600 diff --git a/z2/resources/node_provision.tera.py b/z2/resources/node_provision.tera.py index c145e2b78..349ea69f3 100644 --- a/z2/resources/node_provision.tera.py +++ b/z2/resources/node_provision.tera.py @@ -131,11 +131,12 @@ def health(): Description=Zilliqa Node [Service] +WatchdogSec=60s Type=forking ExecStart=/usr/local/bin/zq2.sh start """ + SECRET_KEY + """ ExecStop=/usr/local/bin/zq2.sh stop RemainAfterExit=yes -Restart=on-failure +Restart=on-failure,on-watchdog RestartSec=10 Environment="RUST_LOG=zilliqa=debug" diff --git a/z2/src/chain/node.rs b/z2/src/chain/node.rs index 08d3a656e..d548fcf4d 100644 --- a/z2/src/chain/node.rs +++ b/z2/src/chain/node.rs @@ -515,6 +515,7 @@ impl ChainNode { let role_name = self.role.to_string(); let eth_chain_id = self.eth_chain_id.to_string(); let bootstrap_public_ip = selected_bootstrap.machine.external_address; + let remote_api_url = self.chain()?.get_endpoint()?; // API gateway let whitelisted_evm_contract_addresses = self.chain()?.get_whitelisted_evm_contracts(); // 4201 is the publically exposed port - We don't expose everything there. let public_api = if self.role == NodeRole::Api { @@ -544,6 +545,7 @@ impl ChainNode { ); ctx.insert("api_servers", &api_servers); ctx.insert("enable_ots_indices", &enable_ots_indices); + ctx.insert("remote_api_url", remote_api_url); Ok(Tera::one_off(spec_config, &ctx, false)?) } diff --git a/z2/src/setup.rs b/z2/src/setup.rs index 31e32dc41..317fbd6d0 100644 --- a/z2/src/setup.rs +++ b/z2/src/setup.rs @@ -21,9 +21,9 @@ use zilliqa::{ block_request_limit_default, consensus_timeout_default, empty_block_timeout_default, eth_chain_id_default, failed_request_sleep_duration_default, local_address_default, max_blocks_in_flight_default, minimum_time_left_for_empty_block_default, - scilla_address_default, scilla_ext_libs_path_default, scilla_stdlib_dir_default, - state_rpc_limit_default, total_native_token_supply_default, Amount, ApiServer, - ConsensusConfig, GenesisDeposit, + remote_api_url_default, scilla_address_default, scilla_ext_libs_path_default, + scilla_stdlib_dir_default, state_rpc_limit_default, total_native_token_supply_default, + Amount, ApiServer, ConsensusConfig, GenesisDeposit, }, transaction::EvmGas, }; @@ -544,6 +544,7 @@ impl Setup { block_request_batch_size: block_request_batch_size_default(), state_rpc_limit: state_rpc_limit_default(), failed_request_sleep_duration: failed_request_sleep_duration_default(), + remote_api_url: remote_api_url_default(), enable_ots_indices: false, }; println!("🧩 Node {node_index} has RPC port {port}"); diff --git a/zilliqa/Cargo.toml b/zilliqa/Cargo.toml index da907ee44..c16586b40 100644 --- a/zilliqa/Cargo.toml +++ b/zilliqa/Cargo.toml @@ -85,6 +85,8 @@ serde_repr = "0.1.19" thiserror = "2.0.6" lru-mem = "0.3.0" opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] } +systemd = "0.10.0" +url = "2.5.4" [dev-dependencies] alloy = { version = "0.6.4", default-features = false, features = ["network", "rand", "signers", "signer-local"] } diff --git a/zilliqa/src/cfg.rs b/zilliqa/src/cfg.rs index fc29a2a5d..fb7fa6b4d 100644 --- a/zilliqa/src/cfg.rs +++ b/zilliqa/src/cfg.rs @@ -4,6 +4,7 @@ use alloy::primitives::Address; use libp2p::{Multiaddr, PeerId}; use rand::{distributions::Alphanumeric, Rng}; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; +use url::Url; use crate::{ crypto::{Hash, NodePublicKey}, @@ -110,6 +111,10 @@ pub struct NodeConfig { /// Defaults to 10 seconds. #[serde(default = "failed_request_sleep_duration_default")] pub failed_request_sleep_duration: Duration, + /// Point to API gateway - used to check for block progress. + /// Defaults to localhost. + #[serde(default = "remote_api_url_default")] + pub remote_api_url: Url, /// Enable additional indices used by some Otterscan APIs. Enabling this will use more disk space and block processing will take longer. #[serde(default)] pub enable_ots_indices: bool, @@ -131,6 +136,7 @@ impl Default for NodeConfig { block_request_batch_size: block_request_batch_size_default(), state_rpc_limit: state_rpc_limit_default(), failed_request_sleep_duration: failed_request_sleep_duration_default(), + remote_api_url: remote_api_url_default(), enable_ots_indices: false, } } @@ -175,6 +181,10 @@ pub fn state_cache_size_default() -> usize { 256 * 1024 * 1024 // 256 MiB } +pub fn remote_api_url_default() -> Url { + Url::parse("http://localhost:4201").unwrap() +} + pub fn eth_chain_id_default() -> u64 { 700 + 0x8000 } diff --git a/zilliqa/src/node_launcher.rs b/zilliqa/src/node_launcher.rs index 995191ab8..adfe5f462 100644 --- a/zilliqa/src/node_launcher.rs +++ b/zilliqa/src/node_launcher.rs @@ -6,6 +6,7 @@ use std::{ use anyhow::{anyhow, Result}; use http::{header, Method}; +use jsonrpsee::core::client::ClientT; use libp2p::{futures::StreamExt, PeerId}; use node::Node; use opentelemetry::KeyValue; @@ -34,6 +35,14 @@ use crate::{ p2p_node::{LocalMessageTuple, OutboundMessageTuple}, }; +// 3 samples should be sufficient to determine if a node is stuck. +const WATCHDOG_THRESHOLD: u64 = 3; +#[derive(Default, Debug)] +pub struct WatchDogDebounce { + pub count: u64, + pub value: u64, +} + pub struct NodeLauncher { pub node: Arc>, pub config: NodeConfig, @@ -45,6 +54,7 @@ pub struct NodeLauncher { /// Channel used to steer next sleep time pub reset_timeout_receiver: UnboundedReceiverStream, node_launched: bool, + watchdog: WatchDogDebounce, } // If the `fake_response_channel` feature is enabled, swap out the libp2p ResponseChannel for a `u64`. In our @@ -157,6 +167,7 @@ impl NodeLauncher { reset_timeout_receiver, node_launched: false, config, + watchdog: WatchDogDebounce::default(), }; let input_channels = NodeInputChannels { broadcasts: broadcasts_sender, @@ -169,6 +180,75 @@ impl NodeLauncher { Ok((launcher, input_channels)) } + async fn handle_watchdog(&mut self) -> Result<()> { + // If watchdog is disabled, then do nothing. + if systemd::daemon::watchdog_enabled(false).unwrap_or_default() == 0 { + return Ok(()); + } + + // 1. Collect quick sample + let self_highest = self + .node + .lock() + .unwrap() + .db + .get_highest_canonical_block_number()? + .ok_or_else(|| anyhow!("can't find highest block num in database!"))?; + + tracing::debug!("WDT check {self_highest}"); + + // 1.5 Debounce + if self.watchdog.value == self_highest { + self.watchdog.count += 1; + } else { + self.watchdog.value = self_highest; + self.watchdog.count = 0; + } + + // 2. Internal check to see if node is possibly stuck. + if self.watchdog.count >= WATCHDOG_THRESHOLD { + // 3. External check to see if others are stuck too. + let client = jsonrpsee::http_client::HttpClientBuilder::default() + .request_timeout(Duration::from_secs(5)) // fast call + .build(self.config.remote_api_url.as_str())?; + + let result: String = client + .request("eth_blockNumber", jsonrpsee::rpc_params![]) + .await + // do not restart due to network/upstream errors, check again later. + .unwrap_or_else(|e| { + tracing::error!( + "WDT remote call to {} failed: {e}", + self.config.remote_api_url, + ); + "0x0".to_string() + }); + + let remote_highest = result + .strip_prefix("0x") + .map(|s| u64::from_str_radix(s, 16).unwrap_or_default()) + .unwrap_or_default(); + + // 4. If self < others for > threshold, then we're stuck + if self_highest < remote_highest { + tracing::warn!(?self_highest, ?remote_highest, "WDT node stuck at"); + return Ok(()); + } else { + tracing::warn!( + ?self_highest, + ?remote_highest, + "WDT network possibly stalled at" + ) + } + } + + // 4. Reset systemd watchdog + if !systemd::daemon::notify(false, [(systemd::daemon::STATE_WATCHDOG, "1")].iter())? { + tracing::error!("Failed to notify systemd."); + } + Ok(()) + } + pub async fn start_shard_node(&mut self) -> Result<()> { if self.node_launched { return Err(anyhow!("Node already running!")); @@ -177,6 +257,14 @@ impl NodeLauncher { let sleep = time::sleep(Duration::from_millis(5)); tokio::pin!(sleep); + // Schedule a systemd watchdog handler + let wdt_dur = Duration::from_micros( + systemd::daemon::watchdog_enabled(false)?.max(60_000_000) / WATCHDOG_THRESHOLD, + ); + let watchdog = time::sleep(wdt_dur); + tokio::pin!(watchdog); + tracing::info!("Watchdog checks every {:?}", wdt_dur); + self.node_launched = true; let meter = opentelemetry::global::meter("zilliqa"); @@ -263,6 +351,22 @@ impl NodeLauncher { let (_source, _message) = message.expect("message stream should be infinite"); todo!("Local messages will need to be handled once cross-shard messaging is implemented"); } + + () = &mut watchdog => { + let attributes = vec![ + KeyValue::new(MESSAGING_OPERATION_NAME, "handle"), + KeyValue::new(MESSAGING_SYSTEM, "tokio_channel"), + KeyValue::new(MESSAGING_DESTINATION_NAME, "watchdog"), + ]; + let start = SystemTime::now(); + self.handle_watchdog().await?; + watchdog.as_mut().reset(Instant::now() + wdt_dur); + messaging_process_duration.record( + start.elapsed().map_or(0.0, |d| d.as_secs_f64()), + &attributes, + ); + } + () = &mut sleep => { let attributes = vec![ KeyValue::new(MESSAGING_OPERATION_NAME, "handle"), @@ -275,7 +379,7 @@ impl NodeLauncher { self.node.lock().unwrap().consensus.tick().unwrap(); // No messages for a while, so check if consensus wants to timeout self.node.lock().unwrap().handle_timeout().unwrap(); - sleep.as_mut().reset(Instant::now() + Duration::from_millis(500)); + sleep.as_mut().reset(Instant::now() + self.config.consensus.empty_block_timeout / 2); messaging_process_duration.record( start.elapsed().map_or(0.0, |d| d.as_secs_f64()), &attributes, diff --git a/zilliqa/src/p2p_node.rs b/zilliqa/src/p2p_node.rs index 112a53a09..a4e71befa 100644 --- a/zilliqa/src/p2p_node.rs +++ b/zilliqa/src/p2p_node.rs @@ -231,6 +231,7 @@ impl P2pNode { if let Some((peer, address)) = &self.config.bootstrap_address { if self.swarm.local_peer_id() != peer { + tracing::info!("Dialling {peer} at {}", address); self.swarm.dial( DialOpts::peer_id(*peer) .override_role() // hole-punch diff --git a/zilliqa/tests/it/main.rs b/zilliqa/tests/it/main.rs index 975298ee5..cf5beef6f 100644 --- a/zilliqa/tests/it/main.rs +++ b/zilliqa/tests/it/main.rs @@ -68,9 +68,10 @@ use zilliqa::{ allowed_timestamp_skew_default, block_request_batch_size_default, block_request_limit_default, eth_chain_id_default, failed_request_sleep_duration_default, max_blocks_in_flight_default, minimum_time_left_for_empty_block_default, - scilla_address_default, scilla_ext_libs_path_default, scilla_stdlib_dir_default, - state_cache_size_default, state_rpc_limit_default, total_native_token_supply_default, - Amount, ApiServer, Checkpoint, ConsensusConfig, GenesisDeposit, NodeConfig, + remote_api_url_default, scilla_address_default, scilla_ext_libs_path_default, + scilla_stdlib_dir_default, state_cache_size_default, state_rpc_limit_default, + total_native_token_supply_default, Amount, ApiServer, Checkpoint, ConsensusConfig, + GenesisDeposit, NodeConfig, }, crypto::{SecretKey, TransactionPublicKey}, db, @@ -357,6 +358,7 @@ impl Network { block_request_batch_size: block_request_batch_size_default(), state_rpc_limit: state_rpc_limit_default(), failed_request_sleep_duration: failed_request_sleep_duration_default(), + remote_api_url: remote_api_url_default(), enable_ots_indices: true, }; @@ -480,6 +482,7 @@ impl Network { block_request_batch_size: block_request_batch_size_default(), state_rpc_limit: state_rpc_limit_default(), failed_request_sleep_duration: failed_request_sleep_duration_default(), + remote_api_url: remote_api_url_default(), enable_ots_indices: true, }; diff --git a/zilliqa/tests/it/persistence.rs b/zilliqa/tests/it/persistence.rs index b65879445..1958d3a3d 100644 --- a/zilliqa/tests/it/persistence.rs +++ b/zilliqa/tests/it/persistence.rs @@ -13,7 +13,7 @@ use zilliqa::{ allowed_timestamp_skew_default, block_request_batch_size_default, block_request_limit_default, consensus_timeout_default, eth_chain_id_default, failed_request_sleep_duration_default, max_blocks_in_flight_default, - minimum_time_left_for_empty_block_default, scilla_address_default, + minimum_time_left_for_empty_block_default, remote_api_url_default, scilla_address_default, scilla_ext_libs_path_default, scilla_stdlib_dir_default, state_cache_size_default, state_rpc_limit_default, total_native_token_supply_default, ApiServer, Checkpoint, ConsensusConfig, NodeConfig, @@ -132,6 +132,7 @@ async fn block_and_tx_data_persistence(mut network: Network) { block_request_batch_size: block_request_batch_size_default(), state_rpc_limit: state_rpc_limit_default(), failed_request_sleep_duration: failed_request_sleep_duration_default(), + remote_api_url: remote_api_url_default(), enable_ots_indices: true, }; let mut rng = network.rng.lock().unwrap();