Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve overlay connection #581

Merged
merged 11 commits into from
Feb 4, 2025
Merged

Improve overlay connection #581

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
10b31a2
timeout if listening to overlay takes too long, break if no proper me…
gianfra-t Jan 31, 2025
61d72a0
allow to pass an array of nodes
gianfra-t Jan 31, 2025
40c947d
fix config parse unit tests
gianfra-t Jan 31, 2025
e49d1e7
format
gianfra-t Jan 31, 2025
b4bdafa
safe unwrap timeout result
gianfra-t Jan 31, 2025
38c2e00
format
gianfra-t Jan 31, 2025
80511b5
return ConfgError
gianfra-t Jan 31, 2025
787e5c7
map_err to ok_or for endpoint error selection
gianfra-t Jan 31, 2025
d65748a
remove conditional definition on rand
gianfra-t Feb 3, 2025
23bf576
fmt, temporary remove serde rename for testing
gianfra-t Feb 3, 2025
d798d0c
rollback serde renaming
gianfra-t Feb 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions clients/stellar-relay-lib/resources/config/mainnet/stellar_relay_config_mainnet_iowa.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
{
"connection_info": {
"address": "154.12.251.105",
"port": 11625
"endpoints": [
{ "address": "154.12.251.105", "port": 11625 }
]
},
"node_info": {
"ledger_version": 22,
Expand Down
30 changes: 30 additions & 0 deletions clients/stellar-relay-lib/resources/config/testnet/stellar_relay_config_sdftest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{
"connection_info": {
"endpoints": [
{
"address": "54.166.220.249",
"port": 11625
},
{
"address": "54.159.138.198",
"port": 11625
},
{
"address": "44.223.45.116",
"port": 11625
}
]
},
"node_info": {
"ledger_version": 22,
"overlay_version": 35,
"overlay_min_version": 33,
"version_str": "stellar-core 22.0.0 (721fd0a654d5e82d38c748a91053e530a475193d)",
"is_pub_net": false
},
"stellar_history_archive_urls": [
"http://history.stellar.org/prd/core-testnet/core_testnet_001",
"http://history.stellar.org/prd/core-testnet/core_testnet_002",
"http://history.stellar.org/prd/core-testnet/core_testnet_003"
]
}
8 changes: 6 additions & 2 deletions clients/stellar-relay-lib/resources/config/testnet/stellar_relay_config_sdftest1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
{
"connection_info": {
"address": "54.166.220.249",
"port": 11625
"endpoints": [
{
"address": "54.166.220.249",
"port": 11625
}
]
},
"node_info": {
"ledger_version": 22,
Expand Down
8 changes: 6 additions & 2 deletions clients/stellar-relay-lib/resources/config/testnet/stellar_relay_config_sdftest2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
{
"connection_info": {
"address": "54.159.138.198",
"port": 11625
"endpoints": [
{
"address": "54.159.138.198",
"port": 11625
}
]
},
"node_info": {
"ledger_version": 22,
Expand Down
8 changes: 6 additions & 2 deletions clients/stellar-relay-lib/resources/config/testnet/stellar_relay_config_sdftest3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
{
"connection_info": {
"address": "44.223.45.116",
"port": 11625
"endpoints": [
{
"address": "44.223.45.116",
"port": 11625
}
]
},
"node_info": {
"ledger_version": 22,
Expand Down
60 changes: 44 additions & 16 deletions clients/stellar-relay-lib/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};
use serde_with::{serde_as, BytesOrString};
use std::fmt::Debug;
use substrate_stellar_sdk::SecretKey;
use rand::seq::SliceRandom;

/// The configuration structure of the StellarOverlay.
/// It configures both the ConnectionInfo and the NodeInfo.
Expand Down Expand Up @@ -51,13 +52,15 @@ impl StellarOverlayConfig {
tracing::info!(
"connection_info(): Connecting to Stellar overlay network using public key: {public_key}"
);
let endpoint = cfg.endpoints.choose(&mut rand::thread_rng()).expect("No endpoints found in config for connecting to overlay ");

let address = std::str::from_utf8(&cfg.address)
let address = std::str::from_utf8(&endpoint.address)
.map_err(|e| Error::ConfigError(format!("Address: {:?}", e)))?;
let port = endpoint.port;

Ok(ConnectionInfo::new_with_timeout(
address,
cfg.port,
port,
secret_key,
cfg.auth_cert_expiration,
cfg.recv_tx_msgs,
Expand All @@ -80,13 +83,19 @@ pub struct NodeInfoCfg {
pub is_pub_net: bool,
}

/// The config structure of the ConnectionInfo
#[serde_as]
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ConnectionInfoCfg {
pub struct ConnectionEndpoint {
#[serde_as(as = "BytesOrString")]
pub address: Vec<u8>,
pub port: u32,
}

#[serde_as]
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ConnectionInfoCfg {
#[serde(default)]
pub endpoints: Vec<ConnectionEndpoint>,

#[serde(default = "ConnectionInfoCfg::default_auth_cert_exp")]
pub auth_cert_expiration: u64,
Expand Down Expand Up @@ -160,8 +169,9 @@ mod test {
fn missing_fields_in_connection_info_config() {
// missing port
let json = r#"
{
"address": "1.2.3.4"
"endpoints": [
{ "address": "1.2.3.4" }
],
"recv_scp_msgs": true,
"remote_called_us": false
}
Expand All @@ -172,7 +182,9 @@ mod test {
// missing address
let json = r#"
{
"port": 11625,
"endpoints": [
{"port": 11625}
],
"auth_cert_expiration": 0,
"recv_tx_msgs": false,
"recv_scp_msgs": true
Expand Down Expand Up @@ -227,9 +239,13 @@ mod test {
fn stellar_relay_config_conversion_successful() {
let json = r#"
{
"connection_info": {
"address": "1.2.3.4",
"port": 11625,
"connection_info":{
"endpoints": [
{
"address": "1.2.3.4",
"port": 11625
}
],
"auth_cert_expiration": 0,
"recv_scp_msgs": true
},
Expand All @@ -254,7 +270,11 @@ mod test {
let json = r#"
{
"connection_info": {
"port": 11625
"endpoints": [
{
"port": 11625
}
]
},
"node_info": {
"ledger_version": 19,
Expand All @@ -271,8 +291,12 @@ mod test {
let json = r#"
{
"connection_info": {
"address": "1.2.3.4",
"port": 11625
"endpoints": [
{
"address": "1.2.3.4",
"port": 11625
}
],
"auth_cert_expiration": 0,
"recv_scp_msgs": true
},
Expand All @@ -290,9 +314,13 @@ mod test {
// missing stellar_history_base_url
let json = r#"
{
"connection_info": {
"address": "1.2.3.4",
"port": 11625,
"connection_info": {
"endpoints": [
{
"address": "1.2.3.4",
"port": 11625
}
],
"auth_cert_expiration": 0,
"recv_scp_msgs": true
},
Expand Down
8 changes: 6 additions & 2 deletions clients/vault/resources/config/mainnet/stellar_relay_config_frankfurt.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
{
"connection_info": {
"address": "85.190.254.217",
"port": 11625
"endpoints": [
{
"address": "85.190.254.217",
"port": 11625
}
]
},
"node_info": {
"ledger_version": 22,
Expand Down
8 changes: 6 additions & 2 deletions clients/vault/resources/config/mainnet/stellar_relay_config_iowa.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
{
"connection_info": {
"address": "154.12.251.105",
"port": 11625
"endpoints": [
{
"address": "154.12.251.105",
"port": 11625
}
]
},
"node_info": {
"ledger_version": 22,
Expand Down
33 changes: 33 additions & 0 deletions clients/vault/resources/config/mainnet/stellar_relay_config_mainnet.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
{
"connection_info": {
"endpoints": [
{
"address": "85.190.254.217",
"port": 11625
},
{
"address": "154.12.251.105",
"port": 11625
},
{
"address": "194.233.85.232",
"port": 11625
}
]
},
"node_info": {
"ledger_version": 22,
"overlay_version": 35,
"overlay_min_version": 33,
"version_str": "stellar-core 22.0.0 (721fd0a654d5e82d38c748a91053e530a475193d)",
"is_pub_net": true
},
"stellar_history_archive_urls": [
"http://history.stellar.org/prd/core-live/core_live_001",
"http://history.stellar.org/prd/core-live/core_live_002",
"http://history.stellar.org/prd/core-live/core_live_003",
"https://stellar-history-de-fra.satoshipay.io",
"https://stellar-history-sg-sin.satoshipay.io",
"https://stellar-history-us-iowa.satoshipay.io"
]
}
8 changes: 6 additions & 2 deletions clients/vault/resources/config/mainnet/stellar_relay_config_singapore.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
{
"connection_info": {
"address": "194.233.85.232",
"port": 11625
"endpoints": [
{
"address": "194.233.85.232",
"port": 11625
}
]
},
"node_info": {
"ledger_version": 22,
Expand Down
30 changes: 30 additions & 0 deletions clients/vault/resources/config/testnet/stellar_relay_config_sdftest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{
"connection_info": {
"endpoints": [
{
"address": "54.166.220.249",
"port": 11625
},
{
"address": "54.159.138.198",
"port": 11625
},
{
"address": "44.223.45.116",
"port": 11625
}
]
},
"node_info": {
"ledger_version": 22,
"overlay_version": 35,
"overlay_min_version": 33,
"version_str": "stellar-core 22.0.0 (721fd0a654d5e82d38c748a91053e530a475193d)",
"is_pub_net": false
},
"stellar_history_archive_urls": [
"http://history.stellar.org/prd/core-testnet/core_testnet_001",
"http://history.stellar.org/prd/core-testnet/core_testnet_002",
"http://history.stellar.org/prd/core-testnet/core_testnet_003"
]
}
26 changes: 19 additions & 7 deletions clients/vault/src/oracle/agent.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use tokio::{
sync::RwLock,
time::{sleep, timeout, Instant},
};

use tracing::error;
use runtime::ShutdownSender;
use stellar_relay_lib::{
connect_to_stellar_overlay_network, sdk::types::StellarMessage, StellarOverlayConfig,
Expand All @@ -22,6 +22,8 @@ use wallet::Slot;

/// The interval to check if we are still receiving messages from Stellar Relay
const STELLAR_RELAY_HEALTH_CHECK_IN_SECS: u64 = 300;
/// The waiting time for reading messages from the overlay.
static STELLAR_MESSAGES_TIMEOUT_IN_SECS: u64 = 60;

pub struct OracleAgent {
pub collector: ArcRwLock<ScpMessageCollector>,
Expand Down Expand Up @@ -148,18 +150,24 @@ pub async fn listen_for_stellar_messages(
let health_check_interval = Duration::from_secs(STELLAR_RELAY_HEALTH_CHECK_IN_SECS);

let mut next_time = Instant::now() + health_check_interval;
let mut last_valid_message_time = Instant::now();
loop {
let collector = oracle_agent.collector.clone();

match overlay_conn.listen().await {
Ok(None) => {},
Ok(Some(StellarMessage::ErrorMsg(e))) => {
if last_valid_message_time < (Instant::now() - health_check_interval) { break }

match timeout(
Duration::from_secs(STELLAR_MESSAGES_TIMEOUT_IN_SECS),
overlay_conn.listen(),
).await {
Ok(Ok(None)) => {},
Ok(Ok(Some(StellarMessage::ErrorMsg(e)))) => {
tracing::error!(
"listen_for_stellar_messages(): received error message from Stellar: {e:?}"
);
break
},
Ok(Some(msg)) => {
Ok(Ok(Some(msg))) => {
last_valid_message_time = Instant::now();
if Instant::now() >= next_time {
tracing::info!("listen_for_stellar_messages(): health check: received message from Stellar");
next_time += health_check_interval;
Expand All @@ -174,10 +182,14 @@ pub async fn listen_for_stellar_messages(
}
},
// connection got lost
Err(e) => {
Ok(Err(e)) => {
tracing::error!("listen_for_stellar_messages(): encounter error in overlay: {e:?}");
break
},
Err(_) => {
error!("listen_for_stellar_messages(): overlay_conn.listen() timed out");
break
},
}
}

Expand Down
Loading
Loading