From b56b56fdf549f0b1214ba3973e0e8e04ac44ed6d Mon Sep 17 00:00:00 2001 From: Oluwatobi Bamidele Date: Mon, 2 Dec 2024 23:52:31 +0100 Subject: [PATCH 1/2] chore: added new value to stack --- src/config.rs | 2 ++ src/defaults.rs | 2 ++ src/secondbrain.rs | 2 ++ src/sphinxv2.rs | 3 +++ 4 files changed, 9 insertions(+) diff --git a/src/config.rs b/src/config.rs index d685718a..3664854e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -65,6 +65,7 @@ pub struct Stack { pub ready: bool, pub ip: Option, pub auto_update: Option>, + pub auto_restart: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub custom_2b_domain: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -312,6 +313,7 @@ impl Stack { ready: self.ready, ip: self.ip.clone(), auto_update: self.auto_update.clone(), + auto_restart: self.auto_restart.clone(), custom_2b_domain: self.custom_2b_domain.clone(), global_mem_limit: self.global_mem_limit, backup_services: self.backup_services.clone(), diff --git a/src/defaults.rs b/src/defaults.rs index 6aca868c..c76d9b89 100644 --- a/src/defaults.rs +++ b/src/defaults.rs @@ -177,6 +177,7 @@ impl Default for Stack { ready: false, ip: env_no_empty("IP"), auto_update: None, + auto_restart: None, custom_2b_domain: env_no_empty("NAV_BOLTWALL_SHARED_HOST"), global_mem_limit: None, backup_services: None, @@ -334,6 +335,7 @@ pub fn llama_only(network: &str, host: Option) -> Stack { ready: false, ip: None, auto_update: None, + auto_restart: None, custom_2b_domain: None, global_mem_limit: None, backup_services: None, diff --git a/src/secondbrain.rs b/src/secondbrain.rs index 5582e8f2..fbd0e27a 100644 --- a/src/secondbrain.rs +++ b/src/secondbrain.rs @@ -26,6 +26,7 @@ pub fn only_second_brain(network: &str, host: Option, lightning_provider "boltwall".to_string(), "navfiber".to_string(), ]), + auto_restart: None, custom_2b_domain: env_no_empty("NAV_BOLTWALL_SHARED_HOST"), global_mem_limit: None, backup_services: Some(vec!["boltwall".to_string(), "neo4j".to_string()]), @@ -126,6 +127,7 @@ pub fn default_local_stack(host: Option, network: &str, nodes: Vec ready: false, ip: None, auto_update: None, + auto_restart: None, custom_2b_domain: None, global_mem_limit: None, backup_services: None, diff --git a/src/sphinxv2.rs b/src/sphinxv2.rs index 3505f86d..40326856 100644 --- a/src/sphinxv2.rs +++ b/src/sphinxv2.rs @@ -74,6 +74,7 @@ pub fn sphinxv2_only(network: &str, host: Option) -> Stack { ready: false, ip: env_no_empty("IP"), auto_update: None, + auto_restart: None, custom_2b_domain: None, global_mem_limit: None, backup_services: None, @@ -111,6 +112,7 @@ pub fn sphinxv1_only(network: &str, host: Option) -> Stack { ready: false, ip: env_no_empty("IP"), auto_update: None, + auto_restart: None, custom_2b_domain: None, global_mem_limit: None, backup_services: Some(vec!["mixer".to_string(), "tribes".to_string()]), @@ -130,6 +132,7 @@ pub fn config_only(host: Option) -> Stack { ready: false, ip: env_no_empty("IP"), auto_update: None, + auto_restart: None, custom_2b_domain: None, global_mem_limit: None, backup_services: None, From a12bb3f97183ed6beb05e9a88aa61d69599aa712 Mon Sep 17 00:00:00 2001 From: Oluwatobi Bamidele Date: Tue, 3 Dec 2024 21:17:22 +0100 Subject: [PATCH 2/2] chore: auto restart services --- src/auto_restart_cron.rs | 75 ++++++++++++++++++++++++++++++++++++++++ src/bin/stack/mod.rs | 9 ++++- src/lib.rs | 1 + 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 src/auto_restart_cron.rs diff --git a/src/auto_restart_cron.rs b/src/auto_restart_cron.rs new file mode 100644 index 00000000..36c90efc --- /dev/null +++ b/src/auto_restart_cron.rs @@ -0,0 +1,75 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use anyhow::{anyhow, Error, Result}; +use bollard::Docker; +use tokio_cron_scheduler::{Job, JobScheduler}; + +use crate::{config::STATE, dock::restart_node_container, utils::getenv}; + +pub static RESTART_SERVICES: AtomicBool = AtomicBool::new(false); + +pub async fn auto_restart_cron( + proj: String, + docker: Docker, + auto_restart_services: Vec, +) -> Result { + log::info!("Auto Restart Services"); + let sched = JobScheduler::new().await?; + + let cron_time = match getenv("AUTO_RESTART_CRON_TIME") { + Ok(env) => env, + Err(_) => "@daily".to_string(), + }; + + sched + .add(Job::new_async(cron_time.as_str(), |_uuid, _l| { + Box::pin(async move { + if !RESTART_SERVICES.load(Ordering::Relaxed) { + RESTART_SERVICES.store(true, Ordering::Relaxed); + } + }) + })?) + .await?; + + sched.start().await?; + + tokio::spawn(async move { + loop { + let go = RESTART_SERVICES.load(Ordering::Relaxed); + if go { + if let Err(e) = + auto_restart_services_handler(&proj, &docker, auto_restart_services.clone()) + .await + { + log::error!("Error auto restarting services: {:?}", e); + } + + RESTART_SERVICES.store(false, Ordering::Relaxed); + } + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + } + }); + + Ok(sched) +} + +async fn auto_restart_services_handler( + proj: &str, + docker: &Docker, + auto_restart_services: Vec, +) -> Result<(), Error> { + let mut state = STATE.lock().await; + let mut err_vec: Vec = Vec::new(); + for service in auto_restart_services { + log::info!("About to auto restart {}", service); + match restart_node_container(docker, &service, &mut state, proj).await { + Ok(()) => {} + Err(err) => err_vec.push(format!("{}: {}", service, err.to_string())), + } + } + if err_vec.is_empty() { + return Ok(()); + } + + Err(anyhow!(err_vec.join("\n"))) +} diff --git a/src/bin/stack/mod.rs b/src/bin/stack/mod.rs index 99873728..3eeed6fa 100644 --- a/src/bin/stack/mod.rs +++ b/src/bin/stack/mod.rs @@ -1,5 +1,6 @@ use anyhow::Result; use rocket::tokio; +use sphinx_swarm::auto_restart_cron::auto_restart_cron; use sphinx_swarm::backup::backup_and_delete_volumes_cron; use sphinx_swarm::builder; use sphinx_swarm::config::{load_config_file, put_config_file, Stack}; @@ -59,7 +60,7 @@ async fn main() -> Result<()> { handler::hydrate_clients(clients).await; if let Some(nn) = stack.auto_update { - let cron_handler_res = builder::auto_updater(proj, docker, nn).await; + let cron_handler_res = builder::auto_updater(proj, docker.clone(), nn).await; if let Err(e) = cron_handler_res { log::error!("CRON failed {:?}", e); } @@ -71,6 +72,12 @@ async fn main() -> Result<()> { log::info!("BACKUP is not set!!") } + if let Some(auto_restart_services) = stack.auto_restart { + auto_restart_cron(proj.to_string(), docker, auto_restart_services).await?; + } else { + log::info!("Auto Restart not set") + } + tokio::signal::ctrl_c().await?; builder::shutdown_now(); diff --git a/src/lib.rs b/src/lib.rs index 126d16f4..63a63462 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ pub mod app_login; pub mod auth; +pub mod auto_restart_cron; pub mod backup; pub mod builder; pub mod cmd;