diff --git a/src/auto_restart_cron.rs b/src/auto_restart_cron.rs new file mode 100644 index 00000000..36c90efc --- /dev/null +++ b/src/auto_restart_cron.rs @@ -0,0 +1,75 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use anyhow::{anyhow, Error, Result}; +use bollard::Docker; +use tokio_cron_scheduler::{Job, JobScheduler}; + +use crate::{config::STATE, dock::restart_node_container, utils::getenv}; + +pub static RESTART_SERVICES: AtomicBool = AtomicBool::new(false); + +pub async fn auto_restart_cron( + proj: String, + docker: Docker, + auto_restart_services: Vec, +) -> Result { + log::info!("Auto Restart Services"); + let sched = JobScheduler::new().await?; + + let cron_time = match getenv("AUTO_RESTART_CRON_TIME") { + Ok(env) => env, + Err(_) => "@daily".to_string(), + }; + + sched + .add(Job::new_async(cron_time.as_str(), |_uuid, _l| { + Box::pin(async move { + if !RESTART_SERVICES.load(Ordering::Relaxed) { + RESTART_SERVICES.store(true, Ordering::Relaxed); + } + }) + })?) + .await?; + + sched.start().await?; + + tokio::spawn(async move { + loop { + let go = RESTART_SERVICES.load(Ordering::Relaxed); + if go { + if let Err(e) = + auto_restart_services_handler(&proj, &docker, auto_restart_services.clone()) + .await + { + log::error!("Error auto restarting services: {:?}", e); + } + + RESTART_SERVICES.store(false, Ordering::Relaxed); + } + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + } + }); + + Ok(sched) +} + +async fn auto_restart_services_handler( + proj: &str, + docker: &Docker, + auto_restart_services: Vec, +) -> Result<(), Error> { + let mut state = STATE.lock().await; + let mut err_vec: Vec = Vec::new(); + for service in auto_restart_services { + log::info!("About to auto restart {}", service); + match restart_node_container(docker, &service, &mut state, proj).await { + Ok(()) => {} + Err(err) => err_vec.push(format!("{}: {}", service, err.to_string())), + } + } + if err_vec.is_empty() { + return Ok(()); + } + + Err(anyhow!(err_vec.join("\n"))) +} diff --git a/src/bin/stack/mod.rs b/src/bin/stack/mod.rs index 99873728..3eeed6fa 100644 --- a/src/bin/stack/mod.rs +++ b/src/bin/stack/mod.rs @@ -1,5 +1,6 @@ use anyhow::Result; use rocket::tokio; +use sphinx_swarm::auto_restart_cron::auto_restart_cron; use sphinx_swarm::backup::backup_and_delete_volumes_cron; use sphinx_swarm::builder; use sphinx_swarm::config::{load_config_file, put_config_file, Stack}; @@ -59,7 +60,7 @@ async fn main() -> Result<()> { handler::hydrate_clients(clients).await; if let Some(nn) = stack.auto_update { - let cron_handler_res = builder::auto_updater(proj, docker, nn).await; + let cron_handler_res = builder::auto_updater(proj, docker.clone(), nn).await; if let Err(e) = cron_handler_res { log::error!("CRON failed {:?}", e); } @@ -71,6 +72,12 @@ async fn main() -> Result<()> { log::info!("BACKUP is not set!!") } + if let Some(auto_restart_services) = stack.auto_restart { + auto_restart_cron(proj.to_string(), docker, auto_restart_services).await?; + } else { + log::info!("Auto Restart not set") + } + tokio::signal::ctrl_c().await?; builder::shutdown_now(); diff --git a/src/lib.rs b/src/lib.rs index 126d16f4..63a63462 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ pub mod app_login; pub mod auth; +pub mod auto_restart_cron; pub mod backup; pub mod builder; pub mod cmd;