Merge pull request #422 from stakwork/feat/auto-restart

Feat/auto restart

src/auto_restart_cron.rs

@@ -0,0 +1,75 @@
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use anyhow::{anyhow, Error, Result};
+use bollard::Docker;
+use tokio_cron_scheduler::{Job, JobScheduler};
+
+use crate::{config::STATE, dock::restart_node_container, utils::getenv};
+
+pub static RESTART_SERVICES: AtomicBool = AtomicBool::new(false);
+
+pub async fn auto_restart_cron(
+    proj: String,
+    docker: Docker,
+    auto_restart_services: Vec<String>,
+) -> Result<JobScheduler> {
+    log::info!("Auto Restart Services");
+    let sched = JobScheduler::new().await?;
+
+    let cron_time = match getenv("AUTO_RESTART_CRON_TIME") {
+        Ok(env) => env,
+        Err(_) => "@daily".to_string(),
+    };
+
+    sched
+        .add(Job::new_async(cron_time.as_str(), |_uuid, _l| {
+            Box::pin(async move {
+                if !RESTART_SERVICES.load(Ordering::Relaxed) {
+                    RESTART_SERVICES.store(true, Ordering::Relaxed);
+                }
+            })
+        })?)
+        .await?;
+
+    sched.start().await?;
+
+    tokio::spawn(async move {
+        loop {
+            let go = RESTART_SERVICES.load(Ordering::Relaxed);
+            if go {
+                if let Err(e) =
+                    auto_restart_services_handler(&proj, &docker, auto_restart_services.clone())
+                        .await
+                {
+                    log::error!("Error auto restarting services: {:?}", e);
+                }
+
+                RESTART_SERVICES.store(false, Ordering::Relaxed);
+            }
+            tokio::time::sleep(std::time::Duration::from_secs(10)).await;
+        }
+    });
+
+    Ok(sched)
+}
+
+async fn auto_restart_services_handler(
+    proj: &str,
+    docker: &Docker,
+    auto_restart_services: Vec<String>,
+) -> Result<(), Error> {
+    let mut state = STATE.lock().await;
+    let mut err_vec: Vec<String> = Vec::new();
+    for service in auto_restart_services {
+        log::info!("About to auto restart {}", service);
+        match restart_node_container(docker, &service, &mut state, proj).await {
+            Ok(()) => {}
+            Err(err) => err_vec.push(format!("{}: {}", service, err.to_string())),
+        }
+    }
+    if err_vec.is_empty() {
+        return Ok(());
+    }
+
+    Err(anyhow!(err_vec.join("\n")))
+}

src/bin/stack/mod.rs

@@ -1,5 +1,6 @@
 use anyhow::Result;
 use rocket::tokio;
+use sphinx_swarm::auto_restart_cron::auto_restart_cron;
 use sphinx_swarm::backup::backup_and_delete_volumes_cron;
 use sphinx_swarm::builder;
 use sphinx_swarm::config::{load_config_file, put_config_file, Stack};
@@ -59,7 +60,7 @@ async fn main() -> Result<()> {
     handler::hydrate_clients(clients).await;
 
     if let Some(nn) = stack.auto_update {
-        let cron_handler_res = builder::auto_updater(proj, docker, nn).await;
+        let cron_handler_res = builder::auto_updater(proj, docker.clone(), nn).await;
         if let Err(e) = cron_handler_res {
             log::error!("CRON failed {:?}", e);
         }
@@ -71,6 +72,12 @@ async fn main() -> Result<()> {
         log::info!("BACKUP is not set!!")
     }
 
+    if let Some(auto_restart_services) = stack.auto_restart {
+        auto_restart_cron(proj.to_string(), docker, auto_restart_services).await?;
+    } else {
+        log::info!("Auto Restart not set")
+    }
+
     tokio::signal::ctrl_c().await?;
 
     builder::shutdown_now();

src/config.rs

@@ -65,6 +65,7 @@ pub struct Stack {
     pub ready: bool,
     pub ip: Option<String>,
     pub auto_update: Option<Vec<String>>,
+    pub auto_restart: Option<Vec<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub custom_2b_domain: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -312,6 +313,7 @@ impl Stack {
             ready: self.ready,
             ip: self.ip.clone(),
             auto_update: self.auto_update.clone(),
+            auto_restart: self.auto_restart.clone(),
             custom_2b_domain: self.custom_2b_domain.clone(),
             global_mem_limit: self.global_mem_limit,
             backup_services: self.backup_services.clone(),

src/defaults.rs

@@ -177,6 +177,7 @@ impl Default for Stack {
             ready: false,
             ip: env_no_empty("IP"),
             auto_update: None,
+            auto_restart: None,
             custom_2b_domain: env_no_empty("NAV_BOLTWALL_SHARED_HOST"),
             global_mem_limit: None,
             backup_services: None,
@@ -334,6 +335,7 @@ pub fn llama_only(network: &str, host: Option<String>) -> Stack {
         ready: false,
         ip: None,
         auto_update: None,
+        auto_restart: None,
         custom_2b_domain: None,
         global_mem_limit: None,
         backup_services: None,

src/lib.rs

@@ -1,5 +1,6 @@
 pub mod app_login;
 pub mod auth;
+pub mod auto_restart_cron;
 pub mod backup;
 pub mod builder;
 pub mod cmd;

src/secondbrain.rs

@@ -26,6 +26,7 @@ pub fn only_second_brain(network: &str, host: Option<String>, lightning_provider
             "boltwall".to_string(),
             "navfiber".to_string(),
         ]),
+        auto_restart: None,
         custom_2b_domain: env_no_empty("NAV_BOLTWALL_SHARED_HOST"),
         global_mem_limit: None,
         backup_services: Some(vec!["boltwall".to_string(), "neo4j".to_string()]),
@@ -126,6 +127,7 @@ pub fn default_local_stack(host: Option<String>, network: &str, nodes: Vec<Node>
         ready: false,
         ip: None,
         auto_update: None,
+        auto_restart: None,
         custom_2b_domain: None,
         global_mem_limit: None,
         backup_services: None,

src/sphinxv2.rs

@@ -74,6 +74,7 @@ pub fn sphinxv2_only(network: &str, host: Option<String>) -> Stack {
         ready: false,
         ip: env_no_empty("IP"),
         auto_update: None,
+        auto_restart: None,
         custom_2b_domain: None,
         global_mem_limit: None,
         backup_services: None,
@@ -111,6 +112,7 @@ pub fn sphinxv1_only(network: &str, host: Option<String>) -> Stack {
         ready: false,
         ip: env_no_empty("IP"),
         auto_update: None,
+        auto_restart: None,
         custom_2b_domain: None,
         global_mem_limit: None,
         backup_services: Some(vec!["mixer".to_string(), "tribes".to_string()]),
@@ -130,6 +132,7 @@ pub fn config_only(host: Option<String>) -> Stack {
         ready: false,
         ip: env_no_empty("IP"),
         auto_update: None,
+        auto_restart: None,
         custom_2b_domain: None,
         global_mem_limit: None,
         backup_services: None,