Allow for continuing after registry failure

src/ddb/nix.rs

@@ -5,6 +5,7 @@ use log::{debug, error, info, warn};
 use serde::Deserialize;
 use std::collections::HashMap;
 use tokio::process::Command;
+use anyhow::Result;
 
 #[derive(Deserialize, Debug, Clone)]
 struct Package {
@@ -13,7 +14,7 @@ struct Package {
     version: Option<String>,
 }
 
-pub async fn get_store(rev: &str) -> HashMap<String, Store> {
+pub async fn get_store(rev: &str) -> Result<HashMap<String, Store>> {
     let nixpath = Command::new("nix-instantiate")
         .arg("--eval")
         .arg("-E")
@@ -71,6 +72,14 @@ pub async fn get_store(rev: &str) -> HashMap<String, Store> {
         .expect("failed to execute process");
     }
 
+    if !output.status.success() {
+        error!(
+            "failed to eval revision: {}",
+            rev
+        );
+        return Err(anyhow::anyhow!("failed to eval revision: {}", rev));
+    }
+
     let output: HashMap<String, Package> =
         serde_json::from_slice(&output.stdout).expect("failed to parse nix-instantiate output");
 
@@ -95,5 +104,5 @@ pub async fn get_store(rev: &str) -> HashMap<String, Store> {
 
     info!("nix-instantiate: got {} store paths", store.len());
 
-    return store;
+    return Ok(store);
 }

src/main.rs

@@ -6,6 +6,7 @@ use aws_config::meta::region::RegionProviderChain;
 use aws_config::BehaviorVersion;
 use clap::{Parser, Subcommand};
 use libsnow_generators::ddb::batch_put::batch_store_put;
+use libsnow_generators::revisions::add_failed_revision;
 use libsnow_generators::{
     ddb::nix::get_store,
     revisions::{get_revisions, update_markers},
@@ -85,7 +86,7 @@ async fn main() -> Result<()> {
                 }
             }
         }
-        Commands::Ddb { table }=> {
+        Commands::Ddb { table } => {
             let client = aws_sdk_dynamodb::Client::new(&config);
             for (i, (channel, revs)) in revision.iter().enumerate() {
                 for (j, r) in revs.iter().enumerate() {
@@ -97,41 +98,51 @@ async fn main() -> Result<()> {
                         i + 1,
                         revision.len()
                     );
-                    let storeset = get_store(r.split('.').last().context("Failed to get revision")?).await;
-
-                    // Read processed
-                    let prevpaths =
-                        fs::read_to_string(format!("{}/{}/store-paths", args.processed, channel))
-                            .unwrap_or_default();
-                    let paths = prevpaths.split("\n").collect::<Vec<&str>>();
-
-                    // Write to processed
-                    let mut file =
-                        fs::File::create(format!("{}/{}/store-paths", args.processed, channel))?;
-
-                    let new_storeset = storeset
-                        .iter()
-                        .filter(|(k, _v)| !paths.contains(&k.as_str()))
-                        .map(|(k, v)| (k.to_string(), v.clone()))
-                        .collect::<std::collections::HashMap<String, _>>();
-
-                    debug!(
-                        "Total paths: {}. New paths: {}",
-                        paths.len(),
-                        new_storeset.len()
-                    );
-
-                    batch_store_put(&client, &new_storeset, &table).await?;
-
-                    file.write_all(
-                        storeset
-                            .keys()
-                            .map(String::to_string)
-                            .collect::<Vec<_>>()
-                            .join("\n")
-                            .as_bytes(),
-                    )
-                    .expect("Failed to write to store-paths file");
+                    let storeset =
+                        get_store(r.split('.').last().context("Failed to get revision")?).await;
+
+                    if let Ok(storeset) = storeset {
+                        // Read processed
+                        let prevpaths = fs::read_to_string(format!(
+                            "{}/{}/store-paths",
+                            args.processed, channel
+                        ))
+                        .unwrap_or_default();
+                        let paths = prevpaths.split("\n").collect::<Vec<&str>>();
+
+                        // Write to processed
+                        let mut file = fs::File::create(format!(
+                            "{}/{}/store-paths",
+                            args.processed, channel
+                        ))?;
+
+                        let new_storeset = storeset
+                            .iter()
+                            .filter(|(k, _v)| !paths.contains(&k.as_str()))
+                            .map(|(k, v)| (k.to_string(), v.clone()))
+                            .collect::<std::collections::HashMap<String, _>>();
+
+                        debug!(
+                            "Total paths: {}. New paths: {}",
+                            paths.len(),
+                            new_storeset.len()
+                        );
+
+                        batch_store_put(&client, &new_storeset, &table).await?;
+
+                        file.write_all(
+                            storeset
+                                .keys()
+                                .map(String::to_string)
+                                .collect::<Vec<_>>()
+                                .join("\n")
+                                .as_bytes(),
+                        )
+                        .expect("Failed to write to store-paths file");
+                    } else {
+                        error!("Failed to eval revision: {}", r);
+                        add_failed_revision(&format!("{}/{}", &args.processed, channel), r)?;
+                    }
                 }
             }
         }

src/revisions.rs

@@ -131,3 +131,12 @@ pub fn update_markers(dir: &str, revs: HashMap<String, Vec<String>>) -> Result<(
     }
     Ok(())
 }
+
+pub fn add_failed_revision(dir: &str, rev: &str) -> Result<()> {
+    let mut file = fs::OpenOptions::new()
+        .create(true)
+        .append(true)
+        .open(format!("{}/failed", dir))?;
+    file.write_all(format!("{}\n", rev).as_bytes())?;
+    Ok(())
+}