reduce ddb update freq

snowfallorg · Jul 29, 2024 · b3b41ff · b3b41ff
1 parent 71ba5d5
commit b3b41ff
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 11 deletions.
diff --git a/src/main.rs b/src/main.rs
@@ -28,7 +28,7 @@ enum Commands {
         #[arg(short, long)]
         /// DynamoDB table to upload to
         table: String,
-    }
+    },
 }
 
 #[derive(Parser, Debug)]
@@ -63,14 +63,13 @@ async fn main() -> Result<()> {
         .load()
         .await;
 
-    let revision = get_revisions(&args.processed)
-        .await
-        .expect("Failed to get revisions");
-
-    info!("Got revisions: {:#?}", revision);
-
     match args.command {
         Commands::S3 { upload, bucket } => {
+            let revision = get_revisions(&args.processed, true)
+                .await
+                .expect("Failed to get revisions");
+
+            info!("Got revisions: {:#?}", revision);
             let client = aws_sdk_s3::Client::new(&config);
             for (i, (channel, revs)) in revision.iter().enumerate() {
                 for (j, r) in revs.iter().enumerate() {
@@ -85,8 +84,14 @@ async fn main() -> Result<()> {
                     create_db(&client, channel, r, upload, &bucket).await?;
                 }
             }
+            let _ = update_markers(&args.processed, revision)?;
         }
         Commands::Ddb { table } => {
+            let revision = get_revisions(&args.processed, false)
+                .await
+                .expect("Failed to get revisions");
+            info!("Got revisions: {:#?}", revision);
+
             let client = aws_sdk_dynamodb::Client::new(&config);
             for (i, (channel, revs)) in revision.iter().enumerate() {
                 for (j, r) in revs.iter().enumerate() {
@@ -145,10 +150,8 @@ async fn main() -> Result<()> {
                     }
                 }
             }
+            let _ = update_markers(&args.processed, revision)?;
         }
     }
-
-    let _ = update_markers(&args.processed, revision)?;
-
     Ok(())
 }
diff --git a/src/revisions.rs b/src/revisions.rs
@@ -18,7 +18,7 @@ struct Content {
     pub last_modified: String,
 }
 
-pub async fn get_revisions(dir: &str) -> Result<HashMap<String, Vec<String>>> {
+pub async fn get_revisions(dir: &str, include_nixpkgs: bool) -> Result<HashMap<String, Vec<String>>> {
     let mut out = HashMap::new();
 
     for entry in std::fs::read_dir(dir)? {
@@ -52,6 +52,9 @@ pub async fn get_revisions(dir: &str) -> Result<HashMap<String, Vec<String>>> {
                 }
             }
         } else if path.is_dir() && path.ends_with("nixpkgs") {
+            if !include_nixpkgs {
+                continue;
+            }
             let channel = path
                 .file_name()
                 .context("Failed to read file name")?