From b3b41ffe4e7a4f9288ce8ec457cbf60a850a2ea5 Mon Sep 17 00:00:00 2001
From: Victor Fuentes <vmfuentes64@gmail.com>
Date: Sun, 28 Jul 2024 20:19:31 -0700
Subject: [PATCH] reduce ddb update freq

---
 src/main.rs      | 23 +++++++++++++----------
 src/revisions.rs |  5 ++++-
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index c7d8a81..868ecd6 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -28,7 +28,7 @@ enum Commands {
         #[arg(short, long)]
         /// DynamoDB table to upload to
         table: String,
-    }
+    },
 }
 
 #[derive(Parser, Debug)]
@@ -63,14 +63,13 @@ async fn main() -> Result<()> {
         .load()
         .await;
 
-    let revision = get_revisions(&args.processed)
-        .await
-        .expect("Failed to get revisions");
-
-    info!("Got revisions: {:#?}", revision);
-
     match args.command {
         Commands::S3 { upload, bucket } => {
+            let revision = get_revisions(&args.processed, true)
+                .await
+                .expect("Failed to get revisions");
+
+            info!("Got revisions: {:#?}", revision);
             let client = aws_sdk_s3::Client::new(&config);
             for (i, (channel, revs)) in revision.iter().enumerate() {
                 for (j, r) in revs.iter().enumerate() {
@@ -85,8 +84,14 @@ async fn main() -> Result<()> {
                     create_db(&client, channel, r, upload, &bucket).await?;
                 }
             }
+            let _ = update_markers(&args.processed, revision)?;
         }
         Commands::Ddb { table } => {
+            let revision = get_revisions(&args.processed, false)
+                .await
+                .expect("Failed to get revisions");
+            info!("Got revisions: {:#?}", revision);
+
             let client = aws_sdk_dynamodb::Client::new(&config);
             for (i, (channel, revs)) in revision.iter().enumerate() {
                 for (j, r) in revs.iter().enumerate() {
@@ -145,10 +150,8 @@ async fn main() -> Result<()> {
                     }
                 }
             }
+            let _ = update_markers(&args.processed, revision)?;
         }
     }
-
-    let _ = update_markers(&args.processed, revision)?;
-
     Ok(())
 }
diff --git a/src/revisions.rs b/src/revisions.rs
index 8364561..aa6af7a 100644
--- a/src/revisions.rs
+++ b/src/revisions.rs
@@ -18,7 +18,7 @@ struct Content {
     pub last_modified: String,
 }
 
-pub async fn get_revisions(dir: &str) -> Result<HashMap<String, Vec<String>>> {
+pub async fn get_revisions(dir: &str, include_nixpkgs: bool) -> Result<HashMap<String, Vec<String>>> {
     let mut out = HashMap::new();
 
     for entry in std::fs::read_dir(dir)? {
@@ -52,6 +52,9 @@ pub async fn get_revisions(dir: &str) -> Result<HashMap<String, Vec<String>>> {
                 }
             }
         } else if path.is_dir() && path.ends_with("nixpkgs") {
+            if !include_nixpkgs {
+                continue;
+            }
             let channel = path
                 .file_name()
                 .context("Failed to read file name")?