handle 404s & update URLs

Signed-off-by: Sebastian Hoß <[email protected]>
metio · Apr 20, 2024 · e7d5cff · e7d5cff
1 parent 5157e8b
commit e7d5cff
Show file tree

Hide file tree

Showing 2 changed files with 104 additions and 82 deletions.
diff --git a/code-generator/src/bin/crd_v1_fetcher.rs b/code-generator/src/bin/crd_v1_fetcher.rs
@@ -5,42 +5,47 @@ use std::{env, fs};
 
 use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
 use k8s_openapi::serde::Deserialize;
-use reqwest::blocking::get;
+use reqwest::blocking::Client;
 use serde_yaml::Value;
 
 use code_generator::catalog;
 
 fn main() {
     let args: Vec<String> = env::args().collect();
     let root = concat!(env!("CARGO_MANIFEST_DIR"), "/..");
+    let client = Client::new();
 
     for source in catalog::CRD_V1_SOURCES {
         for url in source.urls {
             if (args.len() == 2 && url.contains(&args[1])) || args.len() < 2 {
                 let raw_url = gitlab_url(github_url(url));
                 println!("Downloading {}", raw_url);
-                if let Ok(response) = get(raw_url) {
-                    if let Ok(content) = response.text() {
-                        for crd in parse_crds(content) {
-                            let directory = format!(
-                                "{}/crd-catalog/{}/{}/{}",
-                                root,
-                                source.project_name,
-                                crd.spec.group,
-                                crd.spec.versions[0].name
-                            );
-                            let file = format!("{}/{}.yaml", directory, crd.spec.names.plural);
+                if let Ok(response) = client.get(raw_url).send() {
+                    if response.status().is_success() {
+                        if let Ok(content) = response.text() {
+                            for crd in parse_crds(content) {
+                                let directory = format!(
+                                    "{}/crd-catalog/{}/{}/{}",
+                                    root,
+                                    source.project_name,
+                                    crd.spec.group,
+                                    crd.spec.versions[0].name
+                                );
+                                let file = format!("{}/{}.yaml", directory, crd.spec.names.plural);
 
-                            fs::create_dir_all(directory).unwrap_or_else(|why| {
-                                println!("! {:?}", why);
-                            });
-
-                            if let Ok(data) = serde_yaml::to_string(&crd) {
-                                fs::write(file, data).unwrap_or_else(|why| {
+                                fs::create_dir_all(directory).unwrap_or_else(|why| {
                                     println!("! {:?}", why);
                                 });
+
+                                if let Ok(data) = serde_yaml::to_string(&crd) {
+                                    fs::write(file, data).unwrap_or_else(|why| {
+                                        println!("! {:?}", why);
+                                    });
+                                }
                             }
                         }
+                    } else {
+                        println!("  Failed with status code {}", response.status().as_str());
                     }
                 }
             }