From 6525dc8f4545d9ffe6ae012a44c2317cdac3368b Mon Sep 17 00:00:00 2001 From: "Dotan J. Nahum" Date: Sun, 12 May 2024 16:00:54 +0300 Subject: [PATCH] Replace consulrs (#271) replace consulrs --- .gitignore | 1 + Cargo.lock | 211 +++++++++--------- teller-cli/tests/flow_test.rs | 5 +- teller-providers/Cargo.toml | 6 +- .../src/providers/hashicorp_consul.rs | 202 ++++++----------- .../src/providers/hashicorp_vault.rs | 8 +- teller-providers/src/providers/test_utils.rs | 5 +- 7 files changed, 193 insertions(+), 245 deletions(-) diff --git a/.gitignore b/.gitignore index 1ce36153..ad72ad7c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ notes/ tmp/ dist/ .DS_Store +*.profraw diff --git a/Cargo.lock b/Cargo.lock index 5cc35035..bea06f6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,6 +95,12 @@ version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "async-trait" version = "0.1.80" @@ -744,38 +750,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "consulrs" -version = "0.1.0" -source = "git+https://github.com/jmgilman/consulrs?rev=a14fddbdf3695e2e338145134c4b42f823e03370#a14fddbdf3695e2e338145134c4b42f823e03370" -dependencies = [ - "async-trait", - "base64 0.13.1", - "consulrs_derive", - "derive_builder 0.10.2", - "http 0.2.12", - "reqwest", - "rustify", - "rustify_derive", - "serde", - "serde_json", - "serde_with 1.14.0", - "thiserror", - "tracing", - "url 2.5.0", -] - -[[package]] -name = "consulrs_derive" -version = "0.1.0" -source = "git+https://github.com/jmgilman/consulrs?rev=a14fddbdf3695e2e338145134c4b42f823e03370#a14fddbdf3695e2e338145134c4b42f823e03370" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "synstructure", -] - [[package]] name = "content_inspector" version = "0.2.4" @@ -819,6 +793,15 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -897,16 +880,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "darling" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f2c43f534ea4b0b049015d00269734195e6d3f0f6635cb692251aca6f9f8b3c" -dependencies = [ - "darling_core 0.12.4", - "darling_macro 0.12.4", -] - [[package]] name = "darling" version = "0.13.4" @@ -937,20 +910,6 @@ dependencies = [ "darling_macro 0.20.8", ] -[[package]] -name = "darling_core" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e91455b86830a1c21799d94524df0845183fa55bafd9aa137b01c7d1065fa36" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", -] - [[package]] name = "darling_core" version = "0.13.4" @@ -993,17 +952,6 @@ dependencies = [ "syn 2.0.59", ] -[[package]] -name = "darling_macro" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a" -dependencies = [ - "darling_core 0.12.4", - "quote", - "syn 1.0.109", -] - [[package]] name = "darling_macro" version = "0.13.4" @@ -1047,15 +995,6 @@ dependencies = [ "serde", ] -[[package]] -name = "derive_builder" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d13202debe11181040ae9063d739fa32cfcaaebe2275fe387703460ae2365b30" -dependencies = [ - "derive_builder_macro 0.10.2", -] - [[package]] name = "derive_builder" version = "0.11.2" @@ -1074,18 +1013,6 @@ dependencies = [ "derive_builder_macro 0.12.0", ] -[[package]] -name = "derive_builder_core" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66e616858f6187ed828df7c64a6d71720d83767a7f19740b2d1b6fe6327b36e5" -dependencies = [ - "darling 0.12.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "derive_builder_core" version = "0.11.2" @@ -1110,16 +1037,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derive_builder_macro" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58a94ace95092c5acb1e97a7e846b310cfbd499652f72297da7493f618a98d73" -dependencies = [ - "derive_builder_core 0.10.2", - "syn 1.0.109", -] - [[package]] name = "derive_builder_macro" version = "0.11.2" @@ -1319,6 +1236,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "flate2" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2400,6 +2327,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.36" @@ -2558,7 +2491,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 0.25.4", "winreg", ] @@ -2577,6 +2510,26 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rs-consul" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d10191da32e31f7370fbdb5b35df6375de04edc2bd53d527f37568cb3559054" +dependencies = [ + "base64 0.22.0", + "futures", + "http 0.2.12", + "hyper", + "hyper-rustls 0.24.2", + "quick-error", + "serde", + "serde_json", + "slog-scope", + "smart-default", + "tokio", + "ureq", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -3028,6 +2981,23 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-scope" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" +dependencies = [ + "arc-swap", + "lazy_static", + "slog", +] + [[package]] name = "slug" version = "0.1.5" @@ -3044,6 +3014,17 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "smart-default" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.59", +] + [[package]] name = "snapbox" version = "0.4.17" @@ -3279,15 +3260,17 @@ dependencies = [ "aws-config", "aws-sdk-secretsmanager", "aws-sdk-ssm", - "consulrs", + "base64 0.22.0", "crc32c", "dockertest-server", "dotenvy", "fs-err", "google-secretmanager1", "home", + "hyper", "insta", "lazy_static", + "rs-consul", "rustify", "serde", "serde_derive", @@ -3753,6 +3736,25 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f214ce18d8b2cbe84ed3aa6486ed3f5b285cf8d8fbdbce9f3f767a724adc35" +dependencies = [ + "base64 0.21.7", + "flate2", + "log", + "once_cell", + "rustls 0.22.3", + "rustls-pki-types", + "rustls-webpki 0.102.2", + "serde", + "serde_json", + "url 2.5.0", + "webpki-roots 0.26.1", +] + [[package]] name = "url" version = "1.7.2" @@ -3953,6 +3955,15 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "webpki-roots" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/teller-cli/tests/flow_test.rs b/teller-cli/tests/flow_test.rs index 981c9b24..fb0fc143 100644 --- a/teller-cli/tests/flow_test.rs +++ b/teller-cli/tests/flow_test.rs @@ -46,7 +46,7 @@ fn build_providers() -> Test { #[test] #[cfg(not(windows))] fn providers_smoke_test() { - use std::env; + use std::{env, time::Duration}; if env::var("RUNNER_OS").unwrap_or_default() == "macOS" { return; @@ -58,6 +58,9 @@ fn providers_smoke_test() { let user = "linus"; let password = "torvalds123"; let vault_server: VaultServer = instance.server(); + // banner is not enough for vault, we have to wait for the image to stabilize + tokio::time::sleep(Duration::from_secs(2)).await; + let localstack_server: LocalStackServer = instance.server(); fs::write( diff --git a/teller-providers/Cargo.toml b/teller-providers/Cargo.toml index d23dea33..807ae4a2 100644 --- a/teller-providers/Cargo.toml +++ b/teller-providers/Cargo.toml @@ -27,7 +27,7 @@ aws_secretsmanager = ["aws", "dep:aws-sdk-secretsmanager"] google_secretmanager = ["dep:google-secretmanager1", "dep:crc32c"] hashicorp_vault = ["dep:vaultrs", "dep:rustify"] dotenv = ["dep:dotenvy"] -hashicorp_consul = ["dep:consulrs"] +hashicorp_consul = ["dep:rs-consul"] aws = ["dep:aws-config"] [dependencies] @@ -42,6 +42,8 @@ strum = { workspace = true } thiserror = { workspace = true } fs-err = "2.9.0" home = "0.5.5" +hyper = "0.14" +base64 = "0.22.0" # gcp google-secretmanager1 = { version = "5.0.2", optional = true } crc32c = { version = "0.6", optional = true } @@ -57,7 +59,7 @@ dotenvy = { version = "0.15.7", optional = true } vaultrs = { version = "0.7.2", optional = true } rustify = { version = "0.5.3", optional = true } # HashiCorp Consul -consulrs = { git = "https://github.com/jmgilman/consulrs", optional = true, rev = "a14fddbdf3695e2e338145134c4b42f823e03370" } +rs-consul = { version = "0.6.0", optional = true } [dev-dependencies] insta = { workspace = true } diff --git a/teller-providers/src/providers/hashicorp_consul.rs b/teller-providers/src/providers/hashicorp_consul.rs index 9fdafe8e..40c967c7 100644 --- a/teller-providers/src/providers/hashicorp_consul.rs +++ b/teller-providers/src/providers/hashicorp_consul.rs @@ -17,15 +17,7 @@ use std::env; use async_trait::async_trait; -use consulrs::{ - api::kv::requests::{ - DeleteKeyRequestBuilder, ReadKeyRequestBuilder, ReadKeysRequestBuilder, - SetKeyRequestBuilder, - }, - client::{ConsulClient, ConsulClientSettingsBuilder}, - error::ClientError, - kv as ConsulKV, -}; +use rs_consul::{Consul, ConsulError}; use serde_derive::{Deserialize, Serialize}; use super::ProviderKind; @@ -44,50 +36,29 @@ pub struct HashiCorpConsulOptions { pub dc: Option, } -fn xerr(pm: &PathMap, e: ClientError) -> Error { +fn to_err(pm: &PathMap, e: ConsulError) -> Error { match e { - ClientError::RestClientError { source } => match source { - rustify::errors::ClientError::ServerResponseError { code, content } => { - match (code, content.clone()) { - (404, Some(content)) - if content.contains("Invalid path for a versioned K/V secrets") => - { - Error::PathError( - pm.path.clone(), - "missing or incompatible protocol version".to_string(), - ) - } - (404, _) => Error::NotFound { - path: pm.path.clone(), - msg: "not found".to_string(), - }, - _ => Error::Message(format!("code: {code}, {content:?}")), - } + ConsulError::UnexpectedResponseCode(hyper::http::StatusCode::NOT_FOUND, _) => { + Error::NotFound { + path: pm.path.clone(), + msg: "not found".to_string(), } - _ => Error::Any(Box::from(source)), - }, - ClientError::APIError { - code: 404, - message: _, - } => Error::NotFound { - path: pm.path.clone(), - msg: "not found".to_string(), - }, + } _ => Error::Any(Box::from(e)), } } pub struct HashiCorpConsul { - pub client: ConsulClient, + pub consul: Consul, opts: HashiCorpConsulOptions, pub name: String, } impl HashiCorpConsul { #[must_use] - pub fn with_client(name: &str, client: ConsulClient) -> Self { + pub fn with_client(name: &str, client: Consul) -> Self { Self { - client, + consul: client, opts: HashiCorpConsulOptions::default(), name: name.to_string(), } @@ -119,58 +90,19 @@ impl HashiCorpConsul { ) .unwrap_or_default(); - let settings = ConsulClientSettingsBuilder::default() - .address(address) - .token(token) - .build() - .map_err(Box::from)?; - - let client = ConsulClient::new(settings).map_err(Box::from)?; - Ok(Self { - client, + consul: Consul::new(rs_consul::Config { + address, + token: Some(token), + #[allow(clippy::default_trait_access)] + hyper_builder: Default::default(), + }), opts, name: name.to_string(), }) } } -impl HashiCorpConsul { - fn prepare_get_builder_request(&self) -> ReadKeyRequestBuilder { - let mut opts: ReadKeyRequestBuilder = ReadKeyRequestBuilder::default(); - if let Some(dc) = self.opts.dc.as_ref() { - opts.dc(dc.to_string()); - } - opts.recurse(true); - opts - } - - fn prepare_put_builder_request(&self) -> SetKeyRequestBuilder { - let mut opts: SetKeyRequestBuilder = SetKeyRequestBuilder::default(); - if let Some(dc) = self.opts.dc.as_ref() { - opts.dc(dc.to_string()); - } - opts - } - - fn prepare_delete_builder_request(&self) -> DeleteKeyRequestBuilder { - let mut opts: DeleteKeyRequestBuilder = DeleteKeyRequestBuilder::default(); - if let Some(dc) = self.opts.dc.as_ref() { - opts.dc(dc.to_string()); - } - opts.recurse(false); - opts - } - - fn prepare_keys_builder_request(&self) -> ReadKeysRequestBuilder { - let mut opts: ReadKeysRequestBuilder = ReadKeysRequestBuilder::default(); - if let Some(dc) = self.opts.dc.as_ref() { - opts.dc(dc.to_string()); - } - opts.recurse(false); - opts - } -} #[async_trait] impl Provider for HashiCorpConsul { fn kind(&self) -> ProviderInfo { @@ -181,26 +113,24 @@ impl Provider for HashiCorpConsul { } async fn get(&self, pm: &PathMap) -> Result> { - let res = ConsulKV::read( - &self.client, - &pm.path, - Some(&mut self.prepare_get_builder_request()), - ) - .await - .map_err(|e| xerr(pm, e))?; + let res = self + .consul + .read_key(rs_consul::ReadKeyRequest { + key: &pm.path, + datacenter: &self.opts.dc.clone().unwrap_or_default(), + recurse: false, + ..Default::default() + }) + .await + .map_err(|e| to_err(pm, e))?; let mut results = vec![]; - for kv_pair in res.response { - let kv_value = kv_pair.value.ok_or_else(|| Error::NotFound { + for kv_pair in res { + let val = kv_pair.value.ok_or_else(|| Error::NotFound { path: pm.path.to_string(), msg: "value not found".to_string(), })?; - let val: String = kv_value.try_into().map_err(|e| Error::GetError { - path: pm.path.to_string(), - msg: format!("could not decode Base64 value. err: {e:?}"), - })?; - let (_, key) = kv_pair.key.rsplit_once('/').unwrap_or(("", &kv_pair.key)); results.push(KV::from_value(&val, key, key, pm, self.kind())); @@ -211,42 +141,35 @@ impl Provider for HashiCorpConsul { async fn put(&self, pm: &PathMap, kvs: &[KV]) -> Result<()> { for kv in kvs { - ConsulKV::set( - &self.client, - &format!("{}/{}", pm.path, kv.key), - kv.value.as_bytes(), - Some(&mut self.prepare_put_builder_request()), - ) - .await - .map_err(|e| Error::PutError { - path: pm.path.to_string(), - msg: format!( - "could not put value in key {}. err: {:?}", - kv.key.as_str(), - e - ), - })?; + self.consul + .create_or_update_key( + rs_consul::CreateOrUpdateKeyRequest { + key: &format!("{}/{}", pm.path, kv.key), + datacenter: &self.opts.dc.clone().unwrap_or_default(), + ..Default::default() + }, + kv.value.as_bytes().to_vec(), + ) + .await + .map_err(|e| to_err(pm, e))?; } Ok(()) } async fn del(&self, pm: &PathMap) -> Result<()> { let keys = if pm.keys.is_empty() { - ConsulKV::keys( - &self.client, - pm.path.as_str(), - Some(&mut self.prepare_keys_builder_request()), - ) - .await - .map_err(|e| Error::DeleteError { - path: pm.path.to_string(), - msg: format!( - "could not get keys in path: {}. err: {:?}", - pm.path.as_str(), - e - ), - })? - .response + self.consul + .read_key(rs_consul::ReadKeyRequest { + key: &pm.path, + datacenter: &self.opts.dc.clone().unwrap_or_default(), + recurse: true, + ..Default::default() + }) + .await + .map_err(|e| to_err(pm, e))? + .iter() + .map(|resp| resp.key.clone()) + .collect::>() } else { pm.keys .keys() @@ -255,16 +178,14 @@ impl Provider for HashiCorpConsul { }; for key in keys { - ConsulKV::delete( - &self.client, - key.as_str(), - Some(&mut self.prepare_delete_builder_request()), - ) - .await - .map_err(|e| Error::DeleteError { - path: pm.path.to_string(), - msg: format!("could not delete key: {}. err: {:?}", pm.path.as_str(), e), - })?; + self.consul + .delete_key(rs_consul::DeleteKeyRequest { + key: &key, + datacenter: &self.opts.dc.clone().unwrap_or_default(), + ..Default::default() + }) + .await + .map_err(|e| to_err(pm, e))?; } Ok(()) @@ -285,6 +206,8 @@ mod tests { #[test] #[cfg(not(windows))] fn sanity_test() { + use std::time::Duration; + if env::var("RUNNER_OS").unwrap_or_default() == "macOS" { return; } @@ -304,6 +227,9 @@ mod tests { "address": server.external_url(), }); + // banner is not enough, we have to wait for the image to stabilize + tokio::time::sleep(Duration::from_secs(2)).await; + let p = Box::new( super::HashiCorpConsul::new( "hashicorp_consul", diff --git a/teller-providers/src/providers/hashicorp_vault.rs b/teller-providers/src/providers/hashicorp_vault.rs index c1d90fc3..6de3a68e 100644 --- a/teller-providers/src/providers/hashicorp_vault.rs +++ b/teller-providers/src/providers/hashicorp_vault.rs @@ -275,17 +275,16 @@ mod tests { use super::*; use crate::providers::test_utils; - const PORT: u32 = 9200; - #[test] #[cfg(not(windows))] fn sanity_test() { + use std::time::Duration; + if env::var("RUNNER_OS").unwrap_or_default() == "macOS" { return; } let config = VaultServerConfig::builder() - .port(PORT) .version("1.8.2".into()) .build() .unwrap(); @@ -300,6 +299,9 @@ mod tests { "token": server.token }); + // banner is not enough, we have to wait for the image to stabilize + tokio::time::sleep(Duration::from_secs(2)).await; + let p = Box::new( super::Hashivault::new( "hashicorp_vault", diff --git a/teller-providers/src/providers/test_utils.rs b/teller-providers/src/providers/test_utils.rs index 83e92556..bfad3633 100644 --- a/teller-providers/src/providers/test_utils.rs +++ b/teller-providers/src/providers/test_utils.rs @@ -149,6 +149,7 @@ impl ProviderTest { for (root_path, keys) in path_tree { let path_map = PathMap::from_path(&self.get_key_path(root_path)); let res = self.provider.as_ref().put(&path_map, keys).await; + println!("validate_put: {res:?}"); assert!(res.is_ok()); assert_debug_snapshot!(format!("[put-{}]", root_path.replace('/', "_"),), res); } @@ -166,6 +167,8 @@ impl ProviderTest { .as_ref() .get(&PathMap::from_path(&self.get_key_path(root_path))) .await; + + println!("validate_get: {res:?}"); assert!(res.is_ok()); let mut res = res.unwrap(); @@ -289,7 +292,7 @@ impl ProviderTest { ]); let delete_keys_res = self.provider.as_ref().del(&path_path).await; - println!("{delete_keys_res:#?}"); + println!("validate_delete_keys: {delete_keys_res:#?}"); assert!(delete_keys_res.is_ok()); assert_debug_snapshot!( format!("[del-keys-{}]", ROOT_PATH_A.replace('/', "_")),