From 29a9de83a795e61ef991006f8c91ae85c4a4bc94 Mon Sep 17 00:00:00 2001 From: greg Date: Sat, 20 Jan 2024 00:49:45 +0000 Subject: [PATCH 1/3] add support for mult clients. individual client accounts added to each client. add logging during build --- k8s-cluster/src/docker.rs | 160 +++++++++++++++++++++++++++++-------- k8s-cluster/src/genesis.rs | 76 +++++++++++++----- k8s-cluster/src/lib.rs | 6 ++ k8s-cluster/src/main.rs | 80 ++++++++++++------- 4 files changed, 241 insertions(+), 81 deletions(-) diff --git a/k8s-cluster/src/docker.rs b/k8s-cluster/src/docker.rs index 9d1f719f976f42..e9a5a683835148 100644 --- a/k8s-cluster/src/docker.rs +++ b/k8s-cluster/src/docker.rs @@ -1,6 +1,10 @@ use { - crate::{boxed_error, initialize_globals, ValidatorType, SOLANA_ROOT}, + crate::{ + boxed_error, initialize_globals, new_spinner_progress_bar, ValidatorType, BUILD, ROCKET, + SOLANA_ROOT, + }, log::*, + rayon::prelude::*, std::{ error::Error, fs, @@ -32,7 +36,9 @@ impl<'a> DockerConfig<'a> { } pub fn build_image(&self, validator_type: &ValidatorType) -> Result<(), Box> { - match self.create_base_image(validator_type) { + let image_name = format!("{}-{}", validator_type, self.image_config.image_name); + let docker_path = SOLANA_ROOT.join(format!("{}/{}", "docker-build", validator_type)); + match self.create_base_image(image_name, docker_path, validator_type, None) { Ok(res) => { if res.status.success() { info!("Successfully created base Image"); @@ -46,14 +52,39 @@ impl<'a> DockerConfig<'a> { } } + pub fn build_client_images(&self, client_count: i32) -> Result<(), Box> { + for i in 0..client_count { + let image_name = format!("{}-{}-{}", ValidatorType::Client, "image", i); + let docker_path = SOLANA_ROOT.join(format!( + "{}/{}-{}", + "docker-build", + ValidatorType::Client, + i + )); + match self.create_base_image(image_name, docker_path, &ValidatorType::Client, Some(i)) { + Ok(res) => { + if res.status.success() { + info!("Successfully created client base Image: {}", i); + } else { + error!("Failed to build client base image: {}", i); + return Err(boxed_error!(String::from_utf8_lossy(&res.stderr))); + } + } + Err(err) => return Err(err), + } + } + Ok(()) + } + pub fn create_base_image( &self, + image_name: String, + docker_path: PathBuf, validator_type: &ValidatorType, + index: Option, ) -> Result> { - let image_name = format!("{}-{}", validator_type, self.image_config.image_name); - let docker_path = SOLANA_ROOT.join(format!("{}/{}", "docker-build", validator_type)); - - let dockerfile_path = match self.create_dockerfile(validator_type, docker_path, None) { + let dockerfile_path = match self.create_dockerfile(validator_type, docker_path, None, index) + { Ok(res) => res, Err(err) => return Err(err), }; @@ -66,32 +97,66 @@ impl<'a> DockerConfig<'a> { // so we result to using std::process::Command let dockerfile = dockerfile_path.join("Dockerfile"); let context_path = SOLANA_ROOT.display().to_string(); + + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!( + "{BUILD}Building {} docker image...", + validator_type + )); + let command = format!( "docker build -t {}/{}:{} -f {:?} {}", self.image_config.registry, image_name, self.image_config.tag, dockerfile, context_path ); - match Command::new("sh") + let output = match Command::new("sh") .arg("-c") .arg(&command) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) .spawn() .expect("Failed to execute command") .wait_with_output() { Ok(res) => Ok(res), - Err(err) => Err(Box::new(err)), - } + Err(err) => Err(Box::new(err) as Box), + }; + progress_bar.finish_and_clear(); + info!("{} image build complete", validator_type); + + output } - fn insert_client_accounts_if_present(&self) -> String { - if SOLANA_ROOT.join("config-k8s/client-accounts.yml").exists() { - return r#" -COPY --chown=solana:solana ./config-k8s/client-accounts.yml /home/solana - "# - .to_string(); - } - "".to_string() + fn insert_client_accounts_if_present(&self, index: Option) -> String { + let return_string = match index { + Some(i) => { + if SOLANA_ROOT + .join(format!("config-k8s/bench-tps-{}.yml", i)) + .exists() + { + format!( + r#" + COPY --chown=solana:solana ./config-k8s/bench-tps-{}.yml /home/solana/client-accounts.yml + "#, + i + ) + } else { + info!("bench-tps-{} does not exist!", i); + "".to_string() + } + } + None => { + if SOLANA_ROOT.join("config-k8s/client-accounts.yml").exists() { + r#" + COPY --chown=solana:solana ./config-k8s/client-accounts.yml /home/solana + "# + .to_string() + } else { + "".to_string() + } + } + }; + + return_string } pub fn create_dockerfile( @@ -99,6 +164,7 @@ COPY --chown=solana:solana ./config-k8s/client-accounts.yml /home/solana validator_type: &ValidatorType, docker_path: PathBuf, content: Option<&str>, + index: Option, ) -> Result> { if !(validator_type != &ValidatorType::Bootstrap || validator_type != &ValidatorType::Standard @@ -157,7 +223,7 @@ WORKDIR /home/solana let dockerfile = format!( "{}\n{}", dockerfile, - self.insert_client_accounts_if_present() + self.insert_client_accounts_if_present(index) ); debug!("dockerfile: {}", dockerfile); @@ -169,21 +235,18 @@ WORKDIR /home/solana Ok(docker_path) } - pub fn push_image(&self, validator_type: &ValidatorType) -> Result<(), Box> { - let image = format!( - "{}/{}-{}:{}", - self.image_config.registry, - validator_type, - self.image_config.image_name, - self.image_config.tag - ); - - let command = format!("docker push '{}'", image); + fn push_image(image: String, identifier: &str) -> Result<(), Box> { + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!( + "{ROCKET}Pushing {} image to registry...", + identifier + )); + let command = format!("docker push '{}'", image); // | grep \"The push refers\"", image); let output = Command::new("sh") .arg("-c") .arg(&command) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) .spawn() .expect("Failed to execute command") .wait_with_output() @@ -192,6 +255,39 @@ WORKDIR /home/solana if !output.status.success() { return Err(boxed_error!(output.status.to_string())); } + progress_bar.finish_and_clear(); Ok(()) } + + pub fn push_validator_image( + &self, + validator_type: &ValidatorType, + ) -> Result<(), Box> { + info!("Pushing {} image...", validator_type); + let image = format!( + "{}/{}-{}:{}", + self.image_config.registry, + validator_type, + self.image_config.image_name, + self.image_config.tag + ); + + Self::push_image(image, format!("{}", validator_type).as_str()) + } + + pub fn push_client_images(&self, num_clients: i32) -> Result<(), Box> { + info!("Pushing client images..."); + (0..num_clients).into_par_iter().try_for_each(|i| { + let image = format!( + "{}/{}-{}-{}:{}", + self.image_config.registry, + ValidatorType::Client, + "image", + i, + self.image_config.tag + ); + + Self::push_image(image, format!("client-{}", i).as_str()) + }) + } } diff --git a/k8s-cluster/src/genesis.rs b/k8s-cluster/src/genesis.rs index b7a47568de0153..f2047d504b7dec 100644 --- a/k8s-cluster/src/genesis.rs +++ b/k8s-cluster/src/genesis.rs @@ -2,7 +2,10 @@ #![allow(clippy::arithmetic_side_effects)] use { - crate::{add_tag_to_name, boxed_error, initialize_globals, ValidatorType, SOLANA_ROOT}, + crate::{ + add_tag_to_name, boxed_error, initialize_globals, new_spinner_progress_bar, ValidatorType, + SOLANA_ROOT, SUN, WRITING, + }, base64::{engine::general_purpose, Engine as _}, bip39::{Language, Mnemonic, MnemonicType, Seed}, log::*, @@ -247,6 +250,11 @@ impl Genesis { ValidatorType::Bootstrap => format!("{}-validator", validator_type), ValidatorType::Standard => "validator".to_string(), ValidatorType::NonVoting => format!("{}-validator", validator_type), + ValidatorType::Client => { + return Err(boxed_error!( + "Client valdiator_type in generate_accounts not allowed" + )) + } }; let mut filename_prefix_with_optional_tag = filename_prefix; @@ -270,6 +278,24 @@ impl Genesis { Ok(()) } + fn create_client_account( + args: &Vec, + executable_path: &PathBuf, + ) -> Result<(), Box> { + let output = Command::new(executable_path) + .args(args) + .output() + .expect("Failed to execute solana-bench-tps"); + + if !output.status.success() { + return Err(boxed_error!(format!( + "Failed to create client accounts. err: {}", + String::from_utf8_lossy(&output.stderr) + ))); + } + Ok(()) + } + // TODO: only supports one client right now. pub fn create_client_accounts( &mut self, @@ -279,7 +305,16 @@ impl Genesis { build_path: PathBuf, ) -> Result<(), Box> { let client_accounts_file = SOLANA_ROOT.join("config-k8s/client-accounts.yml"); - for i in 0..number_of_clients { + if bench_tps_args.is_some() { + let list_of_bench_tps_args = bench_tps_args.as_ref().unwrap(); + for i in list_of_bench_tps_args.iter() { + info!("bench_tps_arg: {}", i); + } + } + + info!("generating {} client accounts...", number_of_clients); + let _ = (0..number_of_clients).into_par_iter().try_for_each(|i| { + info!("client account: {}", i); let mut args = Vec::new(); let account_path = SOLANA_ROOT.join(format!("config-k8s/bench-tps-{}.yml", i)); args.push("--write-client-keys".to_string()); @@ -290,27 +325,20 @@ impl Genesis { if bench_tps_args.is_some() { let list_of_bench_tps_args = bench_tps_args.as_ref().unwrap(); args.extend(list_of_bench_tps_args.clone()); //can unwrap since we checked is_some() - for i in list_of_bench_tps_args.iter() { - info!("bench_tps_arg: {}", i); - } } - - info!("generating client accounts..."); let executable_path = build_path.join("solana-bench-tps"); - let output = Command::new(executable_path) - .args(&args) - .output() - .expect("Failed to execute solana-bench-tps"); - - if !output.status.success() { - return Err(boxed_error!(format!( - "Failed to create client accounts. err: {}", - String::from_utf8_lossy(&output.stderr) - ))); - } + Self::create_client_account(&args, &executable_path) + }); + + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!("{WRITING}Writing client accounts...")); + for i in 0..number_of_clients { + let account_path = SOLANA_ROOT.join(format!("config-k8s/bench-tps-{}.yml", i)); append_client_accounts_to_file(&account_path, &client_accounts_file)?; } + progress_bar.finish_and_clear(); + info!("client-accounts.yml creation for genesis complete"); // add client accounts file as a primordial account self.primordial_accounts_files.push(client_accounts_file); @@ -335,6 +363,11 @@ impl Genesis { ValidatorType::Bootstrap => format!("{}/{}.json", filename_prefix, account), ValidatorType::Standard => format!("{}-{}-{}.json", filename_prefix, account, i), ValidatorType::NonVoting => format!("{}-{}-{}.json", filename_prefix, account, i), + ValidatorType::Client => { + return Err(boxed_error!( + "Client valdiator_type in generate_account not allowed" + )) + } }; let outfile = self.config_dir.join(&filename); @@ -450,18 +483,25 @@ impl Genesis { debug!("{}", arg); } + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!("{SUN}Building Genesis...")); + let executable_path = build_path.join("solana-genesis"); let output = Command::new(executable_path) .args(&args) .output() .expect("Failed to execute solana-genesis"); + progress_bar.finish_and_clear(); + if !output.status.success() { return Err(boxed_error!(format!( "Failed to create genesis. err: {}", String::from_utf8_lossy(&output.stderr) ))); } + info!("Genesis build complete"); + Ok(()) } diff --git a/k8s-cluster/src/lib.rs b/k8s-cluster/src/lib.rs index 8e5e38642fa426..a7cfa10a064dc3 100644 --- a/k8s-cluster/src/lib.rs +++ b/k8s-cluster/src/lib.rs @@ -42,6 +42,7 @@ pub enum ValidatorType { Bootstrap, Standard, NonVoting, + Client, } impl std::fmt::Display for ValidatorType { @@ -50,6 +51,7 @@ impl std::fmt::Display for ValidatorType { ValidatorType::Bootstrap => write!(f, "bootstrap"), ValidatorType::Standard => write!(f, "validator"), ValidatorType::NonVoting => write!(f, "non-voting"), + ValidatorType::Client => write!(f, "client"), } } } @@ -71,6 +73,10 @@ macro_rules! boxed_error { static TRUCK: Emoji = Emoji("🚚 ", ""); static PACKAGE: Emoji = Emoji("📦 ", ""); +static ROCKET: Emoji = Emoji("🚀 ", ""); +static WRITING: Emoji = Emoji("🖊️ ", ""); +static SUN: Emoji = Emoji("🌞 ", ""); +static BUILD: Emoji = Emoji("👷 ", ""); /// Creates a new process bar for processing that will take an unknown amount of time pub fn new_spinner_progress_bar() -> ProgressBar { diff --git a/k8s-cluster/src/main.rs b/k8s-cluster/src/main.rs index 4b3e75474bb834..777820e71309fc 100644 --- a/k8s-cluster/src/main.rs +++ b/k8s-cluster/src/main.rs @@ -106,6 +106,7 @@ fn parse_matches() -> ArgMatches<'static> { .long("profile-build") .help("Enable Profile Build flags"), ) + //Docker config .arg( Arg::with_name("docker_build") .long("docker-build") @@ -116,30 +117,35 @@ fn parse_matches() -> ArgMatches<'static> { Arg::with_name("registry_name") .long("registry") .takes_value(true) - .required_if("docker_build", "true") + // .required_if("docker_build", "true") + .required(true) .help("Registry to push docker image to"), ) .arg( Arg::with_name("image_name") .long("image-name") .takes_value(true) - .default_value_if("docker_build", None, "k8s-cluster-image") - .requires("docker_build") + // .default_value_if("docker_build", None, "k8s-cluster-image") + .default_value("k8s-cluster-image") + .required(true) .help("Docker image name. Will be prepended with validator_type (bootstrap or validator)"), ) .arg( Arg::with_name("base_image") .long("base-image") .takes_value(true) - .default_value_if("docker_build", None, "ubuntu:20.04") - .requires("docker_build") + // .default_value_if("docker_build", None, "ubuntu:20.04") + .default_value("ubuntu:20.04") + .required(true) .help("Docker base image"), ) .arg( Arg::with_name("image_tag") .long("tag") .takes_value(true) - .default_value_if("docker_build", None, "latest") + // .default_value_if("docker_build", None, "latest") + .required(true) + .default_value("latest") .help("Docker image tag."), ) // Multiple Deployment Config @@ -898,20 +904,16 @@ async fn main() { }; } - // Download validator version and Build docker image - let docker_image_config = if build_config.docker_build() && !setup_config.skip_genesis_build { - Some(DockerImageConfig { - base_image: matches.value_of("base_image").unwrap_or_default(), - image_name: matches.value_of("image_name").unwrap(), - tag: matches.value_of("image_tag").unwrap_or_default(), - registry: matches.value_of("registry_name").unwrap(), - }) - } else { - None + //unwraps are safe here. since their requirement is enforced by argmatches + let docker_image_config = DockerImageConfig { + base_image: matches.value_of("base_image").unwrap_or_default(), + image_name: matches.value_of("image_name").unwrap(), + tag: matches.value_of("image_tag").unwrap_or_default(), + registry: matches.value_of("registry_name").unwrap(), }; - if let Some(config) = docker_image_config { - let docker = DockerConfig::new(config, build_config.deploy_method()); + if build_config.docker_build() && !setup_config.skip_genesis_build { + let docker = DockerConfig::new(docker_image_config.clone(), build_config.deploy_method()); let mut image_types = vec![]; if !no_bootstrap { image_types.push(ValidatorType::Bootstrap); @@ -925,7 +927,7 @@ async fn main() { for image_type in &image_types { match docker.build_image(image_type) { - Ok(_) => info!("Docker image built successfully"), + Ok(_) => info!("{} image built successfully", image_type), Err(err) => { error!("Exiting........ {}", err); return; @@ -935,8 +937,26 @@ async fn main() { // Need to push image to registry so Monogon nodes can pull image from registry to local for image_type in &image_types { - match docker.push_image(image_type) { - Ok(_) => info!("{} image built successfully", image_type), + match docker.push_validator_image(image_type) { + Ok(_) => info!("{} image pushed successfully", image_type), + Err(err) => { + error!("Exiting........ {}", err); + return; + } + } + } + + if client_config.num_clients > 0 { + match docker.build_client_images(client_config.num_clients) { + Ok(_) => info!("Client image built successfully"), + Err(err) => { + error!("Exiting........ {}", err); + return; + } + } + + match docker.push_client_images(client_config.num_clients) { + Ok(_) => info!("Client image pushed successfully"), Err(err) => { error!("Exiting........ {}", err); return; @@ -960,14 +980,6 @@ async fn main() { .value_of("validator_image_name") .expect("Validator image name is required"); - // Just using validator container for client right now. they're all the same image - let client_container_name = matches - .value_of("validator_container_name") - .unwrap_or_default(); - let client_image_name = matches - .value_of("validator_image_name") - .expect("Validator image name is required"); - // Just using validator container for nvv right now. they're all the same image let nvv_container_name = matches .value_of("validator_container_name") @@ -1357,6 +1369,12 @@ async fn main() { thread::sleep(Duration::from_secs(client_config.client_delay_start)); for client_index in 0..client_config.num_clients { + let client_container_name = format!("client-container-{}", client_index); + let client_image_name = format!( + "{}/{}-{}:{}", + docker_image_config.registry, "client-image", client_index, docker_image_config.tag + ); + info!("deploying client: {}", client_index); let client_secret = match kub_controller.create_client_secret(client_index) { Ok(secret) => secret, @@ -1379,9 +1397,9 @@ async fn main() { ); let client_replica_set = match kub_controller.create_client_replica_set( - client_container_name, + &client_container_name, client_index, - client_image_name, + &client_image_name, client_secret.metadata.name.clone(), &label_selector, ) { From b5ab968287f89f5335fdb55a56a280a40e52e01a Mon Sep 17 00:00:00 2001 From: greg Date: Sat, 20 Jan 2024 01:52:01 +0000 Subject: [PATCH 2/3] clean up --- k8s-cluster/src/docker.rs | 2 +- k8s-cluster/src/genesis.rs | 2 -- k8s-cluster/src/lib.rs | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/k8s-cluster/src/docker.rs b/k8s-cluster/src/docker.rs index e9a5a683835148..c2cdfdf77c1074 100644 --- a/k8s-cluster/src/docker.rs +++ b/k8s-cluster/src/docker.rs @@ -241,7 +241,7 @@ WORKDIR /home/solana "{ROCKET}Pushing {} image to registry...", identifier )); - let command = format!("docker push '{}'", image); // | grep \"The push refers\"", image); + let command = format!("docker push '{}'", image); let output = Command::new("sh") .arg("-c") .arg(&command) diff --git a/k8s-cluster/src/genesis.rs b/k8s-cluster/src/genesis.rs index f2047d504b7dec..73bc11ae365553 100644 --- a/k8s-cluster/src/genesis.rs +++ b/k8s-cluster/src/genesis.rs @@ -51,7 +51,6 @@ fn generate_keypair() -> Result { let mnemonic_type = MnemonicType::for_word_count(DEFAULT_WORD_COUNT).unwrap(); let mnemonic = Mnemonic::new(mnemonic_type, Language::English); let seed = Seed::new(&mnemonic, &passphrase); - // keypair_from_seed(seed.as_bytes()) keypair_from_seed(seed.as_bytes()).map_err(ThreadSafeError::from) } @@ -296,7 +295,6 @@ impl Genesis { Ok(()) } - // TODO: only supports one client right now. pub fn create_client_accounts( &mut self, number_of_clients: i32, diff --git a/k8s-cluster/src/lib.rs b/k8s-cluster/src/lib.rs index a7cfa10a064dc3..b656577022f692 100644 --- a/k8s-cluster/src/lib.rs +++ b/k8s-cluster/src/lib.rs @@ -142,7 +142,6 @@ pub async fn download_to_temp( .build()?; let response = client.get(url.as_str()).send().await?; - // let file_name: PathBuf = SOLANA_ROOT.join("solana-release.tar.bz2"); let file_name: PathBuf = SOLANA_ROOT.join(file_name); let mut out = File::create(file_name).expect("failed to create file"); let mut content = Cursor::new(response.bytes().await?); From 00cc049f22f990983c7e92fce71e987089d82144 Mon Sep 17 00:00:00 2001 From: greg Date: Sat, 20 Jan 2024 01:58:56 +0000 Subject: [PATCH 3/3] clean up post ci/run-local.sh --- k8s-cluster/src/docker.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/k8s-cluster/src/docker.rs b/k8s-cluster/src/docker.rs index c2cdfdf77c1074..94323649beb3c6 100644 --- a/k8s-cluster/src/docker.rs +++ b/k8s-cluster/src/docker.rs @@ -127,7 +127,7 @@ impl<'a> DockerConfig<'a> { } fn insert_client_accounts_if_present(&self, index: Option) -> String { - let return_string = match index { + match index { Some(i) => { if SOLANA_ROOT .join(format!("config-k8s/bench-tps-{}.yml", i)) @@ -140,7 +140,6 @@ impl<'a> DockerConfig<'a> { i ) } else { - info!("bench-tps-{} does not exist!", i); "".to_string() } } @@ -154,9 +153,7 @@ impl<'a> DockerConfig<'a> { "".to_string() } } - }; - - return_string + } } pub fn create_dockerfile(