From 90f8db80c80946e843e63f7e0255e3851ca8d177 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 14 Nov 2023 19:44:00 +0000 Subject: [PATCH 1/2] enable metrics for k8s-cluster --- k8s-cluster/README.md | 16 +++++ k8s-cluster/src/kubernetes.rs | 94 +++++++++++++++++++++++-- k8s-cluster/src/main.rs | 63 ++++++++++++++++- k8s-cluster/src/scripts/common.sh | 18 ++++- k8s-cluster/src/scripts/init-metrics.sh | 1 + net/init-metrics.sh | 8 ++- 6 files changed, 193 insertions(+), 7 deletions(-) create mode 120000 k8s-cluster/src/scripts/init-metrics.sh diff --git a/k8s-cluster/README.md b/k8s-cluster/README.md index 44573fb04935c8..67f3c69132b0f7 100644 --- a/k8s-cluster/README.md +++ b/k8s-cluster/README.md @@ -66,6 +66,22 @@ cargo run --bin solana-k8s -- --tag v2 ``` +## Metrics are now supported as of 11/14/23!! ~woo~ +1) Setup metrics database: +``` +cd k8s-cluster/src/scripts +./init-metrics -c +# enter password when promted +``` +2) add the following to your `solana-k8s` command from above +``` +--metrics-host https://internal-metrics.solana.com # need the `https://` here +--metrics-port 8086 +--metrics-db # from (1) +--metrics-username # from (1) +--metrics-password # from (1) +``` + Verify validators have deployed: ``` kubectl get pods -n diff --git a/k8s-cluster/src/kubernetes.rs b/k8s-cluster/src/kubernetes.rs index 803a461771454e..946c5673de6530 100644 --- a/k8s-cluster/src/kubernetes.rs +++ b/k8s-cluster/src/kubernetes.rs @@ -1,5 +1,5 @@ use { - crate::SOLANA_ROOT, + crate::{boxed_error, SOLANA_ROOT}, base64::{engine::general_purpose, Engine as _}, k8s_openapi::{ api::{ @@ -88,11 +88,45 @@ pub struct ClientConfig { pub num_nodes: Option, } +#[derive(Clone, Debug)] +pub struct Metrics { + pub host: String, + pub port: String, + pub database: String, + pub username: String, + password: String, +} + +impl Metrics { + pub fn new( + host: String, + port: String, + database: String, + username: String, + password: String, + ) -> Self { + Metrics { + host, + port, + database, + username, + password, + } + } + pub fn to_env_string(&self) -> String { + format!( + "host={}:{},db={},u={},p={}", + self.host, self.port, self.database, self.username, self.password + ) + } +} + pub struct Kubernetes<'a> { client: Client, namespace: &'a str, validator_config: &'a mut ValidatorConfig<'a>, client_config: ClientConfig, + pub metrics: Option, } impl<'a> Kubernetes<'a> { @@ -100,12 +134,14 @@ impl<'a> Kubernetes<'a> { namespace: &'a str, validator_config: &'a mut ValidatorConfig<'a>, client_config: ClientConfig, + metrics: Option, ) -> Kubernetes<'a> { Kubernetes { client: Client::try_default().await.unwrap(), namespace, validator_config, client_config, + metrics, } } @@ -233,7 +269,7 @@ impl<'a> Kubernetes<'a> { secret_name: Option, label_selector: &BTreeMap, ) -> Result> { - let env_var = vec![EnvVar { + let mut env_var = vec![EnvVar { name: "MY_POD_IP".to_string(), value_from: Some(EnvVarSource { field_ref: Some(ObjectFieldSelector { @@ -245,6 +281,10 @@ impl<'a> Kubernetes<'a> { ..Default::default() }]; + if let Some(_) = &self.metrics { + env_var.push(self.get_metrics_env_var_secret()) + } + let accounts_volume = Some(vec![Volume { name: "bootstrap-accounts-volume".into(), secret: Some(SecretVolumeSource { @@ -349,6 +389,46 @@ impl<'a> Kubernetes<'a> { secrets_api.create(&PostParams::default(), secret).await } + pub fn create_metrics_secret(&self) -> Result> { + let mut data = BTreeMap::new(); + if let Some(metrics) = &self.metrics { + data.insert( + "SOLANA_METRICS_CONFIG".to_string(), + ByteString(metrics.to_env_string().into_bytes()), + ); + } else { + return Err(boxed_error!(format!( + "Called create_metrics_secret() but metrics were not provided." + ))); + } + + let secret = Secret { + metadata: ObjectMeta { + name: Some("solana-metrics-secret".to_string()), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }; + + Ok(secret) + } + + pub fn get_metrics_env_var_secret(&self) -> EnvVar { + EnvVar { + name: "SOLANA_METRICS_CONFIG".to_string(), + value_from: Some(k8s_openapi::api::core::v1::EnvVarSource { + secret_key_ref: Some(k8s_openapi::api::core::v1::SecretKeySelector { + name: Some("solana-metrics-secret".to_string()), + key: "SOLANA_METRICS_CONFIG".to_string(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } + } + pub fn create_bootstrap_secret(&self, secret_name: &str) -> Result> { let faucet_key_path = SOLANA_ROOT.join("config-k8s"); let faucet_keypair = @@ -608,7 +688,10 @@ impl<'a> Kubernetes<'a> { secret_name: Option, label_selector: &BTreeMap, ) -> Result> { - let env_vars = self.set_non_bootstrap_environment_variables(); + let mut env_vars = self.set_non_bootstrap_environment_variables(); + if let Some(_) = &self.metrics { + env_vars.push(self.get_metrics_env_var_secret()) + } let accounts_volume = Some(vec![Volume { name: format!("validator-accounts-volume-{}", validator_index), @@ -656,7 +739,10 @@ impl<'a> Kubernetes<'a> { secret_name: Option, label_selector: &BTreeMap, ) -> Result> { - let env_vars = self.set_non_bootstrap_environment_variables(); + let mut env_vars = self.set_non_bootstrap_environment_variables(); + if let Some(_) = &self.metrics { + env_vars.push(self.get_metrics_env_var_secret()) + } let accounts_volume = Some(vec![Volume { name: format!("client-accounts-volume-{}", client_index), diff --git a/k8s-cluster/src/main.rs b/k8s-cluster/src/main.rs index 28edb13d956cca..11bae7b8e97bc8 100644 --- a/k8s-cluster/src/main.rs +++ b/k8s-cluster/src/main.rs @@ -8,7 +8,7 @@ use { DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL, }, get_solana_root, initialize_globals, - kubernetes::{ClientConfig, Kubernetes, ValidatorConfig}, + kubernetes::{ClientConfig, Kubernetes, Metrics, ValidatorConfig}, ledger_helper::LedgerHelper, release::{BuildConfig, Deploy}, ValidatorType, @@ -364,6 +364,37 @@ fn parse_matches() -> ArgMatches<'static> { .takes_value(true) .help("Client Config. Optional: Wait for NUM nodes to converge: --num-nodes "), ) + .arg( + Arg::with_name("metrics_host") + .long("metrics-host") + .takes_value(true) + .requires_all(&["metrics_port", "metrics_db", "metrics_username", "metrics_password"]) + .help("Metrics Config. Optional: specify metrics host. e.g. https://internal-metrics.solana.com"), + ) + .arg( + Arg::with_name("metrics_port") + .long("metrics-port") + .takes_value(true) + .help("Client Config. Optional: specify metrics port. e.g. 8086"), + ) + .arg( + Arg::with_name("metrics_db") + .long("metrics-db") + .takes_value(true) + .help("Client Config. Optional: specify metrics database. e.g. k8s-cluster-"), + ) + .arg( + Arg::with_name("metrics_username") + .long("metrics-username") + .takes_value(true) + .help("Client Config. Optional: specify metrics username"), + ) + .arg( + Arg::with_name("metrics_password") + .long("metrics-password") + .takes_value(true) + .help("Client Config. Optional: Specify metrics password"), + ) .get_matches() } @@ -565,11 +596,24 @@ async fn main() { info!("Runtime Config: {}", validator_config); + let metrics = if matches.is_present("metrics_host") { + Some(Metrics::new( + matches.value_of("metrics_host").unwrap().to_string(), + matches.value_of("metrics_port").unwrap().to_string(), + matches.value_of("metrics_db").unwrap().to_string(), + matches.value_of("metrics_username").unwrap().to_string(), + matches.value_of("metrics_password").unwrap().to_string(), + )) + } else { + None + }; + // Check if namespace exists let mut kub_controller = Kubernetes::new( setup_config.namespace, &mut validator_config, client_config.clone(), + metrics, ) .await; match kub_controller.namespace_exists().await { @@ -746,6 +790,23 @@ async fn main() { .value_of("validator_image_name") .expect("Validator image name is required"); + if let Some(_) = &kub_controller.metrics { + let metrics_secret = match kub_controller.create_metrics_secret() { + Ok(secret) => secret, + Err(err) => { + error!("Failed to create metrics secret! {}", err); + return; + } + }; + match kub_controller.deploy_secret(&metrics_secret).await { + Ok(_) => (), + Err(err) => { + error!("{}", err); + return; + } + } + }; + let bootstrap_secret = match kub_controller.create_bootstrap_secret("bootstrap-accounts-secret") { Ok(secret) => secret, diff --git a/k8s-cluster/src/scripts/common.sh b/k8s-cluster/src/scripts/common.sh index ccfa74d166fb7e..a936131b841db4 100755 --- a/k8s-cluster/src/scripts/common.sh +++ b/k8s-cluster/src/scripts/common.sh @@ -74,7 +74,23 @@ solana_cli=$(solana_program) export RUST_BACKTRACE=1 echo "solana command: $solana_validator" -echo "post solana command" + +# https://gist.github.com/cdown/1163649 +urlencode() { + declare s="$1" + declare l=$((${#s} - 1)) + for i in $(seq 0 $l); do + declare c="${s:$i:1}" + case $c in + [a-zA-Z0-9.~_-]) + echo -n "$c" + ;; + *) + printf '%%%02X' "'$c" + ;; + esac + done +} default_arg() { declare name=$1 diff --git a/k8s-cluster/src/scripts/init-metrics.sh b/k8s-cluster/src/scripts/init-metrics.sh new file mode 120000 index 00000000000000..c33703d7d21c9a --- /dev/null +++ b/k8s-cluster/src/scripts/init-metrics.sh @@ -0,0 +1 @@ +../../../net/init-metrics.sh \ No newline at end of file diff --git a/net/init-metrics.sh b/net/init-metrics.sh index f79cd065b17cfa..2aab2cb20485ea 100755 --- a/net/init-metrics.sh +++ b/net/init-metrics.sh @@ -88,6 +88,12 @@ else SOLANA_METRICS_CONFIG="host=$host,db=$netBasename,u=scratch_writer,p=topsecret" fi -echo "export SOLANA_METRICS_CONFIG=\"$SOLANA_METRICS_CONFIG\"" >> "$configFile" +# Skip echo into config file if running from `k8s-cluster`` repo +real_metrics_script_path=$(readlink -f "$0") # real path of the script +invoked_path="$0" #invoked path +full_invoked_path="$(pwd)${invoked_path:1}" +if [[ "$real_metrics_script_path" == "$full_invoked_path" ]]; then + echo "export SOLANA_METRICS_CONFIG=\"$SOLANA_METRICS_CONFIG\"" >> "$configFile" +fi exit 0 From 30c5741454ddd63bca431af51dc31b72a8e915b8 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 14 Nov 2023 19:48:40 +0000 Subject: [PATCH 2/2] fix some code quality stuff --- k8s-cluster/src/kubernetes.rs | 6 +++--- k8s-cluster/src/main.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s-cluster/src/kubernetes.rs b/k8s-cluster/src/kubernetes.rs index 946c5673de6530..d9d7c4ab4a98a3 100644 --- a/k8s-cluster/src/kubernetes.rs +++ b/k8s-cluster/src/kubernetes.rs @@ -281,7 +281,7 @@ impl<'a> Kubernetes<'a> { ..Default::default() }]; - if let Some(_) = &self.metrics { + if self.metrics.is_some() { env_var.push(self.get_metrics_env_var_secret()) } @@ -689,7 +689,7 @@ impl<'a> Kubernetes<'a> { label_selector: &BTreeMap, ) -> Result> { let mut env_vars = self.set_non_bootstrap_environment_variables(); - if let Some(_) = &self.metrics { + if self.metrics.is_some() { env_vars.push(self.get_metrics_env_var_secret()) } @@ -740,7 +740,7 @@ impl<'a> Kubernetes<'a> { label_selector: &BTreeMap, ) -> Result> { let mut env_vars = self.set_non_bootstrap_environment_variables(); - if let Some(_) = &self.metrics { + if self.metrics.is_some() { env_vars.push(self.get_metrics_env_var_secret()) } diff --git a/k8s-cluster/src/main.rs b/k8s-cluster/src/main.rs index 11bae7b8e97bc8..40e92d61516902 100644 --- a/k8s-cluster/src/main.rs +++ b/k8s-cluster/src/main.rs @@ -790,7 +790,7 @@ async fn main() { .value_of("validator_image_name") .expect("Validator image name is required"); - if let Some(_) = &kub_controller.metrics { + if kub_controller.metrics.is_some() { let metrics_secret = match kub_controller.create_metrics_secret() { Ok(secret) => secret, Err(err) => {