Merge pull request #35 from gregcusack/kco-metrics

Add in metrics

k8s-cluster/README.md

@@ -66,6 +66,22 @@ cargo run --bin solana-k8s --
     --tag v2
 ```
 
+## Metrics are now supported as of 11/14/23!! ~woo~
+1) Setup metrics database:
+```
+cd k8s-cluster/src/scripts
+./init-metrics -c <database-name> <metrics-username>
+# enter password when promted
+```
+2) add the following to your `solana-k8s` command from above
+```
+--metrics-host https://internal-metrics.solana.com # need the `https://` here
+--metrics-port 8086
+--metrics-db <database-name>            # from (1)
+--metrics-username <metrics-username>   # from (1)
+--metrics-password <metrics-password>   # from (1)
+```
+
 Verify validators have deployed:
 ```
 kubectl get pods -n <namespace>

k8s-cluster/src/kubernetes.rs

@@ -1,5 +1,5 @@
 use {
-    crate::SOLANA_ROOT,
+    crate::{boxed_error, SOLANA_ROOT},
     base64::{engine::general_purpose, Engine as _},
     k8s_openapi::{
         api::{
@@ -88,24 +88,60 @@ pub struct ClientConfig {
     pub num_nodes: Option<u64>,
 }
 
+#[derive(Clone, Debug)]
+pub struct Metrics {
+    pub host: String,
+    pub port: String,
+    pub database: String,
+    pub username: String,
+    password: String,
+}
+
+impl Metrics {
+    pub fn new(
+        host: String,
+        port: String,
+        database: String,
+        username: String,
+        password: String,
+    ) -> Self {
+        Metrics {
+            host,
+            port,
+            database,
+            username,
+            password,
+        }
+    }
+    pub fn to_env_string(&self) -> String {
+        format!(
+            "host={}:{},db={},u={},p={}",
+            self.host, self.port, self.database, self.username, self.password
+        )
+    }
+}
+
 pub struct Kubernetes<'a> {
     client: Client,
     namespace: &'a str,
     validator_config: &'a mut ValidatorConfig<'a>,
     client_config: ClientConfig,
+    pub metrics: Option<Metrics>,
 }
 
 impl<'a> Kubernetes<'a> {
     pub async fn new(
         namespace: &'a str,
         validator_config: &'a mut ValidatorConfig<'a>,
         client_config: ClientConfig,
+        metrics: Option<Metrics>,
     ) -> Kubernetes<'a> {
         Kubernetes {
             client: Client::try_default().await.unwrap(),
             namespace,
             validator_config,
             client_config,
+            metrics,
         }
     }
 
@@ -233,7 +269,7 @@ impl<'a> Kubernetes<'a> {
         secret_name: Option<String>,
         label_selector: &BTreeMap<String, String>,
     ) -> Result<ReplicaSet, Box<dyn Error>> {
-        let env_var = vec![EnvVar {
+        let mut env_var = vec![EnvVar {
             name: "MY_POD_IP".to_string(),
             value_from: Some(EnvVarSource {
                 field_ref: Some(ObjectFieldSelector {
@@ -245,6 +281,10 @@ impl<'a> Kubernetes<'a> {
             ..Default::default()
         }];
 
+        if self.metrics.is_some() {
+            env_var.push(self.get_metrics_env_var_secret())
+        }
+
         let accounts_volume = Some(vec![Volume {
             name: "bootstrap-accounts-volume".into(),
             secret: Some(SecretVolumeSource {
@@ -349,6 +389,46 @@ impl<'a> Kubernetes<'a> {
         secrets_api.create(&PostParams::default(), secret).await
     }
 
+    pub fn create_metrics_secret(&self) -> Result<Secret, Box<dyn Error>> {
+        let mut data = BTreeMap::new();
+        if let Some(metrics) = &self.metrics {
+            data.insert(
+                "SOLANA_METRICS_CONFIG".to_string(),
+                ByteString(metrics.to_env_string().into_bytes()),
+            );
+        } else {
+            return Err(boxed_error!(format!(
+                "Called create_metrics_secret() but metrics were not provided."
+            )));
+        }
+
+        let secret = Secret {
+            metadata: ObjectMeta {
+                name: Some("solana-metrics-secret".to_string()),
+                ..Default::default()
+            },
+            data: Some(data),
+            ..Default::default()
+        };
+
+        Ok(secret)
+    }
+
+    pub fn get_metrics_env_var_secret(&self) -> EnvVar {
+        EnvVar {
+            name: "SOLANA_METRICS_CONFIG".to_string(),
+            value_from: Some(k8s_openapi::api::core::v1::EnvVarSource {
+                secret_key_ref: Some(k8s_openapi::api::core::v1::SecretKeySelector {
+                    name: Some("solana-metrics-secret".to_string()),
+                    key: "SOLANA_METRICS_CONFIG".to_string(),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }),
+            ..Default::default()
+        }
+    }
+
     pub fn create_bootstrap_secret(&self, secret_name: &str) -> Result<Secret, Box<dyn Error>> {
         let faucet_key_path = SOLANA_ROOT.join("config-k8s");
         let faucet_keypair =
@@ -608,7 +688,10 @@ impl<'a> Kubernetes<'a> {
         secret_name: Option<String>,
         label_selector: &BTreeMap<String, String>,
     ) -> Result<ReplicaSet, Box<dyn Error>> {
-        let env_vars = self.set_non_bootstrap_environment_variables();
+        let mut env_vars = self.set_non_bootstrap_environment_variables();
+        if self.metrics.is_some() {
+            env_vars.push(self.get_metrics_env_var_secret())
+        }
 
         let accounts_volume = Some(vec![Volume {
             name: format!("validator-accounts-volume-{}", validator_index),
@@ -656,7 +739,10 @@ impl<'a> Kubernetes<'a> {
         secret_name: Option<String>,
         label_selector: &BTreeMap<String, String>,
     ) -> Result<ReplicaSet, Box<dyn Error>> {
-        let env_vars = self.set_non_bootstrap_environment_variables();
+        let mut env_vars = self.set_non_bootstrap_environment_variables();
+        if self.metrics.is_some() {
+            env_vars.push(self.get_metrics_env_var_secret())
+        }
 
         let accounts_volume = Some(vec![Volume {
             name: format!("client-accounts-volume-{}", client_index),

k8s-cluster/src/main.rs

@@ -8,7 +8,7 @@ use {
             DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL,
         },
         get_solana_root, initialize_globals,
-        kubernetes::{ClientConfig, Kubernetes, ValidatorConfig},
+        kubernetes::{ClientConfig, Kubernetes, Metrics, ValidatorConfig},
         ledger_helper::LedgerHelper,
         release::{BuildConfig, Deploy},
         ValidatorType,
@@ -364,6 +364,37 @@ fn parse_matches() -> ArgMatches<'static> {
                 .takes_value(true)
                 .help("Client Config. Optional: Wait for NUM nodes to converge: --num-nodes <NUM> "),
         )
+        .arg(
+            Arg::with_name("metrics_host")
+                .long("metrics-host")
+                .takes_value(true)
+                .requires_all(&["metrics_port", "metrics_db", "metrics_username", "metrics_password"])
+                .help("Metrics Config. Optional: specify metrics host. e.g. https://internal-metrics.solana.com"),
+        )
+        .arg(
+            Arg::with_name("metrics_port")
+                .long("metrics-port")
+                .takes_value(true)
+                .help("Client Config. Optional: specify metrics port. e.g. 8086"),
+        )
+        .arg(
+            Arg::with_name("metrics_db")
+                .long("metrics-db")
+                .takes_value(true)
+                .help("Client Config. Optional: specify metrics database. e.g. k8s-cluster-<your name>"),
+        )
+        .arg(
+            Arg::with_name("metrics_username")
+                .long("metrics-username")
+                .takes_value(true)
+                .help("Client Config. Optional: specify metrics username"),
+        )
+        .arg(
+            Arg::with_name("metrics_password")
+                .long("metrics-password")
+                .takes_value(true)
+                .help("Client Config. Optional: Specify metrics password"),
+        )
         .get_matches()
 }
 
@@ -565,11 +596,24 @@ async fn main() {
 
     info!("Runtime Config: {}", validator_config);
 
+    let metrics = if matches.is_present("metrics_host") {
+        Some(Metrics::new(
+            matches.value_of("metrics_host").unwrap().to_string(),
+            matches.value_of("metrics_port").unwrap().to_string(),
+            matches.value_of("metrics_db").unwrap().to_string(),
+            matches.value_of("metrics_username").unwrap().to_string(),
+            matches.value_of("metrics_password").unwrap().to_string(),
+        ))
+    } else {
+        None
+    };
+
     // Check if namespace exists
     let mut kub_controller = Kubernetes::new(
         setup_config.namespace,
         &mut validator_config,
         client_config.clone(),
+        metrics,
     )
     .await;
     match kub_controller.namespace_exists().await {
@@ -746,6 +790,23 @@ async fn main() {
         .value_of("validator_image_name")
         .expect("Validator image name is required");
 
+    if kub_controller.metrics.is_some() {
+        let metrics_secret = match kub_controller.create_metrics_secret() {
+            Ok(secret) => secret,
+            Err(err) => {
+                error!("Failed to create metrics secret! {}", err);
+                return;
+            }
+        };
+        match kub_controller.deploy_secret(&metrics_secret).await {
+            Ok(_) => (),
+            Err(err) => {
+                error!("{}", err);
+                return;
+            }
+        }
+    };
+
     let bootstrap_secret = match kub_controller.create_bootstrap_secret("bootstrap-accounts-secret")
     {
         Ok(secret) => secret,

k8s-cluster/src/scripts/common.sh

@@ -74,7 +74,23 @@ solana_cli=$(solana_program)
 export RUST_BACKTRACE=1
 
 echo "solana command: $solana_validator"
-echo "post solana command"
+
+# https://gist.github.com/cdown/1163649
+urlencode() {
+  declare s="$1"
+  declare l=$((${#s} - 1))
+  for i in $(seq 0 $l); do
+    declare c="${s:$i:1}"
+    case $c in
+      [a-zA-Z0-9.~_-])
+        echo -n "$c"
+        ;;
+      *)
+        printf '%%%02X' "'$c"
+        ;;
+    esac
+  done
+}
 
 default_arg() {
   declare name=$1

k8s-cluster/src/scripts/init-metrics.sh

@@ -0,0 +1 @@
+../../../net/init-metrics.sh

net/init-metrics.sh

@@ -88,6 +88,12 @@ else
   SOLANA_METRICS_CONFIG="host=$host,db=$netBasename,u=scratch_writer,p=topsecret"
 fi
 
-echo "export SOLANA_METRICS_CONFIG=\"$SOLANA_METRICS_CONFIG\"" >> "$configFile"
+# Skip echo into config file if running from `k8s-cluster`` repo
+real_metrics_script_path=$(readlink -f "$0") # real path of the script
+invoked_path="$0" #invoked path
+full_invoked_path="$(pwd)${invoked_path:1}"
+if [[ "$real_metrics_script_path" == "$full_invoked_path" ]]; then
+  echo "export SOLANA_METRICS_CONFIG=\"$SOLANA_METRICS_CONFIG\"" >> "$configFile"
+fi
 
 exit 0