rm default --require-tower

k8s-cluster/README.md

@@ -82,6 +82,7 @@ cd k8s-cluster/src/scripts
 --metrics-password <metrics-password>   # from (1)
 ```
 
+## Verifying Deployment
 Verify validators have deployed:
 ```
 kubectl get pods -n <namespace>
@@ -98,10 +99,10 @@ solana -ul validators # should see `--num-validators`+1 current validators (incl
 ```
 ^ if you ran the tar deployment, you should see the Stake by Version as well read `<release-channel>` in the `solana -ul validators` output.
 
-TODO:
-- we currently write binary to file for genesis, then read it back to verify it. and then read it again to convert into a base64 string and then converted into binrary and then converted into a GenesisConfig lol. So need to fix
+## Trouble Shooting
+1) The default `solana-validator` command includes `--require-tower`, as a result if your pods restart for whatever reason after the validator was running for some time, the validator will not properly boot. Deploy your experiment with `--skip-require-tower`
 
 ### Notes
-- Have tested deployments of up to 500 validators
-- Additional validator commandline flags are coming....stay tuned
-- Once again, we assume you are logged into docker and you are pulling from a public repo (Monogon hosts need to access)
+- Have tested deployments of up to 1200 validators
+- Once again, we assume you are logged into docker and you are pulling from a public repo (Monogon hosts need to access)
+

k8s-cluster/src/kubernetes.rs

@@ -35,7 +35,7 @@ pub struct ValidatorConfig<'a> {
     pub max_ledger_size: Option<u64>,
     pub skip_poh_verify: bool,
     pub no_snapshot_fetch: bool,
-    pub skip_require_tower: bool,
+    pub require_tower: bool,
     pub enable_full_rpc: bool,
 }
 
@@ -56,7 +56,7 @@ impl<'a> std::fmt::Display for ValidatorConfig<'a> {
              max_ledger_size: {:?}\n\
              skip_poh_verify: {}\n\
              no_snapshot_fetch: {}\n\
-             skip_require_tower: {}\n\
+             require_tower: {}\n\
              enable_full_rpc: {}",
             self.tpu_enable_udp,
             self.tpu_disable_quic,
@@ -70,7 +70,7 @@ impl<'a> std::fmt::Display for ValidatorConfig<'a> {
             self.max_ledger_size,
             self.skip_poh_verify,
             self.no_snapshot_fetch,
-            self.skip_require_tower,
+            self.require_tower,
             self.enable_full_rpc,
         )
     }
@@ -168,8 +168,8 @@ impl<'a> Kubernetes<'a> {
         if self.validator_config.no_snapshot_fetch {
             flags.push("--no-snapshot-fetch".to_string());
         }
-        if self.validator_config.skip_require_tower {
-            flags.push("--skip-require-tower".to_string());
+        if self.validator_config.require_tower {
+            flags.push("--require-tower".to_string());
         }
         if self.validator_config.enable_full_rpc {
             flags.push("--enable-rpc-transaction-history".to_string());

k8s-cluster/src/main.rs

@@ -276,9 +276,10 @@ fn parse_matches() -> ArgMatches<'static> {
                 .help("Validator config. If set, disables booting validators from a snapshot"),
         )
         .arg(
-            Arg::with_name("skip_require_tower")
-                .long("skip-require-tower")
-                .help("Validator config. Disable require tower"),
+            Arg::with_name("require_tower")
+                .long("require-tower")
+                .help("Validator config. Refuse to start if saved tower state is not found.
+                Off by default since validator won't restart if the pod restarts"),
         )
         .arg(
             Arg::with_name("full_rpc")
@@ -573,7 +574,7 @@ async fn main() {
         },
         skip_poh_verify: matches.is_present("skip_poh_verify"),
         no_snapshot_fetch: matches.is_present("no_snapshot_fetch"),
-        skip_require_tower: matches.is_present("skip_require_tower"),
+        require_tower: matches.is_present("require_tower"),
         enable_full_rpc: matches.is_present("enable_full_rpc"),
     };
 

k8s-cluster/src/scripts/bootstrap-startup-script.sh

@@ -22,7 +22,6 @@ else
 fi
 
 no_restart=0
-maybeRequireTower=true
 
 echo "PROGRAM: $program"
 
@@ -101,8 +100,8 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --accounts-db-skip-shrink ]]; then
       args+=("$1")
       shift
-    elif [[ $1 == --skip-require-tower ]]; then # Done
-      maybeRequireTower=false
+    elif [[ $1 == --require-tower ]]; then # Done
+      args+=("$1")
       shift
     elif [[ $1 = --log-messages-bytes-limit ]]; then # not enabled in net.sh
       args+=("$1" "$2")
@@ -129,10 +128,6 @@ ledger_dir=/home/solana/ledger
   exit 1
 }
 
-if [[ $maybeRequireTower = true ]]; then
-  args+=(--require-tower)
-fi
-
 args+=(
   --no-os-network-limits-test \
   --no-wait-for-vote-to-start-leader \

k8s-cluster/src/scripts/validator-startup-script.sh

@@ -45,7 +45,6 @@ EOF
   exit 1
 }
 
-maybeRequireTower=true
 echo "pre positional args"
 positional_args=()
 while [[ -n $1 ]]; do
@@ -177,8 +176,8 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --accounts-db-skip-shrink ]]; then
       args+=("$1")
       shift
-    elif [[ $1 == --skip-require-tower ]]; then
-      maybeRequireTower=false
+    elif [[ $1 == --require-tower ]]; then
+      args+=("$1")
       shift
     elif [[ $1 = -h ]]; then
       usage "$@"
@@ -246,11 +245,6 @@ default_arg --allow-private-addr
 default_arg --gossip-port 8001
 default_arg --rpc-port 8899
 
-if [[ $maybeRequireTower = true ]]; then
-  default_arg --require-tower
-fi
-echo "post default args"
-
 program=
 if [[ -n $SOLANA_CUDA ]]; then
   program="solana-validator --cuda"