From 5e89c6213ec376a45f0c86bd18364a4e63d645db Mon Sep 17 00:00:00 2001 From: Kjeld Schouten Date: Thu, 7 Nov 2024 23:51:00 +0100 Subject: [PATCH] fix(clustertool): fix kubeconfig not boing loaded correctly and put some more silence into healthchecks --- clustertool/pkg/gencmd/bootstrap.go | 4 ++-- clustertool/pkg/nodestatus/health.go | 17 +++++++++++------ clustertool/pkg/nodestatus/status.go | 6 ++++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/clustertool/pkg/gencmd/bootstrap.go b/clustertool/pkg/gencmd/bootstrap.go index 65366d910e46..dd75eb721702 100644 --- a/clustertool/pkg/gencmd/bootstrap.go +++ b/clustertool/pkg/gencmd/bootstrap.go @@ -53,9 +53,9 @@ func RunBootstrap(args []string) { log.Info().Msgf("Bootstrap: waiting for VIP %v to come online...", helper.TalEnv["VIP_IP"]) nodestatus.WaitForHealth(helper.TalEnv["VIP_IP"], []string{"running"}) - log.Info().Msgf("Bootstrap: Configuring kubectl for VIP: %v", helper.TalEnv["VIP_IP"]) + log.Info().Msgf("Bootstrap: Configuring kubeconfig/kubectl for VIP: %v", helper.TalEnv["VIP_IP"]) // Ensure kubeconfig is loaded - kubeconfigcmds := GenPlain("health", helper.TalEnv["VIP_IP"], extraArgs) + kubeconfigcmds := GenPlain("kubeconfig", helper.TalEnv["VIP_IP"], extraArgs) ExecCmd(kubeconfigcmds[0]) // Desired pod names diff --git a/clustertool/pkg/nodestatus/health.go b/clustertool/pkg/nodestatus/health.go index 8b562ecf18b5..6a0263712ad8 100644 --- a/clustertool/pkg/nodestatus/health.go +++ b/clustertool/pkg/nodestatus/health.go @@ -15,8 +15,8 @@ func CheckHealth(node string, status string, silent bool) error { if err != nil { errstring := "healthcheck failed. status: " + string(out) + " error: " + err.Error() if !silent { - log.Info().Msgf("Healthcheck: check on node : failed %v", node) - log.Info().Msgf("failed with error: %s", errstring) + log.Error().Msgf("Healthcheck: check on node : failed %v", node) + log.Error().Msgf("failed with error: %s", errstring) } log.Error().Err(err).Str("node", node).Msg("Healthcheck failed") return errors.New(errstring) @@ -36,17 +36,22 @@ func CheckHealth(node string, status string, silent bool) error { response := "Healthcheck: WARN detected node " + node + " in mode " + "maintenance" + ".\nLikely a new node, so trying commands anyway. Continuing..." log.Warn().Msg(response) } else if status == "" && strings.Contains(out, "running") { - _, err = CheckReadyStatus(node) + _, err = CheckReadyStatus(node, silent) if err != nil { + errstring := "healthcheck failed. status: " + string(out) + " error: " + err.Error() - log.Error().Err(err).Str("node", node).Msg("Healthcheck failed while checking readiness") + + if !silent { + log.Error().Err(err).Str("node", node).Msg("Healthcheck failed while checking readiness") + } return errors.New(errstring) } } else { if !silent { log.Info().Msgf("Healthcheck: check on node : failed %v", node) + log.Error().Str("node", node).Msg("Healthcheck failed with unexpected status") } - log.Error().Str("node", node).Msg("Healthcheck failed with unexpected status") + return errors.New("healthcheck failed") } log.Debug().Str("node", node).Msg("Health check completed successfully") @@ -83,7 +88,7 @@ func WaitForHealth(node string, status []string) (string, error) { log.Debug().Str("node", node).Str("check", check).Msg("Performing initial health check") err := CheckHealth(node, check, true) if err == nil { - log.Info().Str("node", node).Str("status", check).Msg("Initial health check passed") + log.Debug().Str("node", node).Str("status", check).Msg("Initial health check passed") return check, nil } } diff --git a/clustertool/pkg/nodestatus/status.go b/clustertool/pkg/nodestatus/status.go index 313f0c944e85..52ff7d510d7b 100644 --- a/clustertool/pkg/nodestatus/status.go +++ b/clustertool/pkg/nodestatus/status.go @@ -75,7 +75,7 @@ func CheckStatus(node string) (string, error) { return string(out), nil } -func CheckReadyStatus(node string) (string, error) { +func CheckReadyStatus(node string, silent bool) (string, error) { log.Info().Str("node", node).Msg("Checking node readiness status") argsslice := append(baseStatusCMD(node), "-o", "jsonpath={.spec.status.ready}") @@ -83,7 +83,9 @@ func CheckReadyStatus(node string) (string, error) { if err != nil { errstring := "status: " + string(out) + " error: " + err.Error() - log.Error().Msg(errstring) + if !silent { + log.Error().Msg(errstring) + } return "ERROR", errors.New(errstring) } if strings.Contains(string(out), "true") {