Refactor logs gather flow to support multiple clusters (#452)

- Refactor logs gathering flow to support logs gather from multiple clusters. This is used in multi cluster scenario - Add "getSecret" function to fetch certs that are used in multicluster test. - Add name cluster to be specified during log gathering. Signed-off-by: Maxim Babushkin <[email protected]>
istio-ecosystem · Oct 28, 2024 · ec13475 · ec13475
1 parent 1ac8e14
commit ec13475
Show file tree

Hide file tree

Showing 11 changed files with 70 additions and 44 deletions.
diff --git a/tests/e2e/controlplane/control_plane_suite_test.go b/tests/e2e/controlplane/control_plane_suite_test.go
@@ -63,5 +63,5 @@ func setup() {
 	cl, err = k8sclient.InitK8sClient("")
 	Expect(err).NotTo(HaveOccurred())
 
-	k = kubectl.New()
+	k = kubectl.New("clControlPlane")
 }
diff --git a/tests/e2e/controlplane/control_plane_test.go b/tests/e2e/controlplane/control_plane_test.go
@@ -295,7 +295,7 @@ spec:
 
 		AfterAll(func(ctx SpecContext) {
 			if CurrentSpecReport().Failed() {
-				common.LogDebugInfo()
+				common.LogDebugInfo(k)
 				debugInfoLogged = true
 			}
 
@@ -314,7 +314,7 @@ spec:
 
 	AfterAll(func() {
 		if CurrentSpecReport().Failed() && !debugInfoLogged {
-			common.LogDebugInfo()
+			common.LogDebugInfo(k)
 			debugInfoLogged = true
 		}
 

diff --git a/tests/e2e/dualstack/dualstack_suite_test.go b/tests/e2e/dualstack/dualstack_suite_test.go
@@ -63,5 +63,5 @@ func setup() {
 	cl, err = k8sclient.InitK8sClient("")
 	Expect(err).NotTo(HaveOccurred())
 
-	k = kubectl.New()
+	k = kubectl.New("clDualStack")
 }
diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go
@@ -280,7 +280,7 @@ spec:
 
 		AfterAll(func(ctx SpecContext) {
 			if CurrentSpecReport().Failed() {
-				common.LogDebugInfo()
+				common.LogDebugInfo(k)
 				debugInfoLogged = true
 			}
 
@@ -295,7 +295,7 @@ spec:
 
 	AfterAll(func() {
 		if CurrentSpecReport().Failed() && !debugInfoLogged {
-			common.LogDebugInfo()
+			common.LogDebugInfo(k)
 			debugInfoLogged = true
 		}
 

diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go
@@ -253,7 +253,8 @@ spec:
 
 				AfterAll(func(ctx SpecContext) {
 					if CurrentSpecReport().Failed() {
-						common.LogDebugInfo()
+						common.LogDebugInfo(k1)
+						common.LogDebugInfo(k2)
 						debugInfoLogged = true
 					}
 
@@ -277,7 +278,8 @@ spec:
 
 	AfterAll(func(ctx SpecContext) {
 		if CurrentSpecReport().Failed() && !debugInfoLogged {
-			common.LogDebugInfo()
+			common.LogDebugInfo(k1)
+			common.LogDebugInfo(k2)
 			debugInfoLogged = true
 		}
 

diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go
@@ -295,7 +295,8 @@ spec:
 
 				AfterAll(func(ctx SpecContext) {
 					if CurrentSpecReport().Failed() {
-						common.LogDebugInfo()
+						common.LogDebugInfo(k1)
+						common.LogDebugInfo(k2)
 						debugInfoLogged = true
 					}
 
@@ -319,7 +320,8 @@ spec:
 
 	AfterAll(func(ctx SpecContext) {
 		if CurrentSpecReport().Failed() && !debugInfoLogged {
-			common.LogDebugInfo()
+			common.LogDebugInfo(k1)
+			common.LogDebugInfo(k2)
 			debugInfoLogged = true
 		}
 

diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go
@@ -99,6 +99,6 @@ func setup(t *testing.T) {
 	exposeIstiodYAML = fmt.Sprintf("%s/docs/multicluster/expose-istiod.yaml", baseRepoDir)
 
 	// Initialize kubectl utilities, one for each cluster
-	k1 = kubectl.New().WithKubeconfig(kubeconfig)
-	k2 = kubectl.New().WithKubeconfig(kubeconfig2)
+	k1 = kubectl.New("clPrimary").WithKubeconfig(kubeconfig)
+	k2 = kubectl.New("clRemote").WithKubeconfig(kubeconfig2)
 }
diff --git a/tests/e2e/operator/operator_install_test.go b/tests/e2e/operator/operator_install_test.go
@@ -100,14 +100,14 @@ var _ = Describe("Operator", Ordered, func() {
 
 		AfterAll(func() {
 			if CurrentSpecReport().Failed() {
-				common.LogDebugInfo()
+				common.LogDebugInfo(k)
 			}
 		})
 	})
 
 	AfterAll(func() {
 		if CurrentSpecReport().Failed() {
-			common.LogDebugInfo()
+			common.LogDebugInfo(k)
 		}
 
 		if skipDeploy {

diff --git a/tests/e2e/operator/operator_suite_test.go b/tests/e2e/operator/operator_suite_test.go
@@ -62,5 +62,5 @@ func setup() {
 		GinkgoWriter.Println("Running on Kubernetes")
 	}
 
-	k = kubectl.New()
+	k = kubectl.New("clOperator")
 }
diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go
@@ -57,8 +57,6 @@ var (
 	// - 1.24-alpha.feabc1234
 	// matching only the version before first '_' which is used in the downstream builds, e.g. "1.23.2_ossm_tp.2"
 	istiodVersionRegex = regexp.MustCompile(`Version:"([^"_]*)[^"]*"`)
-
-	k = kubectl.New()
 )
 
 // GetObject returns the object with the given key
@@ -131,73 +129,80 @@ func CheckNamespaceEmpty(ctx SpecContext, cl client.Client, ns string) {
 	}).Should(BeEmpty(), "No Services should be present in the namespace")
 }
 
-func LogDebugInfo() {
+func LogDebugInfo(k kubectl.Kubectl) {
 	// General debugging information to help diagnose the failure
 	// TODO: Add the creation of file with this information to be attached to the test report
 
 	GinkgoWriter.Println()
-	GinkgoWriter.Println("The test run has failures and the debug information is as follows:")
+	GinkgoWriter.Println("The test run has failures and the debug information is as follows from cluster: %q:", k.GetClusterName())
+	GinkgoWriter.Println("=========================================================")
+	logOperatorDebugInfo(k)
 	GinkgoWriter.Println("=========================================================")
-	logOperatorDebugInfo()
+	logIstioDebugInfo(k)
 	GinkgoWriter.Println("=========================================================")
-	logIstioDebugInfo()
+	logCNIDebugInfo(k)
 	GinkgoWriter.Println("=========================================================")
-	logCNIDebugInfo()
+	logCertsDebugInfo(k)
 	GinkgoWriter.Println("=========================================================")
 }
 
-func logOperatorDebugInfo() {
-	k := k.WithNamespace(OperatorNamespace)
+func logOperatorDebugInfo(k kubectl.Kubectl) {
+	k = k.WithNamespace(OperatorNamespace)
 	operator, err := k.GetYAML("deployment", deploymentName)
-	logDebugElement("Operator Deployment YAML", operator, err)
+	logDebugElement("=====Operator Deployment YAML=====", operator, err)
 
 	logs, err := k.Logs("deploy/"+deploymentName, ptr.Of(120*time.Second))
-	logDebugElement("Operator logs", logs, err)
+	logDebugElement("=====Operator logs=====", logs, err)
 
 	events, err := k.GetEvents()
-	logDebugElement("Events in "+OperatorNamespace, events, err)
+	logDebugElement("=====Events in "+OperatorNamespace+"=====", events, err)
 
 	// Temporary information to gather more details about failure
 	pods, err := k.GetPods("", "-o wide")
-	logDebugElement("Pods in "+OperatorNamespace, pods, err)
+	logDebugElement("=====Pods in "+OperatorNamespace+"=====", pods, err)
 
 	describe, err := k.Describe("deployment", deploymentName)
-	logDebugElement("Operator Deployment describe", describe, err)
+	logDebugElement("=====Operator Deployment describe=====", describe, err)
 }
 
-func logIstioDebugInfo() {
+func logIstioDebugInfo(k kubectl.Kubectl) {
 	resource, err := k.GetYAML("istio", istioName)
-	logDebugElement("Istio YAML", resource, err)
+	logDebugElement("=====Istio YAML=====", resource, err)
 
 	output, err := k.WithNamespace(controlPlaneNamespace).GetPods("", "-o wide")
-	logDebugElement("Pods in "+controlPlaneNamespace, output, err)
+	logDebugElement("=====Pods in "+controlPlaneNamespace+"=====", output, err)
 
 	logs, err := k.WithNamespace(controlPlaneNamespace).Logs("deploy/istiod", ptr.Of(120*time.Second))
-	logDebugElement("Istiod logs", logs, err)
+	logDebugElement("=====Istiod logs=====", logs, err)
 
 	events, err := k.WithNamespace(controlPlaneNamespace).GetEvents()
-	logDebugElement("Events in "+controlPlaneNamespace, events, err)
+	logDebugElement("=====Events in "+controlPlaneNamespace+"=====", events, err)
 }
 
-func logCNIDebugInfo() {
+func logCNIDebugInfo(k kubectl.Kubectl) {
 	resource, err := k.GetYAML("istiocni", istioCniName)
-	logDebugElement("IstioCNI YAML", resource, err)
+	logDebugElement("=====IstioCNI YAML=====", resource, err)
 
 	ds, err := k.WithNamespace(istioCniNamespace).GetYAML("daemonset", "istio-cni-node")
-	logDebugElement("Istio CNI DaemonSet YAML", ds, err)
+	logDebugElement("=====Istio CNI DaemonSet YAML=====", ds, err)
 
 	events, err := k.WithNamespace(istioCniNamespace).GetEvents()
-	logDebugElement("Events in "+istioCniNamespace, events, err)
+	logDebugElement("=====Events in "+istioCniNamespace+"=====", events, err)
 
 	// Temporary information to gather more details about failure
 	pods, err := k.WithNamespace(istioCniNamespace).GetPods("", "-o wide")
-	logDebugElement("Pods in "+istioCniNamespace, pods, err)
+	logDebugElement("=====Pods in "+istioCniNamespace+"=====", pods, err)
 
 	describe, err := k.WithNamespace(istioCniNamespace).Describe("daemonset", "istio-cni-node")
-	logDebugElement("Istio CNI DaemonSet describe", describe, err)
+	logDebugElement("=====Istio CNI DaemonSet describe=====", describe, err)
 
 	logs, err := k.WithNamespace(istioCniNamespace).Logs("daemonset/istio-cni-node", ptr.Of(120*time.Second))
-	logDebugElement("Istio CNI logs", logs, err)
+	logDebugElement("=====Istio CNI logs=====", logs, err)
+}
+
+func logCertsDebugInfo(k kubectl.Kubectl) {
+	certs, err := k.WithNamespace(controlPlaneNamespace).GetSecret("cacerts")
+	logDebugElement("=====CA certs=====", certs, err)
 }
 
 func logDebugElement(caption string, info string, err error) {
@@ -211,7 +216,7 @@ func logDebugElement(caption string, info string, err error) {
 }
 
 func GetVersionFromIstiod() (*semver.Version, error) {
-	k := kubectl.New()
+	k := kubectl.New("testCluster")
 	output, err := k.WithNamespace(controlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version")
 	if err != nil {
 		return nil, fmt.Errorf("error getting version from istiod: %w", err)

diff --git a/tests/e2e/util/kubectl/kubectl.go b/tests/e2e/util/kubectl/kubectl.go
@@ -25,14 +25,15 @@ import (
 )
 
 type Kubectl struct {
+	name       string
 	binary     string
 	namespace  string
 	kubeconfig string
 }
 
 // New creates a new kubectl.Kubectl
-func New() Kubectl {
-	return Kubectl{}.WithBinary(os.Getenv("COMMAND"))
+func New(name string) Kubectl {
+	return Kubectl{name: name}.WithBinary(os.Getenv("COMMAND"))
 }
 
 func (k Kubectl) build(cmd string) string {
@@ -217,6 +218,11 @@ func (k Kubectl) ForceDelete(kind, name string) error {
 	return k.Delete(kind, name)
 }
 
+// Gets cluster name defined during initialization
+func (k Kubectl) GetClusterName() string {
+	return k.name
+}
+
 // GetYAML returns the yaml of a resource
 func (k Kubectl) GetYAML(kind, name string) (string, error) {
 	cmd := k.build(fmt.Sprintf(" get %s %s -o yaml", kind, name))
@@ -250,6 +256,17 @@ func (k Kubectl) GetInternalIP(label string) (string, error) {
 	return output, nil
 }
 
+// GetSecret returns the secret of a namespace
+func (k Kubectl) GetSecret(secret string) (string, error) {
+	cmd := k.build(fmt.Sprintf(" get secret %s -o yaml", secret))
+	output, err := k.executeCommand(cmd)
+	if err != nil {
+		return "", fmt.Errorf("error getting secret: %w, output %s", err, output)
+	}
+
+	return output, nil
+}
+
 // Exec executes a command in the pod or specific container
 func (k Kubectl) Exec(pod, container, command string) (string, error) {
 	cmd := k.build(fmt.Sprintf(" exec %s %s -- %s", pod, containerFlag(container), command))