From 118a03b63f71af6f1fe9d836a8c68fee7279e811 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Mon, 13 Jan 2025 13:36:55 -0600 Subject: [PATCH] Add additional support to performance suite for SNO/CRC performance: skip if test pod are unschedulable --- tests/globalhelper/pod.go | 14 ++++++++++++++ tests/performance/tests/exclusive_cpu_pools.go | 12 ++++++++++-- tests/performance/tests/rt_app_no_exec_probes.go | 12 +++++++++++- .../rt_exclusive_cpu_pool_scheduling_policy.go | 14 ++++++++++++++ tests/performance/tests/rt_isolated_cpu_pool.go | 13 ++++++++++++- .../shared-cpu-pool-non-rt-scheduling-policy.go | 10 ++++++++-- 6 files changed, 69 insertions(+), 6 deletions(-) diff --git a/tests/globalhelper/pod.go b/tests/globalhelper/pod.go index 00e0b57e0..afa7608fa 100644 --- a/tests/globalhelper/pod.go +++ b/tests/globalhelper/pod.go @@ -75,6 +75,20 @@ func CreateAndWaitUntilPodIsReady(pod *corev1.Pod, timeout time.Duration) error return fmt.Errorf("failed to create pod %q (ns %s): %w", pod.Name, pod.Namespace, err) } + // Loop through pod conditions to check if pod is schedulable + // If it is not schedulable, we return an error + podSchedulable := false + for _, condition := range createdPod.Status.Conditions { + if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionTrue { + podSchedulable = true + break + } + } + + if !podSchedulable { + return fmt.Errorf("pod %s is not schedulable", createdPod.Name) + } + Eventually(func() bool { status, err := isPodReady(createdPod.Namespace, createdPod.Name) if err != nil { diff --git a/tests/performance/tests/exclusive_cpu_pools.go b/tests/performance/tests/exclusive_cpu_pools.go index 641671d3d..4f93b3d85 100644 --- a/tests/performance/tests/exclusive_cpu_pools.go +++ b/tests/performance/tests/exclusive_cpu_pools.go @@ -2,6 +2,7 @@ package tests import ( "runtime" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -56,6 +57,9 @@ var _ = Describe("performance-exclusive-cpu-pool", func() { globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels) err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Start exclusive-cpu-pool test") @@ -70,7 +74,6 @@ var _ = Describe("performance-exclusive-cpu-pool", func() { }) It("One pod with one exclusive container, and one shared container", func() { - By("Define pod") testPod := tshelper.DefineExclusivePod(tsparams.TestPodName, randomNamespace, globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels) @@ -78,6 +81,9 @@ var _ = Describe("performance-exclusive-cpu-pool", func() { tshelper.RedefinePodWithSharedContainer(testPod, 0) err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Start exclusive-cpu-pool test") @@ -92,7 +98,6 @@ var _ = Describe("performance-exclusive-cpu-pool", func() { }) It("One pod with only shared containers", func() { - By("Define pod") testPod := tshelper.DefineExclusivePod(tsparams.TestPodName, randomNamespace, globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels) @@ -100,6 +105,9 @@ var _ = Describe("performance-exclusive-cpu-pool", func() { pod.RedefineWithCPUResources(testPod, "0.75", "0.5") err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Start exclusive-cpu-pool test") diff --git a/tests/performance/tests/rt_app_no_exec_probes.go b/tests/performance/tests/rt_app_no_exec_probes.go index d7ae6ecb2..fb428ad96 100644 --- a/tests/performance/tests/rt_app_no_exec_probes.go +++ b/tests/performance/tests/rt_app_no_exec_probes.go @@ -1,6 +1,8 @@ package tests import ( + "strings" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/redhat-best-practices-for-k8s/certsuite-qe/tests/globalhelper" @@ -46,6 +48,9 @@ var _ = Describe("performance-rt-apps-no-exec-probes", func() { tsparams.RtImageName, tsparams.CertsuiteTargetPodLabels) err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, 2*tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) command := "chrt -f -p 50 1" // To change the scheduling policy of the container start process to FIFO scheduling @@ -71,6 +76,9 @@ var _ = Describe("performance-rt-apps-no-exec-probes", func() { pod.RedefineWithLivenessProbe(testPod) err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, 2*tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) command := "chrt -f -p 50 1" // To change the scheduling policy of the container start process to FIFO scheduling @@ -89,7 +97,6 @@ var _ = Describe("performance-rt-apps-no-exec-probes", func() { }) It("One non-Rt exclusive pod with no exec probes", func() { - By("Define pod") testPod := pod.DefinePod(tsparams.TestPodName, randomNamespace, globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels) @@ -99,6 +106,9 @@ var _ = Describe("performance-rt-apps-no-exec-probes", func() { By("Create and wait until pod is ready") err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Assert pod has modified resources") diff --git a/tests/performance/tests/rt_exclusive_cpu_pool_scheduling_policy.go b/tests/performance/tests/rt_exclusive_cpu_pool_scheduling_policy.go index b1a334dd9..233e316ef 100644 --- a/tests/performance/tests/rt_exclusive_cpu_pool_scheduling_policy.go +++ b/tests/performance/tests/rt_exclusive_cpu_pool_scheduling_policy.go @@ -1,6 +1,8 @@ package tests import ( + "strings" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/redhat-best-practices-for-k8s/certsuite-qe/tests/globalhelper" @@ -45,6 +47,9 @@ var _ = Describe("performance-exclusive-cpu-pool-rt-scheduling-policy", func() { tsparams.RtImageName, tsparams.CertsuiteTargetPodLabels) err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Start exclusive-cpu-pool-rt-scheduling-policy test") @@ -67,6 +72,9 @@ var _ = Describe("performance-exclusive-cpu-pool-rt-scheduling-policy", func() { tsparams.RtImageName, tsparams.CertsuiteTargetPodLabels) err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Change to rt scheduling policy") @@ -94,6 +102,9 @@ var _ = Describe("performance-exclusive-cpu-pool-rt-scheduling-policy", func() { tsparams.RtImageName, tsparams.CertsuiteTargetPodLabels) err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Change to rt scheduling policy") @@ -120,6 +131,9 @@ var _ = Describe("performance-exclusive-cpu-pool-rt-scheduling-policy", func() { globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels) err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Start exclusive-cpu-pool-rt-scheduling-policy test") diff --git a/tests/performance/tests/rt_isolated_cpu_pool.go b/tests/performance/tests/rt_isolated_cpu_pool.go index 991f77520..3f6a03d1b 100644 --- a/tests/performance/tests/rt_isolated_cpu_pool.go +++ b/tests/performance/tests/rt_isolated_cpu_pool.go @@ -1,6 +1,8 @@ package tests import ( + "strings" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/redhat-best-practices-for-k8s/certsuite-qe/tests/globalhelper" @@ -61,6 +63,9 @@ var _ = Describe("performance-isolated-cpu-pool-rt-scheduling-policy", Serial, f Expect(err).To(BeNil()) err = globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Change to rt scheduling policy") @@ -101,7 +106,10 @@ var _ = Describe("performance-isolated-cpu-pool-rt-scheduling-policy", Serial, f Expect(err).To(BeNil()) err = globalhelper.CreateAndWaitUntilPodIsReady(testPod, 2*tsparams.WaitingTime) - Expect(err).NotTo(HaveOccurred()) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } + Expect(err).ToNot(HaveOccurred()) By("Start isolated-cpu-pool-rt-scheduling-policy test") err = globalhelper.LaunchTests( @@ -122,6 +130,9 @@ var _ = Describe("performance-isolated-cpu-pool-rt-scheduling-policy", Serial, f globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels) err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Start isolated-cpu-pool-rt-scheduling-policy test") diff --git a/tests/performance/tests/shared-cpu-pool-non-rt-scheduling-policy.go b/tests/performance/tests/shared-cpu-pool-non-rt-scheduling-policy.go index e0be83fee..d49668583 100644 --- a/tests/performance/tests/shared-cpu-pool-non-rt-scheduling-policy.go +++ b/tests/performance/tests/shared-cpu-pool-non-rt-scheduling-policy.go @@ -1,6 +1,8 @@ package tests import ( + "strings" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/redhat-best-practices-for-k8s/certsuite-qe/tests/globalhelper" @@ -40,12 +42,14 @@ var _ = Describe("performance-shared-cpu-pool-non-rt-scheduling-policy", func() }) It("One pod with container running in shared cpu pool", func() { - By("Define pod") testPod := pod.DefinePod(tsparams.TestPodName, randomNamespace, globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels) err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Start shared-cpu-pool-non-rt-scheduling-policy test") @@ -62,7 +66,6 @@ var _ = Describe("performance-shared-cpu-pool-non-rt-scheduling-policy", func() }) It("One pod with container running in exclusive cpu pool", func() { - By("Define pod") testPod := pod.DefinePod(tsparams.TestPodName, randomNamespace, globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels) @@ -71,6 +74,9 @@ var _ = Describe("performance-shared-cpu-pool-non-rt-scheduling-policy", func() pod.RedefineWithMemoryResources(testPod, "512Mi", "512Mi") err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime) + if strings.Contains(err.Error(), "not schedulable") { + Skip("This test cannot run because the pod is not schedulable due to insufficient resources") + } Expect(err).ToNot(HaveOccurred()) By("Start shared-cpu-pool-non-rt-scheduling-policy test")