Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix glooctl check test flake #10615

Merged
merged 5 commits into from
Feb 14, 2025
Merged

Fix glooctl check test flake #10615

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e94e36f
fix, add obvious comments
arianaw66 Feb 14, 2025
2bf74f2
improve glooctl check output
arianaw66 Feb 14, 2025
288cb86
changelog
arianaw66 Feb 14, 2025
f3b558b
handle output types
arianaw66 Feb 14, 2025
4757ecc
oops
arianaw66 Feb 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions changelog/v1.19.0-beta8/fix_glooctl-check_test-flake.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
changelog:
- type: NON_USER_FACING
issueLink: https://github.com/solo-io/solo-projects/issues/6619
description: fixes flaky enterprise glooctl check tests
resolvesIssue: false
7 changes: 7 additions & 0 deletions projects/gloo/cli/pkg/cmd/check/gloo_stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ func ResourcesSyncedOverXds(stats, deploymentName string) bool {
fmt.Println(resourcesOutOfSyncMessage(outOfSyncResources))
return false
}

if len(metrics) == 0 {
fmt.Println("No xds metrics to check")
} else {
fmt.Println("OK")
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will break glooctl output types right? I think we do that in the out of sync but this will make it always escape the output formats

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good call. How about now?

return true
}

Expand All @@ -69,6 +75,7 @@ func RateLimitIsConnected(stats string) bool {
}

func checkXdsMetrics(ctx context.Context, printer printers.P, opts *options.Options, deployments *appsv1.DeploymentList) error {
printer.AppendCheck("Checking xds metrics... ")
errMessage := "Problem while checking for gloo xds errors"
if deployments == nil {
fmt.Println("Skipping due to an error in checking deployments")
Expand Down
36 changes: 21 additions & 15 deletions test/kubernetes/e2e/features/glooctl/check_suite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// TODO(nfuden): Fix clusterloadassignment issues that forced xds-metrics to be excluded.
// Consider https://github.com/envoyproxy/envoy/issues/7529#issuecomment-1227724217

var _ e2e.NewSuiteFunc = NewCheckSuite

// checkSuite contains the set of tests to validate the behavior of `glooctl check`
Expand All @@ -23,9 +26,7 @@ type checkSuite struct {
testInstallation *e2e.TestInstallation
}

// NewChecksuite for glooctl check validation
// TODO(nfuden): Fix clusterloadassignment issues that forced xds-metrics to be excluded.
// Consider https://github.com/envoyproxy/envoy/issues/7529#issuecomment-1227724217
// NewCheckSuite for glooctl check validation
func NewCheckSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite {
return &checkSuite{
ctx: ctx,
Expand All @@ -34,6 +35,7 @@ func NewCheckSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.Te
}

func (s *checkSuite) TestCheck() {
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics")
s.NoError(err)
Expand All @@ -50,6 +52,7 @@ func (s *checkSuite) TestCheck() {
}

func (s *checkSuite) TestCheckExclude() {
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
for excludeKey, expectedOutput := range checkCommonGlooGatewayOutputByKey {
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", fmt.Sprintf("xds-metrics,%s", excludeKey))
Expand All @@ -68,6 +71,7 @@ func (s *checkSuite) TestCheckExclude() {
}

func (s *checkSuite) TestCheckReadOnly() {
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace, "--read-only", "-x", "xds-metrics")
s.NoError(err)
Expand All @@ -92,11 +96,12 @@ func (s *checkSuite) TestCheckReadOnly() {
func (s *checkSuite) TestCheckKubeContext() {
// When passing an invalid kube-context, `glooctl check` should succeed
_, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", "invalid-context")
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", "invalid-context", "-x", "xds-metrics")
s.Error(err)
s.Contains(err.Error(), "Could not get kubernetes client: Error retrieving Kubernetes configuration: context \"invalid-context\" does not exist")

// When passing the kube-context of the running cluster, `glooctl check` should succeed
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
s.NoError(err)
Expand All @@ -110,7 +115,7 @@ func (s *checkSuite) TestCheckTimeout() {
s.NoError(err)

_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace,
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
"-c", shortTimeoutValues.Name())
s.Error(err)
s.Contains(err.Error(), "context deadline exceeded")
Expand All @@ -121,38 +126,36 @@ func (s *checkSuite) TestCheckTimeout() {
_, err = normalTimeoutValues.Write([]byte(`checkTimeoutSeconds: 300s`))
s.NoError(err)

// Note: This test does not use `"-x", "xds-metrics"`, so it will require the xds-metrics check to pass with no errors
// In gloo-ee the prometheus metrics must be enabled for the check to pass
// exclude xds-metrics due to Envoy bug: see TODO at top of file
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace,
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
"-c", normalTimeoutValues.Name())
s.NoError(err)
}

func (s *checkSuite) TestCheckNamespace() {
// namespace does not exist
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "not-gloo-system")
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "not-gloo-system", "-x", "xds-metrics")
s.Error(err)
s.Contains(output, "Could not communicate with kubernetes cluster: namespaces \"not-gloo-system\" not found")

// gloo not in namespace
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "default")
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "default", "-x", "xds-metrics")
s.Error(err)
s.Contains(output, "Warning: The provided label selector (gloo) applies to no pods")

// pod does not exist
// Note: This test does not use `"-x", "xds-metrics"`, so it will require the xds-metrics check to pass with no errors
// In gloo-ee the prometheus metrics must be enabled for the check to pass
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace,
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
"-p", "not-gloo")
s.NoError(err)
s.Contains(output, "Warning: The provided label selector (not-gloo) applies to no pods")
s.Contains(output, "No problems detected.")

// resource namespace does not exist
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace,
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
"-r", "not-gloo-system")
s.Error(err)
s.Contains(output, fmt.Sprintf("No namespaces specified are currently being watched (defaulting to '%s' namespace)", s.testInstallation.Metadata.InstallNamespace))
Expand Down Expand Up @@ -192,6 +195,7 @@ func (s *checkSuite) TestNoGateways() {
Namespace: s.testInstallation.Metadata.InstallNamespace,
}, gomega.Equal(0))

// exclude xds-metrics check due to Envoy bug: see TODO at top of file
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
s.Error(err)
Expand Down Expand Up @@ -219,6 +223,7 @@ func (s *checkSuite) TestEdgeGatewayScaled() {
Namespace: s.testInstallation.Metadata.InstallNamespace,
}, gomega.Equal(0))

// exclude xds-metrics check due to Envoy bug: see TODO at top of file
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
s.NoError(err)
Expand Down Expand Up @@ -252,9 +257,10 @@ func (s *checkSuite) TestEdgeResourceError() {
s.NoError(err)

// Run check. This needs to run in eventually to get all errors reported
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
gomega.Eventually(func() error {
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext)
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
return err
}, time.Minute*2, time.Second*10).Should(gomega.SatisfyAll(
gomega.MatchError(gomega.ContainSubstring(fmt.Sprintf("* Found rejected virtual service by '%s': default reject-me-too (Reason: 2 errors occurred:", s.testInstallation.Metadata.InstallNamespace))),
Expand Down
Loading