e2e test of managed service deployment

test/e2e/clusterdeployment/constants.go

@@ -24,6 +24,9 @@ const (
 	// debugging of test failures.
 	EnvVarNoCleanup = "NO_CLEANUP"
 
+	EnvVarServiceNamespace = "INGRESS_SERVICE_NAMESPACE"
+	EnvVarServiceName      = "INGRESS_SERVICE_NAME"
+
 	// AWS
 	EnvVarAWSAccessKeyID            = "AWS_ACCESS_KEY_ID"
 	EnvVarAWSSecretAccessKey        = "AWS_SECRET_ACCESS_KEY"

test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl

@@ -17,3 +17,8 @@ spec:
       instanceType: ${AWS_INSTANCE_TYPE:=t3.small}
     worker:
       instanceType: ${AWS_INSTANCE_TYPE:=t3.small}
+  serviceSpec:
+    services:
+      - template: ingress-nginx-4-11-0
+        name: ${INGRESS_SERVICE_NAME}
+        namespace: ${INGRESS_SERVICE_NAMESPACE}

test/e2e/clusterdeployment/servicevalidator.go

@@ -0,0 +1,85 @@
+// Copyright 2024
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clusterdeployment
+
+import (
+	"context"
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+
+	"github.com/K0rdent/kcm/test/e2e/kubeclient"
+)
+
+type ManagedServiceResource struct {
+	// ResourceNameSuffix is the suffix added to the resource name
+	ResourceNameSuffix string
+
+	// ValidationFunc is the validation function for the resource
+	ValidationFunc resourceValidationFunc
+}
+
+func (m ManagedServiceResource) resourceFullName(serviceName string) string {
+	if m.ResourceNameSuffix == "" {
+		return serviceName
+	}
+	return fmt.Sprintf("%s-%s", serviceName, m.ResourceNameSuffix)
+}
+
+type ServiceValidator struct {
+	// managedClusterName is the name of managed cluster
+	managedClusterName string
+
+	// managedServiceName is the name of managed service
+	managedServiceName string
+
+	// template being validated
+	template Template
+
+	// namespace is a namespace of deployed service
+	namespace string
+
+	// resourcesToValidate is a map of resource names and corresponding resources definitions
+	resourcesToValidate map[string]ManagedServiceResource
+}
+
+func NewServiceValidator(clusterName, serviceName, namespace string) *ServiceValidator {
+	return &ServiceValidator{
+		managedClusterName:  clusterName,
+		managedServiceName:  serviceName,
+		namespace:           namespace,
+		resourcesToValidate: make(map[string]ManagedServiceResource),
+	}
+}
+
+func (v *ServiceValidator) WithResourceValidation(resourceName string, resource ManagedServiceResource) *ServiceValidator {
+	v.resourcesToValidate[resourceName] = resource
+	return v
+}
+
+func (v *ServiceValidator) Validate(ctx context.Context, kc *kubeclient.KubeClient) error {
+	clusterKubeClient := kc.NewFromCluster(ctx, v.namespace, v.managedClusterName)
+
+	for resourceName, resource := range v.resourcesToValidate {
+		resourceFullName := resource.resourceFullName(v.managedServiceName)
+		err := resource.ValidationFunc(ctx, clusterKubeClient, resourceFullName)
+		if err != nil {
+			_, _ = fmt.Fprintf(GinkgoWriter, "[%s/%s] validation error: %v\n", v.template, resourceName, err)
+			return err
+		}
+		_, _ = fmt.Fprintf(GinkgoWriter, "[%s/%s] validation succeeded\n", v.template, resourceName)
+	}
+	return nil
+}

test/e2e/clusterdeployment/validate_deployed.go

@@ -273,3 +273,22 @@ func validateCCM(ctx context.Context, kc *kubeclient.KubeClient, clusterName str
 
 	return fmt.Errorf("%s Service does not yet have an external hostname", service.Name)
 }
+
+func ValidateService(ctx context.Context, kc *kubeclient.KubeClient, name string) error {
+	_, err := kc.Client.CoreV1().Services(kc.Namespace).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func ValidateDeployment(ctx context.Context, kc *kubeclient.KubeClient, name string) error {
+	dep, err := kc.Client.AppsV1().Deployments(kc.Namespace).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if *dep.Spec.Replicas != dep.Status.ReadyReplicas {
+		return fmt.Errorf("deployment %s has %d ready replicas, expected %d", name, dep.Status.ReadyReplicas, *dep.Spec.Replicas)
+	}
+	return nil
+}

test/e2e/provider_aws_test.go

@@ -77,6 +77,8 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order
 		// Deploy standalone with an xlarge instance since it will also be
 		// hosting the hosted cluster.
 		GinkgoT().Setenv(clusterdeployment.EnvVarAWSInstanceType, "t3.xlarge")
+		GinkgoT().Setenv(clusterdeployment.EnvVarServiceNamespace, "default")
+		GinkgoT().Setenv(clusterdeployment.EnvVarServiceName, "managed-ingress-nginx")
 
 		templateBy(clusterdeployment.TemplateAWSStandaloneCP, "creating a ClusterDeployment")
 		sd := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAWSStandaloneCP)
@@ -95,6 +97,20 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order
 			return deploymentValidator.Validate(context.Background(), kc)
 		}).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
 
+		// validating service included in the cluster deployment is deployed
+		serviceDeployedValidator := clusterdeployment.NewServiceValidator(clusterName, "default", "managed-ingress-nginx").
+			WithResourceValidation("service", clusterdeployment.ManagedServiceResource{
+				ResourceNameSuffix: "",
+				ValidationFunc:     clusterdeployment.ValidateService,
+			}).
+			WithResourceValidation("deployment", clusterdeployment.ManagedServiceResource{
+				ResourceNameSuffix: "",
+				ValidationFunc:     clusterdeployment.ValidateDeployment,
+			})
+		Eventually(func() error {
+			return serviceDeployedValidator.Validate(context.Background(), kc)
+		}).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
 		templateBy(clusterdeployment.TemplateAWSHostedCP, "installing controller and templates on standalone cluster")
 
 		// Download the KUBECONFIG for the standalone cluster and load it