[e2e] Add test cases for pre-created VPC (#933)

This change is to add a new end-to-end test for the pre-created VPC, it includes
these steps,
1. Create a VPC/LBS/Attachment on NSX
2. Create a Namespace on vCenter or K8s, which depends on if the setup is a wcp
testbed or not. Note, it planned to run the test on wcp testbed, please provide
a valid pair of vc-user and vc-password which has the permission to create
instance on vCenter
3. Create LoadBalancer typed Service inside the Namespace
4. Create server Pod for the LoadBalancer typed Service
5. Create client Pod in the same Namespace
6. Verified the traffic from the client Pod to the LoadBalancer's external IP
7. Delete LoadBalancer typed Service and verified that the realted NSX resources
are removed as expected
8. Delete the Namespace from vCenter or K8s
9. Delete the VPC from NSX

go.mod

@@ -65,13 +65,16 @@ require (
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/spdystream v0.4.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect

go.sum

@@ -3,6 +3,8 @@ github.com/agiledragon/gomonkey/v2 v2.11.0 h1:5oxSgA+tC1xuGsrIorR+sYiziYltmJyEZ9
 github.com/agiledragon/gomonkey/v2 v2.11.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY=
 github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU=
 github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -68,6 +70,8 @@ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -89,13 +93,17 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8=
+github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA=
 github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
 github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=

test/e2e/framework.go

@@ -15,14 +15,19 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-
-	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
+	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/utils/ptr"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/client/clientset/versioned"
+	"github.com/vmware-tanzu/nsx-operator/pkg/config"
+	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 	"github.com/vmware-tanzu/nsx-operator/test/e2e/providers"
@@ -81,6 +86,7 @@ type TestData struct {
 	clientset    clientset.Interface
 	crdClientset versioned.Interface
 	nsxClient    *NSXClient
+	vcClient     *vcClient
 }
 
 var testData *TestData
@@ -101,21 +107,29 @@ func initProvider() error {
 	return nil
 }
 
-func NewTestData(nsxConfig string) error {
+func NewTestData(nsxConfig string, vcUser string, vcPassword string) error {
 	testData = &TestData{}
 	err := testData.createClients()
 	if err != nil {
 		return err
 	}
-	err = testData.createNSXClients(nsxConfig)
+	config.UpdateConfigFilePath(nsxConfig)
+	cf, err := config.NewNSXOperatorConfigFromFile()
+	if err != nil {
+		return err
+	}
+	err = testData.createNSXClients(cf)
 	if err != nil {
 		return err
 	}
+	if vcUser != "" && vcPassword != "" {
+		testData.vcClient = newVcClient(cf.VCEndPoint, cf.HttpsPort, vcUser, vcPassword)
+	}
 	return nil
 }
 
-func (data *TestData) createNSXClients(nsxConfig string) error {
-	nsxClient, err := NewNSXClient(nsxConfig)
+func (data *TestData) createNSXClients(cf *config.NSXOperatorConfig) error {
+	nsxClient, err := NewNSXClient(cf)
 	if err != nil {
 		return err
 	}
@@ -257,12 +271,15 @@ func collectClusterInfo() error {
 }
 
 // createNamespace creates the provided namespace.
-func (data *TestData) createNamespace(namespace string) error {
+func (data *TestData) createNamespace(namespace string, mutators ...func(ns *corev1.Namespace)) error {
 	ns := corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: namespace,
 		},
 	}
+	for _, mutator := range mutators {
+		mutator(&ns)
+	}
 	if ns, err := data.clientset.CoreV1().Namespaces().Create(context.TODO(), &ns, metav1.CreateOptions{}); err != nil {
 		// Ignore error if the namespace already exists
 		if !errors.IsAlreadyExists(err) {
@@ -334,8 +351,12 @@ func (data *TestData) deploymentWaitForNames(timeout time.Duration, namespace, d
 	return podNames, nil
 }
 
-// Temporarily disable traffic check
-/*
+type PodIPs struct {
+	ipv4      *net.IP
+	ipv6      *net.IP
+	ipStrings []string
+}
+
 // podWaitFor polls the K8s apiServer until the specified Pod is found (in the test Namespace) and
 // the condition predicate is met (or until the provided timeout expires).
 func (data *TestData) podWaitFor(timeout time.Duration, name, namespace string, condition PodCondition) (*corev1.Pod, error) {
@@ -364,6 +385,7 @@ func (data *TestData) podWaitForIPs(timeout time.Duration, name, namespace strin
 		return pod.Status.Phase == corev1.PodRunning, nil
 	})
 	if err != nil {
+		log.Error(err, "Failed to wait for Pod becoming RUNNING phase", "Pod", name)
 		return nil, err
 	}
 	// According to the K8s API documentation (https://godoc.org/k8s.io/api/core/v1#PodStatus),
@@ -381,6 +403,7 @@ func (data *TestData) podWaitForIPs(timeout time.Duration, name, namespace strin
 	}
 	ips, err := parsePodIPs(podIPStrings)
 	if err != nil {
+		log.Error(err, "Failed to parse Pod's IP", "Pod", name)
 		return nil, err
 	}
 
@@ -399,6 +422,7 @@ func (data *TestData) podWaitForIPs(timeout time.Duration, name, namespace strin
 	return ips, nil
 }
 
+/*
 // deploymentWaitForIPsOrNames polls the K8s apiServer until the specified Pod in deployment has an IP address
 func (data *TestData) deploymentWaitForIPsOrNames(timeout time.Duration, namespace, deployment string) ([]string, []string, error) {
 	podIPStrings := sets.NewString()
@@ -431,6 +455,7 @@ func (data *TestData) deploymentWaitForIPsOrNames(timeout time.Duration, namespa
 	}
 	return podIPStrings.List(), podNames, nil
 }
+*/
 
 func parsePodIPs(podIPStrings sets.Set[string]) (*PodIPs, error) {
 	ips := new(PodIPs)
@@ -464,7 +489,7 @@ func parsePodIPs(podIPStrings sets.Set[string]) (*PodIPs, error) {
 // stdout and stderr as strings. An error either indicates that the command couldn't be run or that
 // the command returned a non-zero error code.
 func (data *TestData) runCommandFromPod(namespace string, podName string, containerName string, cmd []string) (stdout string, stderr string, err error) {
-	log.Info("Running '%s' in Pod '%s/%s' container '%s'", strings.Join(cmd, " "), namespace, podName, containerName)
+	log.Info("Running command in Pod's container", "Namespace", namespace, "Pod", podName, "Container", containerName, "Command", cmd)
 	request := data.clientset.CoreV1().RESTClient().Post().
 		Namespace(namespace).
 		Resource("pods").
@@ -487,14 +512,15 @@ func (data *TestData) runCommandFromPod(namespace string, podName string, contai
 		Stdout: &stdoutB,
 		Stderr: &stderrB,
 	}); err != nil {
-		log.Info("Error when running command '%s' in Pod '%s/%s' container '%s': %v", strings.Join(cmd, " "), namespace, podName, containerName, err)
+		log.Error(err, "Failed to run command in Pod's container", "Namespace", namespace, "Pod", podName, "Container", containerName, "Command", cmd)
 		return stdoutB.String(), stderrB.String(), err
 	}
 	outStr, errStr := stdoutB.String(), stderrB.String()
-	log.Info("Command '%s' in Pod '%s/%s' container '%s' returned with output: '%s' and error: '%s'", strings.Join(cmd, " "), namespace, podName, containerName, outStr, errStr)
+	log.Info("Successfully run command in Pod's container", "Namespace", namespace, "Pod", podName, "Container", containerName, "Command", cmd, "stdOut", outStr, "stdErr", errStr)
 	return stdoutB.String(), stderrB.String(), nil
 }
 
+/*
 func (data *TestData) runPingCommandFromPod(namespace string, podName string, targetPodIPs *PodIPs, count int) error {
 	var cmd []string
 	if targetPodIPs.ipv4 != nil {
@@ -690,3 +716,102 @@ func (data *TestData) waitForResourceExistByPath(pathPolicy string, shouldExist
 	})
 	return err
 }
+
+func (data *TestData) createService(namespace, serviceName string, port, targetPort int32, protocol corev1.Protocol, selector map[string]string,
+	serviceType corev1.ServiceType, mutators ...func(service *corev1.Service)) (*corev1.Service, error) {
+	ipFamilies := []corev1.IPFamily{corev1.IPv4Protocol}
+
+	service := corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      serviceName,
+			Namespace: namespace,
+			Labels: map[string]string{
+				"nsx-op-e2e": serviceName,
+				"app":        serviceName,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			SessionAffinity: corev1.ServiceAffinityNone,
+			Ports: []corev1.ServicePort{{
+				Port:       port,
+				TargetPort: intstr.FromInt32(targetPort),
+				Protocol:   protocol,
+			}},
+			Type:       serviceType,
+			Selector:   selector,
+			IPFamilies: ipFamilies,
+		},
+	}
+	for _, mutator := range mutators {
+		mutator(&service)
+	}
+	return data.clientset.CoreV1().Services(namespace).Create(context.TODO(), &service, metav1.CreateOptions{})
+}
+
+func (data *TestData) createPod(namespace, podName, containerName, image string, protocol corev1.Protocol, containerPort int32,
+	mutators ...func(pod *corev1.Pod)) (*corev1.Pod, error) {
+	pod := corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        podName,
+			Namespace:   namespace,
+			Annotations: map[string]string{},
+			Labels: map[string]string{
+				"nsx-op-e2e": podName,
+				"app":        podName,
+			},
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:            containerName,
+					Image:           image,
+					ImagePullPolicy: corev1.PullIfNotPresent,
+					Ports: []corev1.ContainerPort{
+						{
+							Protocol:      protocol,
+							ContainerPort: containerPort,
+						},
+					},
+				},
+			},
+			RestartPolicy: corev1.RestartPolicyNever,
+			HostNetwork:   false,
+			// Set it to 1s for immediate shutdown to reduce test run time and to avoid affecting subsequent tests.
+			TerminationGracePeriodSeconds: ptr.To[int64](1),
+		},
+	}
+	for _, mutator := range mutators {
+		mutator(&pod)
+	}
+	return data.clientset.CoreV1().Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{})
+}
+
+func (data *TestData) serviceWaitFor(readyTime time.Duration, namespace string, name string, conditionFunc func(svc *corev1.Service) (bool, error)) (*corev1.Service, error) {
+	err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, readyTime, false, func(ctx context.Context) (bool, error) {
+		if svc, err := data.clientset.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{}); err != nil {
+			if errors.IsNotFound(err) {
+				return false, nil
+			}
+			return false, fmt.Errorf("error when getting Service '%s/%s': %v", namespace, name, err)
+		} else {
+			return conditionFunc(svc)
+		}
+	})
+	if err != nil {
+		return nil, err
+	}
+	return data.clientset.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+}
+
+func (data *TestData) deleteService(nsName string, svcName string) error {
+	ctx := context.TODO()
+	err := data.clientset.CoreV1().Services(nsName).Delete(ctx, svcName, metav1.DeleteOptions{})
+	if err != nil {
+		log.Error(err, "Failed to delete Service", "namespace", nsName, "name", svcName)
+	}
+	return err
+}
+
+func (data *TestData) useWCPSetup() bool {
+	return data.vcClient != nil
+}

test/e2e/main_test.go

@@ -39,7 +39,7 @@ func testMain(m *testing.M) int {
 
 	log.Info("Creating clientSets")
 
-	if err := NewTestData(testOptions.operatorConfigPath); err != nil {
+	if err := NewTestData(testOptions.operatorConfigPath, testOptions.vcUser, testOptions.vcPassword); err != nil {
 		log.Error(err, "Error when creating client")
 		return 1
 	}

test/e2e/nsxclient.go

@@ -14,13 +14,8 @@ type NSXClient struct {
 	*nsx.Client
 }
 
-func NewNSXClient(configFile string) (*NSXClient, error) {
+func NewNSXClient(cf *config.NSXOperatorConfig) (*NSXClient, error) {
 	// nsxClient is used to interact with NSX API.
-	config.UpdateConfigFilePath(configFile)
-	cf, err := config.NewNSXOperatorConfigFromFile()
-	if err != nil {
-		return nil, err
-	}
 	client := nsx.GetClient(cf)
 	if client == nil {
 		return nil, fmt.Errorf("failed to get nsx client")