e2e: add test case for specifying mac address when nad plugin is macv…

…lan (#4836) Signed-off-by: zhangzujian <[email protected]>
kubeovn · Dec 17, 2024 · 517f0e4 · 517f0e4
1 parent 9291e1f
commit 517f0e4
Show file tree

Hide file tree

Showing 26 changed files with 150 additions and 517 deletions.
diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko
@@ -1247,7 +1247,7 @@ spec:
     - name: $podName
       image: $imageID
       imagePullPolicy: IfNotPresent
-      command: ["sh", "-c", "sleep infinity"]
+      command: ["sleep", "infinity"]
   nodeSelector:
     kubernetes.io/hostname: $nodeID
 EOF
@@ -1271,7 +1271,7 @@ spec:
     - name: $podName
       image: $imageID
       imagePullPolicy: IfNotPresent
-      command: ["sh", "-c", "sleep infinity"]
+      command: ["sleep", "infinity"]
   nodeSelector:
     kubernetes.io/hostname: $nodeID
 EOF

diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go
@@ -7,6 +7,8 @@ import (
 	"strings"
 	"unicode"
 
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"github.com/ovn-org/libovsdb/ovsdb"
 	"github.com/scylladb/go-set/strset"
 	corev1 "k8s.io/api/core/v1"
@@ -963,7 +965,7 @@ func (c *Controller) getVMLsps() []string {
 			vmLsp := ovs.PodNameToPortName(vm.Name, ns.Name, util.OvnProvider)
 			vmLsps = append(vmLsps, vmLsp)
 
-			attachNets, err := util.ParsePodNetworkAnnotation(vm.Spec.Template.ObjectMeta.Annotations[util.AttachmentNetworkAnnotation], vm.Namespace)
+			attachNets, err := nadutils.ParseNetworkAnnotation(vm.Spec.Template.ObjectMeta.Annotations[nadv1.NetworkAttachmentAnnot], vm.Namespace)
 			if err != nil {
 				klog.Errorf("failed to get attachment subnet of vm %s, %v", vm.Name, err)
 				continue

diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go
@@ -13,6 +13,8 @@ import (
 	"sync"
 	"time"
 
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"github.com/scylladb/go-set/strset"
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
 	multustypes "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types"
@@ -1466,20 +1468,20 @@ type kubeovnNet struct {
 }
 
 func (c *Controller) getPodAttachmentNet(pod *v1.Pod) ([]*kubeovnNet, error) {
-	var multusNets []*multustypes.NetworkSelectionElement
+	var multusNets []*nadv1.NetworkSelectionElement
 	defaultAttachNetworks := pod.Annotations[util.DefaultNetworkAnnotation]
 	if defaultAttachNetworks != "" {
-		attachments, err := util.ParsePodNetworkAnnotation(defaultAttachNetworks, pod.Namespace)
+		attachments, err := nadutils.ParseNetworkAnnotation(defaultAttachNetworks, pod.Namespace)
 		if err != nil {
 			klog.Errorf("failed to parse default attach net for pod '%s', %v", pod.Name, err)
 			return nil, err
 		}
 		multusNets = attachments
 	}
 
-	attachNetworks := pod.Annotations[util.AttachmentNetworkAnnotation]
+	attachNetworks := pod.Annotations[nadv1.NetworkAttachmentAnnot]
 	if attachNetworks != "" {
-		attachments, err := util.ParsePodNetworkAnnotation(attachNetworks, pod.Namespace)
+		attachments, err := nadutils.ParseNetworkAnnotation(attachNetworks, pod.Namespace)
 		if err != nil {
 			klog.Errorf("failed to parse attach net for pod '%s', %v", pod.Name, err)
 			return nil, err

diff --git a/pkg/controller/service_lb.go b/pkg/controller/service_lb.go
@@ -83,8 +83,8 @@ func (c *Controller) genLbSvcDeployment(svc *corev1.Service, nad *nadv1.NetworkA
 	attachSubnetAnnotation := fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, providerName)
 	attachIPAnnotation := fmt.Sprintf(util.IPAddressAnnotationTemplate, providerName)
 	podAnnotations := map[string]string{
-		util.AttachmentNetworkAnnotation: fmt.Sprintf("%s/%s", attachmentNs, attachmentName),
-		attachSubnetAnnotation:           svc.Annotations[attachSubnetAnnotation],
+		nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", attachmentNs, attachmentName),
+		attachSubnetAnnotation:       svc.Annotations[attachSubnetAnnotation],
 	}
 	if svc.Spec.LoadBalancerIP != "" {
 		podAnnotations[attachIPAnnotation] = svc.Spec.LoadBalancerIP
@@ -146,8 +146,8 @@ func (c *Controller) updateLbSvcDeployment(svc *corev1.Service, dp *v1.Deploymen
 	attachSubnetAnnotation := fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, providerName)
 	attachIPAnnotation := fmt.Sprintf(util.IPAddressAnnotationTemplate, providerName)
 	podAnnotations := map[string]string{
-		util.AttachmentNetworkAnnotation: fmt.Sprintf("%s/%s", attachmentNs, attachmentName),
-		attachSubnetAnnotation:           svc.Annotations[attachSubnetAnnotation],
+		nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", attachmentNs, attachmentName),
+		attachSubnetAnnotation:       svc.Annotations[attachSubnetAnnotation],
 	}
 	if svc.Spec.LoadBalancerIP != "" {
 		podAnnotations[attachIPAnnotation] = svc.Spec.LoadBalancerIP

diff --git a/pkg/controller/vpc_dns.go b/pkg/controller/vpc_dns.go
@@ -13,6 +13,7 @@ import (
 	"strings"
 	"text/template"
 
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	v1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -371,7 +372,7 @@ func (c *Controller) genVpcDNSSlr(vpcName, namespace string) (*kubeovnv1.SwitchL
 func setVpcDNSInterface(dp *v1.Deployment, subnetName string, defaultSubnet *kubeovnv1.Subnet) {
 	annotations := dp.Spec.Template.Annotations
 	annotations[util.LogicalSwitchAnnotation] = subnetName
-	annotations[util.AttachmentNetworkAnnotation] = fmt.Sprintf("%s/%s", corev1.NamespaceDefault, nadName)
+	annotations[nadv1.NetworkAttachmentAnnot] = fmt.Sprintf("%s/%s", corev1.NamespaceDefault, nadName)
 	annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, nadProvider)] = util.DefaultSubnet
 
 	setVpcDNSRoute(dp, defaultSubnet.Spec.Gateway)

diff --git a/pkg/controller/vpc_egress_gateway.go b/pkg/controller/vpc_egress_gateway.go
@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -314,7 +315,7 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa
 		klog.Error(err)
 		return attachmentNetworkName, nil, nil, nil, err
 	}
-	annotations[util.AttachmentNetworkAnnotation] = attachmentNetworkName
+	annotations[nadv1.NetworkAttachmentAnnot] = attachmentNetworkName
 	annotations[util.LogicalSwitchAnnotation] = intSubnet.Name
 	if len(gw.Spec.InternalIPs) != 0 {
 		// set internal IPs

diff --git a/pkg/controller/vpc_lb.go b/pkg/controller/vpc_lb.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	v1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -103,9 +104,9 @@ func (c *Controller) genVpcLbDeployment(vpc *kubeovnv1.Vpc) (*v1.Deployment, err
 	}
 
 	podAnnotations := map[string]string{
-		util.VpcAnnotation:               vpc.Name,
-		util.LogicalSwitchAnnotation:     defaultSubnet.Name,
-		util.AttachmentNetworkAnnotation: fmt.Sprintf(`[{"name": "%s", "default-route": ["%s"]}]`, util.VpcLbNetworkAttachment, strings.ReplaceAll(gateway, ",", `" ,"`)),
+		util.VpcAnnotation:           vpc.Name,
+		util.LogicalSwitchAnnotation: defaultSubnet.Name,
+		nadv1.NetworkAttachmentAnnot: fmt.Sprintf(`[{"name": "%s", "default-route": ["%s"]}]`, util.VpcLbNetworkAttachment, strings.ReplaceAll(gateway, ",", `" ,"`)),
 	}
 
 	deployment := &v1.Deployment{

diff --git a/pkg/controller/vpc_nat_gateway.go b/pkg/controller/vpc_nat_gateway.go
@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	v1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -696,7 +697,7 @@ func (c *Controller) setNatGwAPIAccess(annotations map[string]string, externalNe
 
 	// Attach the NADs to the Pod by adding them to the special annotation
 	attachmentAnnotation := fmt.Sprintf("%s, %s", externalNetworkAttachment, apiNetworkAttachment)
-	annotations[util.AttachmentNetworkAnnotation] = attachmentAnnotation
+	annotations[nadv1.NetworkAttachmentAnnot] = attachmentAnnotation
 
 	// Set the network route to the API, so we can reach it
 	return c.setNatGwAPIRoute(annotations, namespace, name)
@@ -752,10 +753,10 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 
 	externalNetworkNad := util.GetNatGwExternalNetwork(gw.Spec.ExternalSubnets)
 	podAnnotations := map[string]string{
-		util.VpcNatGatewayAnnotation:     gw.Name,
-		util.AttachmentNetworkAnnotation: fmt.Sprintf("%s/%s", c.config.PodNamespace, externalNetworkNad),
-		util.LogicalSwitchAnnotation:     gw.Spec.Subnet,
-		util.IPAddressAnnotation:         gw.Spec.LanIP,
+		util.VpcNatGatewayAnnotation: gw.Name,
+		nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", c.config.PodNamespace, externalNetworkNad),
+		util.LogicalSwitchAnnotation: gw.Spec.Subnet,
+		util.IPAddressAnnotation:     gw.Spec.LanIP,
 	}
 
 	// Add an interface that can reach the API server, we need access to it to probe Kube-OVN resources

diff --git a/pkg/daemon/controller.go b/pkg/daemon/controller.go
@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"time"
 
+	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"github.com/scylladb/go-set/strset"
 	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -497,7 +498,7 @@ func (c *Controller) enqueuePod(oldObj, newObj interface{}) {
 		c.podQueue.Add(key)
 	}
 
-	attachNets, err := util.ParsePodNetworkAnnotation(newPod.Annotations[util.AttachmentNetworkAnnotation], newPod.Namespace)
+	attachNets, err := nadutils.ParsePodNetworkAnnotation(newPod)
 	if err != nil {
 		return
 	}

diff --git a/pkg/daemon/controller_linux.go b/pkg/daemon/controller_linux.go
@@ -12,6 +12,7 @@ import (
 	"strings"
 	"syscall"
 
+	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"github.com/kubeovn/felix/ipsets"
 	"github.com/kubeovn/go-iptables/iptables"
 	"github.com/vishvananda/netlink"
@@ -574,7 +575,7 @@ func (c *Controller) handlePod(key string) error {
 	}
 
 	// set multus-nic bandwidth
-	attachNets, err := util.ParsePodNetworkAnnotation(pod.Annotations[util.AttachmentNetworkAnnotation], pod.Namespace)
+	attachNets, err := nadutils.ParsePodNetworkAnnotation(pod)
 	if err != nil {
 		klog.Error(err)
 		return err

diff --git a/pkg/daemon/controller_windows.go b/pkg/daemon/controller_windows.go
@@ -5,6 +5,7 @@ import (
 	"net"
 	"strings"
 
+	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	"github.com/scylladb/go-set/strset"
 	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -178,7 +179,7 @@ func (c *Controller) handlePod(key string) error {
 	}
 
 	// set multus-nic bandwidth
-	attachNets, err := util.ParsePodNetworkAnnotation(pod.Annotations[util.AttachmentNetworkAnnotation], pod.Namespace)
+	attachNets, err := nadutils.ParsePodNetworkAnnotation(pod)
 	if err != nil {
 		klog.Error(err)
 		return err

diff --git a/pkg/util/const.go b/pkg/util/const.go
@@ -171,7 +171,6 @@ const (
 	TCPIPHeaderLength  = 40
 
 	OvnProvider                         = "ovn"
-	AttachmentNetworkAnnotation         = "k8s.v1.cni.cncf.io/networks"
 	DefaultNetworkAnnotation            = "v1.multus-cni.io/default-network"
 	AttachNetworkResourceNameAnnotation = "k8s.v1.cni.cncf.io/resourceName"
 

diff --git a/pkg/util/k8s.go b/pkg/util/k8s.go
@@ -11,7 +11,7 @@ import (
 	"strings"
 	"time"
 
-	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -98,16 +98,15 @@ func PodAttachmentIPs(pod *v1.Pod, networkName string) ([]string, error) {
 		return nil, errors.New("programmatic error: pod is nil")
 	}
 
-	if pod.Annotations[nadv1.NetworkStatusAnnot] == "" {
-		return nil, fmt.Errorf("pod %s/%s has no network status annotation", pod.Namespace, pod.Name)
-	}
-	var status []nadv1.NetworkStatus
-	if err := json.Unmarshal([]byte(pod.Annotations[nadv1.NetworkStatusAnnot]), &status); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal network status annotation of pod %s/%s: %w", pod.Namespace, pod.Name, err)
+	statuses, err := nadutils.GetNetworkStatus(pod)
+	if err != nil {
+		klog.Error(err)
+		return nil, fmt.Errorf("failed to get network status for pod %s/%s: %w", pod.Namespace, pod.Name, err)
 	}
-	for _, s := range status {
-		if s.Name == networkName {
-			return s.IPs, nil
+
+	for _, status := range statuses {
+		if status.Name == networkName {
+			return status.IPs, nil
 		}
 	}