From d10251fd741033c7983fab2e3e5ac60a11be62b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=A5=96=E5=BB=BA?= Date: Mon, 30 Dec 2024 14:41:43 +0800 Subject: [PATCH] ipam: use ip provided by nad annotation when providing IPAM for other CNI plugins (#4883) Signed-off-by: zhangzujian --- pkg/controller/pod.go | 10 +- test/e2e/framework/iproute/iproute.go | 10 ++ test/e2e/multus/e2e_test.go | 147 ++++++++++++++++++++++---- 3 files changed, 139 insertions(+), 28 deletions(-) diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 917f959c669..e2c7c664794 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -1546,13 +1546,9 @@ func (c *Controller) getPodAttachmentNet(pod *v1.Pod) ([]*kubeovnNet, error) { Subnet: subnet, IsDefault: isDefault, AllowLiveMigration: allowLiveMigration, + MacRequest: attach.MacRequest, + IPRequest: strings.Join(attach.IPRequest, ","), } - - if len(attach.IPRequest) != 0 { - ret.IPRequest = strings.Join(attach.IPRequest, ",") - } - - ret.MacRequest = attach.MacRequest result = append(result, ret) } else { providerName = fmt.Sprintf("%s.%s", attach.Name, attach.Namespace) @@ -1562,6 +1558,8 @@ func (c *Controller) getPodAttachmentNet(pod *v1.Pod) ([]*kubeovnNet, error) { Type: providerTypeIPAM, ProviderName: providerName, Subnet: subnet, + MacRequest: attach.MacRequest, + IPRequest: strings.Join(attach.IPRequest, ","), }) break } diff --git a/test/e2e/framework/iproute/iproute.go b/test/e2e/framework/iproute/iproute.go index ccab6c1097a..a62da2c8e1e 100644 --- a/test/e2e/framework/iproute/iproute.go +++ b/test/e2e/framework/iproute/iproute.go @@ -61,6 +61,16 @@ func (l *Link) NonLinkLocalAddresses() []string { return result } +func (l *Link) NonLinkLocalIPs() []string { + var result []string + for _, addr := range l.AddrInfo { + if !net.ParseIP(addr.Local).IsLinkLocalUnicast() { + result = append(result, addr.Local) + } + } + return result +} + type Route struct { Type string `json:"type"` Dst string `json:"dst"` diff --git a/test/e2e/multus/e2e_test.go b/test/e2e/multus/e2e_test.go index daf3065a859..85d4e2da2d6 100644 --- a/test/e2e/multus/e2e_test.go +++ b/test/e2e/multus/e2e_test.go @@ -304,8 +304,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() { gateway := pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, provider)] framework.ExpectIPInCIDR(ip, cidr) framework.ExpectIPInCIDR(gateway, cidr) - framework.ExpectNotHaveKey(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider)) - // framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac) + framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac) ipName := ovs.PodNameToPortName(podName, namespaceName, provider) ginkgo.By("Validating IP resource " + ipName) @@ -438,8 +437,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() { gateway := pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, provider)] framework.ExpectIPInCIDR(ip, cidr) framework.ExpectIPInCIDR(gateway, cidr) - framework.ExpectNotHaveKey(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider)) - // framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac) + framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac) ipName := ovs.PodNameToPortName(podName, namespaceName, provider) ginkgo.By("Validating IP resource " + ipName) @@ -506,6 +504,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() { } provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider) + ginkgo.By("Creating network attachment definition " + nadName) nad := framework.MakeOVNNetworkAttachmentDefinition(nadName, namespaceName, provider, nil) nad = nadClient.Create(nad) @@ -515,35 +514,139 @@ var _ = framework.SerialDescribe("[group:multus]", func() { subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", provider, nil, nil, nil) subnet = subnetClient.CreateSync(subnet) + ginkgo.By("Generating k8s.v1.cni.cncf.io/networks annotation") + mac := util.GenerateMac() + ips := strings.Split(framework.RandomIPs(subnet.Spec.CIDRBlock, "", 1), ",") + networks := []*nadv1.NetworkSelectionElement{{ + Name: nad.Name, + Namespace: nad.Namespace, + MacRequest: mac, + IPRequest: ips, + }} + networksAnnotation, err := json.Marshal(networks) + framework.ExpectNoError(err) + framework.Logf("networks annotation: %s", string(networksAnnotation)) + ginkgo.By("Creating pod " + podName) - mac := "00:00:00:11:22:33" - randomIP := framework.RandomIPs(subnet.Spec.CIDRBlock, "", 1) - - randomIPArray := strings.Split(randomIP, ",") - var requestIPString string - for i, ip := range randomIPArray { - if i == len(randomIPArray)-1 { - requestIPString += fmt.Sprintf(`"%s"`, ip) - } else { - requestIPString += fmt.Sprintf(`"%s",`, ip) + annotations := map[string]string{nadv1.NetworkAttachmentAnnot: string(networksAnnotation)} + cmd := []string{"sleep", "infinity"} + pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) + pod = podClient.CreateSync(pod) + + ginkgo.By("Validating pod annotations") + framework.ExpectHaveKey(pod.Annotations, nadv1.NetworkStatusAnnot) + framework.Logf("pod network status:\n%s", pod.Annotations[nadv1.NetworkStatusAnnot]) + framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac) + framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.IPAddressAnnotationTemplate, provider), strings.Join(ips, ",")) + + ginkgo.By("Getting attachment interface name") + statuses, err := nadutils.GetNetworkStatus(pod) + framework.ExpectNoError(err) + var ifaceName string + nadKey := cache.MetaObjectToName(nad).String() + for _, status := range statuses { + if status.Name == nadKey { + framework.ExpectEqual(status.Mac, mac) + ifaceName = status.Interface + break } } + framework.ExpectNotEmpty(ifaceName) + + ginkgo.By("Validating pod ip and mac") + links, err := iproute.AddressShow(ifaceName, func(cmd ...string) ([]byte, []byte, error) { + return framework.KubectlExec(pod.Namespace, pod.Name, cmd...) + }) + framework.ExpectNoError(err) + framework.ExpectHaveLen(links, 1) + framework.ExpectEqual(links[0].Address, mac) + framework.ExpectConsistOf(links[0].NonLinkLocalIPs(), ips) + }) + + framework.ConformanceIt("should be able to provide IPAM for macvlan with ip provided by k8s.v1.cni.cncf.io/networks annotation", func() { + if f.VersionPriorTo(1, 13) { + ginkgo.Skip("this feature was introduced in v1.13") + } - framework.Logf("requestIPString: %s", requestIPString) - annotations := map[string]string{nadv1.NetworkAttachmentAnnot: fmt.Sprintf(`[{"name": "%s", "namespace": "%s", "mac": "%s", "ips": [%s]}]`, nad.Name, nad.Namespace, mac, requestIPString)} - annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, provider)] = subnetName + provider := fmt.Sprintf("%s.%s", nadName, namespaceName) + + ginkgo.By("Creating subnet " + subnetName) + subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil) + subnet.Spec.Provider = provider + subnet = subnetClient.CreateSync(subnet) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + ginkgo.By("Generating networks annotation") + mac := util.GenerateMac() + ips := strings.Split(framework.RandomIPs(subnet.Spec.CIDRBlock, "", 1), ",") + networks := []*nadv1.NetworkSelectionElement{{ + Name: nad.Name, + Namespace: nad.Namespace, + MacRequest: mac, + IPRequest: ips, + }} + networksAnnotation, err := json.Marshal(networks) + framework.ExpectNoError(err) + framework.Logf("networks annotation: %s", string(networksAnnotation)) - cmd := []string{"sh", "-c", "sleep infinity"} + ginkgo.By("Creating pod " + podName) + annotations := map[string]string{nadv1.NetworkAttachmentAnnot: string(networksAnnotation)} + cmd := []string{"sleep", "infinity"} pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) pod = podClient.CreateSync(pod) ginkgo.By("Validating pod annotations") framework.ExpectHaveKey(pod.Annotations, nadv1.NetworkStatusAnnot) framework.Logf("pod network status:\n%s", pod.Annotations[nadv1.NetworkStatusAnnot]) - retMac := pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, provider)] - retIP := pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, provider)] + statuses, err := nadutils.GetNetworkStatus(pod) + framework.ExpectNoError(err) + var ifaceName string + nadKey := cache.MetaObjectToName(nad).String() + for _, status := range statuses { + if status.Name == nadKey { + framework.ExpectEqual(status.Mac, mac) + ifaceName = status.Interface + break + } + } + framework.ExpectNotEmpty(ifaceName) + cidr := pod.Annotations[fmt.Sprintf(util.CidrAnnotationTemplate, provider)] + ip := pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, provider)] + gateway := pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, provider)] + framework.ExpectIPInCIDR(ip, cidr) + framework.ExpectIPInCIDR(gateway, cidr) + framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac) + + ipName := ovs.PodNameToPortName(podName, namespaceName, provider) + ginkgo.By("Validating IP resource " + ipName) + ipCR := ipClient.Get(ipName) + framework.ExpectEqual(ipCR.Spec.Subnet, subnetName) + framework.ExpectEqual(ipCR.Spec.PodName, podName) + framework.ExpectEqual(ipCR.Spec.Namespace, namespaceName) + framework.ExpectEqual(ipCR.Spec.NodeName, pod.Spec.NodeName) + framework.ExpectEqual(ipCR.Spec.IPAddress, ip) + framework.ExpectEmpty(ipCR.Spec.MacAddress) + ipv4, ipv6 := util.SplitStringIP(ip) + framework.ExpectEqual(ipCR.Spec.V4IPAddress, ipv4) + framework.ExpectEqual(ipCR.Spec.V6IPAddress, ipv6) + framework.ExpectHaveKeyWithValue(ipCR.Labels, subnetName, "") + framework.ExpectHaveKeyWithValue(ipCR.Labels, util.SubnetNameLabel, subnetName) + framework.ExpectHaveKeyWithValue(ipCR.Labels, util.NodeNameLabel, pod.Spec.NodeName) + if !f.VersionPriorTo(1, 13) { + framework.ExpectHaveKeyWithValue(ipCR.Labels, util.IPReservedLabel, "false") + } - framework.ExpectEqual(mac, retMac) - framework.ExpectEqual(strings.Join(randomIPArray, ","), retIP) + ginkgo.By("Validating pod ip and mac") + links, err := iproute.AddressShow(ifaceName, func(cmd ...string) ([]byte, []byte, error) { + return framework.KubectlExec(namespaceName, podName, cmd...) + }) + framework.ExpectNoError(err) + framework.ExpectHaveLen(links, 1) + framework.ExpectEqual(links[0].Address, mac) + framework.ExpectConsistOf(links[0].NonLinkLocalIPs(), ips) }) })