Skip to content

Commit

Permalink
ipam: use ip provided by nad annotation when providing IPAM for other…
Browse files Browse the repository at this point in the history 
… CNI plugins (#4883)

Signed-off-by: zhangzujian <[email protected]>
  • Loading branch information
@zhangzujian
zhangzujian authored Dec 30, 2024
1 parent bad1bdb commit d10251f
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 28 deletions.
10 changes: 4 additions & 6 deletions pkg/controller/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1546,13 +1546,9 @@ func (c *Controller) getPodAttachmentNet(pod *v1.Pod) ([]*kubeovnNet, error) {
Subnet: subnet,
IsDefault: isDefault,
AllowLiveMigration: allowLiveMigration,
MacRequest: attach.MacRequest,
IPRequest: strings.Join(attach.IPRequest, ","),
}

if len(attach.IPRequest) != 0 {
ret.IPRequest = strings.Join(attach.IPRequest, ",")
}

ret.MacRequest = attach.MacRequest
result = append(result, ret)
} else {
providerName = fmt.Sprintf("%s.%s", attach.Name, attach.Namespace)
Expand All @@ -1562,6 +1558,8 @@ func (c *Controller) getPodAttachmentNet(pod *v1.Pod) ([]*kubeovnNet, error) {
Type: providerTypeIPAM,
ProviderName: providerName,
Subnet: subnet,
MacRequest: attach.MacRequest,
IPRequest: strings.Join(attach.IPRequest, ","),
})
break
}
Expand Down
10 changes: 10 additions & 0 deletions test/e2e/framework/iproute/iproute.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,16 @@ func (l *Link) NonLinkLocalAddresses() []string {
return result
}

func (l *Link) NonLinkLocalIPs() []string {
var result []string
for _, addr := range l.AddrInfo {
if !net.ParseIP(addr.Local).IsLinkLocalUnicast() {
result = append(result, addr.Local)
}
}
return result
}

type Route struct {
Type string `json:"type"`
Dst string `json:"dst"`
Expand Down
147 changes: 125 additions & 22 deletions test/e2e/multus/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,8 +304,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() {
gateway := pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, provider)]
framework.ExpectIPInCIDR(ip, cidr)
framework.ExpectIPInCIDR(gateway, cidr)
framework.ExpectNotHaveKey(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider))
// framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac)
framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac)

ipName := ovs.PodNameToPortName(podName, namespaceName, provider)
ginkgo.By("Validating IP resource " + ipName)
Expand Down Expand Up @@ -438,8 +437,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() {
gateway := pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, provider)]
framework.ExpectIPInCIDR(ip, cidr)
framework.ExpectIPInCIDR(gateway, cidr)
framework.ExpectNotHaveKey(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider))
// framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac)
framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac)

ipName := ovs.PodNameToPortName(podName, namespaceName, provider)
ginkgo.By("Validating IP resource " + ipName)
Expand Down Expand Up @@ -506,6 +504,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() {
}

provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider)

ginkgo.By("Creating network attachment definition " + nadName)
nad := framework.MakeOVNNetworkAttachmentDefinition(nadName, namespaceName, provider, nil)
nad = nadClient.Create(nad)
Expand All @@ -515,35 +514,139 @@ var _ = framework.SerialDescribe("[group:multus]", func() {
subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", provider, nil, nil, nil)
subnet = subnetClient.CreateSync(subnet)

ginkgo.By("Generating k8s.v1.cni.cncf.io/networks annotation")
mac := util.GenerateMac()
ips := strings.Split(framework.RandomIPs(subnet.Spec.CIDRBlock, "", 1), ",")
networks := []*nadv1.NetworkSelectionElement{{
Name: nad.Name,
Namespace: nad.Namespace,
MacRequest: mac,
IPRequest: ips,
}}
networksAnnotation, err := json.Marshal(networks)
framework.ExpectNoError(err)
framework.Logf("networks annotation: %s", string(networksAnnotation))

ginkgo.By("Creating pod " + podName)
mac := "00:00:00:11:22:33"
randomIP := framework.RandomIPs(subnet.Spec.CIDRBlock, "", 1)

randomIPArray := strings.Split(randomIP, ",")
var requestIPString string
for i, ip := range randomIPArray {
if i == len(randomIPArray)-1 {
requestIPString += fmt.Sprintf(`"%s"`, ip)
} else {
requestIPString += fmt.Sprintf(`"%s",`, ip)
annotations := map[string]string{nadv1.NetworkAttachmentAnnot: string(networksAnnotation)}
cmd := []string{"sleep", "infinity"}
pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil)
pod = podClient.CreateSync(pod)

ginkgo.By("Validating pod annotations")
framework.ExpectHaveKey(pod.Annotations, nadv1.NetworkStatusAnnot)
framework.Logf("pod network status:\n%s", pod.Annotations[nadv1.NetworkStatusAnnot])
framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac)
framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.IPAddressAnnotationTemplate, provider), strings.Join(ips, ","))

ginkgo.By("Getting attachment interface name")
statuses, err := nadutils.GetNetworkStatus(pod)
framework.ExpectNoError(err)
var ifaceName string
nadKey := cache.MetaObjectToName(nad).String()
for _, status := range statuses {
if status.Name == nadKey {
framework.ExpectEqual(status.Mac, mac)
ifaceName = status.Interface
break
}
}
framework.ExpectNotEmpty(ifaceName)

ginkgo.By("Validating pod ip and mac")
links, err := iproute.AddressShow(ifaceName, func(cmd ...string) ([]byte, []byte, error) {
return framework.KubectlExec(pod.Namespace, pod.Name, cmd...)
})
framework.ExpectNoError(err)
framework.ExpectHaveLen(links, 1)
framework.ExpectEqual(links[0].Address, mac)
framework.ExpectConsistOf(links[0].NonLinkLocalIPs(), ips)
})

framework.ConformanceIt("should be able to provide IPAM for macvlan with ip provided by k8s.v1.cni.cncf.io/networks annotation", func() {
if f.VersionPriorTo(1, 13) {
ginkgo.Skip("this feature was introduced in v1.13")
}

framework.Logf("requestIPString: %s", requestIPString)
annotations := map[string]string{nadv1.NetworkAttachmentAnnot: fmt.Sprintf(`[{"name": "%s", "namespace": "%s", "mac": "%s", "ips": [%s]}]`, nad.Name, nad.Namespace, mac, requestIPString)}
annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, provider)] = subnetName
provider := fmt.Sprintf("%s.%s", nadName, namespaceName)

ginkgo.By("Creating subnet " + subnetName)
subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil)
subnet.Spec.Provider = provider
subnet = subnetClient.CreateSync(subnet)

ginkgo.By("Creating network attachment definition " + nadName)
nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil)
nad = nadClient.Create(nad)
framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config)

ginkgo.By("Generating networks annotation")
mac := util.GenerateMac()
ips := strings.Split(framework.RandomIPs(subnet.Spec.CIDRBlock, "", 1), ",")
networks := []*nadv1.NetworkSelectionElement{{
Name: nad.Name,
Namespace: nad.Namespace,
MacRequest: mac,
IPRequest: ips,
}}
networksAnnotation, err := json.Marshal(networks)
framework.ExpectNoError(err)
framework.Logf("networks annotation: %s", string(networksAnnotation))

cmd := []string{"sh", "-c", "sleep infinity"}
ginkgo.By("Creating pod " + podName)
annotations := map[string]string{nadv1.NetworkAttachmentAnnot: string(networksAnnotation)}
cmd := []string{"sleep", "infinity"}
pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil)
pod = podClient.CreateSync(pod)

ginkgo.By("Validating pod annotations")
framework.ExpectHaveKey(pod.Annotations, nadv1.NetworkStatusAnnot)
framework.Logf("pod network status:\n%s", pod.Annotations[nadv1.NetworkStatusAnnot])
retMac := pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, provider)]
retIP := pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, provider)]
statuses, err := nadutils.GetNetworkStatus(pod)
framework.ExpectNoError(err)
var ifaceName string
nadKey := cache.MetaObjectToName(nad).String()
for _, status := range statuses {
if status.Name == nadKey {
framework.ExpectEqual(status.Mac, mac)
ifaceName = status.Interface
break
}
}
framework.ExpectNotEmpty(ifaceName)
cidr := pod.Annotations[fmt.Sprintf(util.CidrAnnotationTemplate, provider)]
ip := pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, provider)]
gateway := pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, provider)]
framework.ExpectIPInCIDR(ip, cidr)
framework.ExpectIPInCIDR(gateway, cidr)
framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.MacAddressAnnotationTemplate, provider), mac)

ipName := ovs.PodNameToPortName(podName, namespaceName, provider)
ginkgo.By("Validating IP resource " + ipName)
ipCR := ipClient.Get(ipName)
framework.ExpectEqual(ipCR.Spec.Subnet, subnetName)
framework.ExpectEqual(ipCR.Spec.PodName, podName)
framework.ExpectEqual(ipCR.Spec.Namespace, namespaceName)
framework.ExpectEqual(ipCR.Spec.NodeName, pod.Spec.NodeName)
framework.ExpectEqual(ipCR.Spec.IPAddress, ip)
framework.ExpectEmpty(ipCR.Spec.MacAddress)
ipv4, ipv6 := util.SplitStringIP(ip)
framework.ExpectEqual(ipCR.Spec.V4IPAddress, ipv4)
framework.ExpectEqual(ipCR.Spec.V6IPAddress, ipv6)
framework.ExpectHaveKeyWithValue(ipCR.Labels, subnetName, "")
framework.ExpectHaveKeyWithValue(ipCR.Labels, util.SubnetNameLabel, subnetName)
framework.ExpectHaveKeyWithValue(ipCR.Labels, util.NodeNameLabel, pod.Spec.NodeName)
if !f.VersionPriorTo(1, 13) {
framework.ExpectHaveKeyWithValue(ipCR.Labels, util.IPReservedLabel, "false")
}

framework.ExpectEqual(mac, retMac)
framework.ExpectEqual(strings.Join(randomIPArray, ","), retIP)
ginkgo.By("Validating pod ip and mac")
links, err := iproute.AddressShow(ifaceName, func(cmd ...string) ([]byte, []byte, error) {
return framework.KubectlExec(namespaceName, podName, cmd...)
})
framework.ExpectNoError(err)
framework.ExpectHaveLen(links, 1)
framework.ExpectEqual(links[0].Address, mac)
framework.ExpectConsistOf(links[0].NonLinkLocalIPs(), ips)
})
})

0 comments on commit d10251f

Please sign in to comment.