Skip to content

Commit

Permalink
Support multiple IPPools in the namespace (#4777)
Browse files Browse the repository at this point in the history 
Signed-off-by: Karol Szwaj <[email protected]>
  • Loading branch information
@cnvergence
cnvergence authored Dec 30, 2024
1 parent 5eb316b commit 010b701
Show file tree
Hide file tree
Showing 5 changed files with 156 additions and 21 deletions.
20 changes: 10 additions & 10 deletions pkg/controller/namespace.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,14 @@ func (c *Controller) handleAddNamespace(key string) error {
}
namespace := cachedNs.DeepCopy()

var ls, ippool string
var lss, cidrs, excludeIps []string
var ls string
var lss, cidrs, excludeIps, ipPoolsAnnotation []string
subnets, err := c.subnetsLister.List(labels.Everything())
if err != nil {
klog.Errorf("failed to list subnets %v", err)
return err
}
ippools, err := c.ippoolLister.List(labels.Everything())
ipPoolList, err := c.ippoolLister.List(labels.Everything())
if err != nil {
klog.Errorf("failed to list ippools: %v", err)
return err
Expand Down Expand Up @@ -151,10 +151,9 @@ func (c *Controller) handleAddNamespace(key string) error {
}
}

for _, p := range ippools {
if slices.Contains(p.Spec.Namespaces, key) {
ippool = p.Name
break
for _, ipPool := range ipPoolList {
if slices.Contains(ipPool.Spec.Namespaces, key) {
ipPoolsAnnotation = append(ipPoolsAnnotation, ipPool.Name)
}
}

Expand Down Expand Up @@ -195,7 +194,7 @@ func (c *Controller) handleAddNamespace(key string) error {
if namespace.Annotations[util.LogicalSwitchAnnotation] == strings.Join(lss, ",") &&
namespace.Annotations[util.CidrAnnotation] == strings.Join(cidrs, ";") &&
namespace.Annotations[util.ExcludeIpsAnnotation] == strings.Join(excludeIps, ";") &&
namespace.Annotations[util.IPPoolAnnotation] == ippool {
namespace.Annotations[util.IPPoolAnnotation] == strings.Join(ipPoolsAnnotation, ",") {
return nil
}

Expand All @@ -204,10 +203,11 @@ func (c *Controller) handleAddNamespace(key string) error {
util.CidrAnnotation: strings.Join(cidrs, ";"),
util.ExcludeIpsAnnotation: strings.Join(excludeIps, ";"),
}
if ippool == "" {

if len(ipPoolsAnnotation) == 0 {
patch[util.IPPoolAnnotation] = nil
} else {
patch[util.IPPoolAnnotation] = ippool
patch[util.IPPoolAnnotation] = strings.Join(ipPoolsAnnotation, ",")
}

if err = util.PatchAnnotations(c.config.KubeClient.CoreV1().Namespaces(), key, patch); err != nil {
Expand Down
32 changes: 31 additions & 1 deletion pkg/controller/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1643,8 +1643,38 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
klog.Errorf("failed to get namespace %s: %v", pod.Namespace, err)
return "", "", "", podNet.Subnet, err
}

if len(ns.Annotations) != 0 {
ippoolStr = ns.Annotations[util.IPPoolAnnotation]
if ipPoolList, ok := ns.Annotations[util.IPPoolAnnotation]; ok {
for _, ipPoolName := range strings.Split(ipPoolList, ",") {
ippool, err := c.ippoolLister.Get(ipPoolName)
if err != nil {
klog.Errorf("failed to get ippool %s: %v", ipPoolName, err)
return "", "", "", podNet.Subnet, err
}

switch podNet.Subnet.Spec.Protocol {
case kubeovnv1.ProtocolDual:
if ippool.Status.V4AvailableIPs.Int64() == 0 || ippool.Status.V6AvailableIPs.Int64() == 0 {
continue
}
case kubeovnv1.ProtocolIPv4:
if ippool.Status.V4AvailableIPs.Int64() == 0 {
continue
}

default:
if ippool.Status.V6AvailableIPs.Int64() == 0 {
continue
}
}

if ippool.Spec.Subnet == podNet.Subnet.Name {
ippoolStr = ippool.Name
break
}
}
}
}
}

Expand Down
7 changes: 4 additions & 3 deletions test/e2e/framework/namespace.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,11 +112,12 @@ func (c *NamespaceClient) WaitToDisappear(name string, _, timeout time.Duration)
return nil
}

func MakeNamespace(name string, labels map[string]string) *corev1.Namespace {
func MakeNamespace(name string, labels, annotations map[string]string) *corev1.Namespace {
namespace := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: labels,
Name: name,
Labels: labels,
Annotations: annotations,
},
}
return namespace
Expand Down
112 changes: 108 additions & 4 deletions test/e2e/kube-ovn/ipam/ipam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ var _ = framework.Describe("[group:ipam]", func() {
var stsClient *framework.StatefulSetClient
var subnetClient *framework.SubnetClient
var ippoolClient *framework.IPPoolClient
var namespaceName, subnetName, ippoolName, podName, deployName, stsName string
var namespaceName, subnetName, subnetName2, ippoolName, ippoolName2, podName, deployName, stsName, stsName2 string
var subnet *apiv1.Subnet
var cidr string

Expand All @@ -41,10 +41,14 @@ var _ = framework.Describe("[group:ipam]", func() {
ippoolClient = f.IPPoolClient()
namespaceName = f.Namespace.Name
subnetName = "subnet-" + framework.RandomSuffix()
subnetName2 = "subnet2-" + framework.RandomSuffix()
ippoolName = "ippool-" + framework.RandomSuffix()
ippoolName2 = "ippool2-" + framework.RandomSuffix()
podName = "pod-" + framework.RandomSuffix()
deployName = "deploy-" + framework.RandomSuffix()
stsName = "sts-" + framework.RandomSuffix()
stsName2 = "sts2-" + framework.RandomSuffix()

cidr = framework.RandomCIDR(f.ClusterIPFamily)

ginkgo.By("Creating subnet " + subnetName)
Expand All @@ -58,14 +62,17 @@ var _ = framework.Describe("[group:ipam]", func() {
ginkgo.By("Deleting deployment " + deployName)
deployClient.DeleteSync(deployName)

ginkgo.By("Deleting statefulset " + stsName)
ginkgo.By("Deleting statefulset " + stsName + " and " + stsName2)
stsClient.DeleteSync(stsName)
stsClient.DeleteSync(stsName2)

ginkgo.By("Deleting ippool " + ippoolName)
ginkgo.By("Deleting ippool " + ippoolName + " and " + ippoolName2)
ippoolClient.DeleteSync(ippoolName)
ippoolClient.DeleteSync(ippoolName2)

ginkgo.By("Deleting subnet " + subnetName)
ginkgo.By("Deleting subnet " + subnetName + " and " + subnetName2)
subnetClient.DeleteSync(subnetName)
subnetClient.DeleteSync(subnetName2)
})

framework.ConformanceIt("should allocate static ipv4 and mac for pod", func() {
Expand Down Expand Up @@ -509,4 +516,101 @@ var _ = framework.Describe("[group:ipam]", func() {
deploy = deployClient.PatchSync(deploy, patchedDeploy)
checkFn()
})

framework.ConformanceIt("should allocate right IPs for the statefulset when there are multiple IP Pools added to its namespace", func() {
f.SkipVersionPriorTo(1, 14, "Multiple IP Pools per namespace support was introduced in v1.14")
replicas := 1
ipsCount := 12

ginkgo.By("Creating a new subnet " + subnetName2)
testCidr := framework.RandomCIDR(f.ClusterIPFamily)
testSubnet := framework.MakeSubnet(subnetName2, "", testCidr, "", "", "", nil, nil, []string{namespaceName})
subnetClient.CreateSync(testSubnet)

ginkgo.By("Creating IPPool resources ")
ipsRange1 := framework.RandomIPPool(cidr, ipsCount)
ipsRange2 := framework.RandomIPPool(testCidr, ipsCount)
ippool1 := framework.MakeIPPool(ippoolName, subnetName, ipsRange1, []string{namespaceName})
ippool2 := framework.MakeIPPool(ippoolName2, subnetName2, ipsRange2, []string{namespaceName})
ippoolClient.CreateSync(ippool1)
ippoolClient.CreateSync(ippool2)

ginkgo.By("Creating statefulset " + stsName + " with logical switch annotation and no ippool annotation")
labels := map[string]string{"app": stsName}
sts := framework.MakeStatefulSet(stsName, stsName, int32(replicas), labels, framework.PauseImage)
sts.Spec.Template.Annotations = map[string]string{util.LogicalSwitchAnnotation: subnetName}
sts = stsClient.CreateSync(sts)

ginkgo.By("Getting pods for statefulset " + stsName)
pods := stsClient.GetPods(sts)
framework.ExpectHaveLen(pods.Items, replicas)

for _, pod := range pods.Items {
framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, util.CidrAnnotation, subnet.Spec.CIDRBlock)
framework.ExpectHaveKeyWithValue(pod.Annotations, util.GatewayAnnotation, subnet.Spec.Gateway)
framework.ExpectHaveKeyWithValue(pod.Annotations, util.LogicalSwitchAnnotation, subnetName)
framework.ExpectIPInCIDR(pod.Annotations[util.IPAddressAnnotation], subnet.Spec.CIDRBlock)
framework.ExpectMAC(pod.Annotations[util.MacAddressAnnotation])
framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
}
})

framework.ConformanceIt("should allocate right IPs for the statefulset when there are multiple ippools added in the namespace and there are no available ips in the first ippool", func() {
f.SkipVersionPriorTo(1, 14, "Multiple IP Pools per namespace support was introduced in v1.14")
replicas := 1
ipsCount := 1

ginkgo.By("Creating IPPool resources ")
ipsRange1 := framework.RandomIPPool(cidr, ipsCount)
ipsRange2 := framework.RandomIPPool(cidr, ipsCount)
ippool1 := framework.MakeIPPool(ippoolName, subnetName, ipsRange1, []string{namespaceName})
ippool2 := framework.MakeIPPool(ippoolName2, subnetName, ipsRange2, []string{namespaceName})
ippoolClient.CreateSync(ippool1)
ippoolClient.CreateSync(ippool2)

ginkgo.By("Creating first statefulset " + stsName + " with logical switch annotation and no ippool annotation")
sts := framework.MakeStatefulSet(stsName, stsName, int32(replicas), map[string]string{"app": stsName}, framework.PauseImage)
sts.Spec.Template.Annotations = map[string]string{util.LogicalSwitchAnnotation: subnetName}
sts = stsClient.CreateSync(sts)

ginkgo.By("Getting pods for the first statefulset " + stsName)
pods := stsClient.GetPods(sts)
framework.ExpectHaveLen(pods.Items, replicas)

for _, pod := range pods.Items {
framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, util.CidrAnnotation, subnet.Spec.CIDRBlock)
framework.ExpectHaveKeyWithValue(pod.Annotations, util.GatewayAnnotation, subnet.Spec.Gateway)
framework.ExpectHaveKeyWithValue(pod.Annotations, util.LogicalSwitchAnnotation, subnetName)
framework.ExpectIPInCIDR(pod.Annotations[util.IPAddressAnnotation], subnet.Spec.CIDRBlock)
framework.ExpectMAC(pod.Annotations[util.MacAddressAnnotation])
framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
for _, ip := range util.PodIPs(pod) {
framework.ExpectContainElement(append(ipsRange1, ipsRange2...), ip)
}
}

ginkgo.By("Creating second statefulset " + stsName2 + " with logical switch annotation and no ippool annotation")
sts2 := framework.MakeStatefulSet(stsName2, stsName2, int32(replicas), map[string]string{"app": stsName2}, framework.PauseImage)
sts2.Spec.Template.Annotations = map[string]string{util.LogicalSwitchAnnotation: subnetName}
sts2 = stsClient.CreateSync(sts2)

ginkgo.By("Getting pods for the second statefulset " + stsName2)
pods2 := stsClient.GetPods(sts2)
framework.ExpectHaveLen(pods2.Items, replicas)

for _, pod := range pods2.Items {
framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, util.CidrAnnotation, subnet.Spec.CIDRBlock)
framework.ExpectHaveKeyWithValue(pod.Annotations, util.GatewayAnnotation, subnet.Spec.Gateway)
framework.ExpectHaveKeyWithValue(pod.Annotations, util.LogicalSwitchAnnotation, subnetName)
framework.ExpectIPInCIDR(pod.Annotations[util.IPAddressAnnotation], subnet.Spec.CIDRBlock)
framework.ExpectMAC(pod.Annotations[util.MacAddressAnnotation])
framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
for _, ip := range util.PodIPs(pod) {
framework.ExpectContainElement(append(ipsRange1, ipsRange2...), ip)
}
}
})
})
6 changes: 3 additions & 3 deletions test/e2e/kube-ovn/subnet/subnet-selectors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,15 @@ var _ = framework.Describe("[group:subnet]", func() {
cidr = framework.RandomCIDR(f.ClusterIPFamily)

ginkgo.By("Creating namespace " + ns1Name)
ns1 = framework.MakeNamespace(ns1Name, map[string]string{projectKey: ns1Name})
ns1 = framework.MakeNamespace(ns1Name, map[string]string{projectKey: ns1Name}, nil)
ns1 = nsClient.Create(ns1)

ginkgo.By("Creating namespace " + ns2Name)
ns2 = framework.MakeNamespace(ns2Name, map[string]string{projectKey: ns2Name})
ns2 = framework.MakeNamespace(ns2Name, map[string]string{projectKey: ns2Name}, nil)
ns2 = nsClient.Create(ns2)

ginkgo.By("Creating namespace " + ns3Name)
ns3 = framework.MakeNamespace(ns3Name, nil)
ns3 = framework.MakeNamespace(ns3Name, nil, nil)
ns3 = nsClient.Create(ns3)

ns1MatchLabels := map[string]string{projectKey: ns1Name}
Expand Down

0 comments on commit 010b701

Please sign in to comment.