feat(pool): ensure secondary ip exists before giving it to Pod

Makefile

@@ -58,6 +58,10 @@ vip-controller:
 	$(DOCKER_CMD) push $(VIP_CONTROLLER_IMAGE)
 	@echo "Build done: $(VIP_CONTROLLER_IMAGE)"
 
+.PHONY: release-ip
+release-ip:
+	CGO_ENABLED=0 GOOS="linux" GOARCH="amd64" go build -o ./bin/release-ip ./cmd/release-ip
+
 .PHONY: fmt
 fmt:
 	@command -v goimports >/dev/null || { echo "ERROR: goimports not installed"; exit 1; }

cmd/release-ip/main.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ucloud/ucloud-sdk-go/ucloud"
+	"github.com/ucloud/uk8s-cni-vpc/pkg/iputils"
+	"github.com/ucloud/uk8s-cni-vpc/pkg/uapi"
+)
+
+func releaseIp(ip string) error {
+	macAddr, err := iputils.GetNodeMacAddress("")
+	if err != nil {
+		return fmt.Errorf("failed to get mac addr: %w", err)
+	}
+
+	uapiClient, err := uapi.NewClient()
+	if err != nil {
+		return fmt.Errorf("failed to init uapi client: %w", err)
+	}
+
+	vpcClient, err := uapiClient.VPCClient()
+	if err != nil {
+		return fmt.Errorf("failed to init vpc client: %w", err)
+	}
+
+	req := vpcClient.NewDeleteSecondaryIpRequest()
+	req.VPCId = ucloud.String(uapiClient.VPCID())
+	req.SubnetId = ucloud.String(uapiClient.SubnetID())
+	req.Ip = ucloud.String(ip)
+	req.Mac = ucloud.String(macAddr)
+
+	_, err = vpcClient.DeleteSecondaryIp(req)
+	return err
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Println("Usage: release-ip <ip>")
+		os.Exit(1)
+	}
+
+	ip := os.Args[1]
+	err := releaseIp(ip)
+	if err != nil {
+		fmt.Printf("failed to release ip %v: %v\n", ip, err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("released ip %v successfully\n", ip)
+}

pkg/ipamd/pool.go

@@ -122,35 +122,50 @@ func (s *ipamServer) getPodIp(r *rpc.AddPodNetworkRequest) (*rpc.PodNetwork, err
 	podName := r.GetPodName()
 	podNS := r.GetPodNamespace()
 	sandboxId := r.GetSandboxID()
-	enable, pod, err := s.podEnableStaticIP(podName, podNS)
+	enableStatic, pod, err := s.podEnableStaticIP(podName, podNS)
 	if err != nil {
 		return nil, err
 	}
 	var pn *rpc.PodNetwork
-	if enable {
+	if enableStatic {
 		pn, err = s.assignStaticPodIP(pod, sandboxId)
 	} else {
 		pn, err = s.assignPodIP()
 	}
 	if err != nil {
 		return nil, err
 	}
+
+	ulog.Infof("Check IP %s status in VPC", pn.VPCIP)
+	// In some cases, the IP is deleted in VPC but still remain in the pool. If we give it to
+	// the Pod, the Pod network will be unavailable.
+	// So this check must be done before we returning IP. If the IP does not exist, returns
+	// error to make kubelet retries to get another one.
+	ok, err := s.checkSecondaryIpExist(pn.VPCIP, s.hostMacAddr)
+	if err != nil {
+		if !enableStatic {
+			s.putIpToPool(pn)
+		}
+		return nil, fmt.Errorf("check ip %v status in vpc error: %v", pn.VPCIP, err)
+	}
+	if !ok {
+		return nil, fmt.Errorf("ip %v does not exist on current node, we will try to use another one", pn.VPCIP)
+	}
+
 	if !pn.Recycled && pn.VPCIP != "" {
 		// We need to detect IP conflict before using it.
 		// See: https://www.rfc-editor.org/rfc/rfc5227
 		err = s.checkIPConflict(pn.VPCIP)
 		if err != nil {
 			ulog.Errorf("Detect ip conflict for %s error: %v, we will release it", pn.VPCIP, err)
-			err = s.uapiDeleteSecondaryIp(pn.VPCIP)
-			if err != nil {
-				ulog.Errorf("Release ip %s after conflict error: %v", pn.VPCIP, err)
-				return nil, err
+			delErr := s.uapiDeleteSecondaryIp(pn.VPCIP)
+			if delErr != nil {
+				ulog.Errorf("Release ip %s after conflict error: %v", pn.VPCIP, delErr)
 			}
 			return nil, err
 		}
-	} else {
-		ulog.Infof("IP %s is recycled, no need to detect conflict", pn.VPCIP)
 	}
+
 	return pn, nil
 }
 

pkg/ipamd/uapi.go

@@ -323,8 +323,10 @@ func (s *ipamServer) checkSecondaryIpExist(ip, mac string) (bool, error) {
 		ulog.Errorf("DescribeSecondaryIp %s error: %v, request id %s", ip, err, resp.GetRequestUUID())
 		return false, err
 	}
-	if len(resp.DataSet) > 0 {
-		return true, nil
+	for _, data := range resp.DataSet {
+		if data.Ip == ip {
+			return true, nil
+		}
 	}
 	return false, nil
 }