:support vm live migrate

Signed-off-by: bobz965 <[email protected]>
Signed-off-by: zhangbingbing <[email protected]>

go.mod

@@ -51,7 +51,7 @@ require (
 	k8s.io/pod-security-admission v0.27.4
 	k8s.io/sample-controller v0.27.4
 	k8s.io/utils v0.0.0-20230505201702-9f6742963106
-	kubevirt.io/client-go v0.58.1
+	kubevirt.io/client-go v0.59.0
 	sigs.k8s.io/controller-runtime v0.15.1
 )
 
@@ -268,7 +268,7 @@ require (
 	k8s.io/kubelet v0.27.4 // indirect
 	k8s.io/legacy-cloud-providers v0.0.0 // indirect
 	k8s.io/mount-utils v0.0.0 // indirect
-	kubevirt.io/api v0.58.1 // indirect
+	kubevirt.io/api v0.59.0 // indirect
 	kubevirt.io/containerized-data-importer-api v1.55.2 // indirect
 	kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect
 	moul.io/http2curl v1.0.0 // indirect

go.sum

@@ -2064,8 +2064,8 @@ k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU=
 k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-kubevirt.io/api v0.58.1 h1:+b9fydWSMD4SCOr1VHTEgXOmy5JzYU62ddSuQSWggys=
-kubevirt.io/api v0.58.1/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI=
+kubevirt.io/api v0.59.0 h1:UDsJWklzd0x/w3EQjc48jafZc4p4vVxKUpmBhg2nVRk=
+kubevirt.io/api v0.59.0/go.mod h1:zts/6mioR8vGgvYmQ17Cb9XsUR9e/WjJcdokmrE38wY=
 kubevirt.io/containerized-data-importer-api v1.55.2 h1:AzYnKIUFkKwO6c0uCQZYlAIxfzbiPkJXP29hFhauaQ8=
 kubevirt.io/containerized-data-importer-api v1.55.2/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg=
 kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY=

pkg/controller/init.go

@@ -291,7 +291,7 @@ func (c *Controller) InitIPAM() error {
 		u2oInterconnLrpName := fmt.Sprintf("%s-%s", subnet.Spec.Vpc, subnet.Name)
 		if subnet.Status.U2OInterconnectionIP != "" {
 			if _, _, _, err = c.ipam.GetStaticAddress(u2oInterconnName, u2oInterconnLrpName, subnet.Status.U2OInterconnectionIP, nil, subnet.Name, true); err != nil {
-				klog.Errorf("failed to init subnet u2o interonnection ip to ipam %v", subnet.Name, err)
+				klog.Errorf("failed to init subnet %q u2o interonnection ip to ipam %v", subnet.Name, err)
 			}
 		}
 	}

pkg/controller/pod.go

@@ -641,9 +641,18 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca
 	namespace := pod.Namespace
 	name := pod.Name
 	isVMPod, vmName := isVMPod(pod)
+	var vmInMigrate bool
+	var srcChassisName, dstChassisName string
+	var err error
+	if isVMPod && c.config.EnableKeepVMIP {
+		// only dst pod in not running status could set migration options
+		if vmInMigrate, srcChassisName, dstChassisName, err = c.vmIsMigrating(pod, vmName); err != nil {
+			klog.Error(err)
+			return nil, err
+		}
+	}
 
 	klog.Infof("sync pod %s/%s allocated", namespace, name)
-
 	vipsMap := c.getVirtualIPs(pod, needAllocatePodNets)
 
 	// Avoid create lsp for already running pod in ovn-nb when controller restart
@@ -675,11 +684,23 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca
 		}
 		pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true"
 		if isVMPod && c.config.EnableKeepVMIP {
+			vmKey := fmt.Sprintf("%s/%s", namespace, vmName)
 			pod.Annotations[fmt.Sprintf(util.VMTemplate, podNet.ProviderName)] = vmName
 			if err := c.changeVMSubnet(vmName, namespace, podNet.ProviderName, subnet.Name, pod); err != nil {
-				klog.Errorf("change subnet of pod %s/%s to %s failed: %v", namespace, name, subnet.Name, err)
+				klog.Errorf("vm %s change subnet to %s failed: %v", vmKey, subnet.Name, err)
 				return nil, err
 			}
+			if vmInMigrate {
+				klog.Infof("vm %s is migrating from %s to %s", vmKey, srcChassisName, dstChassisName)
+				if srcChassisName != "" && dstChassisName != "" {
+					// set lsp migration options for vm from src chassis to dst chassis
+					if err = c.setVMLSPMigration(pod, srcChassisName, dstChassisName, podNet); err != nil {
+						klog.Errorf("failed to set migrations for port of vm %s: %v", vmKey, err)
+						return nil, err
+					}
+				}
+				pod.Annotations[util.VMMigratedAnnotation] = "true"
+			}
 		}
 
 		if err := util.ValidatePodCidr(podNet.Subnet.Spec.CIDRBlock, ipStr); err != nil {
@@ -733,10 +754,12 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca
 				DHCPv6OptionsUUID: subnet.Status.DHCPv6OptionsUUID,
 			}
 
-			if err := c.OVNNbClient.CreateLogicalSwitchPort(subnet.Name, portName, ipStr, mac, podName, pod.Namespace, portSecurity, securityGroupAnnotation, vips, podNet.Subnet.Spec.EnableDHCP, dhcpOptions, subnet.Spec.Vpc); err != nil {
-				c.recorder.Eventf(pod, v1.EventTypeWarning, "CreateOVNPortFailed", err.Error())
-				klog.Errorf("%v", err)
-				return nil, err
+			if !vmInMigrate {
+				if err := c.OVNNbClient.CreateLogicalSwitchPort(subnet.Name, portName, ipStr, mac, podName, pod.Namespace, portSecurity, securityGroupAnnotation, vips, podNet.Subnet.Spec.EnableDHCP, dhcpOptions, subnet.Spec.Vpc); err != nil {
+					c.recorder.Eventf(pod, v1.EventTypeWarning, "CreateOVNPortFailed", err.Error())
+					klog.Errorf("%v", err)
+					return nil, err
+				}
 			}
 
 			if pod.Annotations[fmt.Sprintf(util.Layer2ForwardAnnotationTemplate, podNet.ProviderName)] == "true" {
@@ -1603,6 +1626,7 @@ func (c *Controller) validatePodIP(podName, subnetName, ipv4, ipv6 string) (bool
 }
 
 func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, string, string, *kubeovnv1.Subnet, error) {
+
 	podName := c.getNameByPod(pod)
 	key := fmt.Sprintf("%s/%s", pod.Namespace, podName)
 
@@ -1880,6 +1904,121 @@ func isVMPod(pod *v1.Pod) (bool, string) {
 	return false, ""
 }
 
+func (c *Controller) vmIsMigrating(pod *v1.Pod, vmiName string) (bool, string, string, error) {
+	// return migrating, src node, dst node, error
+	// there are two vms when live migration, one is src node vm, the other is dst node vm
+	// these two vms both are not completed pod
+	// the src vm is migrating from the src node to the dst node
+
+	if migrated, ok := pod.Annotations[util.VMMigratedAnnotation]; ok && migrated == "true" {
+		// use pod annotation to trigger the migration once
+		// the annotation will control that dst migration pod will not trigger LSP creation
+		// LSP creation will clean its migration options for migrated pod between vm stop and vm start
+		return true, "", "", nil
+	}
+	vmKey := fmt.Sprintf("%s/%s", pod.Namespace, vmiName)
+	klog.Infof("check vm %s migration state", vmKey)
+	listOpts := metav1.ListOptions{
+		LabelSelector: labels.Set{util.KubevirtVMILabel: vmiName}.AsSelector().String(),
+	}
+	vmimList, err := c.config.KubevirtClient.VirtualMachineInstanceMigration(pod.Namespace).List(&listOpts)
+	if err != nil {
+		err = fmt.Errorf("failed to list migration of vmi %s, %v", vmiName, err)
+		klog.Error(err)
+		return false, "", "", err
+	}
+	if len(vmimList.Items) == 0 {
+		klog.Infof("not migrating vmi %s", vmiName)
+		return false, "", "", nil
+	}
+	var pendingVMIMExist bool
+	for _, vmim := range vmimList.Items {
+		if vmim.Status.Phase != "Failed" && vmim.Status.Phase != "Running" {
+			pendingVMIMExist = true
+			klog.Infof("VirtualMachineInstanceMigration %s is migrating vmi %s", vmim.Name, vmKey)
+			break
+		}
+	}
+	if !pendingVMIMExist {
+		klog.Infof("no migration is migrating vmi %s", vmiName)
+		return false, "", "", nil
+	}
+
+	// find the src pod and dst pod
+	selector := labels.Set{util.KubevirtVMLabel: vmiName}.AsSelector()
+	podList, err := c.podsLister.Pods(pod.Namespace).List(selector)
+	if err != nil {
+		err = fmt.Errorf("failed to list pods of vm %s, %v", vmKey, err)
+		klog.Error(err)
+		return false, "", "", err
+	}
+
+	// src pod and dst pod are both not completed pod
+	notCompletedPods := make([]*v1.Pod, 0)
+	for _, p := range podList {
+		for _, owner := range p.OwnerReferences {
+			// The name of vmi is consistent with vm's name.
+			if owner.Kind == "VirtualMachineInstance" && owner.Name == vmiName {
+				if p.Status.Phase != v1.PodSucceeded && p.Status.Phase != v1.PodFailed {
+					klog.Infof("get not complete pod %s, pod status phase %v", p.Name, p.Status.Phase)
+					notCompletedPods = append(notCompletedPods, p)
+					continue
+				}
+			}
+		}
+	}
+
+	if len(notCompletedPods) != 2 {
+		err = fmt.Errorf("unknown migrate situation, available not complete pod number is %d", len(notCompletedPods))
+		klog.Error(err)
+		return false, "", "", err
+	}
+	var srcPod, dstPod *v1.Pod
+
+	pod0 := notCompletedPods[0]
+	pod1 := notCompletedPods[1]
+	if pod0.CreationTimestamp.After(pod1.CreationTimestamp.Time) {
+		srcPod = pod1
+		dstPod = pod0
+	} else {
+		srcPod = pod0
+		dstPod = pod1
+	}
+	if srcPod.Spec.NodeName == "" {
+		err = fmt.Errorf("failed to get src pod node name")
+		klog.Error(err)
+		return true, "", "", err
+	}
+	if dstPod.Spec.NodeName == "" {
+		err = fmt.Errorf("failed to get dst pod node name")
+		klog.Error(err)
+		return true, "", "", err
+	}
+
+	klog.Infof("src vmi %s is migrating from %s to %s", vmiName, srcPod.Spec.NodeName, dstPod.Spec.NodeName)
+	return true, srcPod.Spec.NodeName, dstPod.Spec.NodeName, nil
+}
+
+func (c *Controller) setVMLSPMigration(pod *v1.Pod, srcChassisName, dstChassisName string, podNet *kubeovnNet) error {
+	// use check ovn-nbctl lsp-set-options migrator-lsp requested-chassis=src,node activation-strategy=rarp
+	// to facilitate the migration of the VM
+	// LSP creation will clean its migration options for migrated pod between vm stop and vm start
+	podName := c.getNameByPod(pod)
+	portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName)
+	if podNet.Type == providerTypeIPAM {
+		klog.Infof("no need set migrate options for the non-ovn port %s", portName)
+		return nil
+	}
+
+	if err := c.OVNNbClient.SetLogicalSwitchPortMigrateOptions(portName, srcChassisName, dstChassisName); err != nil {
+		err = fmt.Errorf("failed to set migrate options for port %s, %v", portName, err)
+		klog.Error(err)
+		return err
+	}
+
+	return nil
+}
+
 func isOwnsByTheVM(vmi metav1.Object) (bool, string) {
 	for _, owner := range vmi.GetOwnerReferences() {
 		if owner.Kind == util.VM && strings.HasPrefix(owner.APIVersion, "kubevirt.io") {

pkg/ovs/interface.go

@@ -61,14 +61,17 @@ type LogicalSwitchPort interface {
 	CreateBareLogicalSwitchPort(lsName, lspName, ip, mac string) error
 	CreateLocalnetLogicalSwitchPort(lsName, lspName, provider string, vlanID int) error
 	CreateVirtualLogicalSwitchPorts(lsName string, ips ...string) error
+	GetLogicalSwitchPortMigrateOptions(lspName string) (string, string, error)
 	// create virtual type logical switch port for allowed-address-pair
 	CreateVirtualLogicalSwitchPort(lspName, lsName, ip string) error
+	CleanLogicalSwitchPortMigrateOptions(lspName string) error
 	// update virtual type logical switch port virtual-parents for allowed-address-pair
 	SetVirtualLogicalSwitchPortVirtualParents(lsName, parents string) error
 	SetLogicalSwitchPortSecurity(portSecurity bool, lspName, mac, ips, vips string) error
 	SetLogicalSwitchPortVirtualParents(lsName, parents string, ips ...string) error
 	SetLogicalSwitchPortArpProxy(lspName string, enableArpProxy bool) error
 	SetLogicalSwitchPortExternalIds(lspName string, externalIds map[string]string) error
+	SetLogicalSwitchPortMigrateOptions(lspName, srcChassisName, dstChassisName string) error
 	SetLogicalSwitchPortVlanTag(lspName string, vlanID int) error
 	SetLogicalSwitchPortsSecurityGroup(sgName, op string) error
 	EnablePortLayer2forward(lspName string) error