Merge pull request #221 from RamLavi/add_guest_agent_ping_wait

checkup: Add guest agent ping wait

pkg/internal/checkup/checkup.go

@@ -124,15 +124,14 @@ func (c *Checkup) Setup(ctx context.Context) (setupErr error) {
 	}()
 
 	var updatedVMIUnderTest *kvcorev1.VirtualMachineInstance
-	updatedVMIUnderTest, err = c.waitForVMIToBoot(setupCtx, c.vmiUnderTest.Name)
+	updatedVMIUnderTest, err = c.waitForVMIToBeReady(setupCtx, c.vmiUnderTest.Name)
 	if err != nil {
 		return err
 	}
 
 	c.vmiUnderTest = updatedVMIUnderTest
-
 	var updatedTrafficGen *kvcorev1.VirtualMachineInstance
-	updatedTrafficGen, err = c.waitForVMIToBoot(setupCtx, c.trafficGen.Name)
+	updatedTrafficGen, err = c.waitForVMIToBeReady(setupCtx, c.trafficGen.Name)
 	if err != nil {
 		return err
 	}
@@ -233,9 +232,9 @@ func (c *Checkup) createVMI(ctx context.Context, vmiToCreate *kvcorev1.VirtualMa
 	return err
 }
 
-func (c *Checkup) waitForVMIToBoot(ctx context.Context, name string) (*kvcorev1.VirtualMachineInstance, error) {
+func (c *Checkup) waitForVMIToBeReady(ctx context.Context, name string) (*kvcorev1.VirtualMachineInstance, error) {
 	vmiFullName := ObjectFullName(c.namespace, name)
-	log.Printf("Waiting for VMI %q to boot...", vmiFullName)
+	log.Printf("Waiting for VMI %q to be ready...", vmiFullName)
 	var updatedVMI *kvcorev1.VirtualMachineInstance
 
 	conditionFn := func(ctx context.Context) (bool, error) {
@@ -246,7 +245,7 @@ func (c *Checkup) waitForVMIToBoot(ctx context.Context, name string) (*kvcorev1.
 		}
 
 		for _, condition := range updatedVMI.Status.Conditions {
-			if condition.Type == kvcorev1.VirtualMachineInstanceAgentConnected && condition.Status == k8scorev1.ConditionTrue {
+			if condition.Type == kvcorev1.VirtualMachineInstanceReady && condition.Status == k8scorev1.ConditionTrue {
 				return true, nil
 			}
 		}
@@ -255,10 +254,10 @@ func (c *Checkup) waitForVMIToBoot(ctx context.Context, name string) (*kvcorev1.
 	}
 	const pollInterval = 5 * time.Second
 	if err := wait.PollImmediateUntilWithContext(ctx, pollInterval, conditionFn); err != nil {
-		return nil, fmt.Errorf("failed to wait for VMI %q to boot: %v", vmiFullName, err)
+		return nil, fmt.Errorf("failed to wait for VMI %q to be ready: %v", vmiFullName, err)
 	}
 
-	log.Printf("VMI %q had successfully booted", vmiFullName)
+	log.Printf("VMI %q has successfully reached ready condition", vmiFullName)
 
 	return updatedVMI, nil
 }

pkg/internal/checkup/checkup_test.go

@@ -417,10 +417,11 @@ func (cs *clientStub) GetVirtualMachineInstance(_ context.Context, namespace, na
 		return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachineinstances"}, name)
 	}
 
-	vmi.Status.Conditions = append(vmi.Status.Conditions, kvcorev1.VirtualMachineInstanceCondition{
-		Type:   kvcorev1.VirtualMachineInstanceAgentConnected,
-		Status: k8scorev1.ConditionTrue,
-	})
+	vmi.Status.Conditions = append(vmi.Status.Conditions,
+		kvcorev1.VirtualMachineInstanceCondition{
+			Type:   kvcorev1.VirtualMachineInstanceReady,
+			Status: k8scorev1.ConditionTrue,
+		})
 
 	return vmi, nil
 }

pkg/internal/checkup/vmi.go

@@ -69,6 +69,7 @@ func newVMIUnderTest(name string, checkupConfig config.Config, configMapName str
 		vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, CloudInit(vmiUnderTestBootCommands(configDiskSerial))),
 		vmi.WithConfigMapVolume(configVolumeName, configMapName),
 		vmi.WithConfigMapDisk(configVolumeName, configDiskSerial),
+		vmi.WithReadinessFileProbe(config.BootScriptReadinessMarkerFileFullPath),
 	)
 
 	return vmi.New(name, optionsToApply...)
@@ -88,6 +89,7 @@ func newTrafficGen(name string, checkupConfig config.Config, configMapName strin
 		vmi.WithCloudInitNoCloudVolume(cloudInitDiskName, CloudInit(trafficGenBootCommands(configDiskSerial))),
 		vmi.WithConfigMapVolume(configVolumeName, configMapName),
 		vmi.WithConfigMapDisk(configVolumeName, configDiskSerial),
+		vmi.WithReadinessFileProbe(config.BootScriptReadinessMarkerFileFullPath),
 	)
 
 	return vmi.New(name, optionsToApply...)
@@ -132,7 +134,7 @@ func generateBootScript() string {
 	sb := strings.Builder{}
 
 	sb.WriteString("#!/bin/bash\n")
-	sb.WriteString("set -e\n")
+	sb.WriteString("set -x\n")
 	sb.WriteString("\n")
 	sb.WriteString("checkup_tuned_adm_set_marker_full_path=" + config.BootScriptTunedAdmSetMarkerFileFullPath + "\n")
 	sb.WriteString("\n")
@@ -141,10 +143,13 @@ func generateBootScript() string {
 	sb.WriteString("  tuned-adm profile cpu-partitioning\n\n")
 	sb.WriteString("  touch $checkup_tuned_adm_set_marker_full_path\n")
 	sb.WriteString("  reboot\n")
+	sb.WriteString("  exit 0\n")
 	sb.WriteString("fi\n")
 	sb.WriteString("\n")
 	sb.WriteString("driverctl set-override " + config.VMIEastNICPCIAddress + " vfio-pci\n")
 	sb.WriteString("driverctl set-override " + config.VMIWestNICPCIAddress + " vfio-pci\n")
+	sb.WriteString("touch " + config.BootScriptReadinessMarkerFileFullPath + "\n")
+	sb.WriteString("chcon -t virt_qemu_ga_exec_t " + config.BootScriptReadinessMarkerFileFullPath + "\n")
 
 	return sb.String()
 }

pkg/internal/checkup/vmi/spec.go

@@ -262,6 +262,23 @@ func WithAffinity(affinity *corev1.Affinity) Option {
 	}
 }
 
+func WithReadinessFileProbe(fileName string) Option {
+	return func(vmi *kvcorev1.VirtualMachineInstance) {
+		var readinessProbeCommand = []string{"cat", fileName}
+		vmi.Spec.ReadinessProbe = &kvcorev1.Probe{
+			Handler: kvcorev1.Handler{
+				Exec: &corev1.ExecAction{
+					Command: readinessProbeCommand,
+				},
+			},
+			FailureThreshold:    30,
+			InitialDelaySeconds: 90,
+			PeriodSeconds:       10,
+			TimeoutSeconds:      10,
+		}
+	}
+}
+
 func Pointer[T any](v T) *T {
 	return &v
 }

pkg/internal/config/config.go

@@ -65,6 +65,7 @@ const (
 	BootScriptName                          = "dpdk-checkup-boot.sh"
 	BootScriptBinDirectory                  = "/usr/bin/"
 	BootScriptTunedAdmSetMarkerFileFullPath = "/var/dpdk-checkup-tuned-adm-set-marker"
+	BootScriptReadinessMarkerFileFullPath   = "/tmp/dpdk-checkup-ready-marker"
 )
 
 var (

vms/traffic-gen/scripts/customize-vm

@@ -40,6 +40,12 @@ set_unsafe_no_io_mmu_mode() {
   echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf
 }
 
+# Enable guest-exec on the qemu-guest-agent configuration
+enable_guest_exec() {
+  sed -i '/^BLACKLIST_RPC=/ { s/guest-exec-status//; s/guest-exec//g }' /etc/sysconfig/qemu-ga
+  sed -i '/^BLACKLIST_RPC=/ { s/,\+/,/g; s/^,\|,$//g }' /etc/sysconfig/qemu-ga
+}
+
 # Install trex package
 install_trex() {
   local TREX_URL=https://trex-tgn.cisco.com/trex/release
@@ -60,4 +66,5 @@ install_trex() {
 disable_services
 setup_hugepages
 set_unsafe_no_io_mmu_mode
+enable_guest_exec
 install_trex

vms/vm-under-test/scripts/customize-vm

@@ -40,6 +40,13 @@ set_unsafe_no_io_mmu_mode() {
   echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf
 }
 
+# Enable guest-exec on the qemu-guest-agent configuration
+enable_guest_exec() {
+  sed -i '/^BLACKLIST_RPC=/ { s/guest-exec-status//; s/guest-exec//g }' /etc/sysconfig/qemu-ga
+  sed -i '/^BLACKLIST_RPC=/ { s/,\+/,/g; s/^,\|,$//g }' /etc/sysconfig/qemu-ga
+}
+
 disable_services
 setup_hugepages
 set_unsafe_no_io_mmu_mode
+enable_guest_exec