vfio: Fix vfio device ordering

If modeVFIO is enabled we need 1st to attach the VFIO control group device /dev/vfio/vfio an 2nd the actuall device(s) afterwards.Sort the devices starting with device #1 being the VFIO control group device and the next the actuall device(s) /dev/vfio/<group> Fixes: kata-containers#7493 Signed-off-by: Zvonko Kaiser <[email protected]>
Amulyam24 · Jul 31, 2023 · cddcde1 · cddcde1
1 parent 61a8eab
commit cddcde1
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 8 deletions.
diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go
@@ -851,6 +851,21 @@ func (c *Container) checkBlockDeviceSupport(ctx context.Context) bool {
 	return false
 }
 
+// Sort the devices starting with device #1 being the VFIO control group
+// device and the next the actuall device(s) e.g. /dev/vfio/<group>
+func sortContainerVFIODevices(devices []ContainerDevice) []ContainerDevice {
+	var vfioDevices []ContainerDevice
+
+	for _, device := range devices {
+		if deviceManager.IsVFIOControlDevice(device.ContainerPath) {
+			vfioDevices = append([]ContainerDevice{device}, vfioDevices...)
+			continue
+		}
+		vfioDevices = append(vfioDevices, device)
+	}
+	return vfioDevices
+}
+
 // create creates and starts a container inside a Sandbox. It has to be
 // called only when a new container, not known by the sandbox, has to be created.
 func (c *Container) create(ctx context.Context) (err error) {
@@ -893,6 +908,13 @@ func (c *Container) create(ctx context.Context) (err error) {
 		}
 		c.devices = cntDevices
 	}
+	// If modeVFIO is enabled we need 1st to attach the VFIO control group
+	// device /dev/vfio/vfio an 2nd the actuall device(s) afterwards.
+	// Sort the devices starting with device #1 being the VFIO control group
+	// device and the next the actuall device(s) /dev/vfio/<group>
+	if modeVFIO {
+		c.devices = sortContainerVFIODevices(c.devices)
+	}
 
 	c.Logger().WithFields(logrus.Fields{
 		"devices": c.devices,

diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
@@ -651,7 +651,8 @@ func (s *Sandbox) coldOrHotPlugVFIO(sandboxConfig *SandboxConfig) (bool, error)
 	hotPlugVFIO := (sandboxConfig.HypervisorConfig.HotPlugVFIO != config.NoPort)
 
 	modeIsGK := (sandboxConfig.VfioMode == config.VFIOModeGuestKernel)
-	modeIsVFIO := (sandboxConfig.VfioMode == config.VFIOModeVFIO)
+	// modeIsVFIO is needed at the container level not the sandbox level.
+	// modeIsVFIO := (sandboxConfig.VfioMode == config.VFIOModeVFIO)
 
 	var vfioDevices []config.DeviceInfo
 	// vhost-user-block device is a PCIe device in Virt, keep track of it
@@ -666,13 +667,6 @@ func (s *Sandbox) coldOrHotPlugVFIO(sandboxConfig *SandboxConfig) (bool, error)
 				continue
 			}
 			isVFIODevice := deviceManager.IsVFIODevice(device.ContainerPath)
-			isVFIOControlDevice := deviceManager.IsVFIOControlDevice(device.ContainerPath)
-			// vfio_mode=vfio needs the VFIO control device add it to the list
-			// of devices to be added to the VM.
-			if modeIsVFIO && isVFIOControlDevice && !hotPlugVFIO {
-				vfioDevices = append(vfioDevices, device)
-			}
-
 			if hotPlugVFIO && isVFIODevice {
 				device.ColdPlug = false
 				device.Port = sandboxConfig.HypervisorConfig.HotPlugVFIO