virtio: add PCI device resolver (#417)

Treat virtio devices offered over VPCI like every other VPCI device and
resolve them from resources.

Eventually MMIO and emulated PCI devices should also be resolved from
resources.

Cargo.lock

@@ -4592,6 +4592,7 @@ dependencies = [
  "storvsp",
  "tpm",
  "uidevices",
+ "virtio",
  "virtio_net",
  "virtio_p9",
  "virtio_pmem",
@@ -7469,11 +7470,13 @@ dependencies = [
  "pal_event",
  "parking_lot",
  "pci_core",
+ "pci_resources",
  "task_control",
  "test_with_tracing",
  "thiserror 2.0.0",
  "tracelimit",
  "tracing",
+ "virtio_resources",
  "vm_resource",
  "vmcore",
  "zerocopy",

openhcl/underhill_core/src/worker.rs

@@ -2678,6 +2678,8 @@ async fn new_underhill_vm(
                 instance_id,
                 resource,
                 &mut chipset_builder,
+                None,
+                None,
                 |device_id| {
                     let device = partition
                         .new_virtual_device()

openvmm/hvlite_core/src/worker/dispatch.rs

@@ -1841,6 +1841,12 @@ impl InitializedVm {
                             .context("VTL2 vmbus not enabled")?,
                     };
 
+                    let vtl = match dev_cfg.vtl {
+                        DeviceVtl::Vtl0 => Vtl::Vtl0,
+                        DeviceVtl::Vtl1 => Vtl::Vtl1,
+                        DeviceVtl::Vtl2 => Vtl::Vtl2,
+                    };
+
                     vmm_core::device_builder::build_vpci_device(
                         &driver_source,
                         &resolver,
@@ -1849,6 +1855,8 @@ impl InitializedVm {
                         dev_cfg.instance_id,
                         dev_cfg.resource,
                         &mut chipset_builder,
+                        partition.clone().into_doorbell_registration(vtl),
+                        Some(&mapper),
                         |device_id| {
                             let hv_device = partition.new_virtual_device(
                                 match dev_cfg.vtl {
@@ -1976,23 +1984,14 @@ impl InitializedVm {
                     },
                 )
                 .await?;
-            let bus = if bus == VirtioBus::Auto {
-                if partition.supports_virtual_devices() {
-                    VirtioBus::Vpci
-                } else {
-                    VirtioBus::Mmio
-                }
-            } else {
-                bus
-            };
             match bus {
                 VirtioBus::Mmio => {
                     let mmio_start = virtio_mmio_start - 0x1000;
                     virtio_mmio_start -= 0x1000;
                     let id = format!("{id}-{mmio_start}");
                     chipset_builder.arc_mutex_device(id).add(|services| {
                         VirtioMmioDevice::new(
-                            device,
+                            device.0,
                             services.new_line(IRQ_LINE_SET, "interrupt", virtio_mmio_irq),
                             partition.clone().into_doorbell_registration(Vtl::Vtl0),
                             mmio_start,
@@ -2020,7 +2019,7 @@ impl InitializedVm {
                         .on_pci_bus(bus)
                         .try_add(|services| {
                             VirtioPciDevice::new(
-                                device,
+                                device.0,
                                 PciInterruptModel::IntX(
                                     PciInterruptPin::IntA,
                                     services.new_line(IRQ_LINE_SET, "interrupt", pci_inta_line),
@@ -2031,19 +2030,6 @@ impl InitializedVm {
                             )
                         })?;
                 }
-                VirtioBus::Vpci => {
-                    add_virtio_vpci(
-                        &driver_source,
-                        &partition,
-                        &vmbus_server,
-                        &mapper,
-                        &id,
-                        &mut chipset_builder,
-                        device,
-                    )
-                    .await?;
-                }
-                VirtioBus::Auto => unreachable!(),
             }
         }
 

openvmm/hvlite_defs/src/config.rs

@@ -307,10 +307,8 @@ pub enum PcatBootDevice {
 
 #[derive(Eq, PartialEq, Debug, Copy, Clone, MeshPayload)]
 pub enum VirtioBus {
-    Auto,
     Mmio,
     Pci,
-    Vpci,
 }
 
 /// Policy for the partition when mapping VTL0 memory late.

openvmm/openvmm_entry/src/cli_args.rs

@@ -24,7 +24,6 @@ use clap::ValueEnum;
 use hvlite_defs::config::DeviceVtl;
 use hvlite_defs::config::Hypervisor;
 use hvlite_defs::config::PcatBootDevice;
-use hvlite_defs::config::VirtioBus;
 use hvlite_defs::config::Vtl2BaseAddressType;
 use hvlite_defs::config::X2ApicConfig;
 use hvlite_defs::config::DEFAULT_PCAT_BOOT_ORDER;
@@ -334,8 +333,8 @@ flags:
     pub virtio_fs_shmem: Vec<(String, String)>,
 
     /// add a virtio_fs device under either the PCI or MMIO bus, or whatever the hypervisor supports (pci | mmio | auto)
-    #[clap(long, value_name = "BUS", default_value = "auto", value_parser = parse_virtio_bus_arg)]
-    pub virtio_fs_bus: VirtioBus,
+    #[clap(long, value_name = "BUS", default_value = "auto")]
+    pub virtio_fs_bus: VirtioBusCli,
 
     /// virtio PMEM device
     #[clap(long, value_name = "PATH")]
@@ -531,14 +530,12 @@ fn parse_fs_arg(opt: &str) -> Result<(String, String), &'static str> {
     Ok((tag.to_owned(), root_path.to_owned()))
 }
 
-fn parse_virtio_bus_arg(opt: &str) -> Result<VirtioBus, &'static str> {
-    Ok(match opt {
-        "auto" => VirtioBus::Auto,
-        "mmio" => VirtioBus::Mmio,
-        "pci" => VirtioBus::Pci,
-        "vpci" => VirtioBus::Vpci,
-        _ => return Err("expected one of [auto, mmio, pci, vpci]"),
-    })
+#[derive(Copy, Clone, clap::ValueEnum)]
+pub enum VirtioBusCli {
+    Auto,
+    Mmio,
+    Pci,
+    Vpci,
 }
 
 #[derive(clap::ValueEnum, Clone, Copy)]

openvmm/openvmm_entry/src/lib.rs

@@ -27,6 +27,7 @@ use cli_args::EndpointConfigCli;
 use cli_args::NicConfigCli;
 use cli_args::SerialConfigCli;
 use cli_args::UefiConsoleModeCli;
+use cli_args::VirtioBusCli;
 use disk_backend_resources::layer::DiskLayerHandle;
 use disk_backend_resources::layer::RamDiskLayerHandle;
 use floppy_resources::FloppyDiskConfig;
@@ -118,12 +119,14 @@ use uidevices_resources::SynthKeyboardHandle;
 use uidevices_resources::SynthMouseHandle;
 use uidevices_resources::SynthVideoHandle;
 use video_core::SharedFramebufferHandle;
+use virtio_resources::VirtioPciDeviceHandle;
 use vm_manifest_builder::BaseChipsetType;
 use vm_manifest_builder::MachineArch;
 use vm_manifest_builder::VmChipsetResult;
 use vm_manifest_builder::VmManifestBuilder;
 use vm_resource::kind::DiskHandleKind;
 use vm_resource::kind::NetEndpointHandleKind;
+use vm_resource::kind::VirtioDeviceHandle;
 use vm_resource::kind::VmbusDeviceHandleKind;
 use vm_resource::IntoResource;
 use vm_resource::Resource;
@@ -1127,24 +1130,50 @@ fn vm_config_from_command_line(
     }
 
     let mut virtio_devices = Vec::new();
+    let mut add_virtio_device = |bus, resource: Resource<VirtioDeviceHandle>| {
+        let bus = match bus {
+            VirtioBusCli::Auto => {
+                // Use VPCI when possible (currently only on Windows and macOS due
+                // to KVM backend limitations).
+                if with_hv && (cfg!(windows) || cfg!(target_os = "macos")) {
+                    None
+                } else {
+                    Some(VirtioBus::Pci)
+                }
+            }
+            VirtioBusCli::Mmio => Some(VirtioBus::Mmio),
+            VirtioBusCli::Pci => Some(VirtioBus::Pci),
+            VirtioBusCli::Vpci => None,
+        };
+        if let Some(bus) = bus {
+            virtio_devices.push((bus, resource));
+        } else {
+            vpci_devices.push(VpciDeviceConfig {
+                vtl: DeviceVtl::Vtl0,
+                instance_id: Guid::new_random(),
+                resource: VirtioPciDeviceHandle(resource).into_resource(),
+            });
+        }
+    };
+
     for cli_cfg in &opt.virtio_net {
         if cli_cfg.underhill {
             anyhow::bail!("use --net uh:[...] to add underhill NICs")
         }
         let vport = parse_endpoint(cli_cfg, &mut nic_index, &mut resources)?;
-        virtio_devices.push((
-            VirtioBus::Auto,
+        add_virtio_device(
+            VirtioBusCli::Auto,
             virtio_resources::net::VirtioNetHandle {
                 max_queues: vport.max_queues,
                 mac_address: vport.mac_address,
                 endpoint: vport.endpoint,
             }
             .into_resource(),
-        ));
+        );
     }
 
     for (tag, root_path) in &opt.virtio_fs {
-        virtio_devices.push((
+        add_virtio_device(
             opt.virtio_fs_bus,
             virtio_resources::fs::VirtioFsHandle {
                 tag: tag.clone(),
@@ -1153,11 +1182,11 @@ fn vm_config_from_command_line(
                 },
             }
             .into_resource(),
-        ));
+        );
     }
 
     for (tag, root_path) in &opt.virtio_fs_shmem {
-        virtio_devices.push((
+        add_virtio_device(
             opt.virtio_fs_bus,
             virtio_resources::fs::VirtioFsHandle {
                 tag: tag.clone(),
@@ -1166,26 +1195,26 @@ fn vm_config_from_command_line(
                 },
             }
             .into_resource(),
-        ));
+        );
     }
 
     for (tag, root_path) in &opt.virtio_9p {
-        virtio_devices.push((
-            VirtioBus::Auto,
+        add_virtio_device(
+            VirtioBusCli::Auto,
             virtio_resources::p9::VirtioPlan9Handle {
                 tag: tag.clone(),
                 root_path: root_path.clone(),
                 debug: opt.virtio_9p_debug,
             }
             .into_resource(),
-        ));
+        );
     }
 
     if let Some(path) = &opt.virtio_pmem {
-        virtio_devices.push((
-            VirtioBus::Auto,
+        add_virtio_device(
+            VirtioBusCli::Auto,
             virtio_resources::pmem::VirtioPmemHandle { path: path.clone() }.into_resource(),
-        ));
+        );
     }
 
     let (vmgs_disk, format_vmgs) = if let Some(path) = &opt.vmgs_file {

openvmm/openvmm_entry/src/ttrpc/mod.rs

@@ -21,6 +21,7 @@ use hvlite_defs::config::MemoryConfig;
 use hvlite_defs::config::ProcessorTopologyConfig;
 use hvlite_defs::config::VirtioBus;
 use hvlite_defs::config::VmbusConfig;
+use hvlite_defs::config::VpciDeviceConfig;
 use hvlite_defs::rpc::VmRpc;
 use hvlite_defs::worker::VmWorkerParameters;
 use hvlite_defs::worker::VM_WORKER;
@@ -56,6 +57,7 @@ use storvsp_resources::ScsiControllerHandle;
 use storvsp_resources::ScsiControllerRequest;
 use storvsp_resources::ScsiDeviceAndPath;
 use unix_socket::UnixListener;
+use virtio_resources::VirtioPciDeviceHandle;
 use vm_manifest_builder::VmManifestBuilder;
 use vm_resource::kind::VmbusDeviceHandleKind;
 use vm_resource::IntoResource;
@@ -540,16 +542,24 @@ impl VmService {
             }
 
             for virtiofs in devices_config.virtiofs_config {
-                config.virtio_devices.push((
-                    VirtioBus::Auto,
-                    virtio_resources::fs::VirtioFsHandle {
-                        tag: virtiofs.tag,
-                        fs: virtio_resources::fs::VirtioFsBackend::HostFs {
-                            root_path: virtiofs.root_path,
-                        },
-                    }
-                    .into_resource(),
-                ));
+                let resource = virtio_resources::fs::VirtioFsHandle {
+                    tag: virtiofs.tag,
+                    fs: virtio_resources::fs::VirtioFsBackend::HostFs {
+                        root_path: virtiofs.root_path,
+                    },
+                }
+                .into_resource();
+                // Use VPCI when possible (currently only on Windows and macOS due
+                // to KVM backend limitations).
+                if cfg!(windows) || cfg!(target_os = "macos") {
+                    config.vpci_devices.push(VpciDeviceConfig {
+                        vtl: DeviceVtl::Vtl0,
+                        instance_id: Guid::new_random(),
+                        resource: VirtioPciDeviceHandle(resource).into_resource(),
+                    });
+                } else {
+                    config.virtio_devices.push((VirtioBus::Pci, resource));
+                }
             }
         }
 

openvmm/openvmm_resources/Cargo.toml

@@ -62,6 +62,7 @@ net_backend.workspace = true
 net_consomme = { workspace = true, optional = true }
 
 # Virtio devices
+virtio.workspace = true
 virtiofs.workspace = true
 virtio_net.workspace = true
 virtio_p9.workspace = true

openvmm/openvmm_resources/src/lib.rs

@@ -57,6 +57,7 @@ vm_resource::register_static_resolvers! {
     // PCI devices
     gdma::resolver::GdmaDeviceResolver,
     nvme::resolver::NvmeControllerResolver,
+    virtio::resolver::VirtioPciResolver,
 
     // SCSI
     scsidisk::resolver::SimpleScsiResolver,

vm/devices/pci/pci_resources/src/lib.rs

@@ -9,8 +9,11 @@
 use chipset_device::mmio::RegisterMmioIntercept;
 use chipset_device_resources::ErasedChipsetDevice;
 use chipset_device_resources::ResolvedChipsetDevice;
+use guestmem::DoorbellRegistration;
 use guestmem::GuestMemory;
+use guestmem::MemoryMapper;
 use pci_core::msi::RegisterMsi;
+use std::sync::Arc;
 use vm_resource::kind::PciDeviceHandleKind;
 use vm_resource::CanResolveTo;
 use vmcore::vm_task::VmTaskDriverSource;
@@ -38,4 +41,8 @@ pub struct ResolvePciDeviceHandleParams<'a> {
     pub driver_source: &'a VmTaskDriverSource,
     /// The VM's guest memory.
     pub guest_memory: &'a GuestMemory,
+    /// An object with which to register doorbell regions.
+    pub doorbell_registration: Option<Arc<dyn DoorbellRegistration>>,
+    /// An object with which to register shared memory regions.
+    pub shared_mem_mapper: Option<&'a dyn MemoryMapper>,
 }

vm/devices/virtio/virtio/Cargo.toml

@@ -9,6 +9,8 @@ rust-version.workspace = true
 [dependencies]
 device_emulators.workspace = true
 pci_core.workspace = true
+pci_resources.workspace = true
+virtio_resources.workspace = true
 
 chipset_device.workspace = true
 guestmem.workspace = true

vm/devices/virtio/virtio/src/lib.rs

@@ -6,6 +6,7 @@
 mod common;
 pub mod queue;
 pub mod resolve;
+pub mod resolver;
 pub mod spec;
 pub mod transport;