Wire up viona params from illumos#16738

Cargo.toml

@@ -23,6 +23,7 @@ default-members = [
 
 exclude = [
   "crates/bhyve-api/header-check",
+  "crates/nvpair/header-check",
   "crates/viona-api/header-check",
   "phd-tests/buildomat",
 ]
@@ -46,6 +47,8 @@ bhyve_api_sys = { path = "crates/bhyve-api/sys" }
 cpuid_utils = { path = "crates/cpuid-utils" }
 cpuid_profile_config = { path = "crates/cpuid-profile-config" }
 dladm = { path = "crates/dladm" }
+nvpair = { path = "crates/nvpair" }
+nvpair_sys = { path = "crates/nvpair/sys" }
 propolis-server-config = { path = "crates/propolis-server-config" }
 propolis_api_types = { path = "crates/propolis-api-types" }
 propolis_types = { path = "crates/propolis-types" }

bin/propolis-server/src/lib/initializer.rs

@@ -713,10 +713,38 @@ impl<'a> MachineInitializer<'a> {
             info!(self.log, "Creating vNIC {}", device_name);
             let bdf: pci::Bdf = nic.device_spec.pci_path.into();
 
+            // Set viona device parameters if possible.
+            //
+            // The values chosen here are tuned to maximize performance when
+            // Propolis is used with OPTE in a full Oxide rack deployment,
+            // although they should not negatively impact use outside those
+            // conditions.  These parameters and their effects (save for
+            // performance delta) are not guest-visible.
+            let params = if virtio::viona::api_version()
+                .expect("can query viona version")
+                >= virtio::viona::ApiVersion::V3
+            {
+                Some(virtio::viona::DeviceParams {
+                    // Use guest memory "loaning", rather than copying and
+                    // allocating entire transmitted packets
+                    copy_data: false,
+                    // Leave room for underlay encapsulation:
+                    // - ethernet: 14
+                    // - IPv6: 40
+                    // - UDP: 8
+                    // - Geneve: 8
+                    // - (and then round up to nearest 8)
+                    header_pad: 72,
+                })
+            } else {
+                None
+            };
+
             let viona = virtio::PciVirtioViona::new(
                 &nic.backend_spec.vnic_name,
                 0x100,
                 &self.machine.hdl,
+                params,
             )
             .with_context(|| {
                 format!("failed to create viona device {device_name:?}")

bin/propolis-standalone/src/config.rs

@@ -122,6 +122,34 @@ struct MemAsyncConfig {
     workers: Option<usize>,
 }
 
+#[derive(Deserialize)]
+pub struct VionaDeviceParams {
+    tx_copy_data: Option<bool>,
+    tx_header_pad: Option<u16>,
+}
+impl VionaDeviceParams {
+    pub fn from_opts(
+        opts: &BTreeMap<String, toml::Value>,
+    ) -> Result<Option<propolis::hw::virtio::viona::DeviceParams>, anyhow::Error>
+    {
+        use propolis::hw::virtio::viona::DeviceParams;
+        let parsed: Self = opt_deser(opts)?;
+        let out = if parsed.tx_copy_data.is_some()
+            || parsed.tx_header_pad.is_some()
+        {
+            let default = DeviceParams::default();
+
+            Some(DeviceParams {
+                copy_data: parsed.tx_copy_data.unwrap_or(default.copy_data),
+                header_pad: parsed.tx_header_pad.unwrap_or(default.header_pad),
+            })
+        } else {
+            None
+        };
+        Ok(out)
+    }
+}
+
 // Try to turn unmatched flattened options into a config struct
 fn opt_deser<'de, T: Deserialize<'de>>(
     value: &BTreeMap<String, toml::Value>,

bin/propolis-standalone/src/main.rs

@@ -1192,8 +1192,24 @@ fn setup_instance(
                         dev.options.get("vnic").unwrap().as_str().unwrap();
                     let bdf = bdf.unwrap();
 
+                    let viona_params =
+                        config::VionaDeviceParams::from_opts(&dev.options)
+                            .expect("viona params are valid");
+
+                    if viona_params.is_some()
+                        && hw::virtio::viona::api_version()
+                            .expect("can query viona version")
+                            < hw::virtio::viona::ApiVersion::V3
+                    {
+                        // lazy cop-out for now
+                        panic!("can't set viona params on too-old version");
+                    }
+
                     let viona = hw::virtio::PciVirtioViona::new(
-                        vnic_name, 0x100, &hdl,
+                        vnic_name,
+                        0x100,
+                        &hdl,
+                        viona_params,
                     )?;
                     guard.inventory.register_instance(&viona, &bdf.to_string());
                     chipset_pci_attach(bdf, viona);

crates/nvpair/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "nvpair"
+version = "0.0.0"
+license = "MPL-2.0"
+edition = "2021"
+
+[lib]
+doctest = false
+
+[dependencies]
+nvpair_sys.workspace = true
+libc.workspace = true

crates/nvpair/header-check/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "nvpair-hdrchk"
+version = "0.0.0"
+license = "MPL-2.0"
+edition = "2021"
+build = "build.rs"
+publish = false
+
+[dependencies]
+nvpair_sys = { path = "../sys" }
+libc = "0.2"
+
+[build-dependencies]
+cc = "1"
+ctest2 = "0.4.7"
+
+[[test]]
+name = "main"
+path = "test/main.rs"
+harness = false

crates/nvpair/header-check/build.rs

@@ -0,0 +1,32 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#![deny(warnings)]
+
+fn main() {
+    let mut cfg = ctest2::TestGenerator::new();
+
+    cfg.header("libnvpair.h");
+
+    cfg.type_name(|ty, is_struct, is_union| match ty {
+        t if t.ends_with("_t") => t.to_string(),
+        t if is_struct => format!("struct {t}"),
+        t if is_union => format!("union {t}"),
+        t => t.to_string(),
+    });
+
+    cfg.skip_const(move |name| match name {
+        _ => false,
+    });
+
+    cfg.skip_struct(|name| match name {
+        _ => false,
+    });
+
+    cfg.skip_field_type(|ty, field| match (ty, field) {
+        _ => false,
+    });
+
+    cfg.generate("../sys/src/lib.rs", "main.rs");
+}

crates/nvpair/header-check/test/main.rs

@@ -0,0 +1,7 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use nvpair_sys::*;
+
+include!(concat!(env!("OUT_DIR"), "/main.rs"));