diff --git a/scripts/lib/libvirt/compute-config b/scripts/lib/libvirt/compute-config index 422f5ba570..ff9226ccd6 100755 --- a/scripts/lib/libvirt/compute-config +++ b/scripts/lib/libvirt/compute-config @@ -47,9 +47,11 @@ def main(): help="Local Repository Directory Target") parser.add_argument("--ipmi", action='store_true', help="Whether to simulate IPMI BMC devices") + parser.add_argument("--pcipassthrough", action='store_true', + help="Whether to simulate PCI passthrough devices") args = parser.parse_args() - print(libvirt_setup.compute_config(args, libvirt_setup.cpuflags())) + print(libvirt_setup.compute_config(args, libvirt_setup.cpuflags(args.pcipassthrough))) if __name__ == "__main__": diff --git a/scripts/lib/libvirt/libvirt_setup.py b/scripts/lib/libvirt/libvirt_setup.py index 122131cce2..594cf9db25 100644 --- a/scripts/lib/libvirt/libvirt_setup.py +++ b/scripts/lib/libvirt/libvirt_setup.py @@ -28,13 +28,13 @@ def remove_files(files): os.remove(f) -def cpuflags(): +def cpuflags(pcipassthrough=False): cpu_template = "cpu-default.xml" cpu_info = readfile("/proc/cpuinfo") if re.search("^CPU architecture.* 8", cpu_info, re.MULTILINE): cpu_template = "cpu-arm64.xml" elif re.search("^vendor_id.*GenuineIntel", cpu_info, re.MULTILINE): - cpu_template = "cpu-intel.xml" + cpu_template = get_intel_cputemplate(pcipassthrough) elif re.search("^vendor_id.*AuthenticAMD", cpu_info, re.MULTILINE): cpu_template = "cpu-amd.xml" elif re.search("^vendor_id.*IBM/S390", cpu_info, re.MULTILINE): @@ -43,6 +43,13 @@ def cpuflags(): return readfile(os.path.join(TEMPLATE_DIR, cpu_template)) +def get_intel_cputemplate(pcipassthrough=False): + cpu_template = "cpu-intel.xml" + if pcipassthrough: + cpu_template = "cpu-intel-pcipassthrough.xml" + return cpu_template + + def hypervisor_has_virtio(libvirt_type): return libvirt_type == "kvm" @@ -257,7 +264,7 @@ def compute_config(args, cpu_flags=cpuflags()): "bus='{0}' target='0' unit='0'/>" # override nic model for ironic setups - if args.ironicnic >= 0: + if args.ironicnic >= 0 and not args.pcipassthrough: configopts['nicmodel'] = 'e1000' controller_raid_volumes = args.controller_raid_volumes @@ -320,7 +327,32 @@ def compute_config(args, cpu_flags=cpuflags()): 'target_address': target_address.format('0x1f')}, configopts)) - if args.ipmi: + machine = "" + machine = get_default_machine(args.emulator) + pciecontrollers = "" + iommudevice = "" + extravolume = "" + if args.pcipassthrough and not args.drbdserial: + volume_template = string.Template( + readfile(os.path.join(TEMPLATE_DIR, "extra-volume.xml"))) + extravolume = volume_template.substitute(merge_dicts({ + 'volume_serial': "{0}-node{1}-extra".format( + args.cloud, + args.nodecounter), + 'source_dev': "{0}/{1}.node{2}-extra".format( + args.vdiskdir, + args.cloud, + args.nodecounter), + 'target_dev': targetdevprefix + ''.join(next(alldevices)), + 'target_address': target_address.format('0x1d')}, + configopts)) + iommudevice = readfile(os.path.join( + TEMPLATE_DIR, 'iommu-device-default.xml')) + machine = "q35" + pciecontrollers = readfile(os.path.join( + TEMPLATE_DIR, 'pcie-root-bridge-default.xml')) + + if args.ipmi and not args.pcipassthrough: values = dict( nodecounter=args.nodecounter ) @@ -329,7 +361,7 @@ def compute_config(args, cpu_flags=cpuflags()): else: ipmi_config = '' - if not hypervisor_has_virtio(libvirt_type): + if not hypervisor_has_virtio(libvirt_type) and not args.pcipassthrough: target_address = target_address.format('0') # map virtio addr to ide: raidvolume = raidvolume.replace("bus='0x17'", "bus='1'") @@ -342,13 +374,16 @@ def compute_config(args, cpu_flags=cpuflags()): nodememory=nodememory, vcpus=args.vcpus, march=get_machine_arch(), - machine=get_default_machine(args.emulator), + machine=machine, osloader=get_os_loader(firmware_type=args.firmwaretype), cpuflags=cpu_flags, consoletype=get_console_type(), raidvolume=raidvolume, cephvolume=cephvolume, drbdvolume=drbdvolume, + pciecontrollers=pciecontrollers, + extravolume=extravolume, + iommudevice=iommudevice, nics=net_interfaces_config(args, configopts["nicmodel"]), maindiskaddress=get_maindisk_address(), videodevices=get_video_devices(), diff --git a/scripts/lib/libvirt/templates/compute-node.xml b/scripts/lib/libvirt/templates/compute-node.xml index 18a07900b2..e141f4aabc 100644 --- a/scripts/lib/libvirt/templates/compute-node.xml +++ b/scripts/lib/libvirt/templates/compute-node.xml @@ -25,6 +25,9 @@ $cpuflags $raidvolume $cephvolume $drbdvolume +$extravolume +$pciecontrollers +$iommudevice $nics $serialdevice diff --git a/scripts/lib/libvirt/templates/cpu-intel-pcipassthrough.xml b/scripts/lib/libvirt/templates/cpu-intel-pcipassthrough.xml new file mode 100644 index 0000000000..240c38c5bd --- /dev/null +++ b/scripts/lib/libvirt/templates/cpu-intel-pcipassthrough.xml @@ -0,0 +1,11 @@ + + + + + + + + Haswell-noTSX + + + diff --git a/scripts/lib/libvirt/templates/iommu-device-default.xml b/scripts/lib/libvirt/templates/iommu-device-default.xml new file mode 100644 index 0000000000..db83c22481 --- /dev/null +++ b/scripts/lib/libvirt/templates/iommu-device-default.xml @@ -0,0 +1,3 @@ + + + diff --git a/scripts/lib/libvirt/templates/pcie-root-bridge-default.xml b/scripts/lib/libvirt/templates/pcie-root-bridge-default.xml new file mode 100644 index 0000000000..9fc31e8c7f --- /dev/null +++ b/scripts/lib/libvirt/templates/pcie-root-bridge-default.xml @@ -0,0 +1,14 @@ + + + + + + +
+ + + + + +
+ diff --git a/scripts/lib/mkcloud-driver-libvirt.sh b/scripts/lib/mkcloud-driver-libvirt.sh index f45ac76788..958a151601 100644 --- a/scripts/lib/mkcloud-driver-libvirt.sh +++ b/scripts/lib/mkcloud-driver-libvirt.sh @@ -285,6 +285,17 @@ function libvirt_do_create_cloud_lvm() done fi + # create an extra volume to test pci passthrough + if [ $extravolumenumber -gt 0 ] ; then + for i in $(nodes ids all) ; do + for n in $(seq 1 $extravolumenumber) ; do + onhost_get_next_pv_device + hdd_size=${extravolume_hdd_size} + _lvcreate $cloud.node$i-extra $hdd_size $cloudvg $next_pv_device + done + done + fi + # create volumes for drbd if [ $drbd_hdd_size != 0 ] ; then for i in `seq 1 2`; do diff --git a/scripts/mkcloud b/scripts/mkcloud index f509e33757..6112481be7 100755 --- a/scripts/mkcloud +++ b/scripts/mkcloud @@ -112,6 +112,12 @@ iscloudver 7plus && : ${controller_node_memory:=12582912} : ${controller_ceph_hdd_size:=25} : ${lonelynode_hdd_size:=20} : ${ironicnode_hdd_size:=20} +: ${want_pci_passthrough:=0} +if [[ $want_pci_passthrough = 1 ]]; then + # Create extra volume to test pci passthrough + : ${extravolumenumber:=1} + : ${extravolume_hdd_size:=1} +fi if [[ $hacloud = 1 ]] && ( [[ "$want_database_sql_engine" == "postgresql" ]] || iscloudver 6minus ) ; then : ${drbd_hdd_size:=15} @@ -739,6 +745,11 @@ function setupnodes localrepos_params="--localreposrc ${localreposdir_src} --localrepotgt ${localreposdir_target}" fi + local pcipassthrough_params="" + if [[ $want_pci_passthrough = 1 ]]; then + pcipassthrough_params="--pcipassthrough" + fi + local ipmi_params="" if [[ $want_ipmi = 1 ]] ; then ipmi_params="--ipmi" @@ -757,6 +768,7 @@ function setupnodes $mac_params \ $ironic_params \ $ipmi_params \ + $pcipassthrough_params \ --cephvolumenumber $cephvolumenumber \ --drbdserial "$drbd_serial"\ --computenodememory $compute_node_memory \