From e9558837a4470fb188a562ab4dc19d38ee16425b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C5=BDupka?= Date: Tue, 14 Jan 2014 11:37:21 +0100 Subject: [PATCH 1/2] virt: qcontainer adds possibility to find device by params. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows to search specific device in qemu container by filter for params. Created due to searching in the QStringDevices. Signed-off-by: Jiří Župka --- virttest/qemu_devices/qcontainer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/virttest/qemu_devices/qcontainer.py b/virttest/qemu_devices/qcontainer.py index c6fe0f5a8..2bc88a7ad 100644 --- a/virttest/qemu_devices/qcontainer.py +++ b/virttest/qemu_devices/qcontainer.py @@ -139,6 +139,23 @@ def get_by_properties(self, filt): out.append(device) return out + def get_by_params(self, filt): + """ + Return list of matching devices + :param filt: filter {'param': 'value', ...} + :type filt: dict + """ + out = [] + for device in self.__devices: + for key, value in filt.iteritems(): + if not key in device.params: + break + if device.params[key] != value: + break + else: + out.append(device) + return out + def __delitem__(self, item): """ Delete specified item from devices list From a597870d057aa2d65615c35831412e3855028b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C5=BDupka?= Date: Fri, 21 Feb 2014 12:35:51 +0100 Subject: [PATCH 2/2] qemu: Adds support for devices with dynamic params to qdevices. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch make qdevice more verbose during qcontainer comparison. Also this patch add support for dynamic params in general devices like QStringDevice and QCustomDevice and for devices classed inherited from them. It sets cmdline with and without dynamic params for QStringDevice. dev = QStringDevice(dev_type="xxx", cmdline="xxx:dynamicparam", cmdline_nd="xxx:DYN") It allows specification which param is dynamic for QCustomDevice dev = QCustomDevice(dev_type="xxx") dev.set_param(name, value, something, dynamic True/False) default is true. Comparison of qcontainer using == works as usual it compare all params without dynamic params. Some of qemu devices needs some kind of preparation before start of qemu vm. Unfortunately there preparatory work are done during creation of devices. For example network tap devices, port redir, etc. It should be divided in some later patch. Signed-off-by: Jiří Župka --- virttest/env_process.py | 1 + virttest/qemu_devices/qcontainer.py | 4 +- virttest/qemu_devices/qdevices.py | 94 ++++++++++++++++++- virttest/qemu_vm.py | 138 ++++++++++++++++++++++++---- virttest/virt_vm.py | 2 + 5 files changed, 212 insertions(+), 27 deletions(-) diff --git a/virttest/env_process.py b/virttest/env_process.py index a1f9f818c..2764b92dd 100644 --- a/virttest/env_process.py +++ b/virttest/env_process.py @@ -113,6 +113,7 @@ def preprocess_vm(test, params, env, name): if vm.needs_restart(name=name, params=params, basedir=test.bindir): + vm.devices = None start_vm = True old_vm.destroy(gracefully=gracefully_kill, free_mac_addresses=False) diff --git a/virttest/qemu_devices/qcontainer.py b/virttest/qemu_devices/qcontainer.py index 2bc88a7ad..0c6b7724a 100644 --- a/virttest/qemu_devices/qcontainer.py +++ b/virttest/qemu_devices/qcontainer.py @@ -673,14 +673,14 @@ def idx_of_next_named_bus(self, bus_pattern): return i i += 1 - def cmdline(self): + def cmdline(self, dynamic=True): """ Creates cmdline arguments for creating all defined devices :return: cmdline of all devices (without qemu-cmd itself) """ out = "" for device in self.__devices: - _out = device.cmdline() + _out = device.cmdline() if dynamic else device.cmdline_nd() if _out: out += " %s" % _out if out: diff --git a/virttest/qemu_devices/qdevices.py b/virttest/qemu_devices/qdevices.py index 95b9bf628..926844d18 100644 --- a/virttest/qemu_devices/qdevices.py +++ b/virttest/qemu_devices/qdevices.py @@ -15,6 +15,7 @@ from virttest import qemu_monitor from virttest import utils_misc import qbuses +import traceback try: # pylint: disable=E0611 @@ -81,6 +82,7 @@ def __init__(self, dev_type="QBaseDevice", params=None, aobject=None, else: for bus in child_bus: self.add_child_bus(bus) + self.dynamic_params = [] self.params = OrderedDict() # various device params (id, name, ...) if params: for key, value in params.iteritems(): @@ -104,13 +106,21 @@ def rm_child_bus(self, bus): self.child_bus.remove(bus) bus.set_device(None) - def set_param(self, option, value, option_type=None): + def set_param(self, option, value, option_type=None, dynamic=False): """ Set device param using qemu notation ("on", "off" instead of bool...) :param option: which option's value to set :param value: new value :param option_type: type of the option (bool) + :param dynamic: if true value is changed to DYN for not_dynamic compare """ + if dynamic: + if option not in self.dynamic_params: + self.dynamic_params.append(option) + else: + if option in self.dynamic_params: + self.dynamic_params.remove(option) + if option_type is bool or isinstance(value, bool): if value in ['yes', 'on', True]: self.params[option] = "on" @@ -123,6 +133,8 @@ def set_param(self, option, value, option_type=None): self.params[option] = value elif value is None and option in self.params: del(self.params[option]) + if option in self.dynamic_params: + self.dynamic_params.remove(option) def get_param(self, option, default=None): """ :return: object param """ @@ -152,11 +164,11 @@ def __str__(self): """ :return: Short string representation of this object. """ return self.str_short() - def __eq__(self, dev2): + def __eq__(self, dev2, dynamic=True): """ :return: True when devs are similar, False when different. """ + check_attrs = ['cmdline_nd', 'hotplug_hmp_nd', 'hotplug_qmp_nd'] try: - for check_attr in ('cmdline', 'hotplug_hmp', - 'hotplug_qmp'): + for check_attr in check_attrs: try: _ = getattr(self, check_attr)() except (DeviceError, NotImplementedError, AttributeError): @@ -168,6 +180,7 @@ def __eq__(self, dev2): if _ != getattr(dev2, check_attr)(): return False except Exception: + logging.error(traceback.format_exc()) return False return True @@ -227,6 +240,13 @@ def cmdline(self): """ :return: cmdline command to define this device """ raise NotImplementedError + def cmdline_nd(self): + """ + :return: cmdline command to define this device + without dynamic parameters + """ + self.cmdline() + # pylint: disable=E0202 def hotplug(self, monitor): """ :return: the output of monitor.cmd() hotplug command """ @@ -306,7 +326,7 @@ class QStringDevice(QBaseDevice): """ def __init__(self, dev_type="dummy", params=None, aobject=None, - parent_bus=None, child_bus=None, cmdline=""): + parent_bus=None, child_bus=None, cmdline="", cmdline_nd=None): """ :param dev_type: type of this component :param params: component's parameters @@ -318,6 +338,9 @@ def __init__(self, dev_type="dummy", params=None, aobject=None, super(QStringDevice, self).__init__(dev_type, params, aobject, parent_bus, child_bus) self._cmdline = cmdline + self._cmdline_nd = cmdline_nd + if cmdline_nd is None: + self._cmdline_nd = cmdline def cmdline(self): """ :return: cmdline command to define this device """ @@ -329,6 +352,19 @@ def cmdline(self): % (details, self.str_long())) + def cmdline_nd(self): + """ + :return: cmdline command to define this device + without dynamic parameters + """ + try: + if self._cmdline_nd: + return self._cmdline_nd % self.params + except KeyError, details: + raise KeyError("Param %s required for cmdline is not present in %s" + % (details, self.str_long())) + + class QCustomDevice(QBaseDevice): """ @@ -366,6 +402,30 @@ def cmdline(self): out = out[:-1] return out + def cmdline_nd(self): + """ + :return: cmdline command to define this device + without dynamic parameters + """ + if self.__backend and self.params.get(self.__backend): + out = "-%s %s," % (self.type, self.params.get(self.__backend)) + params = self.params.copy() + del params[self.__backend] + else: + out = "-%s " % self.type + params = self.params + for key, value in params.iteritems(): + if value != "NO_EQUAL_STRING": + if key in self.dynamic_params: + out += "%s=DYN," % (key,) + else: + out += "%s=%s," % (key, value) + else: + out += "%s," % key + if out[-1] == ',': + out = out[:-1] + return out + class QDrive(QCustomDevice): @@ -599,6 +659,30 @@ def hotplug_qmp(self): """ :return: the hotplug monitor command """ return "device_add", self.params + def hotplug_hmp_nd(self): + """ :return: the hotplug monitor command without dynamic parameters""" + if self.params.get('driver'): + params = self.params.copy() + out = "device_add %s" % params.pop('driver') + for key in self.dynamic_params: + params[key] = "DYN" + params = _convert_args(params) + if params: + out += ",%s" % params + else: + params = self.params.copy() + for key in self.dynamic_params: + params[key] = "DYN" + out = "device_add %s" % _convert_args(params) + return out + + def hotplug_qmp_nd(self): + """ :return: the hotplug monitor command without dynamic parameters""" + params = self.params.copy() + for key in self.dynamic_params: + params[key] = "DYN" + return "device_add", params + def get_children(self): """ Device bus should be removed too """ devices = super(QDevice, self).get_children() diff --git a/virttest/qemu_vm.py b/virttest/qemu_vm.py index 6d5091f7c..face37a7b 100644 --- a/virttest/qemu_vm.py +++ b/virttest/qemu_vm.py @@ -534,7 +534,7 @@ def add_nic(devices, vlan, model=None, mac=None, device_id=None, elif model == "virtio": model = "virtio-net-pci" dev = QDevice(model) - dev.set_param('mac', mac) + dev.set_param('mac', mac, dynamic=True) # only pci domain=0,bus=0,function=0 is supported for now. # # libvirt gains the pci_slot, free_pci_addr here, @@ -553,7 +553,7 @@ def add_nic(devices, vlan, model=None, mac=None, device_id=None, dev = qdevices.QCustomDevice('net', backend='type') dev.set_param('type', 'nic') dev.set_param('model', model) - dev.set_param('macaddr', mac, 'NEED_QUOTE') + dev.set_param('macaddr', mac, 'NEED_QUOTE', True) dev.set_param('id', device_id, 'NEED_QUOTE') if "virtio" in model: if int(queues) > 1: @@ -581,54 +581,66 @@ def add_net(devices, vlan, nettype, ifname=None, tftp=None, if devices.has_option("netdev"): cmd = " -netdev %s,id=%s" % (mode, netdev_id) + cmd_nd = cmd if vhost: if vhost in ["on", "off"]: cmd += ",vhost=%s" % vhost elif vhost == "vhost=on": # Keeps compatibility with old. cmd += ",%s" % vhost + cmd_nd = cmd if vhostfds: if (int(queues) > 1 and 'vhostfds=' in devices.get_help_text()): - cmd += ",vhostfds=%s" % vhostfds + cmd += ",vhostfds=%(vhostfds)s" + cmd_nd += ",vhostfds=DYN" else: txt = "" if int(queues) > 1: txt = "qemu do not support vhost multiqueue," txt += " Fall back to single queue." if 'vhostfd=' in devices.get_help_text(): - cmd += ",vhostfd=%s" % vhostfds.split(":")[0] + cmd += ",vhostfd=%(vhostfd)s" + cmd_nd += ",vhostfd=DYN" else: txt += " qemu do not support vhostfd." if txt: logging.warn(txt) if netdev_extra_params: cmd += "%s" % netdev_extra_params + cmd_nd += "%s" % netdev_extra_params else: cmd = " -net %s,vlan=%d" % (mode, vlan) - + cmd_nd = cmd if mode == "tap": if script: cmd += ",script='%s'" % script cmd += ",downscript='%s'" % (downscript or "no") + cmd_nd = cmd if ifname: cmd += ",ifname='%s'" % ifname + cmd_nd = cmd elif tapfds: - if ((int(queues)) > 1 - and ',fds=' in devices.get_help_text()): - cmd += ",fds=%s" % tapfds + if (int(queues) > 1 and + ',fds=' in devices.get_help_text()): + cmd += ",fds=%(tapfds)s" + cmd_nd += ",fds=DYN" else: - cmd += ",fd=%s" % tapfds + cmd += ",fd=%(tapfd)s" + cmd_nd += ",fd=DYN" elif mode == "user": if tftp and "[,tftp=" in devices.get_help_text(): cmd += ",tftp='%s'" % tftp + cmd_nd = cmd if bootfile and "[,bootfile=" in devices.get_help_text(): cmd += ",bootfile='%s'" % bootfile + cmd_nd = cmd if "[,hostfwd=" in devices.get_help_text(): - for host_port, guest_port in hostfwd: - cmd += ",hostfwd=tcp::%s-:%s" % (host_port, - guest_port) + for i in xrange(len(hostfwd)): + cmd += (",hostfwd=tcp::%%(host_port%d)s" + "-:%%(guest_port%d)s" % (i, i)) + cmd_nd += ",hostfwd=tcp::DYN-:%%(guest_port)ds" - return cmd + return cmd, cmd_nd def add_floppy(devices, filename, index): cmd_list = [" -fda '%s'", " -fdb '%s'"] @@ -1347,12 +1359,32 @@ def add_numa_node(devices, mem=None, cpus=None, nodeid=None): bootindex, queues, vectors, pci_bus) # Handle the '-net tap' or '-net user' or '-netdev' part - cmd = add_net(devices, vlan, nettype, ifname, tftp, - bootp, redirs, netdev_id, netdev_extra, - tapfds, script, downscript, vhost, queues, - vhostfds) + cmd, cmd_nd = add_net(devices, vlan, nettype, ifname, tftp, + bootp, redirs, netdev_id, netdev_extra, + tapfds, script, downscript, vhost, + queues, vhostfds) + + if vhostfds is None: + vhostfds = "" + + if tapfds is None: + tapfds = "" + + net_params = {'netdev_id': netdev_id, + 'vhostfd': vhostfds.split(":")[0], + 'vhostfds': vhostfds, + 'tapfd': tapfds.split(":")[0], + 'tapfds': tapfds, + 'ifname': ifname, + } + + for i, (host_port, guest_port) in enumerate(redirs): + net_params["host_port%d" % i] = host_port + net_params["guest_port%d" % i] = guest_port + # TODO: Is every NIC a PCI device? - devices.insert(StrDev("NET-%s" % nettype, cmdline=cmd)) + devices.insert(StrDev("NET-%s" % nettype, cmdline=cmd, + params=net_params, cmdline_nd=cmd_nd)) else: device_driver = nic_params.get("device_driver", "pci-assign") pci_id = vm.pa_pci_ids[iov] @@ -1779,6 +1811,69 @@ def create_serial_console(self): prompt=self.params.get("shell_prompt", "[\#\$]")) del tmp_serial + def update_system_dependent_devs(self): + # Networking + devices = self.devices + params = self.params + redirs = [] + for redir_name in params.objects("redirs"): + redir_params = params.object_params(redir_name) + guest_port = int(redir_params.get("guest_port")) + host_port = self.redirs.get(guest_port) + redirs += [(host_port, guest_port)] + + for nic in self.virtnet: + nic_params = params.object_params(nic.nic_name) + if nic_params.get('pci_assignable') == "no": + script = nic_params.get("nic_script") + downscript = nic_params.get("nic_downscript") + script_dir = data_dir.get_data_dir() + if script: + script = utils_misc.get_path(script_dir, script) + if downscript: + downscript = utils_misc.get_path(script_dir, + downscript) + # setup nic parameters as needed + # add_netdev if netdev_id not set + nic = self.add_nic(**dict(nic)) + # gather set values or None if unset + netdev_id = nic.get('netdev_id') + # don't force conversion add_nic()/add_net() optional + # parameter + if 'tapfds' in nic: + tapfds = nic.tapfds + else: + tapfds = "" + if 'vhostfds' in nic: + vhostfds = nic.vhostfds + else: + vhostfds = "" + ifname = nic.get('ifname') + # specify the number of MSI-X vectors that the card should + # have this option currently only affects virtio cards + + net_params = {'netdev_id': netdev_id, + 'vhostfd': vhostfds.split(":")[0], + 'vhostfds': vhostfds, + 'tapfd': tapfds.split(":")[0], + 'tapfds': tapfds, + 'ifname': ifname, + } + + for i, (host_port, guest_port) in enumerate(redirs): + net_params["host_port%d" % i] = host_port + net_params["guest_port%d" % i] = guest_port + + + # TODO: Is every NIC a PCI device? + devs = devices.get_by_params({'netdev_id': netdev_id}) + # TODO: Is every NIC a PCI device? + if len(devs) > 1: + logging.error("There are %d devices with netdev_id %s." + " This shouldn't happens." % (len(devs), + netdev_id)) + devs[0].params.update(net_params) + @error.context_aware def create(self, name=None, params=None, root_dir=None, timeout=CREATE_TIMEOUT, migration_mode=None, @@ -1980,6 +2075,9 @@ def create(self, name=None, params=None, root_dir=None, else: raise virt_vm.VMPAError(pa_type) + if (name is None and params is None and root_dir is None + and self.devices is not None): + self.update_system_dependent_devs() # Make qemu command try: self.devices = self.make_create_command() @@ -2066,7 +2164,7 @@ def create(self, name=None, params=None, root_dir=None, # test doesn't need to hold tapfd's open for nic in self.virtnet: - if nic.has_key('tapfds'): # implies bridge/tap + if 'tapfds' in nic: # implies bridge/tap try: for i in nic.tapfds.split(':'): os.close(int(i)) @@ -2078,7 +2176,7 @@ def create(self, name=None, params=None, root_dir=None, # File descriptor is already closed except OSError: pass - if nic.has_key('vhostfds'): + if 'vhostfds' in nic: try: for i in nic.vhostfds.split(':'): os.close(int(i)) diff --git a/virttest/virt_vm.py b/virttest/virt_vm.py index 4a804b6a7..dd5ec1a7c 100644 --- a/virttest/virt_vm.py +++ b/virttest/virt_vm.py @@ -9,6 +9,7 @@ import utils_net import remote import aexpect +import traceback import ppm_utils import data_dir @@ -564,6 +565,7 @@ def needs_restart(self, name, params, basedir): need_restart = (self.make_create_command() != self.make_create_command(name, params, basedir)) except Exception: + logging.error(traceback.format_exc()) need_restart = True if need_restart: logging.debug(