From 9531d26c10613348b53e1846566380be4f15b23c Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Mon, 18 Feb 2019 12:37:23 -0500 Subject: Acceptance tests: drop usage of ":avocado: enable" The Avocado test runner attemps to find its INSTRUMENTED (that is, Python based tests) in a manner that is as safe as possible to the user. Different from plain Python unittest, it won't load or execute test code on an operation such as: $ avocado list tests/acceptance/ Before version 68.0, the logic implemented to identify INSTRUMENTED tests would require either the ":avocado: enable" or ":avocado: recursive" statement as a flag for tests that would not inherit directly from "avocado.Test". This is not necessary anymore, and because of that the boiler plate statements can now be removed. Reference: https://avocado-framework.readthedocs.io/en/68.0/release_notes/68_0.html#users-test-writers Signed-off-by: Cleber Rosa Reviewed-by: Caio Carrara Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Wainer dos Santos Moschetta Message-Id: <20190218173723.26120-1-crosa@redhat.com> Signed-off-by: Cleber Rosa --- tests/acceptance/boot_linux_console.py | 1 - tests/acceptance/linux_initrd.py | 1 - tests/acceptance/version.py | 1 - tests/acceptance/virtio_version.py | 1 - tests/acceptance/vnc.py | 1 - tests/requirements.txt | 2 +- 6 files changed, 1 insertion(+), 6 deletions(-) (limited to 'tests') diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py index 98324f7591..beeb1e59e8 100644 --- a/tests/acceptance/boot_linux_console.py +++ b/tests/acceptance/boot_linux_console.py @@ -18,7 +18,6 @@ class BootLinuxConsole(Test): Boots a x86_64 Linux kernel and checks that the console is operational and the kernel command line is properly passed from QEMU to the kernel - :avocado: enable :avocado: tags=x86_64 """ diff --git a/tests/acceptance/linux_initrd.py b/tests/acceptance/linux_initrd.py index 737355c2ef..5a15fc4347 100644 --- a/tests/acceptance/linux_initrd.py +++ b/tests/acceptance/linux_initrd.py @@ -18,7 +18,6 @@ class LinuxInitrd(Test): """ Checks QEMU evaluates correctly the initrd file passed as -initrd option. - :avocado: enable :avocado: tags=x86_64 """ diff --git a/tests/acceptance/version.py b/tests/acceptance/version.py index 13b0a7440d..67c2192c93 100644 --- a/tests/acceptance/version.py +++ b/tests/acceptance/version.py @@ -14,7 +14,6 @@ from avocado_qemu import Test class Version(Test): """ - :avocado: enable :avocado: tags=quick """ def test_qmp_human_info_version(self): diff --git a/tests/acceptance/virtio_version.py b/tests/acceptance/virtio_version.py index ce990250d8..464d75aa4e 100644 --- a/tests/acceptance/virtio_version.py +++ b/tests/acceptance/virtio_version.py @@ -61,7 +61,6 @@ class VirtioVersionCheck(Test): same device tree created by `disable-modern` and `disable-legacy`. - :avocado: enable :avocado: tags=x86_64 """ diff --git a/tests/acceptance/vnc.py b/tests/acceptance/vnc.py index b1ef9d71b1..064ceabcc1 100644 --- a/tests/acceptance/vnc.py +++ b/tests/acceptance/vnc.py @@ -13,7 +13,6 @@ from avocado_qemu import Test class Vnc(Test): """ - :avocado: enable :avocado: tags=vnc,quick """ def test_no_vnc(self): diff --git a/tests/requirements.txt b/tests/requirements.txt index 64c6e27a94..002ded6a22 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,4 @@ # Add Python module requirements, one per line, to be installed # in the tests/venv Python virtual environment. For more info, # refer to: https://pip.pypa.io/en/stable/user_guide/#id1 -avocado-framework==65.0 +avocado-framework==68.0 -- cgit v1.2.3-55-g7522 From 8f8fd9edba4bd6768da2c8e2bea49ad5c16ced1a Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Wed, 6 Feb 2019 11:29:01 -0500 Subject: Introduce a Python module structure This is a simple move of Python code that wraps common QEMU functionality, and are used by a number of different tests and scripts. By treating that code as a real Python module, we can more easily: * reuse code * have a proper place for the module's own unittests * apply a more consistent style * generate documentation Signed-off-by: Cleber Rosa Reviewed-by: Caio Carrara Reviewed-by: Stefan Hajnoczi Message-Id: <20190206162901.19082-2-crosa@redhat.com> Signed-off-by: Cleber Rosa --- configure | 1 + python/qemu/__init__.py | 518 ++++++++++++++++++++++++++++++ python/qemu/qmp.py | 256 +++++++++++++++ python/qemu/qtest.py | 116 +++++++ scripts/device-crash-test | 2 + scripts/qemu.py | 517 ----------------------------- scripts/qmp/__init__.py | 0 scripts/qmp/qemu-ga-client | 5 +- scripts/qmp/qmp-shell | 4 +- scripts/qmp/qmp.py | 256 --------------- scripts/qtest.py | 115 ------- scripts/render_block_graph.py | 2 + tests/acceptance/avocado_qemu/__init__.py | 5 +- tests/acceptance/virtio_version.py | 2 +- tests/migration/guestperf/engine.py | 7 +- tests/qemu-iotests/235 | 2 +- tests/qemu-iotests/238 | 2 +- tests/qemu-iotests/iotests.py | 4 +- tests/vm/basevm.py | 2 +- 19 files changed, 914 insertions(+), 902 deletions(-) create mode 100644 python/qemu/__init__.py create mode 100644 python/qemu/qmp.py create mode 100644 python/qemu/qtest.py delete mode 100644 scripts/qemu.py delete mode 100644 scripts/qmp/__init__.py delete mode 100644 scripts/qmp/qmp.py delete mode 100644 scripts/qtest.py (limited to 'tests') diff --git a/configure b/configure index a61682c3c7..343dd004e9 100755 --- a/configure +++ b/configure @@ -7604,6 +7604,7 @@ LINKS="$LINKS pc-bios/qemu-icon.bmp" LINKS="$LINKS .gdbinit scripts" # scripts needed by relative path in .gdbinit LINKS="$LINKS tests/acceptance tests/data" LINKS="$LINKS tests/qemu-iotests/check" +LINKS="$LINKS python" for bios_file in \ $source_path/pc-bios/*.bin \ $source_path/pc-bios/*.lid \ diff --git a/python/qemu/__init__.py b/python/qemu/__init__.py new file mode 100644 index 0000000000..38de3e9177 --- /dev/null +++ b/python/qemu/__init__.py @@ -0,0 +1,518 @@ +# QEMU library +# +# Copyright (C) 2015-2016 Red Hat Inc. +# Copyright (C) 2012 IBM Corp. +# +# Authors: +# Fam Zheng +# +# This work is licensed under the terms of the GNU GPL, version 2. See +# the COPYING file in the top-level directory. +# +# Based on qmp.py. +# + +import errno +import logging +import os +import subprocess +import re +import shutil +import socket +import tempfile + +from . import qmp + + +LOG = logging.getLogger(__name__) + +# Mapping host architecture to any additional architectures it can +# support which often includes its 32 bit cousin. +ADDITIONAL_ARCHES = { + "x86_64" : "i386", + "aarch64" : "armhf" +} + +def kvm_available(target_arch=None): + host_arch = os.uname()[4] + if target_arch and target_arch != host_arch: + if target_arch != ADDITIONAL_ARCHES.get(host_arch): + return False + return os.access("/dev/kvm", os.R_OK | os.W_OK) + + +#: Maps machine types to the preferred console device types +CONSOLE_DEV_TYPES = { + r'^clipper$': 'isa-serial', + r'^malta': 'isa-serial', + r'^(pc.*|q35.*|isapc)$': 'isa-serial', + r'^(40p|powernv|prep)$': 'isa-serial', + r'^pseries.*': 'spapr-vty', + r'^s390-ccw-virtio.*': 'sclpconsole', + } + + +class QEMUMachineError(Exception): + """ + Exception called when an error in QEMUMachine happens. + """ + + +class QEMUMachineAddDeviceError(QEMUMachineError): + """ + Exception raised when a request to add a device can not be fulfilled + + The failures are caused by limitations, lack of information or conflicting + requests on the QEMUMachine methods. This exception does not represent + failures reported by the QEMU binary itself. + """ + +class MonitorResponseError(qmp.QMPError): + """ + Represents erroneous QMP monitor reply + """ + def __init__(self, reply): + try: + desc = reply["error"]["desc"] + except KeyError: + desc = reply + super(MonitorResponseError, self).__init__(desc) + self.reply = reply + + +class QEMUMachine(object): + """ + A QEMU VM + + Use this object as a context manager to ensure the QEMU process terminates:: + + with VM(binary) as vm: + ... + # vm is guaranteed to be shut down here + """ + + def __init__(self, binary, args=None, wrapper=None, name=None, + test_dir="/var/tmp", monitor_address=None, + socket_scm_helper=None): + ''' + Initialize a QEMUMachine + + @param binary: path to the qemu binary + @param args: list of extra arguments + @param wrapper: list of arguments used as prefix to qemu binary + @param name: prefix for socket and log file names (default: qemu-PID) + @param test_dir: where to create socket and log file + @param monitor_address: address for QMP monitor + @param socket_scm_helper: helper program, required for send_fd_scm() + @note: Qemu process is not started until launch() is used. + ''' + if args is None: + args = [] + if wrapper is None: + wrapper = [] + if name is None: + name = "qemu-%d" % os.getpid() + self._name = name + self._monitor_address = monitor_address + self._vm_monitor = None + self._qemu_log_path = None + self._qemu_log_file = None + self._popen = None + self._binary = binary + self._args = list(args) # Force copy args in case we modify them + self._wrapper = wrapper + self._events = [] + self._iolog = None + self._socket_scm_helper = socket_scm_helper + self._qmp = None + self._qemu_full_args = None + self._test_dir = test_dir + self._temp_dir = None + self._launched = False + self._machine = None + self._console_device_type = None + self._console_address = None + self._console_socket = None + + # just in case logging wasn't configured by the main script: + logging.basicConfig() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.shutdown() + return False + + # This can be used to add an unused monitor instance. + def add_monitor_telnet(self, ip, port): + args = 'tcp:%s:%d,server,nowait,telnet' % (ip, port) + self._args.append('-monitor') + self._args.append(args) + + def add_fd(self, fd, fdset, opaque, opts=''): + """ + Pass a file descriptor to the VM + """ + options = ['fd=%d' % fd, + 'set=%d' % fdset, + 'opaque=%s' % opaque] + if opts: + options.append(opts) + + # This did not exist before 3.4, but since then it is + # mandatory for our purpose + if hasattr(os, 'set_inheritable'): + os.set_inheritable(fd, True) + + self._args.append('-add-fd') + self._args.append(','.join(options)) + return self + + # Exactly one of fd and file_path must be given. + # (If it is file_path, the helper will open that file and pass its + # own fd) + def send_fd_scm(self, fd=None, file_path=None): + # In iotest.py, the qmp should always use unix socket. + assert self._qmp.is_scm_available() + if self._socket_scm_helper is None: + raise QEMUMachineError("No path to socket_scm_helper set") + if not os.path.exists(self._socket_scm_helper): + raise QEMUMachineError("%s does not exist" % + self._socket_scm_helper) + + # This did not exist before 3.4, but since then it is + # mandatory for our purpose + if hasattr(os, 'set_inheritable'): + os.set_inheritable(self._qmp.get_sock_fd(), True) + if fd is not None: + os.set_inheritable(fd, True) + + fd_param = ["%s" % self._socket_scm_helper, + "%d" % self._qmp.get_sock_fd()] + + if file_path is not None: + assert fd is None + fd_param.append(file_path) + else: + assert fd is not None + fd_param.append(str(fd)) + + devnull = open(os.path.devnull, 'rb') + proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, close_fds=False) + output = proc.communicate()[0] + if output: + LOG.debug(output) + + return proc.returncode + + @staticmethod + def _remove_if_exists(path): + """ + Remove file object at path if it exists + """ + try: + os.remove(path) + except OSError as exception: + if exception.errno == errno.ENOENT: + return + raise + + def is_running(self): + return self._popen is not None and self._popen.poll() is None + + def exitcode(self): + if self._popen is None: + return None + return self._popen.poll() + + def get_pid(self): + if not self.is_running(): + return None + return self._popen.pid + + def _load_io_log(self): + if self._qemu_log_path is not None: + with open(self._qemu_log_path, "r") as iolog: + self._iolog = iolog.read() + + def _base_args(self): + if isinstance(self._monitor_address, tuple): + moncdev = "socket,id=mon,host=%s,port=%s" % ( + self._monitor_address[0], + self._monitor_address[1]) + else: + moncdev = 'socket,id=mon,path=%s' % self._vm_monitor + args = ['-chardev', moncdev, + '-mon', 'chardev=mon,mode=control', + '-display', 'none', '-vga', 'none'] + if self._machine is not None: + args.extend(['-machine', self._machine]) + if self._console_device_type is not None: + self._console_address = os.path.join(self._temp_dir, + self._name + "-console.sock") + chardev = ('socket,id=console,path=%s,server,nowait' % + self._console_address) + device = '%s,chardev=console' % self._console_device_type + args.extend(['-chardev', chardev, '-device', device]) + return args + + def _pre_launch(self): + self._temp_dir = tempfile.mkdtemp(dir=self._test_dir) + if self._monitor_address is not None: + self._vm_monitor = self._monitor_address + else: + self._vm_monitor = os.path.join(self._temp_dir, + self._name + "-monitor.sock") + self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log") + self._qemu_log_file = open(self._qemu_log_path, 'wb') + + self._qmp = qmp.QEMUMonitorProtocol(self._vm_monitor, + server=True) + + def _post_launch(self): + self._qmp.accept() + + def _post_shutdown(self): + if self._qemu_log_file is not None: + self._qemu_log_file.close() + self._qemu_log_file = None + + self._qemu_log_path = None + + if self._console_socket is not None: + self._console_socket.close() + self._console_socket = None + + if self._temp_dir is not None: + shutil.rmtree(self._temp_dir) + self._temp_dir = None + + def launch(self): + """ + Launch the VM and make sure we cleanup and expose the + command line/output in case of exception + """ + + if self._launched: + raise QEMUMachineError('VM already launched') + + self._iolog = None + self._qemu_full_args = None + try: + self._launch() + self._launched = True + except: + self.shutdown() + + LOG.debug('Error launching VM') + if self._qemu_full_args: + LOG.debug('Command: %r', ' '.join(self._qemu_full_args)) + if self._iolog: + LOG.debug('Output: %r', self._iolog) + raise + + def _launch(self): + """ + Launch the VM and establish a QMP connection + """ + devnull = open(os.path.devnull, 'rb') + self._pre_launch() + self._qemu_full_args = (self._wrapper + [self._binary] + + self._base_args() + self._args) + self._popen = subprocess.Popen(self._qemu_full_args, + stdin=devnull, + stdout=self._qemu_log_file, + stderr=subprocess.STDOUT, + shell=False, + close_fds=False) + self._post_launch() + + def wait(self): + """ + Wait for the VM to power off + """ + self._popen.wait() + self._qmp.close() + self._load_io_log() + self._post_shutdown() + + def shutdown(self): + """ + Terminate the VM and clean up + """ + if self.is_running(): + try: + self._qmp.cmd('quit') + self._qmp.close() + except: + self._popen.kill() + self._popen.wait() + + self._load_io_log() + self._post_shutdown() + + exitcode = self.exitcode() + if exitcode is not None and exitcode < 0: + msg = 'qemu received signal %i: %s' + if self._qemu_full_args: + command = ' '.join(self._qemu_full_args) + else: + command = '' + LOG.warn(msg, -exitcode, command) + + self._launched = False + + def qmp(self, cmd, conv_keys=True, **args): + """ + Invoke a QMP command and return the response dict + """ + qmp_args = dict() + for key, value in args.items(): + if conv_keys: + qmp_args[key.replace('_', '-')] = value + else: + qmp_args[key] = value + + return self._qmp.cmd(cmd, args=qmp_args) + + def command(self, cmd, conv_keys=True, **args): + """ + Invoke a QMP command. + On success return the response dict. + On failure raise an exception. + """ + reply = self.qmp(cmd, conv_keys, **args) + if reply is None: + raise qmp.QMPError("Monitor is closed") + if "error" in reply: + raise MonitorResponseError(reply) + return reply["return"] + + def get_qmp_event(self, wait=False): + """ + Poll for one queued QMP events and return it + """ + if len(self._events) > 0: + return self._events.pop(0) + return self._qmp.pull_event(wait=wait) + + def get_qmp_events(self, wait=False): + """ + Poll for queued QMP events and return a list of dicts + """ + events = self._qmp.get_events(wait=wait) + events.extend(self._events) + del self._events[:] + self._qmp.clear_events() + return events + + def event_wait(self, name, timeout=60.0, match=None): + """ + Wait for specified timeout on named event in QMP; optionally filter + results by match. + + The 'match' is checked to be a recursive subset of the 'event'; skips + branch processing on match's value None + {"foo": {"bar": 1}} matches {"foo": None} + {"foo": {"bar": 1}} does not matches {"foo": {"baz": None}} + """ + def event_match(event, match=None): + if match is None: + return True + + for key in match: + if key in event: + if isinstance(event[key], dict): + if not event_match(event[key], match[key]): + return False + elif event[key] != match[key]: + return False + else: + return False + + return True + + # Search cached events + for event in self._events: + if (event['event'] == name) and event_match(event, match): + self._events.remove(event) + return event + + # Poll for new events + while True: + event = self._qmp.pull_event(wait=timeout) + if (event['event'] == name) and event_match(event, match): + return event + self._events.append(event) + + return None + + def get_log(self): + """ + After self.shutdown or failed qemu execution, this returns the output + of the qemu process. + """ + return self._iolog + + def add_args(self, *args): + """ + Adds to the list of extra arguments to be given to the QEMU binary + """ + self._args.extend(args) + + def set_machine(self, machine_type): + """ + Sets the machine type + + If set, the machine type will be added to the base arguments + of the resulting QEMU command line. + """ + self._machine = machine_type + + def set_console(self, device_type=None): + """ + Sets the device type for a console device + + If set, the console device and a backing character device will + be added to the base arguments of the resulting QEMU command + line. + + This is a convenience method that will either use the provided + device type, of if not given, it will used the device type set + on CONSOLE_DEV_TYPES. + + The actual setting of command line arguments will be be done at + machine launch time, as it depends on the temporary directory + to be created. + + @param device_type: the device type, such as "isa-serial" + @raises: QEMUMachineAddDeviceError if the device type is not given + and can not be determined. + """ + if device_type is None: + if self._machine is None: + raise QEMUMachineAddDeviceError("Can not add a console device:" + " QEMU instance without a " + "defined machine type") + for regex, device in CONSOLE_DEV_TYPES.items(): + if re.match(regex, self._machine): + device_type = device + break + if device_type is None: + raise QEMUMachineAddDeviceError("Can not add a console device:" + " no matching console device " + "type definition") + self._console_device_type = device_type + + @property + def console_socket(self): + """ + Returns a socket connected to the console + """ + if self._console_socket is None: + self._console_socket = socket.socket(socket.AF_UNIX, + socket.SOCK_STREAM) + self._console_socket.connect(self._console_address) + return self._console_socket diff --git a/python/qemu/qmp.py b/python/qemu/qmp.py new file mode 100644 index 0000000000..5c8cf6a056 --- /dev/null +++ b/python/qemu/qmp.py @@ -0,0 +1,256 @@ +# QEMU Monitor Protocol Python class +# +# Copyright (C) 2009, 2010 Red Hat Inc. +# +# Authors: +# Luiz Capitulino +# +# This work is licensed under the terms of the GNU GPL, version 2. See +# the COPYING file in the top-level directory. + +import json +import errno +import socket +import logging + + +class QMPError(Exception): + pass + + +class QMPConnectError(QMPError): + pass + + +class QMPCapabilitiesError(QMPError): + pass + + +class QMPTimeoutError(QMPError): + pass + + +class QEMUMonitorProtocol(object): + + #: Logger object for debugging messages + logger = logging.getLogger('QMP') + #: Socket's error class + error = socket.error + #: Socket's timeout + timeout = socket.timeout + + def __init__(self, address, server=False): + """ + Create a QEMUMonitorProtocol class. + + @param address: QEMU address, can be either a unix socket path (string) + or a tuple in the form ( address, port ) for a TCP + connection + @param server: server mode listens on the socket (bool) + @raise socket.error on socket connection errors + @note No connection is established, this is done by the connect() or + accept() methods + """ + self.__events = [] + self.__address = address + self.__sock = self.__get_sock() + self.__sockfile = None + if server: + self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.__sock.bind(self.__address) + self.__sock.listen(1) + + def __get_sock(self): + if isinstance(self.__address, tuple): + family = socket.AF_INET + else: + family = socket.AF_UNIX + return socket.socket(family, socket.SOCK_STREAM) + + def __negotiate_capabilities(self): + greeting = self.__json_read() + if greeting is None or "QMP" not in greeting: + raise QMPConnectError + # Greeting seems ok, negotiate capabilities + resp = self.cmd('qmp_capabilities') + if "return" in resp: + return greeting + raise QMPCapabilitiesError + + def __json_read(self, only_event=False): + while True: + data = self.__sockfile.readline() + if not data: + return + resp = json.loads(data) + if 'event' in resp: + self.logger.debug("<<< %s", resp) + self.__events.append(resp) + if not only_event: + continue + return resp + + def __get_events(self, wait=False): + """ + Check for new events in the stream and cache them in __events. + + @param wait (bool): block until an event is available. + @param wait (float): If wait is a float, treat it as a timeout value. + + @raise QMPTimeoutError: If a timeout float is provided and the timeout + period elapses. + @raise QMPConnectError: If wait is True but no events could be + retrieved or if some other error occurred. + """ + + # Check for new events regardless and pull them into the cache: + self.__sock.setblocking(0) + try: + self.__json_read() + except socket.error as err: + if err[0] == errno.EAGAIN: + # No data available + pass + self.__sock.setblocking(1) + + # Wait for new events, if needed. + # if wait is 0.0, this means "no wait" and is also implicitly false. + if not self.__events and wait: + if isinstance(wait, float): + self.__sock.settimeout(wait) + try: + ret = self.__json_read(only_event=True) + except socket.timeout: + raise QMPTimeoutError("Timeout waiting for event") + except: + raise QMPConnectError("Error while reading from socket") + if ret is None: + raise QMPConnectError("Error while reading from socket") + self.__sock.settimeout(None) + + def connect(self, negotiate=True): + """ + Connect to the QMP Monitor and perform capabilities negotiation. + + @return QMP greeting dict + @raise socket.error on socket connection errors + @raise QMPConnectError if the greeting is not received + @raise QMPCapabilitiesError if fails to negotiate capabilities + """ + self.__sock.connect(self.__address) + self.__sockfile = self.__sock.makefile() + if negotiate: + return self.__negotiate_capabilities() + + def accept(self): + """ + Await connection from QMP Monitor and perform capabilities negotiation. + + @return QMP greeting dict + @raise socket.error on socket connection errors + @raise QMPConnectError if the greeting is not received + @raise QMPCapabilitiesError if fails to negotiate capabilities + """ + self.__sock.settimeout(15) + self.__sock, _ = self.__sock.accept() + self.__sockfile = self.__sock.makefile() + return self.__negotiate_capabilities() + + def cmd_obj(self, qmp_cmd): + """ + Send a QMP command to the QMP Monitor. + + @param qmp_cmd: QMP command to be sent as a Python dict + @return QMP response as a Python dict or None if the connection has + been closed + """ + self.logger.debug(">>> %s", qmp_cmd) + try: + self.__sock.sendall(json.dumps(qmp_cmd).encode('utf-8')) + except socket.error as err: + if err[0] == errno.EPIPE: + return + raise socket.error(err) + resp = self.__json_read() + self.logger.debug("<<< %s", resp) + return resp + + def cmd(self, name, args=None, cmd_id=None): + """ + Build a QMP command and send it to the QMP Monitor. + + @param name: command name (string) + @param args: command arguments (dict) + @param cmd_id: command id (dict, list, string or int) + """ + qmp_cmd = {'execute': name} + if args: + qmp_cmd['arguments'] = args + if cmd_id: + qmp_cmd['id'] = cmd_id + return self.cmd_obj(qmp_cmd) + + def command(self, cmd, **kwds): + """ + Build and send a QMP command to the monitor, report errors if any + """ + ret = self.cmd(cmd, kwds) + if "error" in ret: + raise Exception(ret['error']['desc']) + return ret['return'] + + def pull_event(self, wait=False): + """ + Pulls a single event. + + @param wait (bool): block until an event is available. + @param wait (float): If wait is a float, treat it as a timeout value. + + @raise QMPTimeoutError: If a timeout float is provided and the timeout + period elapses. + @raise QMPConnectError: If wait is True but no events could be + retrieved or if some other error occurred. + + @return The first available QMP event, or None. + """ + self.__get_events(wait) + + if self.__events: + return self.__events.pop(0) + return None + + def get_events(self, wait=False): + """ + Get a list of available QMP events. + + @param wait (bool): block until an event is available. + @param wait (float): If wait is a float, treat it as a timeout value. + + @raise QMPTimeoutError: If a timeout float is provided and the timeout + period elapses. + @raise QMPConnectError: If wait is True but no events could be + retrieved or if some other error occurred. + + @return The list of available QMP events. + """ + self.__get_events(wait) + return self.__events + + def clear_events(self): + """ + Clear current list of pending events. + """ + self.__events = [] + + def close(self): + self.__sock.close() + self.__sockfile.close() + + def settimeout(self, timeout): + self.__sock.settimeout(timeout) + + def get_sock_fd(self): + return self.__sock.fileno() + + def is_scm_available(self): + return self.__sock.family == socket.AF_UNIX diff --git a/python/qemu/qtest.py b/python/qemu/qtest.py new file mode 100644 index 0000000000..eb45824dd0 --- /dev/null +++ b/python/qemu/qtest.py @@ -0,0 +1,116 @@ +# QEMU qtest library +# +# Copyright (C) 2015 Red Hat Inc. +# +# Authors: +# Fam Zheng +# +# This work is licensed under the terms of the GNU GPL, version 2. See +# the COPYING file in the top-level directory. +# +# Based on qmp.py. +# + +import socket +import os + +from . import QEMUMachine + + +class QEMUQtestProtocol(object): + def __init__(self, address, server=False): + """ + Create a QEMUQtestProtocol object. + + @param address: QEMU address, can be either a unix socket path (string) + or a tuple in the form ( address, port ) for a TCP + connection + @param server: server mode, listens on the socket (bool) + @raise socket.error on socket connection errors + @note No connection is established, this is done by the connect() or + accept() methods + """ + self._address = address + self._sock = self._get_sock() + self._sockfile = None + if server: + self._sock.bind(self._address) + self._sock.listen(1) + + def _get_sock(self): + if isinstance(self._address, tuple): + family = socket.AF_INET + else: + family = socket.AF_UNIX + return socket.socket(family, socket.SOCK_STREAM) + + def connect(self): + """ + Connect to the qtest socket. + + @raise socket.error on socket connection errors + """ + self._sock.connect(self._address) + self._sockfile = self._sock.makefile() + + def accept(self): + """ + Await connection from QEMU. + + @raise socket.error on socket connection errors + """ + self._sock, _ = self._sock.accept() + self._sockfile = self._sock.makefile() + + def cmd(self, qtest_cmd): + """ + Send a qtest command on the wire. + + @param qtest_cmd: qtest command text to be sent + """ + self._sock.sendall((qtest_cmd + "\n").encode('utf-8')) + resp = self._sockfile.readline() + return resp + + def close(self): + self._sock.close() + self._sockfile.close() + + def settimeout(self, timeout): + self._sock.settimeout(timeout) + + +class QEMUQtestMachine(QEMUMachine): + '''A QEMU VM''' + + def __init__(self, binary, args=None, name=None, test_dir="/var/tmp", + socket_scm_helper=None): + if name is None: + name = "qemu-%d" % os.getpid() + super(QEMUQtestMachine, + self).__init__(binary, args, name=name, test_dir=test_dir, + socket_scm_helper=socket_scm_helper) + self._qtest = None + self._qtest_path = os.path.join(test_dir, name + "-qtest.sock") + + def _base_args(self): + args = super(QEMUQtestMachine, self)._base_args() + args.extend(['-qtest', 'unix:path=' + self._qtest_path, + '-machine', 'accel=qtest']) + return args + + def _pre_launch(self): + super(QEMUQtestMachine, self)._pre_launch() + self._qtest = QEMUQtestProtocol(self._qtest_path, server=True) + + def _post_launch(self): + super(QEMUQtestMachine, self)._post_launch() + self._qtest.accept() + + def _post_shutdown(self): + super(QEMUQtestMachine, self)._post_shutdown() + self._remove_if_exists(self._qtest_path) + + def qtest(self, cmd): + '''Send a qtest command to guest''' + return self._qtest.cmd(cmd) diff --git a/scripts/device-crash-test b/scripts/device-crash-test index 2a13fa4f84..a6748910ad 100755 --- a/scripts/device-crash-test +++ b/scripts/device-crash-test @@ -25,6 +25,7 @@ check for crashes and unexpected errors. """ from __future__ import print_function +import os import sys import glob import logging @@ -34,6 +35,7 @@ import random import argparse from itertools import chain +sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python')) from qemu import QEMUMachine logger = logging.getLogger('device-crash-test') diff --git a/scripts/qemu.py b/scripts/qemu.py deleted file mode 100644 index 32b00af5cc..0000000000 --- a/scripts/qemu.py +++ /dev/null @@ -1,517 +0,0 @@ -# QEMU library -# -# Copyright (C) 2015-2016 Red Hat Inc. -# Copyright (C) 2012 IBM Corp. -# -# Authors: -# Fam Zheng -# -# This work is licensed under the terms of the GNU GPL, version 2. See -# the COPYING file in the top-level directory. -# -# Based on qmp.py. -# - -import errno -import logging -import os -import subprocess -import qmp.qmp -import re -import shutil -import socket -import tempfile - - -LOG = logging.getLogger(__name__) - -# Mapping host architecture to any additional architectures it can -# support which often includes its 32 bit cousin. -ADDITIONAL_ARCHES = { - "x86_64" : "i386", - "aarch64" : "armhf" -} - -def kvm_available(target_arch=None): - host_arch = os.uname()[4] - if target_arch and target_arch != host_arch: - if target_arch != ADDITIONAL_ARCHES.get(host_arch): - return False - return os.access("/dev/kvm", os.R_OK | os.W_OK) - - -#: Maps machine types to the preferred console device types -CONSOLE_DEV_TYPES = { - r'^clipper$': 'isa-serial', - r'^malta': 'isa-serial', - r'^(pc.*|q35.*|isapc)$': 'isa-serial', - r'^(40p|powernv|prep)$': 'isa-serial', - r'^pseries.*': 'spapr-vty', - r'^s390-ccw-virtio.*': 'sclpconsole', - } - - -class QEMUMachineError(Exception): - """ - Exception called when an error in QEMUMachine happens. - """ - - -class QEMUMachineAddDeviceError(QEMUMachineError): - """ - Exception raised when a request to add a device can not be fulfilled - - The failures are caused by limitations, lack of information or conflicting - requests on the QEMUMachine methods. This exception does not represent - failures reported by the QEMU binary itself. - """ - -class MonitorResponseError(qmp.qmp.QMPError): - """ - Represents erroneous QMP monitor reply - """ - def __init__(self, reply): - try: - desc = reply["error"]["desc"] - except KeyError: - desc = reply - super(MonitorResponseError, self).__init__(desc) - self.reply = reply - - -class QEMUMachine(object): - """ - A QEMU VM - - Use this object as a context manager to ensure the QEMU process terminates:: - - with VM(binary) as vm: - ... - # vm is guaranteed to be shut down here - """ - - def __init__(self, binary, args=None, wrapper=None, name=None, - test_dir="/var/tmp", monitor_address=None, - socket_scm_helper=None): - ''' - Initialize a QEMUMachine - - @param binary: path to the qemu binary - @param args: list of extra arguments - @param wrapper: list of arguments used as prefix to qemu binary - @param name: prefix for socket and log file names (default: qemu-PID) - @param test_dir: where to create socket and log file - @param monitor_address: address for QMP monitor - @param socket_scm_helper: helper program, required for send_fd_scm() - @note: Qemu process is not started until launch() is used. - ''' - if args is None: - args = [] - if wrapper is None: - wrapper = [] - if name is None: - name = "qemu-%d" % os.getpid() - self._name = name - self._monitor_address = monitor_address - self._vm_monitor = None - self._qemu_log_path = None - self._qemu_log_file = None - self._popen = None - self._binary = binary - self._args = list(args) # Force copy args in case we modify them - self._wrapper = wrapper - self._events = [] - self._iolog = None - self._socket_scm_helper = socket_scm_helper - self._qmp = None - self._qemu_full_args = None - self._test_dir = test_dir - self._temp_dir = None - self._launched = False - self._machine = None - self._console_device_type = None - self._console_address = None - self._console_socket = None - - # just in case logging wasn't configured by the main script: - logging.basicConfig() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.shutdown() - return False - - # This can be used to add an unused monitor instance. - def add_monitor_telnet(self, ip, port): - args = 'tcp:%s:%d,server,nowait,telnet' % (ip, port) - self._args.append('-monitor') - self._args.append(args) - - def add_fd(self, fd, fdset, opaque, opts=''): - """ - Pass a file descriptor to the VM - """ - options = ['fd=%d' % fd, - 'set=%d' % fdset, - 'opaque=%s' % opaque] - if opts: - options.append(opts) - - # This did not exist before 3.4, but since then it is - # mandatory for our purpose - if hasattr(os, 'set_inheritable'): - os.set_inheritable(fd, True) - - self._args.append('-add-fd') - self._args.append(','.join(options)) - return self - - # Exactly one of fd and file_path must be given. - # (If it is file_path, the helper will open that file and pass its - # own fd) - def send_fd_scm(self, fd=None, file_path=None): - # In iotest.py, the qmp should always use unix socket. - assert self._qmp.is_scm_available() - if self._socket_scm_helper is None: - raise QEMUMachineError("No path to socket_scm_helper set") - if not os.path.exists(self._socket_scm_helper): - raise QEMUMachineError("%s does not exist" % - self._socket_scm_helper) - - # This did not exist before 3.4, but since then it is - # mandatory for our purpose - if hasattr(os, 'set_inheritable'): - os.set_inheritable(self._qmp.get_sock_fd(), True) - if fd is not None: - os.set_inheritable(fd, True) - - fd_param = ["%s" % self._socket_scm_helper, - "%d" % self._qmp.get_sock_fd()] - - if file_path is not None: - assert fd is None - fd_param.append(file_path) - else: - assert fd is not None - fd_param.append(str(fd)) - - devnull = open(os.path.devnull, 'rb') - proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, close_fds=False) - output = proc.communicate()[0] - if output: - LOG.debug(output) - - return proc.returncode - - @staticmethod - def _remove_if_exists(path): - """ - Remove file object at path if it exists - """ - try: - os.remove(path) - except OSError as exception: - if exception.errno == errno.ENOENT: - return - raise - - def is_running(self): - return self._popen is not None and self._popen.poll() is None - - def exitcode(self): - if self._popen is None: - return None - return self._popen.poll() - - def get_pid(self): - if not self.is_running(): - return None - return self._popen.pid - - def _load_io_log(self): - if self._qemu_log_path is not None: - with open(self._qemu_log_path, "r") as iolog: - self._iolog = iolog.read() - - def _base_args(self): - if isinstance(self._monitor_address, tuple): - moncdev = "socket,id=mon,host=%s,port=%s" % ( - self._monitor_address[0], - self._monitor_address[1]) - else: - moncdev = 'socket,id=mon,path=%s' % self._vm_monitor - args = ['-chardev', moncdev, - '-mon', 'chardev=mon,mode=control', - '-display', 'none', '-vga', 'none'] - if self._machine is not None: - args.extend(['-machine', self._machine]) - if self._console_device_type is not None: - self._console_address = os.path.join(self._temp_dir, - self._name + "-console.sock") - chardev = ('socket,id=console,path=%s,server,nowait' % - self._console_address) - device = '%s,chardev=console' % self._console_device_type - args.extend(['-chardev', chardev, '-device', device]) - return args - - def _pre_launch(self): - self._temp_dir = tempfile.mkdtemp(dir=self._test_dir) - if self._monitor_address is not None: - self._vm_monitor = self._monitor_address - else: - self._vm_monitor = os.path.join(self._temp_dir, - self._name + "-monitor.sock") - self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log") - self._qemu_log_file = open(self._qemu_log_path, 'wb') - - self._qmp = qmp.qmp.QEMUMonitorProtocol(self._vm_monitor, - server=True) - - def _post_launch(self): - self._qmp.accept() - - def _post_shutdown(self): - if self._qemu_log_file is not None: - self._qemu_log_file.close() - self._qemu_log_file = None - - self._qemu_log_path = None - - if self._console_socket is not None: - self._console_socket.close() - self._console_socket = None - - if self._temp_dir is not None: - shutil.rmtree(self._temp_dir) - self._temp_dir = None - - def launch(self): - """ - Launch the VM and make sure we cleanup and expose the - command line/output in case of exception - """ - - if self._launched: - raise QEMUMachineError('VM already launched') - - self._iolog = None - self._qemu_full_args = None - try: - self._launch() - self._launched = True - except: - self.shutdown() - - LOG.debug('Error launching VM') - if self._qemu_full_args: - LOG.debug('Command: %r', ' '.join(self._qemu_full_args)) - if self._iolog: - LOG.debug('Output: %r', self._iolog) - raise - - def _launch(self): - """ - Launch the VM and establish a QMP connection - """ - devnull = open(os.path.devnull, 'rb') - self._pre_launch() - self._qemu_full_args = (self._wrapper + [self._binary] + - self._base_args() + self._args) - self._popen = subprocess.Popen(self._qemu_full_args, - stdin=devnull, - stdout=self._qemu_log_file, - stderr=subprocess.STDOUT, - shell=False, - close_fds=False) - self._post_launch() - - def wait(self): - """ - Wait for the VM to power off - """ - self._popen.wait() - self._qmp.close() - self._load_io_log() - self._post_shutdown() - - def shutdown(self): - """ - Terminate the VM and clean up - """ - if self.is_running(): - try: - self._qmp.cmd('quit') - self._qmp.close() - except: - self._popen.kill() - self._popen.wait() - - self._load_io_log() - self._post_shutdown() - - exitcode = self.exitcode() - if exitcode is not None and exitcode < 0: - msg = 'qemu received signal %i: %s' - if self._qemu_full_args: - command = ' '.join(self._qemu_full_args) - else: - command = '' - LOG.warn(msg, -exitcode, command) - - self._launched = False - - def qmp(self, cmd, conv_keys=True, **args): - """ - Invoke a QMP command and return the response dict - """ - qmp_args = dict() - for key, value in args.items(): - if conv_keys: - qmp_args[key.replace('_', '-')] = value - else: - qmp_args[key] = value - - return self._qmp.cmd(cmd, args=qmp_args) - - def command(self, cmd, conv_keys=True, **args): - """ - Invoke a QMP command. - On success return the response dict. - On failure raise an exception. - """ - reply = self.qmp(cmd, conv_keys, **args) - if reply is None: - raise qmp.qmp.QMPError("Monitor is closed") - if "error" in reply: - raise MonitorResponseError(reply) - return reply["return"] - - def get_qmp_event(self, wait=False): - """ - Poll for one queued QMP events and return it - """ - if len(self._events) > 0: - return self._events.pop(0) - return self._qmp.pull_event(wait=wait) - - def get_qmp_events(self, wait=False): - """ - Poll for queued QMP events and return a list of dicts - """ - events = self._qmp.get_events(wait=wait) - events.extend(self._events) - del self._events[:] - self._qmp.clear_events() - return events - - def event_wait(self, name, timeout=60.0, match=None): - """ - Wait for specified timeout on named event in QMP; optionally filter - results by match. - - The 'match' is checked to be a recursive subset of the 'event'; skips - branch processing on match's value None - {"foo": {"bar": 1}} matches {"foo": None} - {"foo": {"bar": 1}} does not matches {"foo": {"baz": None}} - """ - def event_match(event, match=None): - if match is None: - return True - - for key in match: - if key in event: - if isinstance(event[key], dict): - if not event_match(event[key], match[key]): - return False - elif event[key] != match[key]: - return False - else: - return False - - return True - - # Search cached events - for event in self._events: - if (event['event'] == name) and event_match(event, match): - self._events.remove(event) - return event - - # Poll for new events - while True: - event = self._qmp.pull_event(wait=timeout) - if (event['event'] == name) and event_match(event, match): - return event - self._events.append(event) - - return None - - def get_log(self): - """ - After self.shutdown or failed qemu execution, this returns the output - of the qemu process. - """ - return self._iolog - - def add_args(self, *args): - """ - Adds to the list of extra arguments to be given to the QEMU binary - """ - self._args.extend(args) - - def set_machine(self, machine_type): - """ - Sets the machine type - - If set, the machine type will be added to the base arguments - of the resulting QEMU command line. - """ - self._machine = machine_type - - def set_console(self, device_type=None): - """ - Sets the device type for a console device - - If set, the console device and a backing character device will - be added to the base arguments of the resulting QEMU command - line. - - This is a convenience method that will either use the provided - device type, of if not given, it will used the device type set - on CONSOLE_DEV_TYPES. - - The actual setting of command line arguments will be be done at - machine launch time, as it depends on the temporary directory - to be created. - - @param device_type: the device type, such as "isa-serial" - @raises: QEMUMachineAddDeviceError if the device type is not given - and can not be determined. - """ - if device_type is None: - if self._machine is None: - raise QEMUMachineAddDeviceError("Can not add a console device:" - " QEMU instance without a " - "defined machine type") - for regex, device in CONSOLE_DEV_TYPES.items(): - if re.match(regex, self._machine): - device_type = device - break - if device_type is None: - raise QEMUMachineAddDeviceError("Can not add a console device:" - " no matching console device " - "type definition") - self._console_device_type = device_type - - @property - def console_socket(self): - """ - Returns a socket connected to the console - """ - if self._console_socket is None: - self._console_socket = socket.socket(socket.AF_UNIX, - socket.SOCK_STREAM) - self._console_socket.connect(self._console_address) - return self._console_socket diff --git a/scripts/qmp/__init__.py b/scripts/qmp/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scripts/qmp/qemu-ga-client b/scripts/qmp/qemu-ga-client index e8cb7646a0..30cf8a9a0d 100755 --- a/scripts/qmp/qemu-ga-client +++ b/scripts/qmp/qemu-ga-client @@ -37,10 +37,13 @@ # from __future__ import print_function +import os +import sys import base64 import random -import qmp +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu import qmp class QemuGuestAgent(qmp.QEMUMonitorProtocol): diff --git a/scripts/qmp/qmp-shell b/scripts/qmp/qmp-shell index 770140772d..9fec46e2ed 100755 --- a/scripts/qmp/qmp-shell +++ b/scripts/qmp/qmp-shell @@ -66,7 +66,6 @@ # sent to QEMU, which is useful for debugging and documentation generation. from __future__ import print_function -import qmp import json import ast import readline @@ -76,6 +75,9 @@ import errno import atexit import shlex +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu import qmp + class QMPCompleter(list): def complete(self, text, state): for cmd in self: diff --git a/scripts/qmp/qmp.py b/scripts/qmp/qmp.py deleted file mode 100644 index 5c8cf6a056..0000000000 --- a/scripts/qmp/qmp.py +++ /dev/null @@ -1,256 +0,0 @@ -# QEMU Monitor Protocol Python class -# -# Copyright (C) 2009, 2010 Red Hat Inc. -# -# Authors: -# Luiz Capitulino -# -# This work is licensed under the terms of the GNU GPL, version 2. See -# the COPYING file in the top-level directory. - -import json -import errno -import socket -import logging - - -class QMPError(Exception): - pass - - -class QMPConnectError(QMPError): - pass - - -class QMPCapabilitiesError(QMPError): - pass - - -class QMPTimeoutError(QMPError): - pass - - -class QEMUMonitorProtocol(object): - - #: Logger object for debugging messages - logger = logging.getLogger('QMP') - #: Socket's error class - error = socket.error - #: Socket's timeout - timeout = socket.timeout - - def __init__(self, address, server=False): - """ - Create a QEMUMonitorProtocol class. - - @param address: QEMU address, can be either a unix socket path (string) - or a tuple in the form ( address, port ) for a TCP - connection - @param server: server mode listens on the socket (bool) - @raise socket.error on socket connection errors - @note No connection is established, this is done by the connect() or - accept() methods - """ - self.__events = [] - self.__address = address - self.__sock = self.__get_sock() - self.__sockfile = None - if server: - self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.__sock.bind(self.__address) - self.__sock.listen(1) - - def __get_sock(self): - if isinstance(self.__address, tuple): - family = socket.AF_INET - else: - family = socket.AF_UNIX - return socket.socket(family, socket.SOCK_STREAM) - - def __negotiate_capabilities(self): - greeting = self.__json_read() - if greeting is None or "QMP" not in greeting: - raise QMPConnectError - # Greeting seems ok, negotiate capabilities - resp = self.cmd('qmp_capabilities') - if "return" in resp: - return greeting - raise QMPCapabilitiesError - - def __json_read(self, only_event=False): - while True: - data = self.__sockfile.readline() - if not data: - return - resp = json.loads(data) - if 'event' in resp: - self.logger.debug("<<< %s", resp) - self.__events.append(resp) - if not only_event: - continue - return resp - - def __get_events(self, wait=False): - """ - Check for new events in the stream and cache them in __events. - - @param wait (bool): block until an event is available. - @param wait (float): If wait is a float, treat it as a timeout value. - - @raise QMPTimeoutError: If a timeout float is provided and the timeout - period elapses. - @raise QMPConnectError: If wait is True but no events could be - retrieved or if some other error occurred. - """ - - # Check for new events regardless and pull them into the cache: - self.__sock.setblocking(0) - try: - self.__json_read() - except socket.error as err: - if err[0] == errno.EAGAIN: - # No data available - pass - self.__sock.setblocking(1) - - # Wait for new events, if needed. - # if wait is 0.0, this means "no wait" and is also implicitly false. - if not self.__events and wait: - if isinstance(wait, float): - self.__sock.settimeout(wait) - try: - ret = self.__json_read(only_event=True) - except socket.timeout: - raise QMPTimeoutError("Timeout waiting for event") - except: - raise QMPConnectError("Error while reading from socket") - if ret is None: - raise QMPConnectError("Error while reading from socket") - self.__sock.settimeout(None) - - def connect(self, negotiate=True): - """ - Connect to the QMP Monitor and perform capabilities negotiation. - - @return QMP greeting dict - @raise socket.error on socket connection errors - @raise QMPConnectError if the greeting is not received - @raise QMPCapabilitiesError if fails to negotiate capabilities - """ - self.__sock.connect(self.__address) - self.__sockfile = self.__sock.makefile() - if negotiate: - return self.__negotiate_capabilities() - - def accept(self): - """ - Await connection from QMP Monitor and perform capabilities negotiation. - - @return QMP greeting dict - @raise socket.error on socket connection errors - @raise QMPConnectError if the greeting is not received - @raise QMPCapabilitiesError if fails to negotiate capabilities - """ - self.__sock.settimeout(15) - self.__sock, _ = self.__sock.accept() - self.__sockfile = self.__sock.makefile() - return self.__negotiate_capabilities() - - def cmd_obj(self, qmp_cmd): - """ - Send a QMP command to the QMP Monitor. - - @param qmp_cmd: QMP command to be sent as a Python dict - @return QMP response as a Python dict or None if the connection has - been closed - """ - self.logger.debug(">>> %s", qmp_cmd) - try: - self.__sock.sendall(json.dumps(qmp_cmd).encode('utf-8')) - except socket.error as err: - if err[0] == errno.EPIPE: - return - raise socket.error(err) - resp = self.__json_read() - self.logger.debug("<<< %s", resp) - return resp - - def cmd(self, name, args=None, cmd_id=None): - """ - Build a QMP command and send it to the QMP Monitor. - - @param name: command name (string) - @param args: command arguments (dict) - @param cmd_id: command id (dict, list, string or int) - """ - qmp_cmd = {'execute': name} - if args: - qmp_cmd['arguments'] = args - if cmd_id: - qmp_cmd['id'] = cmd_id - return self.cmd_obj(qmp_cmd) - - def command(self, cmd, **kwds): - """ - Build and send a QMP command to the monitor, report errors if any - """ - ret = self.cmd(cmd, kwds) - if "error" in ret: - raise Exception(ret['error']['desc']) - return ret['return'] - - def pull_event(self, wait=False): - """ - Pulls a single event. - - @param wait (bool): block until an event is available. - @param wait (float): If wait is a float, treat it as a timeout value. - - @raise QMPTimeoutError: If a timeout float is provided and the timeout - period elapses. - @raise QMPConnectError: If wait is True but no events could be - retrieved or if some other error occurred. - - @return The first available QMP event, or None. - """ - self.__get_events(wait) - - if self.__events: - return self.__events.pop(0) - return None - - def get_events(self, wait=False): - """ - Get a list of available QMP events. - - @param wait (bool): block until an event is available. - @param wait (float): If wait is a float, treat it as a timeout value. - - @raise QMPTimeoutError: If a timeout float is provided and the timeout - period elapses. - @raise QMPConnectError: If wait is True but no events could be - retrieved or if some other error occurred. - - @return The list of available QMP events. - """ - self.__get_events(wait) - return self.__events - - def clear_events(self): - """ - Clear current list of pending events. - """ - self.__events = [] - - def close(self): - self.__sock.close() - self.__sockfile.close() - - def settimeout(self, timeout): - self.__sock.settimeout(timeout) - - def get_sock_fd(self): - return self.__sock.fileno() - - def is_scm_available(self): - return self.__sock.family == socket.AF_UNIX diff --git a/scripts/qtest.py b/scripts/qtest.py deleted file mode 100644 index afac3fe900..0000000000 --- a/scripts/qtest.py +++ /dev/null @@ -1,115 +0,0 @@ -# QEMU qtest library -# -# Copyright (C) 2015 Red Hat Inc. -# -# Authors: -# Fam Zheng -# -# This work is licensed under the terms of the GNU GPL, version 2. See -# the COPYING file in the top-level directory. -# -# Based on qmp.py. -# - -import socket -import os -import qemu - - -class QEMUQtestProtocol(object): - def __init__(self, address, server=False): - """ - Create a QEMUQtestProtocol object. - - @param address: QEMU address, can be either a unix socket path (string) - or a tuple in the form ( address, port ) for a TCP - connection - @param server: server mode, listens on the socket (bool) - @raise socket.error on socket connection errors - @note No connection is established, this is done by the connect() or - accept() methods - """ - self._address = address - self._sock = self._get_sock() - self._sockfile = None - if server: - self._sock.bind(self._address) - self._sock.listen(1) - - def _get_sock(self): - if isinstance(self._address, tuple): - family = socket.AF_INET - else: - family = socket.AF_UNIX - return socket.socket(family, socket.SOCK_STREAM) - - def connect(self): - """ - Connect to the qtest socket. - - @raise socket.error on socket connection errors - """ - self._sock.connect(self._address) - self._sockfile = self._sock.makefile() - - def accept(self): - """ - Await connection from QEMU. - - @raise socket.error on socket connection errors - """ - self._sock, _ = self._sock.accept() - self._sockfile = self._sock.makefile() - - def cmd(self, qtest_cmd): - """ - Send a qtest command on the wire. - - @param qtest_cmd: qtest command text to be sent - """ - self._sock.sendall((qtest_cmd + "\n").encode('utf-8')) - resp = self._sockfile.readline() - return resp - - def close(self): - self._sock.close() - self._sockfile.close() - - def settimeout(self, timeout): - self._sock.settimeout(timeout) - - -class QEMUQtestMachine(qemu.QEMUMachine): - '''A QEMU VM''' - - def __init__(self, binary, args=None, name=None, test_dir="/var/tmp", - socket_scm_helper=None): - if name is None: - name = "qemu-%d" % os.getpid() - super(QEMUQtestMachine, - self).__init__(binary, args, name=name, test_dir=test_dir, - socket_scm_helper=socket_scm_helper) - self._qtest = None - self._qtest_path = os.path.join(test_dir, name + "-qtest.sock") - - def _base_args(self): - args = super(QEMUQtestMachine, self)._base_args() - args.extend(['-qtest', 'unix:path=' + self._qtest_path, - '-machine', 'accel=qtest']) - return args - - def _pre_launch(self): - super(QEMUQtestMachine, self)._pre_launch() - self._qtest = QEMUQtestProtocol(self._qtest_path, server=True) - - def _post_launch(self): - super(QEMUQtestMachine, self)._post_launch() - self._qtest.accept() - - def _post_shutdown(self): - super(QEMUQtestMachine, self)._post_shutdown() - self._remove_if_exists(self._qtest_path) - - def qtest(self, cmd): - '''Send a qtest command to guest''' - return self._qtest.cmd(cmd) diff --git a/scripts/render_block_graph.py b/scripts/render_block_graph.py index ed7e581b4f..3e9d282a49 100755 --- a/scripts/render_block_graph.py +++ b/scripts/render_block_graph.py @@ -23,6 +23,8 @@ import sys import subprocess import json from graphviz import Digraph + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python')) from qemu import MonitorResponseError diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py index 1e54fd5932..28bfb8e9d3 100644 --- a/tests/acceptance/avocado_qemu/__init__.py +++ b/tests/acceptance/avocado_qemu/__init__.py @@ -13,9 +13,8 @@ import sys import avocado -SRC_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) -SRC_ROOT_DIR = os.path.abspath(os.path.dirname(SRC_ROOT_DIR)) -sys.path.append(os.path.join(SRC_ROOT_DIR, 'scripts')) +SRC_ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..') +sys.path.append(os.path.join(SRC_ROOT_DIR, 'python')) from qemu import QEMUMachine diff --git a/tests/acceptance/virtio_version.py b/tests/acceptance/virtio_version.py index 464d75aa4e..37fc01ea18 100644 --- a/tests/acceptance/virtio_version.py +++ b/tests/acceptance/virtio_version.py @@ -11,7 +11,7 @@ Check compatibility of virtio device types import sys import os -sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts")) +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu import QEMUMachine from avocado_qemu import Test diff --git a/tests/migration/guestperf/engine.py b/tests/migration/guestperf/engine.py index 398e3f2706..0e304660b8 100644 --- a/tests/migration/guestperf/engine.py +++ b/tests/migration/guestperf/engine.py @@ -24,13 +24,14 @@ import re import sys import time -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts')) -import qemu -import qmp.qmp from guestperf.progress import Progress, ProgressStats from guestperf.report import Report from guestperf.timings import TimingRecord, Timings +sys.path.append(os.path.join(os.path.dirname(__file__), + '..', '..', '..', 'python')) +import qemu + class Engine(object): diff --git a/tests/qemu-iotests/235 b/tests/qemu-iotests/235 index d6edd97ab4..75c203b30c 100755 --- a/tests/qemu-iotests/235 +++ b/tests/qemu-iotests/235 @@ -23,7 +23,7 @@ import os import iotests from iotests import qemu_img_create, qemu_io, file_path, log -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts')) +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu import QEMUMachine diff --git a/tests/qemu-iotests/238 b/tests/qemu-iotests/238 index f81ee1112f..688abc9acb 100755 --- a/tests/qemu-iotests/238 +++ b/tests/qemu-iotests/238 @@ -23,7 +23,7 @@ import os import iotests from iotests import log -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts')) +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu import QEMUMachine diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index b461f53abf..54a31044b4 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -32,8 +32,8 @@ import atexit import io from collections import OrderedDict -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts')) -import qtest +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) +from qemu import qtest # This will not work if arguments contain spaces but is necessary if we diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py index bdca6cb2fc..0556bdcf9e 100755 --- a/tests/vm/basevm.py +++ b/tests/vm/basevm.py @@ -17,7 +17,7 @@ import sys import logging import time import datetime -sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts")) +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu import QEMUMachine, kvm_available import subprocess import hashlib -- cgit v1.2.3-55-g7522 From b7287d428325705ddf9d9b26d3a7dc2d4ffe837f Mon Sep 17 00:00:00 2001 From: Caio Carrara Date: Tue, 12 Feb 2019 17:38:54 -0200 Subject: tests.acceptance: adds multi vm capability for acceptance tests This change adds the possibility to write acceptance tests with multi virtual machine support. It's done keeping the virtual machines objects stored in a test attribute (dictionary). This dictionary shouldn't be accessed directly but through the new method added `get_vm`. This new method accept a list of args (that will be added as virtual machine arguments) and an optional name argument. The name is the key that identify a single virtual machine along the test machines available. If a name without a machine is informed a new machine will be instantiated. The current usage of vm in tests will not be broken by this change since it keeps a property called vm in the base test class. This property only calls the new method `get_vm` with default parameters (no args and 'default' as machine name). Signed-off-by: Caio Carrara Reviewed-by: Cleber Rosa Reviewed-by: Wainer dos Santos Moschetta Message-Id: <20190212193855.13223-2-ccarrara@redhat.com> Signed-off-by: Cleber Rosa --- docs/devel/testing.rst | 41 ++++++++++++++++++++++++++++++- tests/acceptance/avocado_qemu/__init__.py | 25 ++++++++++++++++--- 2 files changed, 61 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst index 3ce171829d..60f897d915 100644 --- a/docs/devel/testing.rst +++ b/docs/devel/testing.rst @@ -633,7 +633,46 @@ instance, available at ``self.vm``. Because many tests will tweak the QEMU command line, launching the QEMUMachine (by using ``self.vm.launch()``) is left to the test writer. -At test "tear down", ``avocado_qemu.Test`` handles the QEMUMachine +The base test class has also support for tests with more than one +QEMUMachine. The way to get machines is through the ``self.get_vm()`` +method which will return a QEMUMachine instance. The ``self.get_vm()`` +method accepts arguments that will be passed to the QEMUMachine creation +and also an optional `name` attribute so you can identify a specific +machine and get it more than once through the tests methods. A simple +and hypothetical example follows: + +.. code:: + + from avocado_qemu import Test + + + class MultipleMachines(Test): + """ + :avocado: enable + """ + def test_multiple_machines(self): + first_machine = self.get_vm() + second_machine = self.get_vm() + self.get_vm(name='third_machine').launch() + + first_machine.launch() + second_machine.launch() + + first_res = first_machine.command( + 'human-monitor-command', + command_line='info version') + + second_res = second_machine.command( + 'human-monitor-command', + command_line='info version') + + third_res = self.get_vm(name='third_machine').command( + 'human-monitor-command', + command_line='info version') + + self.assertEquals(first_res, second_res, third_res) + +At test "tear down", ``avocado_qemu.Test`` handles all the QEMUMachines shutdown. QEMUMachine diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py index 28bfb8e9d3..a66ec72daa 100644 --- a/tests/acceptance/avocado_qemu/__init__.py +++ b/tests/acceptance/avocado_qemu/__init__.py @@ -10,6 +10,7 @@ import os import sys +import uuid import avocado @@ -41,13 +42,29 @@ def pick_default_qemu_bin(): class Test(avocado.Test): def setUp(self): - self.vm = None + self._vms = {} self.qemu_bin = self.params.get('qemu_bin', default=pick_default_qemu_bin()) if self.qemu_bin is None: self.cancel("No QEMU binary defined or found in the source tree") - self.vm = QEMUMachine(self.qemu_bin) + + def _new_vm(self, *args): + vm = QEMUMachine(self.qemu_bin) + if args: + vm.add_args(*args) + return vm + + @property + def vm(self): + return self.get_vm(name='default') + + def get_vm(self, *args, name=None): + if not name: + name = str(uuid.uuid4()) + if self._vms.get(name) is None: + self._vms[name] = self._new_vm(*args) + return self._vms[name] def tearDown(self): - if self.vm is not None: - self.vm.shutdown() + for vm in self._vms.values(): + vm.shutdown() -- cgit v1.2.3-55-g7522 From a7abb537654ee8e98a11ec85dbf0728c03afc3c0 Mon Sep 17 00:00:00 2001 From: Caio Carrara Date: Tue, 12 Feb 2019 17:38:55 -0200 Subject: tests.acceptance: adds simple migration test This change adds the simplest possible migration test. Beyond the test purpose itself it's also useful to exercise the multi virtual machines capabilities from base avocado qemu test class. Signed-off-by: Cleber Rosa Signed-off-by: Caio Carrara Reviewed-by: Cleber Rosa Reviewed-by: Wainer dos Santos Moschetta Message-Id: <20190212193855.13223-3-ccarrara@redhat.com> Signed-off-by: Cleber Rosa --- tests/acceptance/migration.py | 53 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/acceptance/migration.py (limited to 'tests') diff --git a/tests/acceptance/migration.py b/tests/acceptance/migration.py new file mode 100644 index 0000000000..6115cf6c24 --- /dev/null +++ b/tests/acceptance/migration.py @@ -0,0 +1,53 @@ +# Migration test +# +# Copyright (c) 2019 Red Hat, Inc. +# +# Authors: +# Cleber Rosa +# Caio Carrara +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + + +from avocado_qemu import Test + +from avocado.utils import network +from avocado.utils import wait + + +class Migration(Test): + """ + :avocado: enable + """ + + timeout = 10 + + @staticmethod + def migration_finished(vm): + return vm.command('query-migrate')['status'] in ('completed', 'failed') + + def _get_free_port(self): + port = network.find_free_port() + if port is None: + self.cancel('Failed to find a free port') + return port + + + def test_migration_with_tcp_localhost(self): + source_vm = self.get_vm() + dest_uri = 'tcp:localhost:%u' % self._get_free_port() + dest_vm = self.get_vm('-incoming', dest_uri) + dest_vm.launch() + source_vm.launch() + source_vm.qmp('migrate', uri=dest_uri) + wait.wait_for( + self.migration_finished, + timeout=self.timeout, + step=0.1, + args=(source_vm,) + ) + self.assertEqual(dest_vm.command('query-migrate')['status'], 'completed') + self.assertEqual(source_vm.command('query-migrate')['status'], 'completed') + self.assertEqual(dest_vm.command('query-status')['status'], 'running') + self.assertEqual(source_vm.command('query-status')['status'], 'postmigrate') -- cgit v1.2.3-55-g7522 From 3c7156fd3e289b9b5f7a6a69e4dd310b4898afc5 Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Mon, 28 Jan 2019 09:15:11 +0800 Subject: Acceptance tests: use linux-3.6 and set vm memory to 4GiB QEMU have already supported to load up to 4G initrd if the sepcified memory is enough and XLF_CAN_BE_LOADED_ABOVE_4G is set by guest kernel linux-3.6 kernel shipped by Fedora-18 cannot support xldflags so that it cannot support loading more than 2GiB initrd CC: Wainer dos Santos Moschetta CC: Caio Carrara CC: Cleber Rosa CC: Eduardo Habkost CC: Philippe Mathieu-Daudé Signed-off-by: Li Zhijian Reviewed-by: Wainer dos Santos Moschetta Reviewed-by: Cleber Rosa Message-Id: <1548638112-31101-1-git-send-email-lizhijian@cn.fujitsu.com> Signed-off-by: Cleber Rosa --- tests/acceptance/linux_initrd.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/tests/acceptance/linux_initrd.py b/tests/acceptance/linux_initrd.py index 5a15fc4347..e33b5dcec0 100644 --- a/tests/acceptance/linux_initrd.py +++ b/tests/acceptance/linux_initrd.py @@ -23,14 +23,16 @@ class LinuxInitrd(Test): timeout = 60 - def test_with_2gib_file_should_exit_error_msg(self): + def test_with_2gib_file_should_exit_error_msg_with_linux_v3_6(self): """ Pretends to boot QEMU with an initrd file with size of 2GiB and expect it exits with error message. + Fedora-18 shipped with linux-3.6 which have not supported xloadflags + cannot support more than 2GiB initrd. """ - kernel_url = ('https://mirrors.kernel.org/fedora/releases/28/' - 'Everything/x86_64/os/images/pxeboot/vmlinuz') - kernel_hash = '238e083e114c48200f80d889f7e32eeb2793e02a' + kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora/li' + 'nux/releases/18/Fedora/x86_64/os/images/pxeboot/vmlinuz') + kernel_hash = '41464f68efe42b9991250bed86c7081d2ccdbb21' kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash) max_size = 2 * (1024 ** 3) - 1 @@ -38,8 +40,8 @@ class LinuxInitrd(Test): initrd.seek(max_size) initrd.write(b'\0') initrd.flush() - cmd = "%s -kernel %s -initrd %s" % (self.qemu_bin, kernel_path, - initrd.name) + cmd = "%s -kernel %s -initrd %s -m 4096" % ( + self.qemu_bin, kernel_path, initrd.name) res = run(cmd, ignore_status=True) self.assertEqual(res.exit_status, 1) expected_msg = r'.*initrd is too large.*max: \d+, need %s.*' % ( -- cgit v1.2.3-55-g7522 From 8f1c89ec7443e4fa2cf106d8fa1c1c97b6ddeffb Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Mon, 28 Jan 2019 09:15:12 +0800 Subject: Acceptance tests: expect boot to extract 2GiB+ initrd with linux-v4.16 XLF_CAN_BE_LOADED_ABOVE_4G is set on vmlinuz shipped by Fedora-28 so that it's allowed to be loaded below 4 GB address. timeout is updated to 5 minutes as well since we need more time to load a large initrd to the guest CC: Wainer dos Santos Moschetta CC: Caio Carrara CC: Cleber Rosa CC: Eduardo Habkost CC: Philippe Mathieu-Daudé Signed-off-by: Li Zhijian Reviewed-by: Wainer dos Santos Moschetta Reviewed-by: Cleber Rosa Message-Id: <1548638112-31101-2-git-send-email-lizhijian@cn.fujitsu.com> Signed-off-by: Cleber Rosa --- tests/acceptance/linux_initrd.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/acceptance/linux_initrd.py b/tests/acceptance/linux_initrd.py index e33b5dcec0..fbdb48e43f 100644 --- a/tests/acceptance/linux_initrd.py +++ b/tests/acceptance/linux_initrd.py @@ -8,6 +8,7 @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. +import logging import tempfile from avocado.utils.process import run @@ -21,7 +22,7 @@ class LinuxInitrd(Test): :avocado: tags=x86_64 """ - timeout = 60 + timeout = 300 def test_with_2gib_file_should_exit_error_msg_with_linux_v3_6(self): """ @@ -47,3 +48,37 @@ class LinuxInitrd(Test): expected_msg = r'.*initrd is too large.*max: \d+, need %s.*' % ( max_size + 1) self.assertRegex(res.stderr_text, expected_msg) + + def test_with_2gib_file_should_work_with_linux_v4_16(self): + """ + QEMU has supported up to 4 GiB initrd for recent kernel + Expect guest can reach 'Unpacking initramfs...' + """ + kernel_url = ('https://mirrors.kernel.org/fedora/releases/28/' + 'Everything/x86_64/os/images/pxeboot/vmlinuz') + kernel_hash = '238e083e114c48200f80d889f7e32eeb2793e02a' + kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash) + max_size = 2 * (1024 ** 3) + 1 + + with tempfile.NamedTemporaryFile() as initrd: + initrd.seek(max_size) + initrd.write(b'\0') + initrd.flush() + + self.vm.set_machine('pc') + self.vm.set_console() + kernel_command_line = 'console=ttyS0' + self.vm.add_args('-kernel', kernel_path, + '-append', kernel_command_line, + '-initrd', initrd.name, + '-m', '5120') + self.vm.launch() + console = self.vm.console_socket.makefile() + console_logger = logging.getLogger('console') + while True: + msg = console.readline() + console_logger.debug(msg.strip()) + if 'Unpacking initramfs...' in msg: + break + if 'Kernel panic - not syncing' in msg: + self.fail("Kernel panic reached") -- cgit v1.2.3-55-g7522