From 5b9b49d7bd3e0da13e8f6d58578443a11817f56e Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 12 Jan 2018 12:11:43 +0100 Subject: docker: change Fedora base image to fedora:27 Using "fedora:latest" makes behavior different depending on when you actually pulled the image from the docker repository. In my case, the supposedly "latest" image was a Fedora 25 download from 8 months ago, and the new "test-debug" test was failing. Use "27" to improve reproducibility and make it clear when the image is obsolete. Cc: Fam Zheng Cc: Marc-André Lureau Signed-off-by: Paolo Bonzini Message-Id: <1515755504-21341-1-git-send-email-pbonzini@redhat.com> Reviewed-by: Marc-André Lureau Signed-off-by: Fam Zheng --- tests/docker/dockerfiles/fedora.docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index 26ede4f1d6..994a35a332 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -1,4 +1,4 @@ -FROM fedora:latest +FROM fedora:27 ENV PACKAGES \ ccache gettext git tar PyYAML sparse flex bison python3 bzip2 hostname \ glib2-devel pixman-devel zlib-devel SDL-devel libfdt-devel \ -- cgit v1.2.3-55-g7522 From 439b6e5efcd79effc5199cba533fe4b28d75e0f6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 3 Feb 2018 10:39:31 -0500 Subject: test-coroutine: add simple CoMutex test In preparation for adding a similar test using QemuLockable, add a very simple testcase that has two interleaved calls to lock and unlock. Reviewed-by: Stefan Hajnoczi Signed-off-by: Paolo Bonzini Message-Id: <20180203153935.8056-2-pbonzini@redhat.com> Reviewed-by: Richard Henderson Reviewed-by: Fam Zheng Signed-off-by: Fam Zheng --- tests/test-coroutine.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/test-coroutine.c b/tests/test-coroutine.c index 76c646107e..ab8fdf701e 100644 --- a/tests/test-coroutine.c +++ b/tests/test-coroutine.c @@ -175,7 +175,7 @@ static void coroutine_fn c1_fn(void *opaque) qemu_coroutine_enter(c2); } -static void test_co_queue(void) +static void test_no_dangling_access(void) { Coroutine *c1; Coroutine *c2; @@ -195,6 +195,51 @@ static void test_co_queue(void) *c1 = tmp; } +static bool locked; +static int done; + +static void coroutine_fn mutex_fn(void *opaque) +{ + CoMutex *m = opaque; + qemu_co_mutex_lock(m); + assert(!locked); + locked = true; + qemu_coroutine_yield(); + locked = false; + qemu_co_mutex_unlock(m); + done++; +} + +static void do_test_co_mutex(CoroutineEntry *entry, void *opaque) +{ + Coroutine *c1 = qemu_coroutine_create(entry, opaque); + Coroutine *c2 = qemu_coroutine_create(entry, opaque); + + done = 0; + qemu_coroutine_enter(c1); + g_assert(locked); + qemu_coroutine_enter(c2); + + /* Unlock queues c2. It is then started automatically when c1 yields or + * terminates. + */ + qemu_coroutine_enter(c1); + g_assert_cmpint(done, ==, 1); + g_assert(locked); + + qemu_coroutine_enter(c2); + g_assert_cmpint(done, ==, 2); + g_assert(!locked); +} + +static void test_co_mutex(void) +{ + CoMutex m; + + qemu_co_mutex_init(&m); + do_test_co_mutex(mutex_fn, &m); +} + /* * Check that creation, enter, and return work */ @@ -422,7 +467,7 @@ int main(int argc, char **argv) * crash, so skip it. */ if (CONFIG_COROUTINE_POOL) { - g_test_add_func("/basic/co_queue", test_co_queue); + g_test_add_func("/basic/no-dangling-access", test_no_dangling_access); } g_test_add_func("/basic/lifecycle", test_lifecycle); @@ -432,6 +477,7 @@ int main(int argc, char **argv) g_test_add_func("/basic/entered", test_entered); g_test_add_func("/basic/in_coroutine", test_in_coroutine); g_test_add_func("/basic/order", test_order); + g_test_add_func("/locking/co-mutex", test_co_mutex); if (g_test_perf()) { g_test_add_func("/perf/lifecycle", perf_lifecycle); g_test_add_func("/perf/nesting", perf_nesting); -- cgit v1.2.3-55-g7522 From e70372fcaffc99444edce400a5178cb196cddaf7 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 3 Feb 2018 10:39:32 -0500 Subject: lockable: add QemuLockable QemuLockable is a polymorphic lock type that takes an object and knows which function to use for locking and unlocking. The implementation could use C11 _Generic, but since the support is not very widespread I am instead using __builtin_choose_expr and __builtin_types_compatible_p, which are already used by include/qemu/atomic.h. QemuLockable can be used to implement lock guards, or to pass around a lock in such a way that a function can release it and re-acquire it. The next patch will do this for CoQueue. Signed-off-by: Paolo Bonzini Message-Id: <20180203153935.8056-3-pbonzini@redhat.com> Reviewed-by: Richard Henderson Reviewed-by: Stefan Hajnoczi Reviewed-by: Fam Zheng Signed-off-by: Fam Zheng --- include/qemu/compiler.h | 39 ++++++++++++++++++++ include/qemu/coroutine.h | 4 +- include/qemu/lockable.h | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ include/qemu/thread.h | 5 +-- include/qemu/typedefs.h | 4 ++ tests/test-coroutine.c | 25 +++++++++++++ 6 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 include/qemu/lockable.h (limited to 'tests') diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h index 5fcc4f7ec7..2cbe6a4f16 100644 --- a/include/qemu/compiler.h +++ b/include/qemu/compiler.h @@ -114,5 +114,44 @@ #ifndef __has_feature #define __has_feature(x) 0 /* compatibility with non-clang compilers */ #endif +/* Implement C11 _Generic via GCC builtins. Example: + * + * QEMU_GENERIC(x, (float, sinf), (long double, sinl), sin) (x) + * + * The first argument is the discriminator. The last is the default value. + * The middle ones are tuples in "(type, expansion)" format. + */ + +/* First, find out the number of generic cases. */ +#define QEMU_GENERIC(x, ...) \ + QEMU_GENERIC_(typeof(x), __VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) + +/* There will be extra arguments, but they are not used. */ +#define QEMU_GENERIC_(x, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, count, ...) \ + QEMU_GENERIC##count(x, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) + +/* Two more helper macros, this time to extract items from a parenthesized + * list. + */ +#define QEMU_FIRST_(a, b) a +#define QEMU_SECOND_(a, b) b + +/* ... and a final one for the common part of the "recursion". */ +#define QEMU_GENERIC_IF(x, type_then, else_) \ + __builtin_choose_expr(__builtin_types_compatible_p(x, \ + QEMU_FIRST_ type_then), \ + QEMU_SECOND_ type_then, else_) + +/* CPP poor man's "recursion". */ +#define QEMU_GENERIC1(x, a0, ...) (a0) +#define QEMU_GENERIC2(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC1(x, __VA_ARGS__)) +#define QEMU_GENERIC3(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC2(x, __VA_ARGS__)) +#define QEMU_GENERIC4(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC3(x, __VA_ARGS__)) +#define QEMU_GENERIC5(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC4(x, __VA_ARGS__)) +#define QEMU_GENERIC6(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC5(x, __VA_ARGS__)) +#define QEMU_GENERIC7(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC6(x, __VA_ARGS__)) +#define QEMU_GENERIC8(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC7(x, __VA_ARGS__)) +#define QEMU_GENERIC9(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC8(x, __VA_ARGS__)) +#define QEMU_GENERIC10(x, a0, ...) QEMU_GENERIC_IF(x, a0, QEMU_GENERIC9(x, __VA_ARGS__)) #endif /* COMPILER_H */ diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h index ce2eb73670..8a5129741c 100644 --- a/include/qemu/coroutine.h +++ b/include/qemu/coroutine.h @@ -121,7 +121,7 @@ bool qemu_coroutine_entered(Coroutine *co); * Provides a mutex that can be used to synchronise coroutines */ struct CoWaitRecord; -typedef struct CoMutex { +struct CoMutex { /* Count of pending lockers; 0 for a free mutex, 1 for an * uncontended mutex. */ @@ -142,7 +142,7 @@ typedef struct CoMutex { unsigned handoff, sequence; Coroutine *holder; -} CoMutex; +}; /** * Initialises a CoMutex. This must be called before any other operation is used diff --git a/include/qemu/lockable.h b/include/qemu/lockable.h new file mode 100644 index 0000000000..b6ed6c89ec --- /dev/null +++ b/include/qemu/lockable.h @@ -0,0 +1,96 @@ +/* + * Polymorphic locking functions (aka poor man templates) + * + * Copyright Red Hat, Inc. 2017, 2018 + * + * Author: Paolo Bonzini + * + * This work is licensed under the terms of the GNU LGPL, version 2 or later. + * See the COPYING.LIB file in the top-level directory. + * + */ + +#ifndef QEMU_LOCKABLE_H +#define QEMU_LOCKABLE_H + +#include "qemu/coroutine.h" +#include "qemu/thread.h" + +typedef void QemuLockUnlockFunc(void *); + +struct QemuLockable { + void *object; + QemuLockUnlockFunc *lock; + QemuLockUnlockFunc *unlock; +}; + +/* This function gives an error if an invalid, non-NULL pointer type is passed + * to QEMU_MAKE_LOCKABLE. For optimized builds, we can rely on dead-code elimination + * from the compiler, and give the errors already at link time. + */ +#ifdef __OPTIMIZE__ +void unknown_lock_type(void *); +#else +static inline void unknown_lock_type(void *unused) +{ + abort(); +} +#endif + +static inline __attribute__((__always_inline__)) QemuLockable * +qemu_make_lockable(void *x, QemuLockable *lockable) +{ + /* We cannot test this in a macro, otherwise we get compiler + * warnings like "the address of 'm' will always evaluate as 'true'". + */ + return x ? lockable : NULL; +} + +/* Auxiliary macros to simplify QEMU_MAKE_LOCABLE. */ +#define QEMU_LOCK_FUNC(x) ((QemuLockUnlockFunc *) \ + QEMU_GENERIC(x, \ + (QemuMutex *, qemu_mutex_lock), \ + (CoMutex *, qemu_co_mutex_lock), \ + (QemuSpin *, qemu_spin_lock), \ + unknown_lock_type)) + +#define QEMU_UNLOCK_FUNC(x) ((QemuLockUnlockFunc *) \ + QEMU_GENERIC(x, \ + (QemuMutex *, qemu_mutex_unlock), \ + (CoMutex *, qemu_co_mutex_unlock), \ + (QemuSpin *, qemu_spin_unlock), \ + unknown_lock_type)) + +/* In C, compound literals have the lifetime of an automatic variable. + * In C++ it would be different, but then C++ wouldn't need QemuLockable + * either... + */ +#define QEMU_MAKE_LOCKABLE_(x) qemu_make_lockable((x), &(QemuLockable) { \ + .object = (x), \ + .lock = QEMU_LOCK_FUNC(x), \ + .unlock = QEMU_UNLOCK_FUNC(x), \ + }) + +/* QEMU_MAKE_LOCKABLE - Make a polymorphic QemuLockable + * + * @x: a lock object (currently one of QemuMutex, CoMutex, QemuSpin). + * + * Returns a QemuLockable object that can be passed around + * to a function that can operate with locks of any kind. + */ +#define QEMU_MAKE_LOCKABLE(x) \ + QEMU_GENERIC(x, \ + (QemuLockable *, (x)), \ + QEMU_MAKE_LOCKABLE_(x)) + +static inline void qemu_lockable_lock(QemuLockable *x) +{ + x->lock(x->object); +} + +static inline void qemu_lockable_unlock(QemuLockable *x) +{ + x->unlock(x->object); +} + +#endif diff --git a/include/qemu/thread.h b/include/qemu/thread.h index 9af4e945aa..ef7bd16123 100644 --- a/include/qemu/thread.h +++ b/include/qemu/thread.h @@ -4,7 +4,6 @@ #include "qemu/processor.h" #include "qemu/atomic.h" -typedef struct QemuMutex QemuMutex; typedef struct QemuCond QemuCond; typedef struct QemuSemaphore QemuSemaphore; typedef struct QemuEvent QemuEvent; @@ -97,9 +96,9 @@ struct Notifier; void qemu_thread_atexit_add(struct Notifier *notifier); void qemu_thread_atexit_remove(struct Notifier *notifier); -typedef struct QemuSpin { +struct QemuSpin { int value; -} QemuSpin; +}; static inline void qemu_spin_init(QemuSpin *spin) { diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h index 9bd7a834ba..5923849cdd 100644 --- a/include/qemu/typedefs.h +++ b/include/qemu/typedefs.h @@ -19,6 +19,7 @@ typedef struct BusClass BusClass; typedef struct BusState BusState; typedef struct Chardev Chardev; typedef struct CompatProperty CompatProperty; +typedef struct CoMutex CoMutex; typedef struct CPUAddressSpace CPUAddressSpace; typedef struct CPUState CPUState; typedef struct DeviceListener DeviceListener; @@ -86,9 +87,12 @@ typedef struct QEMUBH QEMUBH; typedef struct QemuConsole QemuConsole; typedef struct QemuDmaBuf QemuDmaBuf; typedef struct QEMUFile QEMUFile; +typedef struct QemuLockable QemuLockable; +typedef struct QemuMutex QemuMutex; typedef struct QemuOpt QemuOpt; typedef struct QemuOpts QemuOpts; typedef struct QemuOptsList QemuOptsList; +typedef struct QemuSpin QemuSpin; typedef struct QEMUSGList QEMUSGList; typedef struct QEMUTimer QEMUTimer; typedef struct QEMUTimerListGroup QEMUTimerListGroup; diff --git a/tests/test-coroutine.c b/tests/test-coroutine.c index ab8fdf701e..28e79b3210 100644 --- a/tests/test-coroutine.c +++ b/tests/test-coroutine.c @@ -14,6 +14,7 @@ #include "qemu/osdep.h" #include "qemu/coroutine.h" #include "qemu/coroutine_int.h" +#include "qemu/lockable.h" /* * Check that qemu_in_coroutine() works @@ -210,6 +211,18 @@ static void coroutine_fn mutex_fn(void *opaque) done++; } +static void coroutine_fn lockable_fn(void *opaque) +{ + QemuLockable *x = opaque; + qemu_lockable_lock(x); + assert(!locked); + locked = true; + qemu_coroutine_yield(); + locked = false; + qemu_lockable_unlock(x); + done++; +} + static void do_test_co_mutex(CoroutineEntry *entry, void *opaque) { Coroutine *c1 = qemu_coroutine_create(entry, opaque); @@ -240,6 +253,17 @@ static void test_co_mutex(void) do_test_co_mutex(mutex_fn, &m); } +static void test_co_mutex_lockable(void) +{ + CoMutex m; + CoMutex *null_pointer = NULL; + + qemu_co_mutex_init(&m); + do_test_co_mutex(lockable_fn, QEMU_MAKE_LOCKABLE(&m)); + + g_assert(QEMU_MAKE_LOCKABLE(null_pointer) == NULL); +} + /* * Check that creation, enter, and return work */ @@ -478,6 +502,7 @@ int main(int argc, char **argv) g_test_add_func("/basic/in_coroutine", test_in_coroutine); g_test_add_func("/basic/order", test_order); g_test_add_func("/locking/co-mutex", test_co_mutex); + g_test_add_func("/locking/co-mutex/lockable", test_co_mutex_lockable); if (g_test_perf()) { g_test_add_func("/perf/lifecycle", perf_lifecycle); g_test_add_func("/perf/nesting", perf_nesting); -- cgit v1.2.3-55-g7522 From 4eb995603479f0f7aff14b518f8ada16fe694ca7 Mon Sep 17 00:00:00 2001 From: Fam Zheng Date: Thu, 1 Feb 2018 10:20:46 +0800 Subject: docs: Add docs/devel/testing.rst To make our efforts on QEMU testing easier to consume by contributors, let's add a document. For example, Patchew reports build errors on patches that should be relatively easy to reproduce with a few steps, and it is much nicer if there is such a documentation that it can refer to. This focuses on how to run existing tests and how to write new test cases, without going into the frameworks themselves. The VM based testing section is moved from tests/vm/README which now is a single line pointing to the new doc. Signed-off-by: Fam Zheng Reviewed-by: Stefan Hajnoczi Message-Id: <20180201022046.9425-1-famz@redhat.com> Signed-off-by: Fam Zheng --- docs/devel/testing.rst | 486 +++++++++++++++++++++++++++++++++++++++++++++++++ tests/vm/README | 90 +-------- 2 files changed, 487 insertions(+), 89 deletions(-) create mode 100644 docs/devel/testing.rst (limited to 'tests') diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst new file mode 100644 index 0000000000..0ca1a2d4b5 --- /dev/null +++ b/docs/devel/testing.rst @@ -0,0 +1,486 @@ +=============== +Testing in QEMU +=============== + +This document describes the testing infrastructure in QEMU. + +Testing with "make check" +========================= + +The "make check" testing family includes most of the C based tests in QEMU. For +a quick help, run ``make check-help`` from the source tree. + +The usual way to run these tests is: + +.. code:: + + make check + +which includes QAPI schema tests, unit tests, and QTests. Different sub-types +of "make check" tests will be explained below. + +Before running tests, it is best to build QEMU programs first. Some tests +expect the executables to exist and will fail with obscure messages if they +cannot find them. + +Unit tests +---------- + +Unit tests, which can be invoked with ``make check-unit``, are simple C tests +that typically link to individual QEMU object files and exercise them by +calling exported functions. + +If you are writing new code in QEMU, consider adding a unit test, especially +for utility modules that are relatively stateless or have few dependencies. To +add a new unit test: + +1. Create a new source file. For example, ``tests/foo-test.c``. + +2. Write the test. Normally you would include the header file which exports + the module API, then verify the interface behaves as expected from your + test. The test code should be organized with the glib testing framework. + Copying and modifying an existing test is usually a good idea. + +3. Add the test to ``tests/Makefile.include``. First, name the unit test + program and add it to ``$(check-unit-y)``; then add a rule to build the + executable. Optionally, you can add a magical variable to support ``gcov``. + For example: + +.. code:: + + check-unit-y += tests/foo-test$(EXESUF) + tests/foo-test$(EXESUF): tests/foo-test.o $(test-util-obj-y) + ... + gcov-files-foo-test-y = util/foo.c + +Since unit tests don't require environment variables, the simplest way to debug +a unit test failure is often directly invoking it or even running it under +``gdb``. However there can still be differences in behavior between ``make`` +invocations and your manual run, due to ``$MALLOC_PERTURB_`` environment +variable (which affects memory reclamation and catches invalid pointers better) +and gtester options. If necessary, you can run + +.. code:: + make check-unit V=1 + +and copy the actual command line which executes the unit test, then run +it from the command line. + +QTest +----- + +QTest is a device emulation testing framework. It can be very useful to test +device models; it could also control certain aspects of QEMU (such as virtual +clock stepping), with a special purpose "qtest" protocol. Refer to the +documentation in ``qtest.c`` for more details of the protocol. + +QTest cases can be executed with + +.. code:: + + make check-qtest + +The QTest library is implemented by ``tests/libqtest.c`` and the API is defined +in ``tests/libqtest.h``. + +Consider adding a new QTest case when you are introducing a new virtual +hardware, or extending one if you are adding functionalities to an existing +virtual device. + +On top of libqtest, a higher level library, ``libqos``, was created to +encapsulate common tasks of device drivers, such as memory management and +communicating with system buses or devices. Many virtual device tests use +libqos instead of directly calling into libqtest. + +Steps to add a new QTest case are: + +1. Create a new source file for the test. (More than one file can be added as + necessary.) For example, ``tests/test-foo-device.c``. + +2. Write the test code with the glib and libqtest/libqos API. See also existing + tests and the library headers for reference. + +3. Register the new test in ``tests/Makefile.include``. Add the test executable + name to an appropriate ``check-qtest-*-y`` variable. For example: + + ``check-qtest-generic-y = tests/test-foo-device$(EXESUF)`` + +4. Add object dependencies of the executable in the Makefile, including the + test source file(s) and other interesting objects. For example: + + ``tests/test-foo-device$(EXESUF): tests/test-foo-device.o $(libqos-obj-y)`` + +Debugging a QTest failure is slightly harder than the unit test because the +tests look up QEMU program names in the environment variables, such as +``QTEST_QEMU_BINARY`` and ``QTEST_QEMU_IMG``, and also because it is not easy +to attach gdb to the QEMU process spawned from the test. But manual invoking +and using gdb on the test is still simple to do: find out the actual command +from the output of + +.. code:: + make check-qtest V=1 + +which you can run manually. + +QAPI schema tests +----------------- + +The QAPI schema tests validate the QAPI parser used by QMP, by feeding +predefined input to the parser and comparing the result with the reference +output. + +The input/output data is managed under the ``tests/qapi-schema`` directory. +Each test case includes four files that have a common base name: + + * ``${casename}.json`` - the file contains the JSON input for feeding the + parser + * ``${casename}.out`` - the file contains the expected stdout from the parser + * ``${casename}.err`` - the file contains the expected stderr from the parser + * ``${casename}.exit`` - the expected error code + +Consider adding a new QAPI schema test when you are making a change on the QAPI +parser (either fixing a bug or extending/modifying the syntax). To do this: + +1. Add four files for the new case as explained above. For example: + + ``$EDITOR tests/qapi-schema/foo.{json,out,err,exit}``. + +2. Add the new test in ``tests/Makefile.include``. For example: + + ``qapi-schema += foo.json`` + +check-block +----------- + +``make check-block`` is a legacy command to invoke block layer iotests and is +rarely used. See "QEMU iotests" section below for more information. + +GCC gcov support +---------------- + +``gcov`` is a GCC tool to analyze the testing coverage by instrumenting the +tested code. To use it, configure QEMU with ``--enable-gcov`` option and build. +Then run ``make check`` as usual. There will be additional ``gcov`` output as +the testing goes on, showing the test coverage percentage numbers per analyzed +source file. More detailed reports can be obtained by running ``gcov`` command +on the output files under ``$build_dir/tests/``, please read the ``gcov`` +documentation for more information. + +QEMU iotests +============ + +QEMU iotests, under the directory ``tests/qemu-iotests``, is the testing +framework widely used to test block layer related features. It is higher level +than "make check" tests and 99% of the code is written in bash or Python +scripts. The testing success criteria is golden output comparison, and the +test files are named with numbers. + +To run iotests, make sure QEMU is built successfully, then switch to the +``tests/qemu-iotests`` directory under the build directory, and run ``./check`` +with desired arguments from there. + +By default, "raw" format and "file" protocol is used; all tests will be +executed, except the unsupported ones. You can override the format and protocol +with arguments: + +.. code:: + + # test with qcow2 format + ./check -qcow2 + # or test a different protocol + ./check -nbd + +It's also possible to list test numbers explicitly: + +.. code:: + + # run selected cases with qcow2 format + ./check -qcow2 001 030 153 + +Cache mode can be selected with the "-c" option, which may help reveal bugs +that are specific to certain cache mode. + +More options are supported by the ``./check`` script, run ``./check -h`` for +help. + +Writing a new test case +----------------------- + +Consider writing a tests case when you are making any changes to the block +layer. An iotest case is usually the choice for that. There are already many +test cases, so it is possible that extending one of them may achieve the goal +and save the boilerplate to create one. (Unfortunately, there isn't a 100% +reliable way to find a related one out of hundreds of tests. One approach is +using ``git grep``.) + +Usually an iotest case consists of two files. One is an executable that +produces output to stdout and stderr, the other is the expected reference +output. They are given the same number in file names. E.g. Test script ``055`` +and reference output ``055.out``. + +In rare cases, when outputs differ between cache mode ``none`` and others, a +``.out.nocache`` file is added. In other cases, when outputs differ between +image formats, more than one ``.out`` files are created ending with the +respective format names, e.g. ``178.out.qcow2`` and ``178.out.raw``. + +There isn't a hard rule about how to write a test script, but a new test is +usually a (copy and) modification of an existing case. There are a few +commonly used ways to create a test: + +* A Bash script. It will make use of several environmental variables related + to the testing procedure, and could source a group of ``common.*`` libraries + for some common helper routines. + +* A Python unittest script. Import ``iotests`` and create a subclass of + ``iotests.QMPTestCase``, then call ``iotests.main`` method. The downside of + this approach is that the output is too scarce, and the script is considered + harder to debug. + +* A simple Python script without using unittest module. This could also import + ``iotests`` for launching QEMU and utilities etc, but it doesn't inherit + from ``iotests.QMPTestCase`` therefore doesn't use the Python unittest + execution. This is a combination of 1 and 2. + +Pick the language per your preference since both Bash and Python have +comparable library support for invoking and interacting with QEMU programs. If +you opt for Python, it is strongly recommended to write Python 3 compatible +code. + +Docker based tests +================== + +Introduction +------------ + +The Docker testing framework in QEMU utilizes public Docker images to build and +test QEMU in predefined and widely accessible Linux environments. This makes +it possible to expand the test coverage across distros, toolchain flavors and +library versions. + +Prerequisites +------------- + +Install "docker" with the system package manager and start the Docker service +on your development machine, then make sure you have the privilege to run +Docker commands. Typically it means setting up passwordless ``sudo docker`` +command or login as root. For example: + +.. code:: + + $ sudo yum install docker + $ # or `apt-get install docker` for Ubuntu, etc. + $ sudo systemctl start docker + $ sudo docker ps + +The last command should print an empty table, to verify the system is ready. + +An alternative method to set up permissions is by adding the current user to +"docker" group and making the docker daemon socket file (by default +``/var/run/docker.sock``) accessible to the group: + +.. code:: + + $ sudo groupadd docker + $ sudo usermod $USER -G docker + $ sudo chown :docker /var/run/docker.sock + +Note that any one of above configurations makes it possible for the user to +exploit the whole host with Docker bind mounting or other privileged +operations. So only do it on development machines. + +Quickstart +---------- + +From source tree, type ``make docker`` to see the help. Testing can be started +without configuring or building QEMU (``configure`` and ``make`` are done in +the container, with parameters defined by the make target): + +.. code:: + + make docker-test-build@min-glib + +This will create a container instance using the ``min-glib`` image (the image +is downloaded and initialized automatically), in which the ``test-build`` job +is executed. + +Images +------ + +Along with many other images, the ``min-glib`` image is defined in a Dockerfile +in ``tests/docker/dockefiles/``, called ``min-glib.docker``. ``make docker`` +command will list all the available images. + +To add a new image, simply create a new ``.docker`` file under the +``tests/docker/dockerfiles/`` directory. + +A ``.pre`` script can be added beside the ``.docker`` file, which will be +executed before building the image under the build context directory. This is +mainly used to do necessary host side setup. One such setup is ``binfmt_misc``, +for example, to make qemu-user powered cross build containers work. + +Tests +----- + +Different tests are added to cover various configurations to build and test +QEMU. Docker tests are the executables under ``tests/docker`` named +``test-*``. They are typically shell scripts and are built on top of a shell +library, ``tests/docker/common.rc``, which provides helpers to find the QEMU +source and build it. + +The full list of tests is printed in the ``make docker`` help. + +Tools +----- + +There are executables that are created to run in a specific Docker environment. +This makes it easy to write scripts that have heavy or special dependencies, +but are still very easy to use. + +Currently the only tool is ``travis``, which mimics the Travis-CI tests in a +container. It runs in the ``travis`` image: + +.. code:: + + make docker-travis@travis + +Debugging a Docker test failure +------------------------------- + +When CI tasks, maintainers or yourself report a Docker test failure, follow the +below steps to debug it: + +1. Locally reproduce the failure with the reported command line. E.g. run + ``make docker-test-mingw@fedora J=8``. +2. Add "V=1" to the command line, try again, to see the verbose output. +3. Further add "DEBUG=1" to the command line. This will pause in a shell prompt + in the container right before testing starts. You could either manually + build QEMU and run tests from there, or press Ctrl-D to let the Docker + testing continue. +4. If you press Ctrl-D, the same building and testing procedure will begin, and + will hopefully run into the error again. After that, you will be dropped to + the prompt for debug. + +Options +------- + +Various options can be used to affect how Docker tests are done. The full +list is in the ``make docker`` help text. The frequently used ones are: + +* ``V=1``: the same as in top level ``make``. It will be propagated to the + container and enable verbose output. +* ``J=$N``: the number of parallel tasks in make commands in the container, + similar to the ``-j $N`` option in top level ``make``. (The ``-j`` option in + top level ``make`` will not be propagated into the container.) +* ``DEBUG=1``: enables debug. See the previous "Debugging a Docker test + failure" section. + +VM testing +========== + +This test suite contains scripts that bootstrap various guest images that have +necessary packages to build QEMU. The basic usage is documented in ``Makefile`` +help which is displayed with ``make vm-test``. + +Quickstart +---------- + +Run ``make vm-test`` to list available make targets. Invoke a specific make +command to run build test in an image. For example, ``make vm-build-freebsd`` +will build the source tree in the FreeBSD image. The command can be executed +from either the source tree or the build dir; if the former, ``./configure`` is +not needed. The command will then generate the test image in ``./tests/vm/`` +under the working directory. + +Note: images created by the scripts accept a well-known RSA key pair for SSH +access, so they SHOULD NOT be exposed to external interfaces if you are +concerned about attackers taking control of the guest and potentially +exploiting a QEMU security bug to compromise the host. + +QEMU binary +----------- + +By default, qemu-system-x86_64 is searched in $PATH to run the guest. If there +isn't one, or if it is older than 2.10, the test won't work. In this case, +provide the QEMU binary in env var: ``QEMU=/path/to/qemu-2.10+``. + +Make jobs +--------- + +The ``-j$X`` option in the make command line is not propagated into the VM, +specify ``J=$X`` to control the make jobs in the guest. + +Debugging +--------- + +Add ``DEBUG=1`` and/or ``V=1`` to the make command to allow interactive +debugging and verbose output. If this is not enough, see the next section. + +Manual invocation +----------------- + +Each guest script is an executable script with the same command line options. +For example to work with the netbsd guest, use ``$QEMU_SRC/tests/vm/netbsd``: + +.. code:: + + $ cd $QEMU_SRC/tests/vm + + # To bootstrap the image + $ ./netbsd --build-image --image /var/tmp/netbsd.img + <...> + + # To run an arbitrary command in guest (the output will not be echoed unless + # --debug is added) + $ ./netbsd --debug --image /var/tmp/netbsd.img uname -a + + # To build QEMU in guest + $ ./netbsd --debug --image /var/tmp/netbsd.img --build-qemu $QEMU_SRC + + # To get to an interactive shell + $ ./netbsd --interactive --image /var/tmp/netbsd.img sh + +Adding new guests +----------------- + +Please look at existing guest scripts for how to add new guests. + +Most importantly, create a subclass of BaseVM and implement ``build_image()`` +method and define ``BUILD_SCRIPT``, then finally call ``basevm.main()`` from +the script's ``main()``. + +* Usually in ``build_image()``, a template image is downloaded from a + predefined URL. ``BaseVM._download_with_cache()`` takes care of the cache and + the checksum, so consider using it. + +* Once the image is downloaded, users, SSH server and QEMU build deps should + be set up: + + - Root password set to ``BaseVM.ROOT_PASS`` + - User ``BaseVM.GUEST_USER`` is created, and password set to + ``BaseVM.GUEST_PASS`` + - SSH service is enabled and started on boot, + ``$QEMU_SRC/tests/keys/id_rsa.pub`` is added to ssh's ``authorized_keys`` + file of both root and the normal user + - DHCP client service is enabled and started on boot, so that it can + automatically configure the virtio-net-pci NIC and communicate with QEMU + user net (10.0.2.2) + - Necessary packages are installed to untar the source tarball and build + QEMU + +* Write a proper ``BUILD_SCRIPT`` template, which should be a shell script that + untars a raw virtio-blk block device, which is the tarball data blob of the + QEMU source tree, then configure/build it. Running "make check" is also + recommended. + +Image fuzzer testing +==================== + +An image fuzzer was added to exercise format drivers. Currently only qcow2 is +supported. To start the fuzzer, run + +.. code:: + + tests/image-fuzzer/runner.py -c '[["qemu-img", "info", "$test_img"]]' /tmp/test qcow2 + +Alternatively, some command different from "qemu-img info" can be tested, by +changing the ``-c`` option. diff --git a/tests/vm/README b/tests/vm/README index ae53dce6ee..f9c04cc0e7 100644 --- a/tests/vm/README +++ b/tests/vm/README @@ -1,89 +1 @@ -=== VM test suite to run build in guests === - -== Intro == - -This test suite contains scripts that bootstrap various guest images that have -necessary packages to build QEMU. The basic usage is documented in Makefile -help which is displayed with "make vm-test". - -== Quick start == - -Run "make vm-test" to list available make targets. Invoke a specific make -command to run build test in an image. For example, "make vm-build-freebsd" -will build the source tree in the FreeBSD image. The command can be executed -from either the source tree or the build dir; if the former, ./configure is not -needed. The command will then generate the test image in ./tests/vm/ under the -working directory. - -Note: images created by the scripts accept a well-known RSA key pair for SSH -access, so they SHOULD NOT be exposed to external interfaces if you are -concerned about attackers taking control of the guest and potentially -exploiting a QEMU security bug to compromise the host. - -== QEMU binary == - -By default, qemu-system-x86_64 is searched in $PATH to run the guest. If there -isn't one, or if it is older than 2.10, the test won't work. In this case, -provide the QEMU binary in env var: QEMU=/path/to/qemu-2.10+. - -== Make jobs == - -The "-j$X" option in the make command line is not propagated into the VM, -specify "J=$X" to control the make jobs in the guest. - -== Debugging == - -Add "DEBUG=1" and/or "V=1" to the make command to allow interactive debugging -and verbose output. If this is not enough, see the next section. - -== Manual invocation == - -Each guest script is an executable script with the same command line options. -For example to work with the netbsd guest, use $QEMU_SRC/tests/vm/netbsd: - - $ cd $QEMU_SRC/tests/vm - - # To bootstrap the image - $ ./netbsd --build-image --image /var/tmp/netbsd.img - <...> - - # To run an arbitrary command in guest (the output will not be echoed unless - # --debug is added) - $ ./netbsd --debug --image /var/tmp/netbsd.img uname -a - - # To build QEMU in guest - $ ./netbsd --debug --image /var/tmp/netbsd.img --build-qemu $QEMU_SRC - - # To get to an interactive shell - $ ./netbsd --interactive --image /var/tmp/netbsd.img sh - -== Adding new guests == - -Please look at existing guest scripts for how to add new guests. - -Most importantly, create a subclass of BaseVM and implement build_image() -method and define BUILD_SCRIPT, then finally call basevm.main() from the -script's main(). - - - Usually in build_image(), a template image is downloaded from a predefined - URL. BaseVM._download_with_cache() takes care of the cache and the - checksum, so consider using it. - - - Once the image is downloaded, users, SSH server and QEMU build deps should - be set up: - - * Root password set to BaseVM.ROOT_PASS - * User BaseVM.GUEST_USER is created, and password set to BaseVM.GUEST_PASS - * SSH service is enabled and started on boot, - $QEMU_SRC/tests/keys/id_rsa.pub is added to ssh's "authorized_keys" file - of both root and the normal user - * DHCP client service is enabled and started on boot, so that it can - automatically configure the virtio-net-pci NIC and communicate with QEMU - user net (10.0.2.2) - * Necessary packages are installed to untar the source tarball and build - QEMU - - - Write a proper BUILD_SCRIPT template, which should be a shell script that - untars a raw virtio-blk block device, which is the tarball data blob of the - QEMU source tree, then configure/build it. Running "make check" is also - recommended. +See docs/devel/testing.rst for help. -- cgit v1.2.3-55-g7522