Merge remote-tracking branch 'remotes/famz/tags/build-and-test-automation-pull-request' into staging

# gpg: Signature made Fri 22 Sep 2017 08:28:38 BST
# gpg:                using RSA key 0xCA35624C6A9171C6
# gpg: Good signature from "Fam Zheng <famz@redhat.com>"
# gpg: WARNING: This key is not certified with sufficiently trusted signatures!
# gpg:          It is not certain that the signature belongs to the owner.
# Primary key fingerprint: 5003 7CB7 9706 0F76 F021  AD56 CA35 624C 6A91 71C6

* remotes/famz/tags/build-and-test-automation-pull-request: (36 commits)
  docker: Drop 'set -e' from run script
  docker: Use archive-source.py
  tests: Add README for vm tests
  MAINTAINERS: Add tests/vm entry
  Makefile: Add rules to run vm tests
  tests: Add OpenBSD image
  tests: Add NetBSD image
  tests: Add FreeBSD image
  tests: Add ubuntu.i386 image
  tests: Add vm test lib
  tests: Add a test key pair
  scripts: Add archive-source.sh
  qemu.py: Add "wait()" method
  gitignore: Ignore vm test images
  MAINTAINERS: Fix subsystem name for "Build and test automation"
  buildsys: Move rdma libs to per object
  buildsys: Move brlapi libs to per object
  buildsys: Move usb redir cflags/libs to per object
  buildsys: Move libusb cflags/libs to per object
  buildsys: Move libcacard cflags/libs to per object
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

9 files changed, 136 insertions, 42 deletions

diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include
index aa566aa223..0e4f159619 100644
--- a/tests/docker/Makefile.include
+++ b/tests/docker/Makefile.include
@@ -17,23 +17,13 @@ DOCKER_TOOLS := travis
 TESTS ?= %
 IMAGES ?= %
 
-# Make archive from git repo $1 to tar.gz $2
-make-archive-maybe = $(if $(wildcard $1/*), \
-	$(call quiet-command, \
-		(cd $1; if git diff-index --quiet HEAD -- &>/dev/null; then \
-			git archive -1 HEAD --format=tar.gz; \
-		else \
-			git archive -1 $$(git stash create) --format=tar.gz; \
-		fi) > $2, \
-		"ARCHIVE","$(notdir $2)"))
-
 CUR_TIME := $(shell date +%Y-%m-%d-%H.%M.%S.$$$$)
 DOCKER_SRC_COPY := docker-src.$(CUR_TIME)
 
 $(DOCKER_SRC_COPY):
 	@mkdir $@
-	$(call make-archive-maybe, $(SRC_PATH), $@/qemu.tgz)
-	$(call make-archive-maybe, $(SRC_PATH)/dtc, $@/dtc.tgz)
+	$(call quiet-command, $(SRC_PATH)/scripts/archive-source.sh $@/qemu.tar, \
+		"GEN", "$@/qemu.tar")
 	$(call quiet-command, cp $(SRC_PATH)/tests/docker/run $@/run, \
 		"COPY","RUNNER")
 
@@ -70,6 +60,7 @@ docker-image-debian-ppc64el-cross: docker-image-debian9
 docker-image-debian-s390x-cross: docker-image-debian9
 docker-image-debian-win32-cross: docker-image-debian8-mxe
 docker-image-debian-win64-cross: docker-image-debian8-mxe
+docker-image-travis: NOUSER=1
 
 # Expand all the pre-requistes for each docker image and test combination
 $(foreach i,$(DOCKER_IMAGES), \
@@ -144,6 +135,7 @@ docker-run: docker-qemu-src
 	$(call quiet-command,						\
 		$(SRC_PATH)/tests/docker/docker.py run 			\
 			$(if $(NOUSER),,-u $(shell id -u)) -t 		\
+			--security-opt seccomp=unconfined		\
 			$(if $V,,--rm) 					\
 			$(if $(DEBUG),-i,)				\
 			$(if $(NETWORK),$(if $(subst $(NETWORK),,1),--net=$(NETWORK)),--net=none) \
diff --git a/tests/docker/common.rc b/tests/docker/common.rc
index 6865689bb5..87f5263757 100755
--- a/tests/docker/common.rc
+++ b/tests/docker/common.rc
@@ -11,9 +11,6 @@
 # or (at your option) any later version. See the COPYING file in
 # the top-level directory.
 
-BUILD_DIR=/var/tmp/qemu-build
-mkdir $BUILD_DIR
-
 requires()
 {
     for c in $@; do
@@ -28,11 +25,22 @@ build_qemu()
 {
     config_opts="--enable-werror \
                  ${TARGET_LIST:+--target-list=${TARGET_LIST}} \
-                 --prefix=$PWD/install \
+                 --prefix=$INSTALL_DIR \
                  $QEMU_CONFIGURE_OPTS $EXTRA_CONFIGURE_OPTS \
                  $@"
     echo "Configure options:"
     echo $config_opts
-    $QEMU_SRC/configure $config_opts
-    make $MAKEFLAGS
+    $QEMU_SRC/configure $config_opts && make $MAKEFLAGS
+}
+
+test_fail()
+{
+    echo "$@"
+    exit 1
+}
+
+prep_fail()
+{
+    echo "$@"
+    exit 2
 }
diff --git a/tests/docker/docker.py b/tests/docker/docker.py
index 81c87ee329..08122ca17d 100755
--- a/tests/docker/docker.py
+++ b/tests/docker/docker.py
@@ -263,7 +263,8 @@ class BuildCommand(SubCommand):
         tag = args.tag
 
         dkr = Docker()
-        if dkr.image_matches_dockerfile(tag, dockerfile):
+        if "--no-cache" not in argv and \
+           dkr.image_matches_dockerfile(tag, dockerfile):
             if not args.quiet:
                 print "Image is up to date."
         else:
diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker
index 4eaa8ed2a5..27e8201c54 100644
--- a/tests/docker/dockerfiles/fedora.docker
+++ b/tests/docker/dockerfiles/fedora.docker
@@ -3,6 +3,7 @@ ENV PACKAGES \
     ccache git tar PyYAML sparse flex bison python2 bzip2 hostname \
     glib2-devel pixman-devel zlib-devel SDL-devel libfdt-devel \
     gcc gcc-c++ clang make perl which bc findutils libaio-devel \
+    nettle-devel \
     mingw32-pixman mingw32-glib2 mingw32-gmp mingw32-SDL mingw32-pkg-config \
     mingw32-gtk2 mingw32-gtk3 mingw32-gnutls mingw32-nettle mingw32-libtasn1 \
     mingw32-libjpeg-turbo mingw32-libpng mingw32-curl mingw32-libssh2 \
diff --git a/tests/docker/dockerfiles/travis.docker b/tests/docker/dockerfiles/travis.docker
index 636fa590a5..605b6e429b 100644
--- a/tests/docker/dockerfiles/travis.docker
+++ b/tests/docker/dockerfiles/travis.docker
@@ -1,6 +1,8 @@
 FROM quay.io/travisci/travis-ruby
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
 RUN apt-get update
 RUN apt-get -y build-dep qemu
-RUN apt-get -y build-dep device-tree-compiler
-RUN apt-get -y install python2.7 python-yaml dh-autoreconf gdb strace lsof net-tools
+RUN apt-get -y install device-tree-compiler python2.7 python-yaml dh-autoreconf gdb strace lsof net-tools
 ENV FEATURES pyyaml
diff --git a/tests/docker/dockerfiles/ubuntu.docker b/tests/docker/dockerfiles/ubuntu.docker
index a360a050a2..d73ce02246 100644
--- a/tests/docker/dockerfiles/ubuntu.docker
+++ b/tests/docker/dockerfiles/ubuntu.docker
@@ -1,12 +1,17 @@
-FROM ubuntu:14.04
+FROM ubuntu:16.04
 RUN echo "deb http://archive.ubuntu.com/ubuntu/ trusty universe multiverse" >> \
     /etc/apt/sources.list
 RUN apt-get update
 ENV PACKAGES flex bison \
-    libusb-1.0-0-dev libiscsi-dev librados-dev libncurses5-dev \
+    libusb-1.0-0-dev libiscsi-dev librados-dev libncurses5-dev libncursesw5-dev \
     libseccomp-dev libgnutls-dev libssh2-1-dev  libspice-server-dev \
     libspice-protocol-dev libnss3-dev libfdt-dev \
-    libgtk-3-dev libvte-2.90-dev libsdl1.2-dev libpng12-dev libpixman-1-dev \
+    libgtk-3-dev libvte-2.91-dev libsdl1.2-dev libpng12-dev libpixman-1-dev \
+    libvdeplug-dev liblzo2-dev libsnappy-dev libbz2-dev libxen-dev librdmacm-dev libibverbs-dev \
+    libsasl2-dev libjpeg-turbo8-dev xfslibs-dev libcap-ng-dev libbrlapi-dev libcurl4-gnutls-dev \
+    libbluetooth-dev librbd-dev libaio-dev glusterfs-common libnuma-dev libepoxy-dev libdrm-dev libgbm-dev \
+    libjemalloc-dev libcacard-dev libusbredirhost-dev libnfs-dev libcap-dev libattr1-dev \
+    texinfo \
     git make ccache python-yaml gcc clang sparse
 RUN apt-get -y install $PACKAGES
 RUN dpkg -l $PACKAGES | sort > /packages.txt
diff --git a/tests/docker/run b/tests/docker/run
index c1e4513bce..c8f940de15 100755
--- a/tests/docker/run
+++ b/tests/docker/run
@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash
 #
 # Docker test runner
 #
@@ -11,8 +11,6 @@
 # or (at your option) any later version. See the COPYING file in
 # the top-level directory.
 
-set -e
-
 if test -n "$V"; then
     set -x
 fi
@@ -20,7 +18,7 @@ fi
 BASE="$(dirname $(readlink -e $0))"
 
 # Prepare the environment
-. /etc/profile || true
+. /etc/profile
 export PATH=/usr/lib/ccache:$PATH
 
 if test -n "$J"; then
@@ -32,13 +30,7 @@ export TEST_DIR=/tmp/qemu-test
 mkdir -p $TEST_DIR/{src,build,install}
 
 # Extract the source tarballs
-tar -C $TEST_DIR/src -xzf $BASE/qemu.tgz
-for p in dtc pixman; do
-    if test -f $BASE/$p.tgz; then
-        tar -C $TEST_DIR/src/$p -xzf $BASE/$p.tgz
-        export FEATURES="$FEATURES $p"
-    fi
-done
+tar -C $TEST_DIR/src -xf $BASE/qemu.tar || prep_fail "Failed to untar source"
 
 if test -n "$SHOW_ENV"; then
     if test -f /packages.txt; then
@@ -52,10 +44,12 @@ if test -n "$SHOW_ENV"; then
 fi
 
 export QEMU_SRC="$TEST_DIR/src"
+export BUILD_DIR="$TEST_DIR/build"
+export INSTALL_DIR="$TEST_DIR/install"
 
 cd "$QEMU_SRC/tests/docker"
 
-CMD="$QEMU_SRC/tests/docker/$@"
+CMD="./$@"
 
 if test -z "$DEBUG"; then
     exec $CMD
diff --git a/tests/docker/test-block b/tests/docker/test-block
new file mode 100755
index 0000000000..2ca1ce54f6
--- /dev/null
+++ b/tests/docker/test-block
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Run block test cases
+#
+# Copyright 2017 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This code is licensed under the GPL version 2 or later.  See
+# the COPYING file in the top-level directory.
+
+. ./common.rc
+
+cd "$BUILD_DIR"
+
+build_qemu --target-list=x86_64-softmmu
+cd tests/qemu-iotests
+for t in raw qcow2 nbd luks; do
+    ./check -g quick -$t || test_fail "Test failed: iotests $t"
+done
diff --git a/tests/docker/test-full b/tests/docker/test-full
index 05f0d491d1..d71bf9d275 100755
--- a/tests/docker/test-full
+++ b/tests/docker/test-full
@@ -1,8 +1,8 @@
-#!/bin/bash -e
+#!/bin/bash
 #
-# Compile all the targets.
+# Compile all the targets with as many features enabled as possible
 #
-# Copyright (c) 2016 Red Hat Inc.
+# Copyright 2016, 2017 Red Hat Inc.
 #
 # Authors:
 #  Fam Zheng <famz@redhat.com>
@@ -13,7 +13,77 @@
 
 . common.rc
 
-cd "$BUILD_DIR"
+cd "$BUILD_DIR" || exit 1
 
-build_qemu
-make check $MAKEFLAGS
+build_qemu \
+    --enable-attr \
+    --enable-bluez \
+    --enable-brlapi \
+    --enable-bsd-user \
+    --enable-bzip2 \
+    --enable-cap-ng \
+    --enable-coroutine-pool \
+    --enable-crypto-afalg \
+    --enable-curl \
+    --enable-curses \
+    --enable-debug \
+    --enable-debug-info \
+    --enable-debug-tcg \
+    --enable-docs \
+    --enable-fdt \
+    --enable-gcrypt \
+    --enable-glusterfs \
+    --enable-gnutls \
+    --enable-gprof \
+    --enable-gtk \
+    --enable-guest-agent \
+    --enable-jemalloc \
+    --enable-kvm \
+    --enable-libiscsi \
+    --enable-libnfs \
+    --enable-libssh2 \
+    --enable-libusb \
+    --enable-linux-aio \
+    --enable-linux-user \
+    --enable-live-block-migration \
+    --enable-lzo \
+    --enable-modules \
+    --enable-numa \
+    --enable-opengl \
+    --enable-pie \
+    --enable-profiler \
+    --enable-qom-cast-debug \
+    --enable-rbd \
+    --enable-rdma \
+    --enable-replication \
+    --enable-sdl \
+    --enable-seccomp \
+    --enable-smartcard \
+    --enable-snappy \
+    --enable-spice \
+    --enable-stack-protector \
+    --enable-system \
+    --enable-tcg \
+    --enable-tcg-interpreter \
+    --enable-tools \
+    --enable-tpm \
+    --enable-trace-backend=ftrace \
+    --enable-usb-redir \
+    --enable-user \
+    --enable-vde \
+    --enable-vhost-net \
+    --enable-vhost-scsi \
+    --enable-vhost-user \
+    --enable-vhost-vsock \
+    --enable-virtfs \
+    --enable-vnc \
+    --enable-vnc-jpeg \
+    --enable-vnc-png \
+    --enable-vnc-sasl \
+    --enable-vte \
+    --enable-werror \
+    --enable-xen \
+    --enable-xen-pci-passthrough \
+    --enable-xen-pv-domain-build \
+    --enable-xfsctl \
+&& make check $MAKEFLAGS