Merge remote-tracking branch 'remotes/borntraeger/tags/s390x-20170714' into staging

s390x/kvm/migration/cpumodel: fixes, enhancements and cleanups

- add a network boot rom for s390 (Thomas Huth)
- migration of storage attributes like the CMMA used/unused state
- PCI related enhancements - full support for aen, ais and zpci
- migration support for css with vmstates (Halil Pasic)
- cpu model enhancements for cpu features
- guarded storage support

# gpg: Signature made Fri 14 Jul 2017 11:33:04 BST
# gpg:                using RSA key 0x117BBC80B5A61C7C
# gpg: Good signature from "Christian Borntraeger (IBM) <borntraeger@de.ibm.com>"
# Primary key fingerprint: F922 9381 A334 08F9 DBAB  FBCA 117B BC80 B5A6 1C7C

* remotes/borntraeger/tags/s390x-20170714: (40 commits)
  s390x/gdb: add gs registers
  s390x/arch_dump: also dump guarded storage control block
  s390x/kvm: enable guarded storage
  s390x/kvm: Enable KSS facility for nested virtualization
  s390x/cpumodel: add esop/esop2 to z12 model
  s390x/cpumodel: we are always in zarchitecture mode
  s390x/cpumodel: wire up new hardware features
  s390x/flic: migrate ais states
  s390x/cpumodel: add zpci, aen and ais facilities
  s390x: initialize cpu firstly
  pc-bios/s390: rebuild s390-ccw.img
  pc-bios/s390: add s390-netboot.img
  pc-bios/s390-ccw: Link libnet into the netboot image and do the TFTP load
  pc-bios/s390-ccw: Add virtio-net driver code
  pc-bios/s390-ccw: Add core files for the network bootloading program
  roms/SLOF: Update submodule to latest status
  pc-bios/s390-ccw: Add code for virtio feature negotiation
  pc-bios/s390-ccw: Remove unused structs from virtio.h
  pc-bios/s390-ccw: Move byteswap functions to a separate header
  pc-bios/s390-ccw: Add a write() function for stdio
  ...

Conflicts:
	target/s390x/kvm.c

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

12 files changed, 1090 insertions, 50 deletions

diff --git a/hw/intc/s390_flic.c b/hw/intc/s390_flic.c
index 837158bdaf..6eaf178d79 100644
--- a/hw/intc/s390_flic.c
+++ b/hw/intc/s390_flic.c
@@ -13,8 +13,11 @@
 #include "qemu/osdep.h"
 #include "qemu/error-report.h"
 #include "hw/sysbus.h"
+#include "hw/s390x/ioinst.h"
 #include "hw/s390x/s390_flic.h"
+#include "hw/s390x/css.h"
 #include "trace.h"
+#include "cpu.h"
 #include "hw/qdev.h"
 #include "qapi/error.h"
 #include "hw/s390x/s390-virtio-ccw.h"
@@ -48,7 +51,7 @@ void s390_flic_init(void)
 
 static int qemu_s390_register_io_adapter(S390FLICState *fs, uint32_t id,
                                          uint8_t isc, bool swap,
-                                         bool is_maskable)
+                                         bool is_maskable, uint8_t flags)
 {
     /* nothing to do */
     return 0;
@@ -79,15 +82,91 @@ static int qemu_s390_clear_io_flic(S390FLICState *fs, uint16_t subchannel_id,
     return -ENOSYS;
 }
 
+static int qemu_s390_modify_ais_mode(S390FLICState *fs, uint8_t isc,
+                                     uint16_t mode)
+{
+    QEMUS390FLICState *flic  = QEMU_S390_FLIC(fs);
+
+    switch (mode) {
+    case SIC_IRQ_MODE_ALL:
+        flic->simm &= ~AIS_MODE_MASK(isc);
+        flic->nimm &= ~AIS_MODE_MASK(isc);
+        break;
+    case SIC_IRQ_MODE_SINGLE:
+        flic->simm |= AIS_MODE_MASK(isc);
+        flic->nimm &= ~AIS_MODE_MASK(isc);
+        break;
+    default:
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int qemu_s390_inject_airq(S390FLICState *fs, uint8_t type,
+                                 uint8_t isc, uint8_t flags)
+{
+    QEMUS390FLICState *flic = QEMU_S390_FLIC(fs);
+    bool flag = flags & S390_ADAPTER_SUPPRESSIBLE;
+    uint32_t io_int_word = (isc << 27) | IO_INT_WORD_AI;
+
+    if (flag && (flic->nimm & AIS_MODE_MASK(isc))) {
+        trace_qemu_s390_airq_suppressed(type, isc);
+        return 0;
+    }
+
+    s390_io_interrupt(0, 0, 0, io_int_word);
+
+    if (flag && (flic->simm & AIS_MODE_MASK(isc))) {
+        flic->nimm |= AIS_MODE_MASK(isc);
+        trace_qemu_s390_suppress_airq(isc, "Single-Interruption Mode",
+                                      "NO-Interruptions Mode");
+    }
+
+    return 0;
+}
+
+static void qemu_s390_flic_reset(DeviceState *dev)
+{
+    QEMUS390FLICState *flic = QEMU_S390_FLIC(dev);
+
+    flic->simm = 0;
+    flic->nimm = 0;
+}
+
+bool ais_needed(void *opaque)
+{
+    S390FLICState *s = opaque;
+
+    return s->ais_supported;
+}
+
+static const VMStateDescription qemu_s390_flic_vmstate = {
+    .name = "qemu-s390-flic",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = ais_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(simm, QEMUS390FLICState),
+        VMSTATE_UINT8(nimm, QEMUS390FLICState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void qemu_s390_flic_class_init(ObjectClass *oc, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(oc);
     S390FLICStateClass *fsc = S390_FLIC_COMMON_CLASS(oc);
 
+    dc->reset = qemu_s390_flic_reset;
+    dc->vmsd = &qemu_s390_flic_vmstate;
     fsc->register_io_adapter = qemu_s390_register_io_adapter;
     fsc->io_adapter_map = qemu_s390_io_adapter_map;
     fsc->add_adapter_routes = qemu_s390_add_adapter_routes;
     fsc->release_adapter_routes = qemu_s390_release_adapter_routes;
     fsc->clear_io_irq = qemu_s390_clear_io_flic;
+    fsc->modify_ais_mode = qemu_s390_modify_ais_mode;
+    fsc->inject_airq = qemu_s390_inject_airq;
 }
 
 static Property s390_flic_common_properties[] = {
@@ -98,12 +177,16 @@ static Property s390_flic_common_properties[] = {
 
 static void s390_flic_common_realize(DeviceState *dev, Error **errp)
 {
-    uint32_t max_batch = S390_FLIC_COMMON(dev)->adapter_routes_max_batch;
+    S390FLICState *fs = S390_FLIC_COMMON(dev);
+    uint32_t max_batch = fs->adapter_routes_max_batch;
 
     if (max_batch > ADAPTER_ROUTES_MAX_GSI) {
         error_setg(errp, "flic property adapter_routes_max_batch too big"
                    " (%d > %d)", max_batch, ADAPTER_ROUTES_MAX_GSI);
+        return;
     }
+
+    fs->ais_supported = s390_has_feat(S390_FEAT_ADAPTER_INT_SUPPRESSION);
 }
 
 static void s390_flic_class_init(ObjectClass *oc, void *data)
@@ -138,6 +221,22 @@ static void qemu_s390_flic_register_types(void)
 
 type_init(qemu_s390_flic_register_types)
 
+static bool adapter_info_so_needed(void *opaque)
+{
+    return css_migration_enabled();
+}
+
+const VMStateDescription vmstate_adapter_info_so = {
+    .name = "s390_adapter_info/summary_offset",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = adapter_info_so_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(summary_offset, AdapterInfo),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_adapter_info = {
     .name = "s390_adapter_info",
     .version_id = 1,
@@ -151,6 +250,10 @@ const VMStateDescription vmstate_adapter_info = {
          */
         VMSTATE_END_OF_LIST()
     },
+    .subsections = (const VMStateDescription * []) {
+        &vmstate_adapter_info_so,
+        NULL
+    }
 };
 
 const VMStateDescription vmstate_adapter_routes = {
diff --git a/hw/intc/s390_flic_kvm.c b/hw/intc/s390_flic_kvm.c
index 0bcd49f08b..be3fd00a57 100644
--- a/hw/intc/s390_flic_kvm.c
+++ b/hw/intc/s390_flic_kvm.c
@@ -20,6 +20,7 @@
 #include "sysemu/kvm.h"
 #include "hw/s390x/s390_flic.h"
 #include "hw/s390x/adapter.h"
+#include "hw/s390x/css.h"
 #include "trace.h"
 
 #define FLIC_SAVE_INITIAL_SIZE getpagesize()
@@ -149,6 +150,43 @@ static int kvm_s390_clear_io_flic(S390FLICState *fs, uint16_t subchannel_id,
     return rc ? -errno : 0;
 }
 
+static int kvm_s390_modify_ais_mode(S390FLICState *fs, uint8_t isc,
+                                    uint16_t mode)
+{
+    KVMS390FLICState *flic = KVM_S390_FLIC(fs);
+    struct kvm_s390_ais_req req = {
+        .isc = isc,
+        .mode = mode,
+    };
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_AISM,
+        .addr = (uint64_t)&req,
+    };
+
+    if (!fs->ais_supported) {
+        return -ENOSYS;
+    }
+
+    return ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr) ? -errno : 0;
+}
+
+static int kvm_s390_inject_airq(S390FLICState *fs, uint8_t type,
+                                uint8_t isc, uint8_t flags)
+{
+    KVMS390FLICState *flic = KVM_S390_FLIC(fs);
+    uint32_t id = css_get_adapter_id(type, isc);
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_AIRQ_INJECT,
+        .attr = id,
+    };
+
+    if (!fs->ais_supported) {
+        return -ENOSYS;
+    }
+
+    return ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr) ? -errno : 0;
+}
+
 /**
  * __get_all_irqs - store all pending irqs in buffer
  * @flic: pointer to flic device state
@@ -186,13 +224,14 @@ static int __get_all_irqs(KVMS390FLICState *flic,
 
 static int kvm_s390_register_io_adapter(S390FLICState *fs, uint32_t id,
                                         uint8_t isc, bool swap,
-                                        bool is_maskable)
+                                        bool is_maskable, uint8_t flags)
 {
     struct kvm_s390_io_adapter adapter = {
         .id = id,
         .isc = isc,
         .maskable = is_maskable,
         .swap = swap,
+        .flags = flags,
     };
     KVMS390FLICState *flic = KVM_S390_FLIC(fs);
     int r;
@@ -374,7 +413,84 @@ out:
     return r;
 }
 
+typedef struct KVMS390FLICStateMigTmp {
+    KVMS390FLICState *parent;
+    uint8_t simm;
+    uint8_t nimm;
+} KVMS390FLICStateMigTmp;
+
+static void kvm_flic_ais_pre_save(void *opaque)
+{
+    KVMS390FLICStateMigTmp *tmp = opaque;
+    KVMS390FLICState *flic = tmp->parent;
+    struct kvm_s390_ais_all ais;
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_AISM_ALL,
+        .addr = (uint64_t)&ais,
+        .attr = sizeof(ais),
+    };
+
+    if (ioctl(flic->fd, KVM_GET_DEVICE_ATTR, &attr)) {
+        error_report("Failed to retrieve kvm flic ais states");
+        return;
+    }
+
+    tmp->simm = ais.simm;
+    tmp->nimm = ais.nimm;
+}
+
+static int kvm_flic_ais_post_load(void *opaque, int version_id)
+{
+    KVMS390FLICStateMigTmp *tmp = opaque;
+    KVMS390FLICState *flic = tmp->parent;
+    struct kvm_s390_ais_all ais = {
+        .simm = tmp->simm,
+        .nimm = tmp->nimm,
+    };
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_AISM_ALL,
+        .addr = (uint64_t)&ais,
+    };
+
+    /* This can happen when the user mis-configures its guests in an
+     * incompatible fashion or without a CPU model. For example using
+     * qemu with -cpu host (which is not migration safe) and do a
+     * migration from a host that has AIS to a host that has no AIS.
+     * In that case the target system will reject the migration here.
+     */
+    if (!ais_needed(flic)) {
+        return -ENOSYS;
+    }
+
+    return ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr) ? -errno : 0;
+}
+
+static const VMStateDescription kvm_s390_flic_ais_tmp = {
+    .name = "s390-flic-ais-tmp",
+    .pre_save = kvm_flic_ais_pre_save,
+    .post_load = kvm_flic_ais_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(simm, KVMS390FLICStateMigTmp),
+        VMSTATE_UINT8(nimm, KVMS390FLICStateMigTmp),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription kvm_s390_flic_vmstate_ais = {
+    .name = "s390-flic/ais",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = ais_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_WITH_TMP(KVMS390FLICState, KVMS390FLICStateMigTmp,
+                         kvm_s390_flic_ais_tmp),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription kvm_s390_flic_vmstate = {
+    /* should have been like kvm-s390-flic,
+     * can't change without breaking compat */
     .name = "s390-flic",
     .version_id = FLIC_SAVEVM_VERSION,
     .minimum_version_id = FLIC_SAVEVM_VERSION,
@@ -389,6 +505,10 @@ static const VMStateDescription kvm_s390_flic_vmstate = {
             .flags = VMS_SINGLE,
         },
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription * []) {
+        &kvm_s390_flic_vmstate_ais,
+        NULL
     }
 };
 
@@ -436,7 +556,6 @@ static void kvm_s390_flic_realize(DeviceState *dev, Error **errp)
     test_attr.group = KVM_DEV_FLIC_CLEAR_IO_IRQ;
     flic_state->clear_io_supported = !ioctl(flic_state->fd,
                                             KVM_HAS_DEVICE_ATTR, test_attr);
-
     return;
 fail:
     error_propagate(errp, errp_local);
@@ -445,10 +564,12 @@ fail:
 static void kvm_s390_flic_reset(DeviceState *dev)
 {
     KVMS390FLICState *flic = KVM_S390_FLIC(dev);
+    S390FLICState *fs = S390_FLIC_COMMON(dev);
     struct kvm_device_attr attr = {
         .group = KVM_DEV_FLIC_CLEAR_IRQS,
     };
     int rc = 0;
+    uint8_t isc;
 
     if (flic->fd == -1) {
         return;
@@ -456,6 +577,16 @@ static void kvm_s390_flic_reset(DeviceState *dev)
 
     flic_disable_wait_pfault(flic);
 
+    if (fs->ais_supported) {
+        for (isc = 0; isc <= MAX_ISC; isc++) {
+            rc = kvm_s390_modify_ais_mode(fs, isc, SIC_IRQ_MODE_ALL);
+            if (rc) {
+                error_report("Failed to reset ais mode for isc %d: %s",
+                             isc, strerror(-rc));
+            }
+        }
+    }
+
     rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
     if (rc) {
         trace_flic_reset_failed(errno);
@@ -478,6 +609,8 @@ static void kvm_s390_flic_class_init(ObjectClass *oc, void *data)
     fsc->add_adapter_routes = kvm_s390_add_adapter_routes;
     fsc->release_adapter_routes = kvm_s390_release_adapter_routes;
     fsc->clear_io_irq = kvm_s390_clear_io_flic;
+    fsc->modify_ais_mode = kvm_s390_modify_ais_mode;
+    fsc->inject_airq = kvm_s390_inject_airq;
 }
 
 static const TypeInfo kvm_s390_flic_info = {
diff --git a/hw/intc/trace-events b/hw/intc/trace-events
index 729c1288f1..c586714d89 100644
--- a/hw/intc/trace-events
+++ b/hw/intc/trace-events
@@ -73,6 +73,10 @@ flic_create_device(int err) "flic: create device failed %d"
 flic_no_device_api(int err) "flic: no Device Contral API support %d"
 flic_reset_failed(int err) "flic: reset failed %d"
 
+# hw/intc/s390_flic.c
+qemu_s390_airq_suppressed(uint8_t type, uint8_t isc) "flic: adapter I/O interrupt suppressed (type %x isc %x)"
+qemu_s390_suppress_airq(uint8_t isc, const char *from, const char *to) "flic: for isc %x, suppress airq by modifying ais mode from %s to %s"
+
 # hw/intc/aspeed_vic.c
 aspeed_vic_set_irq(int irq, int level) "Enabling IRQ %d: %d"
 aspeed_vic_update_fiq(int flags) "Raising FIQ: %d"
diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index a8e5575a8a..b2aade2466 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -13,5 +13,7 @@ obj-y += css-bridge.o
 obj-y += ccw-device.o
 obj-y += s390-pci-bus.o s390-pci-inst.o
 obj-y += s390-skeys.o
+obj-y += s390-stattrib.o
 obj-$(CONFIG_KVM) += s390-skeys-kvm.o
+obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
 obj-y += s390-ccw.o
diff --git a/hw/s390x/css-bridge.c b/hw/s390x/css-bridge.c
index 823747fcd7..c4a9735d71 100644
--- a/hw/s390x/css-bridge.c
+++ b/hw/s390x/css-bridge.c
@@ -110,7 +110,7 @@ VirtualCssBus *virtual_css_bus_init(void)
     qbus_set_hotplug_handler(bus, dev, &error_abort);
 
     css_register_io_adapters(CSS_IO_ADAPTER_VIRTIO, true, false,
-                             &error_abort);
+                             0, &error_abort);
 
     return cbus;
  }
diff --git a/hw/s390x/css.c b/hw/s390x/css.c
index cd0b776861..6a42b95cee 100644
--- a/hw/s390x/css.c
+++ b/hw/s390x/css.c
@@ -29,12 +29,45 @@ typedef struct CrwContainer {
     QTAILQ_ENTRY(CrwContainer) sibling;
 } CrwContainer;
 
+static const VMStateDescription vmstate_crw = {
+    .name = "s390_crw",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT16(flags, CRW),
+        VMSTATE_UINT16(rsid, CRW),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static const VMStateDescription vmstate_crw_container = {
+    .name = "s390_crw_container",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(crw, CrwContainer, 0, vmstate_crw, CRW),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 typedef struct ChpInfo {
     uint8_t in_use;
     uint8_t type;
     uint8_t is_virtual;
 } ChpInfo;
 
+static const VMStateDescription vmstate_chp_info = {
+    .name = "s390_chp_info",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(in_use, ChpInfo),
+        VMSTATE_UINT8(type, ChpInfo),
+        VMSTATE_UINT8(is_virtual, ChpInfo),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 typedef struct SubchSet {
     SubchDev *sch[MAX_SCHID + 1];
     unsigned long schids_used[BITS_TO_LONGS(MAX_SCHID + 1)];
@@ -132,6 +165,36 @@ static const VMStateDescription vmstate_sense_id = {
     }
 };
 
+static const VMStateDescription vmstate_orb = {
+    .name = "s390_orb",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(intparm, ORB),
+        VMSTATE_UINT16(ctrl0, ORB),
+        VMSTATE_UINT8(lpm, ORB),
+        VMSTATE_UINT8(ctrl1, ORB),
+        VMSTATE_UINT32(cpa, ORB),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool vmstate_schdev_orb_needed(void *opaque)
+{
+    return css_migration_enabled();
+}
+
+static const VMStateDescription vmstate_schdev_orb = {
+    .name = "s390_subch_dev/orb",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = vmstate_schdev_orb_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(orb, SubchDev, 1, vmstate_orb, ORB),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static int subch_dev_post_load(void *opaque, int version_id);
 static void subch_dev_pre_save(void *opaque);
 
@@ -160,6 +223,10 @@ const VMStateDescription vmstate_subch_dev = {
         VMSTATE_BOOL(ccw_fmt_1, SubchDev),
         VMSTATE_UINT8(ccw_no_data_cnt, SubchDev),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription * []) {
+        &vmstate_schdev_orb,
+        NULL
     }
 };
 
@@ -221,10 +288,24 @@ typedef struct CssImage {
     ChpInfo chpids[MAX_CHPID + 1];
 } CssImage;
 
+static const VMStateDescription vmstate_css_img = {
+    .name = "s390_css_img",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        /* Subchannel sets have no relevant state. */
+        VMSTATE_STRUCT_ARRAY(chpids, CssImage, MAX_CHPID + 1, 0,
+                             vmstate_chp_info, ChpInfo),
+        VMSTATE_END_OF_LIST()
+    }
+
+};
+
 typedef struct IoAdapter {
     uint32_t id;
     uint8_t type;
     uint8_t isc;
+    uint8_t flags;
 } IoAdapter;
 
 typedef struct ChannelSubSys {
@@ -238,10 +319,34 @@ typedef struct ChannelSubSys {
     uint64_t chnmon_area;
     CssImage *css[MAX_CSSID + 1];
     uint8_t default_cssid;
+    /* don't migrate, see css_register_io_adapters */
     IoAdapter *io_adapters[CSS_IO_ADAPTER_TYPE_NUMS][MAX_ISC + 1];
+    /* don't migrate, see get_indicator and IndAddrPtrTmp */
     QTAILQ_HEAD(, IndAddr) indicator_addresses;
 } ChannelSubSys;
 
+static const VMStateDescription vmstate_css = {
+    .name = "s390_css",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_QTAILQ_V(pending_crws, ChannelSubSys, 1, vmstate_crw_container,
+                         CrwContainer, sibling),
+        VMSTATE_BOOL(sei_pending, ChannelSubSys),
+        VMSTATE_BOOL(do_crw_mchk, ChannelSubSys),
+        VMSTATE_BOOL(crws_lost, ChannelSubSys),
+        /* These were kind of migrated by virtio */
+        VMSTATE_UINT8(max_cssid, ChannelSubSys),
+        VMSTATE_UINT8(max_ssid, ChannelSubSys),
+        VMSTATE_BOOL(chnmon_active, ChannelSubSys),
+        VMSTATE_UINT64(chnmon_area, ChannelSubSys),
+        VMSTATE_ARRAY_OF_POINTER_TO_STRUCT(css, ChannelSubSys, MAX_CSSID + 1,
+                0, vmstate_css_img, CssImage),
+        VMSTATE_UINT8(default_cssid, ChannelSubSys),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static ChannelSubSys channel_subsys = {
     .pending_crws = QTAILQ_HEAD_INITIALIZER(channel_subsys.pending_crws),
     .do_crw_mchk = true,
@@ -281,6 +386,10 @@ static int subch_dev_post_load(void *opaque, int version_id)
         css_subch_assign(s->cssid, s->ssid, s->schid, s->devno, s);
     }
 
+    if (css_migration_enabled()) {
+        /* No compat voodoo to do ;) */
+        return 0;
+    }
     /*
      * Hack alert. If we don't migrate the channel subsystem status
      * we still need to find out if the guest enabled mss/mcss-e.
@@ -299,6 +408,11 @@ static int subch_dev_post_load(void *opaque, int version_id)
     return 0;
 }
 
+void css_register_vmstate(void)
+{
+    vmstate_register(NULL, 0, &vmstate_css, &channel_subsys);
+}
+
 IndAddr *get_indicator(hwaddr ind_addr, int len)
 {
     IndAddr *indicator;
@@ -392,10 +506,12 @@ uint32_t css_get_adapter_id(CssIoAdapterType type, uint8_t isc)
  *
  * @swap: an indication if byte swap is needed.
  * @maskable: an indication if the adapter is subject to the mask operation.
+ * @flags: further characteristics of the adapter.
+ *         e.g. suppressible, an indication if the adapter is subject to AIS.
  * @errp: location to store error information.
  */
 void css_register_io_adapters(CssIoAdapterType type, bool swap, bool maskable,
-                              Error **errp)
+                              uint8_t flags, Error **errp)
 {
     uint32_t id;
     int ret, isc;
@@ -413,12 +529,13 @@ void css_register_io_adapters(CssIoAdapterType type, bool swap, bool maskable,
 
     for (isc = 0; isc <= MAX_ISC; isc++) {
         id = (type << 3) | isc;
-        ret = fsc->register_io_adapter(fs, id, isc, swap, maskable);
+        ret = fsc->register_io_adapter(fs, id, isc, swap, maskable, flags);
         if (ret == 0) {
             adapter = g_new0(IoAdapter, 1);
             adapter->id = id;
             adapter->isc = isc;
             adapter->type = type;
+            adapter->flags = flags;
             channel_subsys.io_adapters[type][isc] = adapter;
         } else {
             error_setg_errno(errp, -ret, "Unexpected error %d when "
@@ -517,12 +634,52 @@ void css_conditional_io_interrupt(SubchDev *sch)
     }
 }
 
-void css_adapter_interrupt(uint8_t isc)
+int css_do_sic(CPUS390XState *env, uint8_t isc, uint16_t mode)
+{
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
+    int r;
+
+    if (env->psw.mask & PSW_MASK_PSTATE) {
+        r = -PGM_PRIVILEGED;
+        goto out;
+    }
+
+    trace_css_do_sic(mode, isc);
+    switch (mode) {
+    case SIC_IRQ_MODE_ALL:
+    case SIC_IRQ_MODE_SINGLE:
+        break;
+    default:
+        r = -PGM_OPERAND;
+        goto out;
+    }
+
+    r = fsc->modify_ais_mode(fs, isc, mode) ? -PGM_OPERATION : 0;
+out:
+    return r;
+}
+
+void css_adapter_interrupt(CssIoAdapterType type, uint8_t isc)
 {
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
     uint32_t io_int_word = (isc << 27) | IO_INT_WORD_AI;
+    IoAdapter *adapter = channel_subsys.io_adapters[type][isc];
+
+    if (!adapter) {
+        return;
+    }
 
     trace_css_adapter_interrupt(isc);
-    s390_io_interrupt(0, 0, 0, io_int_word);
+    if (fs->ais_supported) {
+        if (fsc->inject_airq(fs, type, isc, adapter->flags)) {
+            error_report("Failed to inject airq with AIS supported");
+            exit(1);
+        }
+    } else {
+        s390_io_interrupt(0, 0, 0, io_int_word);
+    }
 }
 
 static void sch_handle_clear_func(SubchDev *sch)
@@ -752,7 +909,7 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr,
     return ret;
 }
 
-static void sch_handle_start_func_virtual(SubchDev *sch, ORB *orb)
+static void sch_handle_start_func_virtual(SubchDev *sch)
 {
 
     PMCW *p = &sch->curr_status.pmcw;
@@ -766,10 +923,10 @@ static void sch_handle_start_func_virtual(SubchDev *sch, ORB *orb)
 
     if (!(s->ctrl & SCSW_ACTL_SUSP)) {
         /* Start Function triggered via ssch, i.e. we have an ORB */
+        ORB *orb = &sch->orb;
         s->cstat = 0;
         s->dstat = 0;
         /* Look at the orb and try to execute the channel program. */
-        assert(orb != NULL); /* resume does not pass an orb */
         p->intparm = orb->intparm;
         if (!(orb->lpm & path)) {
             /* Generate a deferred cc 3 condition. */
@@ -783,8 +940,7 @@ static void sch_handle_start_func_virtual(SubchDev *sch, ORB *orb)
         sch->ccw_no_data_cnt = 0;
         suspend_allowed = !!(orb->ctrl0 & ORB_CTRL0_MASK_SPND);
     } else {
-        /* Start Function resumed via rsch, i.e. we don't have an
-         * ORB */
+        /* Start Function resumed via rsch */
         s->ctrl &= ~(SCSW_ACTL_SUSP | SCSW_ACTL_RESUME_PEND);
         /* The channel program had been suspended before. */
         suspend_allowed = true;
@@ -854,13 +1010,14 @@ static void sch_handle_start_func_virtual(SubchDev *sch, ORB *orb)
 
 }
 
-static int sch_handle_start_func_passthrough(SubchDev *sch, ORB *orb)
+static int sch_handle_start_func_passthrough(SubchDev *sch)
 {
 
     PMCW *p = &sch->curr_status.pmcw;
     SCSW *s = &sch->curr_status.scsw;
     int ret;
 
+    ORB *orb = &sch->orb;
     if (!(s->ctrl & SCSW_ACTL_SUSP)) {
         assert(orb != NULL);
         p->intparm = orb->intparm;
@@ -905,7 +1062,7 @@ static int sch_handle_start_func_passthrough(SubchDev *sch, ORB *orb)
  * read/writes) asynchronous later on if we start supporting more than
  * our current very simple devices.
  */
-int do_subchannel_work_virtual(SubchDev *sch, ORB *orb)
+int do_subchannel_work_virtual(SubchDev *sch)
 {
 
     SCSW *s = &sch->curr_status.scsw;
@@ -916,7 +1073,7 @@ int do_subchannel_work_virtual(SubchDev *sch, ORB *orb)
         sch_handle_halt_func(sch);
     } else if (s->ctrl & SCSW_FCTL_START_FUNC) {
         /* Triggered by both ssch and rsch. */
-        sch_handle_start_func_virtual(sch, orb);
+        sch_handle_start_func_virtual(sch);
     } else {
         /* Cannot happen. */
         return 0;
@@ -925,7 +1082,7 @@ int do_subchannel_work_virtual(SubchDev *sch, ORB *orb)
     return 0;
 }
 
-int do_subchannel_work_passthrough(SubchDev *sch, ORB *orb)
+int do_subchannel_work_passthrough(SubchDev *sch)
 {
     int ret;
     SCSW *s = &sch->curr_status.scsw;
@@ -939,7 +1096,7 @@ int do_subchannel_work_passthrough(SubchDev *sch, ORB *orb)
         sch_handle_halt_func(sch);
         ret = 0;
     } else if (s->ctrl & SCSW_FCTL_START_FUNC) {
-        ret = sch_handle_start_func_passthrough(sch, orb);
+        ret = sch_handle_start_func_passthrough(sch);
     } else {
         /* Cannot happen. */
         return -ENODEV;
@@ -948,10 +1105,10 @@ int do_subchannel_work_passthrough(SubchDev *sch, ORB *orb)
     return ret;
 }
 
-static int do_subchannel_work(SubchDev *sch, ORB *orb)
+static int do_subchannel_work(SubchDev *sch)
 {
     if (sch->do_subchannel_work) {
-        return sch->do_subchannel_work(sch, orb);
+        return sch->do_subchannel_work(sch);
     } else {
         return -EINVAL;
     }
@@ -1158,7 +1315,7 @@ int css_do_csch(SubchDev *sch)
     s->ctrl &= ~(SCSW_CTRL_MASK_FCTL | SCSW_CTRL_MASK_ACTL);
     s->ctrl |= SCSW_FCTL_CLEAR_FUNC | SCSW_ACTL_CLEAR_PEND;
 
-    do_subchannel_work(sch, NULL);
+    do_subchannel_work(sch);
     ret = 0;
 
 out:
@@ -1199,7 +1356,7 @@ int css_do_hsch(SubchDev *sch)
     }
     s->ctrl |= SCSW_ACTL_HALT_PEND;
 
-    do_subchannel_work(sch, NULL);
+    do_subchannel_work(sch);
     ret = 0;
 
 out:
@@ -1268,12 +1425,13 @@ int css_do_ssch(SubchDev *sch, ORB *orb)
     if (channel_subsys.chnmon_active) {
         css_update_chnmon(sch);
     }
+    sch->orb = *orb;
     sch->channel_prog = orb->cpa;
     /* Trigger the start function. */
     s->ctrl |= (SCSW_FCTL_START_FUNC | SCSW_ACTL_START_PEND);
     s->flags &= ~SCSW_FLAGS_MASK_PNO;
 
-    ret = do_subchannel_work(sch, orb);
+    ret = do_subchannel_work(sch);
 
 out:
     return ret;
@@ -1552,7 +1710,7 @@ int css_do_rsch(SubchDev *sch)
     }
 
     s->ctrl |= SCSW_ACTL_RESUME_PEND;
-    do_subchannel_work(sch, NULL);
+    do_subchannel_work(sch);
     ret = 0;
 
 out:
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index af702f8840..61cfd2138f 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -500,7 +500,7 @@ static void s390_msi_ctrl_write(void *opaque, hwaddr addr, uint64_t data,
                    0x80 >> ((ind_bit + vec) % 8));
     if (!set_ind_atomic(pbdev->routes.adapter.summary_addr + sum_bit / 8,
                                        0x80 >> (sum_bit % 8))) {
-        css_adapter_interrupt(pbdev->isc);
+        css_adapter_interrupt(CSS_IO_ADAPTER_PCI, pbdev->isc);
     }
 }
 
@@ -579,7 +579,8 @@ static int s390_pcihost_init(SysBusDevice *dev)
     QTAILQ_INIT(&s->pending_sei);
     QTAILQ_INIT(&s->zpci_devs);
 
-    css_register_io_adapters(CSS_IO_ADAPTER_PCI, true, false, &error_abort);
+    css_register_io_adapters(CSS_IO_ADAPTER_PCI, true, false,
+                             S390_ADAPTER_SUPPRESSIBLE, &error_abort);
 
     return 0;
 }
diff --git a/hw/s390x/s390-stattrib-kvm.c b/hw/s390x/s390-stattrib-kvm.c
new file mode 100644
index 0000000000..ff3f89fd2d
--- /dev/null
+++ b/hw/s390x/s390-stattrib-kvm.c
@@ -0,0 +1,190 @@
+/*
+ * s390 storage attributes device -- KVM object
+ *
+ * Copyright 2016 IBM Corp.
+ * Author(s): Claudio Imbrenda <imbrenda@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/boards.h"
+#include "migration/qemu-file.h"
+#include "hw/s390x/storage-attributes.h"
+#include "qemu/error-report.h"
+#include "sysemu/kvm.h"
+#include "exec/ram_addr.h"
+#include "cpu.h"
+
+Object *kvm_s390_stattrib_create(void)
+{
+    if (kvm_enabled() &&
+                kvm_check_extension(kvm_state, KVM_CAP_S390_CMMA_MIGRATION)) {
+        return object_new(TYPE_KVM_S390_STATTRIB);
+    }
+    return NULL;
+}
+
+static void kvm_s390_stattrib_instance_init(Object *obj)
+{
+    KVMS390StAttribState *sas = KVM_S390_STATTRIB(obj);
+
+    sas->still_dirty = 0;
+}
+
+static int kvm_s390_stattrib_read_helper(S390StAttribState *sa,
+                                         uint64_t *start_gfn,
+                                         uint32_t count,
+                                         uint8_t *values,
+                                         uint32_t flags)
+{
+    KVMS390StAttribState *sas = KVM_S390_STATTRIB(sa);
+    int r;
+    struct kvm_s390_cmma_log clog = {
+        .values = (uint64_t)values,
+        .start_gfn = *start_gfn,
+        .count = count,
+        .flags = flags,
+    };
+
+    r = kvm_vm_ioctl(kvm_state, KVM_S390_GET_CMMA_BITS, &clog);
+    if (r < 0) {
+        error_report("KVM_S390_GET_CMMA_BITS failed: %s", strerror(-r));
+        return r;
+    }
+
+    *start_gfn = clog.start_gfn;
+    sas->still_dirty = clog.remaining;
+    return clog.count;
+}
+
+static int kvm_s390_stattrib_get_stattr(S390StAttribState *sa,
+                                        uint64_t *start_gfn,
+                                        uint32_t count,
+                                        uint8_t *values)
+{
+    return kvm_s390_stattrib_read_helper(sa, start_gfn, count, values, 0);
+}
+
+static int kvm_s390_stattrib_peek_stattr(S390StAttribState *sa,
+                                         uint64_t start_gfn,
+                                         uint32_t count,
+                                         uint8_t *values)
+{
+    return kvm_s390_stattrib_read_helper(sa, &start_gfn, count, values,
+                                         KVM_S390_CMMA_PEEK);
+}
+
+static int kvm_s390_stattrib_set_stattr(S390StAttribState *sa,
+                                        uint64_t start_gfn,
+                                        uint32_t count,
+                                        uint8_t *values)
+{
+    KVMS390StAttribState *sas = KVM_S390_STATTRIB(sa);
+    MachineState *machine = MACHINE(qdev_get_machine());
+    unsigned long max = machine->maxram_size / TARGET_PAGE_SIZE;
+
+    if (start_gfn + count > max) {
+        error_report("Out of memory bounds when setting storage attributes");
+        return -1;
+    }
+    if (!sas->incoming_buffer) {
+        sas->incoming_buffer = g_malloc0(max);
+    }
+
+    memcpy(sas->incoming_buffer + start_gfn, values, count);
+
+    return 0;
+}
+
+static void kvm_s390_stattrib_synchronize(S390StAttribState *sa)
+{
+    KVMS390StAttribState *sas = KVM_S390_STATTRIB(sa);
+    MachineState *machine = MACHINE(qdev_get_machine());
+    unsigned long max = machine->maxram_size / TARGET_PAGE_SIZE;
+    unsigned long cx, len = 1 << 19;
+    int r;
+    struct kvm_s390_cmma_log clog = {
+        .flags = 0,
+        .mask = ~0ULL,
+    };
+
+    if (sas->incoming_buffer) {
+        for (cx = 0; cx + len <= max; cx += len) {
+            clog.start_gfn = cx;
+            clog.count = len;
+            clog.values = (uint64_t)(sas->incoming_buffer + cx * len);
+            r = kvm_vm_ioctl(kvm_state, KVM_S390_SET_CMMA_BITS, &clog);
+            if (r) {
+                error_report("KVM_S390_SET_CMMA_BITS failed: %s", strerror(-r));
+                return;
+            }
+        }
+        if (cx < max) {
+            clog.start_gfn = cx;
+            clog.count = max - cx;
+            clog.values = (uint64_t)(sas->incoming_buffer + cx * len);
+            r = kvm_vm_ioctl(kvm_state, KVM_S390_SET_CMMA_BITS, &clog);
+            if (r) {
+                error_report("KVM_S390_SET_CMMA_BITS failed: %s", strerror(-r));
+            }
+        }
+        g_free(sas->incoming_buffer);
+        sas->incoming_buffer = NULL;
+    }
+}
+
+static int kvm_s390_stattrib_set_migrationmode(S390StAttribState *sa, bool val)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MIGRATION,
+        .attr = val,
+        .addr = 0,
+    };
+    return kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
+}
+
+static long long kvm_s390_stattrib_get_dirtycount(S390StAttribState *sa)
+{
+    KVMS390StAttribState *sas = KVM_S390_STATTRIB(sa);
+    uint8_t val[8];
+
+    kvm_s390_stattrib_peek_stattr(sa, 0, 1, val);
+    return sas->still_dirty;
+}
+
+static int kvm_s390_stattrib_get_active(S390StAttribState *sa)
+{
+    return kvm_s390_cmma_active() && sa->migration_enabled;
+}
+
+static void kvm_s390_stattrib_class_init(ObjectClass *oc, void *data)
+{
+    S390StAttribClass *sac = S390_STATTRIB_CLASS(oc);
+
+    sac->get_stattr = kvm_s390_stattrib_get_stattr;
+    sac->peek_stattr = kvm_s390_stattrib_peek_stattr;
+    sac->set_stattr = kvm_s390_stattrib_set_stattr;
+    sac->set_migrationmode = kvm_s390_stattrib_set_migrationmode;
+    sac->get_dirtycount = kvm_s390_stattrib_get_dirtycount;
+    sac->synchronize = kvm_s390_stattrib_synchronize;
+    sac->get_active = kvm_s390_stattrib_get_active;
+}
+
+static const TypeInfo kvm_s390_stattrib_info = {
+    .name          = TYPE_KVM_S390_STATTRIB,
+    .parent        = TYPE_S390_STATTRIB,
+    .instance_init = kvm_s390_stattrib_instance_init,
+    .instance_size = sizeof(KVMS390StAttribState),
+    .class_init    = kvm_s390_stattrib_class_init,
+    .class_size    = sizeof(S390StAttribClass),
+};
+
+static void kvm_s390_stattrib_register_types(void)
+{
+    type_register_static(&kvm_s390_stattrib_info);
+}
+
+type_init(kvm_s390_stattrib_register_types)
diff --git a/hw/s390x/s390-stattrib.c b/hw/s390x/s390-stattrib.c
new file mode 100644
index 0000000000..d14923f099
--- /dev/null
+++ b/hw/s390x/s390-stattrib.c
@@ -0,0 +1,404 @@
+/*
+ * s390 storage attributes device
+ *
+ * Copyright 2016 IBM Corp.
+ * Author(s): Claudio Imbrenda <imbrenda@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/boards.h"
+#include "qmp-commands.h"
+#include "migration/qemu-file.h"
+#include "migration/register.h"
+#include "hw/s390x/storage-attributes.h"
+#include "qemu/error-report.h"
+#include "sysemu/kvm.h"
+#include "exec/ram_addr.h"
+#include "qapi/error.h"
+
+#define CMMA_BLOCK_SIZE  (1 << 10)
+
+#define STATTR_FLAG_EOS     0x01ULL
+#define STATTR_FLAG_MORE    0x02ULL
+#define STATTR_FLAG_ERROR   0x04ULL
+#define STATTR_FLAG_DONE    0x08ULL
+
+static S390StAttribState *s390_get_stattrib_device(void)
+{
+    S390StAttribState *sas;
+
+    sas = S390_STATTRIB(object_resolve_path_type("", TYPE_S390_STATTRIB, NULL));
+    assert(sas);
+    return sas;
+}
+
+void s390_stattrib_init(void)
+{
+    Object *obj;
+
+    obj = kvm_s390_stattrib_create();
+    if (!obj) {
+        obj = object_new(TYPE_QEMU_S390_STATTRIB);
+    }
+
+    object_property_add_child(qdev_get_machine(), TYPE_S390_STATTRIB,
+                              obj, NULL);
+    object_unref(obj);
+
+    qdev_init_nofail(DEVICE(obj));
+}
+
+/* Console commands: */
+
+void hmp_migrationmode(Monitor *mon, const QDict *qdict)
+{
+    S390StAttribState *sas = s390_get_stattrib_device();
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    uint64_t what = qdict_get_int(qdict, "mode");
+    int r;
+
+    r = sac->set_migrationmode(sas, what);
+    if (r < 0) {
+        monitor_printf(mon, "Error: %s", strerror(-r));
+    }
+}
+
+void hmp_info_cmma(Monitor *mon, const QDict *qdict)
+{
+    S390StAttribState *sas = s390_get_stattrib_device();
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    uint64_t addr = qdict_get_int(qdict, "addr");
+    uint64_t buflen = qdict_get_try_int(qdict, "count", 8);
+    uint8_t *vals;
+    int cx, len;
+
+    vals = g_try_malloc(buflen);
+    if (!vals) {
+        monitor_printf(mon, "Error: %s\n", strerror(errno));
+        return;
+    }
+
+    len = sac->peek_stattr(sas, addr / TARGET_PAGE_SIZE, buflen, vals);
+    if (len < 0) {
+        monitor_printf(mon, "Error: %s", strerror(-len));
+        goto out;
+    }
+
+    monitor_printf(mon, "  CMMA attributes, "
+                   "pages %" PRIu64 "+%d (0x%" PRIx64 "):\n",
+                   addr / TARGET_PAGE_SIZE, len, addr & ~TARGET_PAGE_MASK);
+    for (cx = 0; cx < len; cx++) {
+        if (cx % 8 == 7) {
+            monitor_printf(mon, "%02x\n", vals[cx]);
+        } else {
+            monitor_printf(mon, "%02x", vals[cx]);
+        }
+    }
+    monitor_printf(mon, "\n");
+
+out:
+    g_free(vals);
+}
+
+/* Migration support: */
+
+static int cmma_load(QEMUFile *f, void *opaque, int version_id)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    uint64_t count, cur_gfn;
+    int flags, ret = 0;
+    ram_addr_t addr;
+    uint8_t *buf;
+
+    while (!ret) {
+        addr = qemu_get_be64(f);
+        flags = addr & ~TARGET_PAGE_MASK;
+        addr &= TARGET_PAGE_MASK;
+
+        switch (flags) {
+        case STATTR_FLAG_MORE: {
+            cur_gfn = addr / TARGET_PAGE_SIZE;
+            count = qemu_get_be64(f);
+            buf = g_try_malloc(count);
+            if (!buf) {
+                error_report("cmma_load could not allocate memory");
+                ret = -ENOMEM;
+                break;
+            }
+
+            qemu_get_buffer(f, buf, count);
+            ret = sac->set_stattr(sas, cur_gfn, count, buf);
+            if (ret < 0) {
+                error_report("Error %d while setting storage attributes", ret);
+            }
+            g_free(buf);
+            break;
+        }
+        case STATTR_FLAG_ERROR: {
+            error_report("Storage attributes data is incomplete");
+            ret = -EINVAL;
+            break;
+        }
+        case STATTR_FLAG_DONE:
+            /* This is after the last pre-copied value has been sent, nothing
+             * more will be sent after this. Pre-copy has finished, and we
+             * are done flushing all the remaining values. Now the target
+             * system is about to take over. We synchronize the buffer to
+             * apply the actual correct values where needed.
+             */
+             sac->synchronize(sas);
+            break;
+        case STATTR_FLAG_EOS:
+            /* Normal exit */
+            return 0;
+        default:
+            error_report("Unexpected storage attribute flag data: %#x", flags);
+            ret = -EINVAL;
+        }
+    }
+
+    return ret;
+}
+
+static int cmma_save_setup(QEMUFile *f, void *opaque)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    int res;
+    /*
+     * Signal that we want to start a migration, thus needing PGSTE dirty
+     * tracking.
+     */
+    res = sac->set_migrationmode(sas, 1);
+    if (res) {
+        return res;
+    }
+    qemu_put_be64(f, STATTR_FLAG_EOS);
+    return 0;
+}
+
+static void cmma_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
+                             uint64_t *non_postcopiable_pending,
+                             uint64_t *postcopiable_pending)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    long long res = sac->get_dirtycount(sas);
+
+    if (res >= 0) {
+        *non_postcopiable_pending += res;
+    }
+}
+
+static int cmma_save(QEMUFile *f, void *opaque, int final)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    uint8_t *buf;
+    int r, cx, reallen = 0, ret = 0;
+    uint32_t buflen = 1 << 19;   /* 512kB cover 2GB of guest memory */
+    uint64_t start_gfn = sas->migration_cur_gfn;
+
+    buf = g_try_malloc(buflen);
+    if (!buf) {
+        error_report("Could not allocate memory to save storage attributes");
+        return -ENOMEM;
+    }
+
+    while (final ? 1 : qemu_file_rate_limit(f) == 0) {
+        reallen = sac->get_stattr(sas, &start_gfn, buflen, buf);
+        if (reallen < 0) {
+            g_free(buf);
+            return reallen;
+        }
+
+        ret = 1;
+        if (!reallen) {
+            break;
+        }
+        qemu_put_be64(f, (start_gfn << TARGET_PAGE_BITS) | STATTR_FLAG_MORE);
+        qemu_put_be64(f, reallen);
+        for (cx = 0; cx < reallen; cx++) {
+            qemu_put_byte(f, buf[cx]);
+        }
+        if (!sac->get_dirtycount(sas)) {
+            break;
+        }
+    }
+
+    sas->migration_cur_gfn = start_gfn + reallen;
+    g_free(buf);
+    if (final) {
+        qemu_put_be64(f, STATTR_FLAG_DONE);
+    }
+    qemu_put_be64(f, STATTR_FLAG_EOS);
+
+    r = qemu_file_get_error(f);
+    if (r < 0) {
+        return r;
+    }
+
+    return ret;
+}
+
+static int cmma_save_iterate(QEMUFile *f, void *opaque)
+{
+    return cmma_save(f, opaque, 0);
+}
+
+static int cmma_save_complete(QEMUFile *f, void *opaque)
+{
+    return cmma_save(f, opaque, 1);
+}
+
+static void cmma_save_cleanup(void *opaque)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    sac->set_migrationmode(sas, 0);
+}
+
+static bool cmma_active(void *opaque)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    return sac->get_active(sas);
+}
+
+/* QEMU object: */
+
+static void qemu_s390_stattrib_instance_init(Object *obj)
+{
+}
+
+static int qemu_s390_peek_stattr_stub(S390StAttribState *sa, uint64_t start_gfn,
+                                     uint32_t count, uint8_t *values)
+{
+    return 0;
+}
+static void qemu_s390_synchronize_stub(S390StAttribState *sa)
+{
+}
+static int qemu_s390_get_stattr_stub(S390StAttribState *sa, uint64_t *start_gfn,
+                                     uint32_t count, uint8_t *values)
+{
+    return 0;
+}
+static long long qemu_s390_get_dirtycount_stub(S390StAttribState *sa)
+{
+    return 0;
+}
+static int qemu_s390_set_migrationmode_stub(S390StAttribState *sa, bool value)
+{
+    return 0;
+}
+
+static int qemu_s390_get_active(S390StAttribState *sa)
+{
+    return sa->migration_enabled;
+}
+
+static void qemu_s390_stattrib_class_init(ObjectClass *oc, void *data)
+{
+    S390StAttribClass *sa_cl = S390_STATTRIB_CLASS(oc);
+
+    sa_cl->synchronize = qemu_s390_synchronize_stub;
+    sa_cl->get_stattr = qemu_s390_get_stattr_stub;
+    sa_cl->set_stattr = qemu_s390_peek_stattr_stub;
+    sa_cl->peek_stattr = qemu_s390_peek_stattr_stub;
+    sa_cl->set_migrationmode = qemu_s390_set_migrationmode_stub;
+    sa_cl->get_dirtycount = qemu_s390_get_dirtycount_stub;
+    sa_cl->get_active = qemu_s390_get_active;
+}
+
+static const TypeInfo qemu_s390_stattrib_info = {
+    .name          = TYPE_QEMU_S390_STATTRIB,
+    .parent        = TYPE_S390_STATTRIB,
+    .instance_init = qemu_s390_stattrib_instance_init,
+    .instance_size = sizeof(QEMUS390StAttribState),
+    .class_init    = qemu_s390_stattrib_class_init,
+    .class_size    = sizeof(S390StAttribClass),
+};
+
+/* Generic abstract object: */
+
+static void s390_stattrib_realize(DeviceState *dev, Error **errp)
+{
+    bool ambiguous = false;
+
+    object_resolve_path_type("", TYPE_S390_STATTRIB, &ambiguous);
+    if (ambiguous) {
+        error_setg(errp, "storage_attributes device already exists");
+    }
+}
+
+static void s390_stattrib_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->hotpluggable = false;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->realize = s390_stattrib_realize;
+}
+
+static inline bool s390_stattrib_get_migration_enabled(Object *obj, Error **e)
+{
+    S390StAttribState *s = S390_STATTRIB(obj);
+
+    return s->migration_enabled;
+}
+
+static inline void s390_stattrib_set_migration_enabled(Object *obj, bool value,
+                                            Error **errp)
+{
+    S390StAttribState *s = S390_STATTRIB(obj);
+
+    s->migration_enabled = value;
+}
+
+static void s390_stattrib_instance_init(Object *obj)
+{
+    S390StAttribState *sas = S390_STATTRIB(obj);
+    SaveVMHandlers *ops;
+
+    /* ops will always be freed by qemu when unregistering */
+    ops = g_new0(SaveVMHandlers, 1);
+
+    ops->save_setup = cmma_save_setup;
+    ops->save_live_iterate = cmma_save_iterate;
+    ops->save_live_complete_precopy = cmma_save_complete;
+    ops->save_live_pending = cmma_save_pending;
+    ops->save_cleanup = cmma_save_cleanup;
+    ops->load_state = cmma_load;
+    ops->is_active = cmma_active;
+    register_savevm_live(NULL, TYPE_S390_STATTRIB, 0, 0, ops, sas);
+
+    object_property_add_bool(obj, "migration-enabled",
+                             s390_stattrib_get_migration_enabled,
+                             s390_stattrib_set_migration_enabled, NULL);
+    object_property_set_bool(obj, true, "migration-enabled", NULL);
+    sas->migration_cur_gfn = 0;
+}
+
+static const TypeInfo s390_stattrib_info = {
+    .name          = TYPE_S390_STATTRIB,
+    .parent        = TYPE_DEVICE,
+    .instance_init = s390_stattrib_instance_init,
+    .instance_size = sizeof(S390StAttribState),
+    .class_init    = s390_stattrib_class_init,
+    .class_size    = sizeof(S390StAttribClass),
+    .abstract      = true,
+};
+
+static void s390_stattrib_register_types(void)
+{
+    type_register_static(&s390_stattrib_info);
+    type_register_static(&qemu_s390_stattrib_info);
+}
+
+type_init(s390_stattrib_register_types)
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 41ca6668e2..ce3921e4de 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -24,11 +24,13 @@
 #include "qemu/config-file.h"
 #include "s390-pci-bus.h"
 #include "hw/s390x/storage-keys.h"
+#include "hw/s390x/storage-attributes.h"
 #include "hw/compat.h"
 #include "ipl.h"
 #include "hw/s390x/s390-virtio-ccw.h"
 #include "hw/s390x/css-bridge.h"
 #include "migration/register.h"
+#include "cpu_models.h"
 
 static const char *const reset_dev_types[] = {
     TYPE_VIRTUAL_CSS_BRIDGE,
@@ -103,6 +105,8 @@ void s390_memory_init(ram_addr_t mem_size)
 
     /* Initialize storage key device */
     s390_skeys_init();
+    /* Initialize storage attributes device */
+    s390_stattrib_init();
 }
 
 static SaveVMHandlers savevm_gtod = {
@@ -119,6 +123,9 @@ static void ccw_init(MachineState *machine)
     s390_sclp_init();
     s390_memory_init(machine->ram_size);
 
+    /* init CPUs */
+    s390_init_cpus(machine);
+
     s390_flic_init();
 
     /* get a BUS */
@@ -135,9 +142,6 @@ static void ccw_init(MachineState *machine)
     /* register hypercalls */
     virtio_ccw_register_hcalls();
 
-    /* init CPUs */
-    s390_init_cpus(machine);
-
     if (kvm_enabled()) {
         kvm_s390_enable_css_support(s390_cpu_addr2state(0));
     }
@@ -206,6 +210,8 @@ static void ccw_machine_class_init(ObjectClass *oc, void *data)
 
     s390mc->ri_allowed = true;
     s390mc->cpu_model_allowed = true;
+    s390mc->css_migration_enabled = true;
+    s390mc->gs_allowed = true;
     mc->init = ccw_init;
     mc->reset = s390_machine_reset;
     mc->hot_add_cpu = s390_hot_add_cpu;
@@ -252,36 +258,51 @@ static inline void machine_set_dea_key_wrap(Object *obj, bool value,
     ms->dea_key_wrap = value;
 }
 
+static S390CcwMachineClass *current_mc;
+
+static S390CcwMachineClass *get_machine_class(void)
+{
+    if (unlikely(!current_mc)) {
+        /*
+        * No s390 ccw machine was instantiated, we are likely to
+        * be called for the 'none' machine. The properties will
+        * have their after-initialization values.
+        */
+        current_mc = S390_MACHINE_CLASS(
+                     object_class_by_name(TYPE_S390_CCW_MACHINE));
+    }
+    return current_mc;
+}
+
 bool ri_allowed(void)
 {
+    if (!kvm_enabled()) {
+        return false;
+    }
+    /* for "none" machine this results in true */
+    return get_machine_class()->ri_allowed;
+}
+
+bool cpu_model_allowed(void)
+{
+    /* for "none" machine this results in true */
+    return get_machine_class()->cpu_model_allowed;
+}
+
+bool gs_allowed(void)
+{
     if (kvm_enabled()) {
         MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
         if (object_class_dynamic_cast(OBJECT_CLASS(mc),
                                       TYPE_S390_CCW_MACHINE)) {
             S390CcwMachineClass *s390mc = S390_MACHINE_CLASS(mc);
 
-            return s390mc->ri_allowed;
+            return s390mc->gs_allowed;
         }
-        /*
-         * Make sure the "none" machine can have ri, otherwise it won't * be
-         * unlocked in KVM and therefore the host CPU model might be wrong.
-         */
+        /* Make sure the "none" machine can have gs */
         return true;
     }
-    return 0;
-}
-
-bool cpu_model_allowed(void)
-{
-    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
-    if (object_class_dynamic_cast(OBJECT_CLASS(mc),
-                                  TYPE_S390_CCW_MACHINE)) {
-        S390CcwMachineClass *s390mc = S390_MACHINE_CLASS(mc);
-
-        return s390mc->cpu_model_allowed;
-    }
-    /* allow CPU model qmp queries with the "none" machine */
-    return true;
+    return false;
 }
 
 static char *machine_get_loadparm(Object *obj, Error **errp)
@@ -376,6 +397,11 @@ static const TypeInfo ccw_machine_info = {
     },
 };
 
+bool css_migration_enabled(void)
+{
+    return get_machine_class()->css_migration_enabled;
+}
+
 #define DEFINE_CCW_MACHINE(suffix, verstr, latest)                            \
     static void ccw_machine_##suffix##_class_init(ObjectClass *oc,            \
                                                   void *data)                 \
@@ -391,6 +417,7 @@ static const TypeInfo ccw_machine_info = {
     static void ccw_machine_##suffix##_instance_init(Object *obj)             \
     {                                                                         \
         MachineState *machine = MACHINE(obj);                                 \
+        current_mc = S390_MACHINE_CLASS(MACHINE_GET_CLASS(machine));          \
         ccw_machine_##suffix##_instance_options(machine);                     \
     }                                                                         \
     static const TypeInfo ccw_machine_##suffix##_info = {                     \
@@ -406,7 +433,12 @@ static const TypeInfo ccw_machine_info = {
     type_init(ccw_machine_register_##suffix)
 
 #define CCW_COMPAT_2_9 \
-        HW_COMPAT_2_9
+        HW_COMPAT_2_9 \
+        {\
+            .driver   = TYPE_S390_STATTRIB,\
+            .property = "migration-enabled",\
+            .value    = "off",\
+        },
 
 #define CCW_COMPAT_2_8 \
         HW_COMPAT_2_8 \
@@ -476,6 +508,9 @@ static const TypeInfo ccw_machine_info = {
 
 static void ccw_machine_2_10_instance_options(MachineState *machine)
 {
+    if (css_migration_enabled()) {
+        css_register_vmstate();
+    }
 }
 
 static void ccw_machine_2_10_class_options(MachineClass *mc)
@@ -486,12 +521,21 @@ DEFINE_CCW_MACHINE(2_10, "2.10", true);
 static void ccw_machine_2_9_instance_options(MachineState *machine)
 {
     ccw_machine_2_10_instance_options(machine);
+    s390_cpudef_featoff_greater(12, 1, S390_FEAT_ESOP);
+    s390_cpudef_featoff_greater(12, 1, S390_FEAT_SIDE_EFFECT_ACCESS_ESOP2);
+    s390_cpudef_featoff_greater(12, 1, S390_FEAT_ZPCI);
+    s390_cpudef_featoff_greater(12, 1, S390_FEAT_ADAPTER_INT_SUPPRESSION);
+    s390_cpudef_featoff_greater(12, 1, S390_FEAT_ADAPTER_EVENT_NOTIFICATION);
 }
 
 static void ccw_machine_2_9_class_options(MachineClass *mc)
 {
+    S390CcwMachineClass *s390mc = S390_MACHINE_CLASS(mc);
+
+    s390mc->gs_allowed = false;
     ccw_machine_2_10_class_options(mc);
     SET_MACHINE_COMPAT(mc, CCW_COMPAT_2_9);
+    s390mc->css_migration_enabled = false;
 }
 DEFINE_CCW_MACHINE(2_9, "2.9", false);
 
diff --git a/hw/s390x/trace-events b/hw/s390x/trace-events
index 84ea964875..f07e974678 100644
--- a/hw/s390x/trace-events
+++ b/hw/s390x/trace-events
@@ -8,6 +8,7 @@ css_new_image(uint8_t cssid, const char *default_cssid) "CSS: add css image %02x
 css_assign_subch(const char *do_assign, uint8_t cssid, uint8_t ssid, uint16_t schid, uint16_t devno) "CSS: %s %x.%x.%04x (devno %04x)"
 css_io_interrupt(int cssid, int ssid, int schid, uint32_t intparm, uint8_t isc, const char *conditional) "CSS: I/O interrupt on sch %x.%x.%04x (intparm %08x, isc %x) %s"
 css_adapter_interrupt(uint8_t isc) "CSS: adapter I/O interrupt (isc %x)"
+css_do_sic(uint16_t mode, uint8_t isc) "CSS: set interruption mode %x on isc %x"
 
 # hw/s390x/virtio-ccw.c
 virtio_ccw_interpret_ccw(int cssid, int ssid, int schid, int cmd_code) "VIRTIO-CCW: %x.%x.%04x: interpret command %x"
diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
index c07ddb1c94..b1976fdd19 100644
--- a/hw/s390x/virtio-ccw.c
+++ b/hw/s390x/virtio-ccw.c
@@ -1070,7 +1070,7 @@ static void virtio_ccw_notify(DeviceState *d, uint16_t vector)
                                   0x80 >> ((ind_bit + vector) % 8));
             if (!virtio_set_ind_atomic(sch, dev->summary_indicator->addr,
                                        0x01)) {
-                css_adapter_interrupt(dev->thinint_isc);
+                css_adapter_interrupt(CSS_IO_ADAPTER_VIRTIO, dev->thinint_isc);
             }
         } else {
             indicators = address_space_ldq(&address_space_memory,