From 21fa15298d88db2050a713cdf79c10cb0e09146f Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Fri, 2 Sep 2022 13:27:31 -0400 Subject: s390x/pci: add routine to get host function handle from CLP info In order to interface with the underlying host zPCI device, we need to know its function handle. Add a routine to grab this from the vfio CLP capabilities chain. Signed-off-by: Matthew Rosato Reviewed-by: Pierre Morel Message-Id: <20220902172737.170349-3-mjrosato@linux.ibm.com> [thuth: Replace free(info) with g_free(info)] Signed-off-by: Thomas Huth --- include/hw/s390x/s390-pci-vfio.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/hw') diff --git a/include/hw/s390x/s390-pci-vfio.h b/include/hw/s390x/s390-pci-vfio.h index ff708aef50..ae1b126ff7 100644 --- a/include/hw/s390x/s390-pci-vfio.h +++ b/include/hw/s390x/s390-pci-vfio.h @@ -20,6 +20,7 @@ bool s390_pci_update_dma_avail(int fd, unsigned int *avail); S390PCIDMACount *s390_pci_start_dma_count(S390pciState *s, S390PCIBusDevice *pbdev); void s390_pci_end_dma_count(S390pciState *s, S390PCIDMACount *cnt); +bool s390_pci_get_host_fh(S390PCIBusDevice *pbdev, uint32_t *fh); void s390_pci_get_clp_info(S390PCIBusDevice *pbdev); #else static inline bool s390_pci_update_dma_avail(int fd, unsigned int *avail) @@ -33,6 +34,10 @@ static inline S390PCIDMACount *s390_pci_start_dma_count(S390pciState *s, } static inline void s390_pci_end_dma_count(S390pciState *s, S390PCIDMACount *cnt) { } +static inline bool s390_pci_get_host_fh(S390PCIBusDevice *pbdev, uint32_t *fh) +{ + return false; +} static inline void s390_pci_get_clp_info(S390PCIBusDevice *pbdev) { } #endif -- cgit v1.2.3-55-g7522 From dd1d5fd9684beeb0c14c39f497ef2aa9ac683aa7 Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Fri, 2 Sep 2022 13:27:32 -0400 Subject: s390x/pci: enable for load/store interpretation If the ZPCI_OP ioctl reports that is is available and usable, then the underlying KVM host will enable load/store intepretation for any guest device without a SHM bit in the guest function handle. For a device that will be using interpretation support, ensure the guest function handle matches the host function handle; this value is re-checked every time the guest issues a SET PCI FN to enable the guest device as it is the only opportunity to reflect function handle changes. By default, unless interpret=off is specified, interpretation support will always be assumed and exploited if the necessary ioctl and features are available on the host kernel. When these are unavailable, we will silently revert to the interception model; this allows existing guest configurations to work unmodified on hosts with and without zPCI interpretation support, allowing QEMU to choose the best support model available. Signed-off-by: Matthew Rosato Acked-by: Thomas Huth Message-Id: <20220902172737.170349-4-mjrosato@linux.ibm.com> Signed-off-by: Thomas Huth --- hw/s390x/meson.build | 1 + hw/s390x/s390-pci-bus.c | 66 ++++++++++++++++++++++++++++++++++++++++- hw/s390x/s390-pci-inst.c | 16 ++++++++++ hw/s390x/s390-pci-kvm.c | 22 ++++++++++++++ include/hw/s390x/s390-pci-bus.h | 1 + include/hw/s390x/s390-pci-kvm.h | 24 +++++++++++++++ target/s390x/kvm/kvm.c | 7 +++++ target/s390x/kvm/kvm_s390x.h | 1 + 8 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 hw/s390x/s390-pci-kvm.c create mode 100644 include/hw/s390x/s390-pci-kvm.h (limited to 'include/hw') diff --git a/hw/s390x/meson.build b/hw/s390x/meson.build index feefe0717e..f291016fee 100644 --- a/hw/s390x/meson.build +++ b/hw/s390x/meson.build @@ -23,6 +23,7 @@ s390x_ss.add(when: 'CONFIG_KVM', if_true: files( 's390-skeys-kvm.c', 's390-stattrib-kvm.c', 'pv.c', + 's390-pci-kvm.c', )) s390x_ss.add(when: 'CONFIG_TCG', if_true: files( 'tod-tcg.c', diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c index 4b2bdd94b3..156051e6e9 100644 --- a/hw/s390x/s390-pci-bus.c +++ b/hw/s390x/s390-pci-bus.c @@ -16,6 +16,7 @@ #include "qapi/visitor.h" #include "hw/s390x/s390-pci-bus.h" #include "hw/s390x/s390-pci-inst.h" +#include "hw/s390x/s390-pci-kvm.h" #include "hw/s390x/s390-pci-vfio.h" #include "hw/pci/pci_bus.h" #include "hw/qdev-properties.h" @@ -971,12 +972,51 @@ static void s390_pci_update_subordinate(PCIDevice *dev, uint32_t nr) } } +static int s390_pci_interp_plug(S390pciState *s, S390PCIBusDevice *pbdev) +{ + uint32_t idx, fh; + + if (!s390_pci_get_host_fh(pbdev, &fh)) { + return -EPERM; + } + + /* + * The host device is already in an enabled state, but we always present + * the initial device state to the guest as disabled (ZPCI_FS_DISABLED). + * Therefore, mask off the enable bit from the passthrough handle until + * the guest issues a CLP SET PCI FN later to enable the device. + */ + pbdev->fh = fh & ~FH_MASK_ENABLE; + + /* Next, see if the idx is already in-use */ + idx = pbdev->fh & FH_MASK_INDEX; + if (pbdev->idx != idx) { + if (s390_pci_find_dev_by_idx(s, idx)) { + return -EINVAL; + } + /* + * Update the idx entry with the passed through idx + * If the relinquished idx is lower than next_idx, use it + * to replace next_idx + */ + g_hash_table_remove(s->zpci_table, &pbdev->idx); + if (idx < s->next_idx) { + s->next_idx = idx; + } + pbdev->idx = idx; + g_hash_table_insert(s->zpci_table, &pbdev->idx, pbdev); + } + + return 0; +} + static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp) { S390pciState *s = S390_PCI_HOST_BRIDGE(hotplug_dev); PCIDevice *pdev = NULL; S390PCIBusDevice *pbdev = NULL; + int rc; if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_BRIDGE)) { PCIBridge *pb = PCI_BRIDGE(dev); @@ -1022,12 +1062,35 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev, set_pbdev_info(pbdev); if (object_dynamic_cast(OBJECT(dev), "vfio-pci")) { - pbdev->fh |= FH_SHM_VFIO; + /* + * By default, interpretation is always requested; if the available + * facilities indicate it is not available, fallback to the + * interception model. + */ + if (pbdev->interp) { + if (s390_pci_kvm_interp_allowed()) { + rc = s390_pci_interp_plug(s, pbdev); + if (rc) { + error_setg(errp, "Plug failed for zPCI device in " + "interpretation mode: %d", rc); + return; + } + } else { + DPRINTF("zPCI interpretation facilities missing.\n"); + pbdev->interp = false; + } + } pbdev->iommu->dma_limit = s390_pci_start_dma_count(s, pbdev); /* Fill in CLP information passed via the vfio region */ s390_pci_get_clp_info(pbdev); + if (!pbdev->interp) { + /* Do vfio passthrough but intercept for I/O */ + pbdev->fh |= FH_SHM_VFIO; + } } else { pbdev->fh |= FH_SHM_EMUL; + /* Always intercept emulated devices */ + pbdev->interp = false; } if (s390_pci_msix_init(pbdev)) { @@ -1360,6 +1423,7 @@ static Property s390_pci_device_properties[] = { DEFINE_PROP_UINT16("uid", S390PCIBusDevice, uid, UID_UNDEFINED), DEFINE_PROP_S390_PCI_FID("fid", S390PCIBusDevice, fid), DEFINE_PROP_STRING("target", S390PCIBusDevice, target), + DEFINE_PROP_BOOL("interpret", S390PCIBusDevice, interp, true), DEFINE_PROP_END_OF_LIST(), }; diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c index 6d400d4147..651ec38635 100644 --- a/hw/s390x/s390-pci-inst.c +++ b/hw/s390x/s390-pci-inst.c @@ -18,6 +18,8 @@ #include "sysemu/hw_accel.h" #include "hw/s390x/s390-pci-inst.h" #include "hw/s390x/s390-pci-bus.h" +#include "hw/s390x/s390-pci-kvm.h" +#include "hw/s390x/s390-pci-vfio.h" #include "hw/s390x/tod.h" #ifndef DEBUG_S390PCI_INST @@ -246,6 +248,20 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra) goto out; } + /* + * Take this opportunity to make sure we still have an accurate + * host fh. It's possible part of the handle changed while the + * device was disabled to the guest (e.g. vfio hot reset for + * ISM during plug) + */ + if (pbdev->interp) { + /* Take this opportunity to make sure we are sync'd with host */ + if (!s390_pci_get_host_fh(pbdev, &pbdev->fh) || + !(pbdev->fh & FH_MASK_ENABLE)) { + stw_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FH); + goto out; + } + } pbdev->fh |= FH_MASK_ENABLE; pbdev->state = ZPCI_FS_ENABLED; stl_p(&ressetpci->fh, pbdev->fh); diff --git a/hw/s390x/s390-pci-kvm.c b/hw/s390x/s390-pci-kvm.c new file mode 100644 index 0000000000..0f16104a74 --- /dev/null +++ b/hw/s390x/s390-pci-kvm.c @@ -0,0 +1,22 @@ +/* + * s390 zPCI KVM interfaces + * + * Copyright 2022 IBM Corp. + * Author(s): Matthew Rosato + * + * This work is licensed under the terms of the GNU GPL, version 2 or (at + * your option) any later version. See the COPYING file in the top-level + * directory. + */ + +#include "qemu/osdep.h" + +#include "kvm/kvm_s390x.h" +#include "hw/s390x/pv.h" +#include "hw/s390x/s390-pci-kvm.h" +#include "cpu_models.h" + +bool s390_pci_kvm_interp_allowed(void) +{ + return kvm_s390_get_zpci_op() && !s390_is_pv(); +} diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h index da3cde2bb4..a9843dfe97 100644 --- a/include/hw/s390x/s390-pci-bus.h +++ b/include/hw/s390x/s390-pci-bus.h @@ -350,6 +350,7 @@ struct S390PCIBusDevice { IndAddr *indicator; bool pci_unplug_request_processed; bool unplug_requested; + bool interp; QTAILQ_ENTRY(S390PCIBusDevice) link; }; diff --git a/include/hw/s390x/s390-pci-kvm.h b/include/hw/s390x/s390-pci-kvm.h new file mode 100644 index 0000000000..80a2e7d0ca --- /dev/null +++ b/include/hw/s390x/s390-pci-kvm.h @@ -0,0 +1,24 @@ +/* + * s390 PCI KVM interfaces + * + * Copyright 2022 IBM Corp. + * Author(s): Matthew Rosato + * + * This work is licensed under the terms of the GNU GPL, version 2 or (at + * your option) any later version. See the COPYING file in the top-level + * directory. + */ + +#ifndef HW_S390_PCI_KVM_H +#define HW_S390_PCI_KVM_H + +#ifdef CONFIG_KVM +bool s390_pci_kvm_interp_allowed(void); +#else +static inline bool s390_pci_kvm_interp_allowed(void) +{ + return false; +} +#endif + +#endif diff --git a/target/s390x/kvm/kvm.c b/target/s390x/kvm/kvm.c index 7bd8db0e7b..6a8dbadf7e 100644 --- a/target/s390x/kvm/kvm.c +++ b/target/s390x/kvm/kvm.c @@ -157,6 +157,7 @@ static int cap_ri; static int cap_hpage_1m; static int cap_vcpu_resets; static int cap_protected; +static int cap_zpci_op; static bool mem_op_storage_key_support; @@ -362,6 +363,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s) cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ); cap_vcpu_resets = kvm_check_extension(s, KVM_CAP_S390_VCPU_RESETS); cap_protected = kvm_check_extension(s, KVM_CAP_S390_PROTECTED); + cap_zpci_op = kvm_check_extension(s, KVM_CAP_S390_ZPCI_OP); kvm_vm_enable_cap(s, KVM_CAP_S390_USER_SIGP, 0); kvm_vm_enable_cap(s, KVM_CAP_S390_VECTOR_REGISTERS, 0); @@ -2574,3 +2576,8 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +int kvm_s390_get_zpci_op(void) +{ + return cap_zpci_op; +} diff --git a/target/s390x/kvm/kvm_s390x.h b/target/s390x/kvm/kvm_s390x.h index 05a5e1e6f4..aaae8570de 100644 --- a/target/s390x/kvm/kvm_s390x.h +++ b/target/s390x/kvm/kvm_s390x.h @@ -27,6 +27,7 @@ void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu); int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu); int kvm_s390_get_hpage_1m(void); int kvm_s390_get_ri(void); +int kvm_s390_get_zpci_op(void); int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock); int kvm_s390_get_clock_ext(uint8_t *tod_high, uint64_t *tod_clock); int kvm_s390_set_clock(uint8_t tod_high, uint64_t tod_clock); -- cgit v1.2.3-55-g7522 From d0bc7091c2013ad2fa164100cf7b17962370e8ab Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Fri, 2 Sep 2022 13:27:34 -0400 Subject: s390x/pci: enable adapter event notification for interpreted devices Use the associated kvm ioctl operation to enable adapter event notification and forwarding for devices when requested. This feature will be set up with or without firmware assist based upon the 'forwarding_assist' setting. Signed-off-by: Matthew Rosato Message-Id: <20220902172737.170349-6-mjrosato@linux.ibm.com> [thuth: Rename "forwarding_assist" property to "forwarding-assist"] Signed-off-by: Thomas Huth --- hw/s390x/s390-pci-bus.c | 20 +++++++++++++++++--- hw/s390x/s390-pci-inst.c | 40 ++++++++++++++++++++++++++++++++++++++-- hw/s390x/s390-pci-kvm.c | 30 ++++++++++++++++++++++++++++++ include/hw/s390x/s390-pci-bus.h | 1 + include/hw/s390x/s390-pci-kvm.h | 14 ++++++++++++++ 5 files changed, 100 insertions(+), 5 deletions(-) (limited to 'include/hw') diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c index 816d17af99..2f19973913 100644 --- a/hw/s390x/s390-pci-bus.c +++ b/hw/s390x/s390-pci-bus.c @@ -190,7 +190,10 @@ void s390_pci_sclp_deconfigure(SCCB *sccb) rc = SCLP_RC_NO_ACTION_REQUIRED; break; default: - if (pbdev->summary_ind) { + if (pbdev->interp && (pbdev->fh & FH_MASK_ENABLE)) { + /* Interpreted devices were using interrupt forwarding */ + s390_pci_kvm_aif_disable(pbdev); + } else if (pbdev->summary_ind) { pci_dereg_irqs(pbdev); } if (pbdev->iommu->enabled) { @@ -1082,6 +1085,7 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev, } else { DPRINTF("zPCI interpretation facilities missing.\n"); pbdev->interp = false; + pbdev->forwarding_assist = false; } } pbdev->iommu->dma_limit = s390_pci_start_dma_count(s, pbdev); @@ -1090,11 +1094,13 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev, if (!pbdev->interp) { /* Do vfio passthrough but intercept for I/O */ pbdev->fh |= FH_SHM_VFIO; + pbdev->forwarding_assist = false; } } else { pbdev->fh |= FH_SHM_EMUL; /* Always intercept emulated devices */ pbdev->interp = false; + pbdev->forwarding_assist = false; } if (s390_pci_msix_init(pbdev) && !pbdev->interp) { @@ -1244,7 +1250,10 @@ static void s390_pcihost_reset(DeviceState *dev) /* Process all pending unplug requests */ QTAILQ_FOREACH_SAFE(pbdev, &s->zpci_devs, link, next) { if (pbdev->unplug_requested) { - if (pbdev->summary_ind) { + if (pbdev->interp && (pbdev->fh & FH_MASK_ENABLE)) { + /* Interpreted devices were using interrupt forwarding */ + s390_pci_kvm_aif_disable(pbdev); + } else if (pbdev->summary_ind) { pci_dereg_irqs(pbdev); } if (pbdev->iommu->enabled) { @@ -1382,7 +1391,10 @@ static void s390_pci_device_reset(DeviceState *dev) break; } - if (pbdev->summary_ind) { + if (pbdev->interp && (pbdev->fh & FH_MASK_ENABLE)) { + /* Interpreted devices were using interrupt forwarding */ + s390_pci_kvm_aif_disable(pbdev); + } else if (pbdev->summary_ind) { pci_dereg_irqs(pbdev); } if (pbdev->iommu->enabled) { @@ -1428,6 +1440,8 @@ static Property s390_pci_device_properties[] = { DEFINE_PROP_S390_PCI_FID("fid", S390PCIBusDevice, fid), DEFINE_PROP_STRING("target", S390PCIBusDevice, target), DEFINE_PROP_BOOL("interpret", S390PCIBusDevice, interp, true), + DEFINE_PROP_BOOL("forwarding-assist", S390PCIBusDevice, forwarding_assist, + true), DEFINE_PROP_END_OF_LIST(), }; diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c index 651ec38635..20a9bcc7af 100644 --- a/hw/s390x/s390-pci-inst.c +++ b/hw/s390x/s390-pci-inst.c @@ -1066,6 +1066,32 @@ static void fmb_update(void *opaque) timer_mod(pbdev->fmb_timer, t + pbdev->pci_group->zpci_group.mui); } +static int mpcifc_reg_int_interp(S390PCIBusDevice *pbdev, ZpciFib *fib) +{ + int rc; + + rc = s390_pci_kvm_aif_enable(pbdev, fib, pbdev->forwarding_assist); + if (rc) { + DPRINTF("Failed to enable interrupt forwarding\n"); + return rc; + } + + return 0; +} + +static int mpcifc_dereg_int_interp(S390PCIBusDevice *pbdev, ZpciFib *fib) +{ + int rc; + + rc = s390_pci_kvm_aif_disable(pbdev); + if (rc) { + DPRINTF("Failed to disable interrupt forwarding\n"); + return rc; + } + + return 0; +} + int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar, uintptr_t ra) { @@ -1120,7 +1146,12 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar, switch (oc) { case ZPCI_MOD_FC_REG_INT: - if (pbdev->summary_ind) { + if (pbdev->interp) { + if (mpcifc_reg_int_interp(pbdev, &fib)) { + cc = ZPCI_PCI_LS_ERR; + s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); + } + } else if (pbdev->summary_ind) { cc = ZPCI_PCI_LS_ERR; s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); } else if (reg_irqs(env, pbdev, fib)) { @@ -1129,7 +1160,12 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar, } break; case ZPCI_MOD_FC_DEREG_INT: - if (!pbdev->summary_ind) { + if (pbdev->interp) { + if (mpcifc_dereg_int_interp(pbdev, &fib)) { + cc = ZPCI_PCI_LS_ERR; + s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); + } + } else if (!pbdev->summary_ind) { cc = ZPCI_PCI_LS_ERR; s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE); } else { diff --git a/hw/s390x/s390-pci-kvm.c b/hw/s390x/s390-pci-kvm.c index 0f16104a74..9134fe185f 100644 --- a/hw/s390x/s390-pci-kvm.c +++ b/hw/s390x/s390-pci-kvm.c @@ -11,12 +11,42 @@ #include "qemu/osdep.h" +#include + #include "kvm/kvm_s390x.h" #include "hw/s390x/pv.h" +#include "hw/s390x/s390-pci-bus.h" #include "hw/s390x/s390-pci-kvm.h" +#include "hw/s390x/s390-pci-inst.h" #include "cpu_models.h" bool s390_pci_kvm_interp_allowed(void) { return kvm_s390_get_zpci_op() && !s390_is_pv(); } + +int s390_pci_kvm_aif_enable(S390PCIBusDevice *pbdev, ZpciFib *fib, bool assist) +{ + struct kvm_s390_zpci_op args = { + .fh = pbdev->fh, + .op = KVM_S390_ZPCIOP_REG_AEN, + .u.reg_aen.ibv = fib->aibv, + .u.reg_aen.sb = fib->aisb, + .u.reg_aen.noi = FIB_DATA_NOI(fib->data), + .u.reg_aen.isc = FIB_DATA_ISC(fib->data), + .u.reg_aen.sbo = FIB_DATA_AISBO(fib->data), + .u.reg_aen.flags = (assist) ? 0 : KVM_S390_ZPCIOP_REGAEN_HOST + }; + + return kvm_vm_ioctl(kvm_state, KVM_S390_ZPCI_OP, &args); +} + +int s390_pci_kvm_aif_disable(S390PCIBusDevice *pbdev) +{ + struct kvm_s390_zpci_op args = { + .fh = pbdev->fh, + .op = KVM_S390_ZPCIOP_DEREG_AEN + }; + + return kvm_vm_ioctl(kvm_state, KVM_S390_ZPCI_OP, &args); +} diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h index a9843dfe97..5b09f0cf2f 100644 --- a/include/hw/s390x/s390-pci-bus.h +++ b/include/hw/s390x/s390-pci-bus.h @@ -351,6 +351,7 @@ struct S390PCIBusDevice { bool pci_unplug_request_processed; bool unplug_requested; bool interp; + bool forwarding_assist; QTAILQ_ENTRY(S390PCIBusDevice) link; }; diff --git a/include/hw/s390x/s390-pci-kvm.h b/include/hw/s390x/s390-pci-kvm.h index 80a2e7d0ca..933814a402 100644 --- a/include/hw/s390x/s390-pci-kvm.h +++ b/include/hw/s390x/s390-pci-kvm.h @@ -12,13 +12,27 @@ #ifndef HW_S390_PCI_KVM_H #define HW_S390_PCI_KVM_H +#include "hw/s390x/s390-pci-bus.h" +#include "hw/s390x/s390-pci-inst.h" + #ifdef CONFIG_KVM bool s390_pci_kvm_interp_allowed(void); +int s390_pci_kvm_aif_enable(S390PCIBusDevice *pbdev, ZpciFib *fib, bool assist); +int s390_pci_kvm_aif_disable(S390PCIBusDevice *pbdev); #else static inline bool s390_pci_kvm_interp_allowed(void) { return false; } +static inline int s390_pci_kvm_aif_enable(S390PCIBusDevice *pbdev, ZpciFib *fib, + bool assist) +{ + return -EINVAL; +} +static inline int s390_pci_kvm_aif_disable(S390PCIBusDevice *pbdev) +{ + return -EINVAL; +} #endif #endif -- cgit v1.2.3-55-g7522 From 30dcf4f7fd23bef7d72a2454c60881710fd4c785 Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Fri, 2 Sep 2022 13:27:35 -0400 Subject: s390x/pci: let intercept devices have separate PCI groups Let's use the reserved pool of simulated PCI groups to allow intercept devices to have separate groups from interpreted devices as some group values may be different. If we run out of simulated PCI groups, subsequent intercept devices just get the default group. Furthermore, if we encounter any PCI groups from hostdevs that are marked as simulated, let's just assign them to the default group to avoid conflicts between host simulated groups and our own simulated groups. Signed-off-by: Matthew Rosato Reviewed-by: Pierre Morel Message-Id: <20220902172737.170349-7-mjrosato@linux.ibm.com> Signed-off-by: Thomas Huth --- hw/s390x/s390-pci-bus.c | 19 +++++++++++++++++-- hw/s390x/s390-pci-vfio.c | 40 +++++++++++++++++++++++++++++++++++++--- include/hw/s390x/s390-pci-bus.h | 6 +++++- 3 files changed, 59 insertions(+), 6 deletions(-) (limited to 'include/hw') diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c index 2f19973913..977e7daa15 100644 --- a/hw/s390x/s390-pci-bus.c +++ b/hw/s390x/s390-pci-bus.c @@ -748,13 +748,14 @@ static void s390_pci_iommu_free(S390pciState *s, PCIBus *bus, int32_t devfn) object_unref(OBJECT(iommu)); } -S390PCIGroup *s390_group_create(int id) +S390PCIGroup *s390_group_create(int id, int host_id) { S390PCIGroup *group; S390pciState *s = s390_get_phb(); group = g_new0(S390PCIGroup, 1); group->id = id; + group->host_id = host_id; QTAILQ_INSERT_TAIL(&s->zpci_groups, group, link); return group; } @@ -772,12 +773,25 @@ S390PCIGroup *s390_group_find(int id) return NULL; } +S390PCIGroup *s390_group_find_host_sim(int host_id) +{ + S390PCIGroup *group; + S390pciState *s = s390_get_phb(); + + QTAILQ_FOREACH(group, &s->zpci_groups, link) { + if (group->id >= ZPCI_SIM_GRP_START && group->host_id == host_id) { + return group; + } + } + return NULL; +} + static void s390_pci_init_default_group(void) { S390PCIGroup *group; ClpRspQueryPciGrp *resgrp; - group = s390_group_create(ZPCI_DEFAULT_FN_GRP); + group = s390_group_create(ZPCI_DEFAULT_FN_GRP, ZPCI_DEFAULT_FN_GRP); resgrp = &group->zpci_group; resgrp->fr = 1; resgrp->dasm = 0; @@ -825,6 +839,7 @@ static void s390_pcihost_realize(DeviceState *dev, Error **errp) NULL, g_free); s->zpci_table = g_hash_table_new_full(g_int_hash, g_int_equal, NULL, NULL); s->bus_no = 0; + s->next_sim_grp = ZPCI_SIM_GRP_START; QTAILQ_INIT(&s->pending_sei); QTAILQ_INIT(&s->zpci_devs); QTAILQ_INIT(&s->zpci_dma_limit); diff --git a/hw/s390x/s390-pci-vfio.c b/hw/s390x/s390-pci-vfio.c index 08bcc55e85..338f436e87 100644 --- a/hw/s390x/s390-pci-vfio.c +++ b/hw/s390x/s390-pci-vfio.c @@ -150,13 +150,18 @@ static void s390_pci_read_group(S390PCIBusDevice *pbdev, { struct vfio_info_cap_header *hdr; struct vfio_device_info_cap_zpci_group *cap; + S390pciState *s = s390_get_phb(); ClpRspQueryPciGrp *resgrp; VFIOPCIDevice *vpci = container_of(pbdev->pdev, VFIOPCIDevice, pdev); + uint8_t start_gid = pbdev->zpci_fn.pfgid; hdr = vfio_get_device_info_cap(info, VFIO_DEVICE_INFO_CAP_ZPCI_GROUP); - /* If capability not provided, just use the default group */ - if (hdr == NULL) { + /* + * If capability not provided or the underlying hostdev is simulated, just + * use the default group. + */ + if (hdr == NULL || pbdev->zpci_fn.pfgid >= ZPCI_SIM_GRP_START) { trace_s390_pci_clp_cap(vpci->vbasedev.name, VFIO_DEVICE_INFO_CAP_ZPCI_GROUP); pbdev->zpci_fn.pfgid = ZPCI_DEFAULT_FN_GRP; @@ -165,11 +170,40 @@ static void s390_pci_read_group(S390PCIBusDevice *pbdev, } cap = (void *) hdr; + /* + * For an intercept device, let's use an existing simulated group if one + * one was already created for other intercept devices in this group. + * If not, create a new simulated group if any are still available. + * If all else fails, just fall back on the default group. + */ + if (!pbdev->interp) { + pbdev->pci_group = s390_group_find_host_sim(pbdev->zpci_fn.pfgid); + if (pbdev->pci_group) { + /* Use existing simulated group */ + pbdev->zpci_fn.pfgid = pbdev->pci_group->id; + return; + } else { + if (s->next_sim_grp == ZPCI_DEFAULT_FN_GRP) { + /* All out of simulated groups, use default */ + trace_s390_pci_clp_cap(vpci->vbasedev.name, + VFIO_DEVICE_INFO_CAP_ZPCI_GROUP); + pbdev->zpci_fn.pfgid = ZPCI_DEFAULT_FN_GRP; + pbdev->pci_group = s390_group_find(ZPCI_DEFAULT_FN_GRP); + return; + } else { + /* We can assign a new simulated group */ + pbdev->zpci_fn.pfgid = s->next_sim_grp; + s->next_sim_grp++; + /* Fall through to create the new sim group using CLP info */ + } + } + } + /* See if the PCI group is already defined, create if not */ pbdev->pci_group = s390_group_find(pbdev->zpci_fn.pfgid); if (!pbdev->pci_group) { - pbdev->pci_group = s390_group_create(pbdev->zpci_fn.pfgid); + pbdev->pci_group = s390_group_create(pbdev->zpci_fn.pfgid, start_gid); resgrp = &pbdev->pci_group->zpci_group; if (cap->flags & VFIO_DEVICE_INFO_ZPCI_FLAG_REFRESH) { diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h index 5b09f0cf2f..0605fcea24 100644 --- a/include/hw/s390x/s390-pci-bus.h +++ b/include/hw/s390x/s390-pci-bus.h @@ -315,13 +315,16 @@ typedef struct ZpciFmb { QEMU_BUILD_BUG_MSG(offsetof(ZpciFmb, fmt0) != 48, "padding in ZpciFmb"); #define ZPCI_DEFAULT_FN_GRP 0xFF +#define ZPCI_SIM_GRP_START 0xF0 typedef struct S390PCIGroup { ClpRspQueryPciGrp zpci_group; int id; + int host_id; QTAILQ_ENTRY(S390PCIGroup) link; } S390PCIGroup; -S390PCIGroup *s390_group_create(int id); +S390PCIGroup *s390_group_create(int id, int host_id); S390PCIGroup *s390_group_find(int id); +S390PCIGroup *s390_group_find_host_sim(int host_id); struct S390PCIBusDevice { DeviceState qdev; @@ -370,6 +373,7 @@ struct S390pciState { QTAILQ_HEAD(, S390PCIBusDevice) zpci_devs; QTAILQ_HEAD(, S390PCIDMACount) zpci_dma_limit; QTAILQ_HEAD(, S390PCIGroup) zpci_groups; + uint8_t next_sim_grp; }; S390pciState *s390_get_phb(void); -- cgit v1.2.3-55-g7522 From 59d1ce44396e3ad2330dc3261ff3da7ad3a16184 Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Fri, 2 Sep 2022 13:27:37 -0400 Subject: s390x/s390-virtio-ccw: add zpcii-disable machine property The zpcii-disable machine property can be used to force-disable the use of zPCI interpretation facilities for a VM. By default, this setting will be off for machine 7.2 and newer. Signed-off-by: Matthew Rosato Message-Id: <20220902172737.170349-9-mjrosato@linux.ibm.com> Reviewed-by: Thomas Huth [thuth: Fix contextual conflict in ccw_machine_7_1_instance_options()] Signed-off-by: Thomas Huth --- hw/s390x/s390-pci-kvm.c | 4 +++- hw/s390x/s390-virtio-ccw.c | 24 ++++++++++++++++++++++++ include/hw/s390x/s390-virtio-ccw.h | 1 + qemu-options.hx | 8 +++++++- util/qemu-config.c | 4 ++++ 5 files changed, 39 insertions(+), 2 deletions(-) (limited to 'include/hw') diff --git a/hw/s390x/s390-pci-kvm.c b/hw/s390x/s390-pci-kvm.c index 9134fe185f..5eb7fd12e2 100644 --- a/hw/s390x/s390-pci-kvm.c +++ b/hw/s390x/s390-pci-kvm.c @@ -22,7 +22,9 @@ bool s390_pci_kvm_interp_allowed(void) { - return kvm_s390_get_zpci_op() && !s390_is_pv(); + return (kvm_s390_get_zpci_op() && !s390_is_pv() && + !object_property_get_bool(OBJECT(qdev_get_machine()), + "zpcii-disable", NULL)); } int s390_pci_kvm_aif_enable(S390PCIBusDevice *pbdev, ZpciFib *fib, bool assist) diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index e18b816aba..03855c7231 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -645,6 +645,21 @@ static inline void machine_set_dea_key_wrap(Object *obj, bool value, ms->dea_key_wrap = value; } +static inline bool machine_get_zpcii_disable(Object *obj, Error **errp) +{ + S390CcwMachineState *ms = S390_CCW_MACHINE(obj); + + return ms->zpcii_disable; +} + +static inline void machine_set_zpcii_disable(Object *obj, bool value, + Error **errp) +{ + S390CcwMachineState *ms = S390_CCW_MACHINE(obj); + + ms->zpcii_disable = value; +} + static S390CcwMachineClass *current_mc; /* @@ -740,6 +755,13 @@ static inline void s390_machine_initfn(Object *obj) "Up to 8 chars in set of [A-Za-z0-9. ] (lower case chars converted" " to upper case) to pass to machine loader, boot manager," " and guest kernel"); + + object_property_add_bool(obj, "zpcii-disable", + machine_get_zpcii_disable, + machine_set_zpcii_disable); + object_property_set_description(obj, "zpcii-disable", + "disable zPCI interpretation facilties"); + object_property_set_bool(obj, "zpcii-disable", false, NULL); } static const TypeInfo ccw_machine_info = { @@ -804,10 +826,12 @@ DEFINE_CCW_MACHINE(7_2, "7.2", true); static void ccw_machine_7_1_instance_options(MachineState *machine) { static const S390FeatInit qemu_cpu_feat = { S390_FEAT_LIST_QEMU_V7_1 }; + S390CcwMachineState *ms = S390_CCW_MACHINE(machine); ccw_machine_7_2_instance_options(machine); s390_cpudef_featoff_greater(16, 1, S390_FEAT_PAIE); s390_set_qemu_cpu_model(0x8561, 15, 1, qemu_cpu_feat); + ms->zpcii_disable = true; } static void ccw_machine_7_1_class_options(MachineClass *mc) diff --git a/include/hw/s390x/s390-virtio-ccw.h b/include/hw/s390x/s390-virtio-ccw.h index 3331990e02..8a0090a071 100644 --- a/include/hw/s390x/s390-virtio-ccw.h +++ b/include/hw/s390x/s390-virtio-ccw.h @@ -27,6 +27,7 @@ struct S390CcwMachineState { bool aes_key_wrap; bool dea_key_wrap; bool pv; + bool zpcii_disable; uint8_t loadparm[8]; }; diff --git a/qemu-options.hx b/qemu-options.hx index d8b5ce5b43..7c61868b68 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -37,7 +37,8 @@ DEF("machine", HAS_ARG, QEMU_OPTION_machine, \ " memory-encryption=@var{} memory encryption object to use (default=none)\n" " hmat=on|off controls ACPI HMAT support (default=off)\n" " memory-backend='backend-id' specifies explicitly provided backend for main RAM (default=none)\n" - " cxl-fmw.0.targets.0=firsttarget,cxl-fmw.0.targets.1=secondtarget,cxl-fmw.0.size=size[,cxl-fmw.0.interleave-granularity=granularity]\n", + " cxl-fmw.0.targets.0=firsttarget,cxl-fmw.0.targets.1=secondtarget,cxl-fmw.0.size=size[,cxl-fmw.0.interleave-granularity=granularity]\n" + " zpcii-disable=on|off disables zPCI interpretation facilities (default=off)\n", QEMU_ARCH_ALL) SRST ``-machine [type=]name[,prop=value[,...]]`` @@ -157,6 +158,11 @@ SRST :: -machine cxl-fmw.0.targets.0=cxl.0,cxl-fmw.0.targets.1=cxl.1,cxl-fmw.0.size=128G,cxl-fmw.0.interleave-granularity=512k + + ``zpcii-disable=on|off`` + Disables zPCI interpretation facilties on s390-ccw hosts. + This feature can be used to disable hardware virtual assists + related to zPCI devices. The default is off. ERST DEF("M", HAS_ARG, QEMU_OPTION_M, diff --git a/util/qemu-config.c b/util/qemu-config.c index 433488aa56..5325f6bf80 100644 --- a/util/qemu-config.c +++ b/util/qemu-config.c @@ -236,6 +236,10 @@ static QemuOptsList machine_opts = { .help = "Up to 8 chars in set of [A-Za-z0-9. ](lower case chars" " converted to upper case) to pass to machine" " loader, boot manager, and guest kernel", + },{ + .name = "zpcii-disable", + .type = QEMU_OPT_BOOL, + .help = "disable zPCI interpretation facilities", }, { /* End of list */ } } -- cgit v1.2.3-55-g7522