From 14776ab5a12972ea439c7fb2203a4c15a09094b4 Mon Sep 17 00:00:00 2001 From: Tony Nguyen Date: Sat, 24 Aug 2019 04:10:58 +1000 Subject: tcg: TCGMemOp is now accelerator independent MemOp Preparation for collapsing the two byte swaps, adjust_endianness and handle_bswap, along the I/O path. Target dependant attributes are conditionalized upon NEED_CPU_H. Signed-off-by: Tony Nguyen Acked-by: David Gibson Reviewed-by: Richard Henderson Acked-by: Cornelia Huck Message-Id: <81d9cd7d7f5aaadfa772d6c48ecee834e9cf7882.1566466906.git.tony.nguyen@bt.com> Signed-off-by: Richard Henderson --- include/exec/memop.h | 110 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 include/exec/memop.h (limited to 'include/exec') diff --git a/include/exec/memop.h b/include/exec/memop.h new file mode 100644 index 0000000000..7262ca3dfd --- /dev/null +++ b/include/exec/memop.h @@ -0,0 +1,110 @@ +/* + * Constants for memory operations + * + * Authors: + * Richard Henderson + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ + +#ifndef MEMOP_H +#define MEMOP_H + +typedef enum MemOp { + MO_8 = 0, + MO_16 = 1, + MO_32 = 2, + MO_64 = 3, + MO_SIZE = 3, /* Mask for the above. */ + + MO_SIGN = 4, /* Sign-extended, otherwise zero-extended. */ + + MO_BSWAP = 8, /* Host reverse endian. */ +#ifdef HOST_WORDS_BIGENDIAN + MO_LE = MO_BSWAP, + MO_BE = 0, +#else + MO_LE = 0, + MO_BE = MO_BSWAP, +#endif +#ifdef NEED_CPU_H +#ifdef TARGET_WORDS_BIGENDIAN + MO_TE = MO_BE, +#else + MO_TE = MO_LE, +#endif +#endif + + /* + * MO_UNALN accesses are never checked for alignment. + * MO_ALIGN accesses will result in a call to the CPU's + * do_unaligned_access hook if the guest address is not aligned. + * The default depends on whether the target CPU defines + * TARGET_ALIGNED_ONLY. + * + * Some architectures (e.g. ARMv8) need the address which is aligned + * to a size more than the size of the memory access. + * Some architectures (e.g. SPARCv9) need an address which is aligned, + * but less strictly than the natural alignment. + * + * MO_ALIGN supposes the alignment size is the size of a memory access. + * + * There are three options: + * - unaligned access permitted (MO_UNALN). + * - an alignment to the size of an access (MO_ALIGN); + * - an alignment to a specified size, which may be more or less than + * the access size (MO_ALIGN_x where 'x' is a size in bytes); + */ + MO_ASHIFT = 4, + MO_AMASK = 7 << MO_ASHIFT, +#ifdef NEED_CPU_H +#ifdef TARGET_ALIGNED_ONLY + MO_ALIGN = 0, + MO_UNALN = MO_AMASK, +#else + MO_ALIGN = MO_AMASK, + MO_UNALN = 0, +#endif +#endif + MO_ALIGN_2 = 1 << MO_ASHIFT, + MO_ALIGN_4 = 2 << MO_ASHIFT, + MO_ALIGN_8 = 3 << MO_ASHIFT, + MO_ALIGN_16 = 4 << MO_ASHIFT, + MO_ALIGN_32 = 5 << MO_ASHIFT, + MO_ALIGN_64 = 6 << MO_ASHIFT, + + /* Combinations of the above, for ease of use. */ + MO_UB = MO_8, + MO_UW = MO_16, + MO_UL = MO_32, + MO_SB = MO_SIGN | MO_8, + MO_SW = MO_SIGN | MO_16, + MO_SL = MO_SIGN | MO_32, + MO_Q = MO_64, + + MO_LEUW = MO_LE | MO_UW, + MO_LEUL = MO_LE | MO_UL, + MO_LESW = MO_LE | MO_SW, + MO_LESL = MO_LE | MO_SL, + MO_LEQ = MO_LE | MO_Q, + + MO_BEUW = MO_BE | MO_UW, + MO_BEUL = MO_BE | MO_UL, + MO_BESW = MO_BE | MO_SW, + MO_BESL = MO_BE | MO_SL, + MO_BEQ = MO_BE | MO_Q, + +#ifdef NEED_CPU_H + MO_TEUW = MO_TE | MO_UW, + MO_TEUL = MO_TE | MO_UL, + MO_TESW = MO_TE | MO_SW, + MO_TESL = MO_TE | MO_SL, + MO_TEQ = MO_TE | MO_Q, +#endif + + MO_SSIZE = MO_SIZE | MO_SIGN, +} MemOp; + +#endif -- cgit v1.2.3-55-g7522 From 66b9b24375ac215cdcbdf9e14d665395360abff4 Mon Sep 17 00:00:00 2001 From: Tony Nguyen Date: Sat, 24 Aug 2019 04:29:05 +1000 Subject: memory: Introduce size_memop The memory_region_dispatch_{read|write} operand "unsigned size" is being converted into a "MemOp op". Introduce no-op size_memop to aid preparatory conversion of interfaces. Once interfaces are converted, size_memop will be implemented to return a MemOp from size in bytes. Signed-off-by: Tony Nguyen Reviewed-by: Richard Henderson Message-Id: <35b8ee74020f67cf40848fb7d5f127cf96c851d6.1566466906.git.tony.nguyen@bt.com> Signed-off-by: Richard Henderson --- include/exec/memop.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/exec') diff --git a/include/exec/memop.h b/include/exec/memop.h index 7262ca3dfd..dfd76a1604 100644 --- a/include/exec/memop.h +++ b/include/exec/memop.h @@ -107,4 +107,14 @@ typedef enum MemOp { MO_SSIZE = MO_SIZE | MO_SIGN, } MemOp; +/* Size in bytes to MemOp. */ +static inline unsigned size_memop(unsigned size) +{ + /* + * FIXME: No-op to aid conversion of memory_region_dispatch_{read|write} + * "unsigned size" operand into a "MemOp op". + */ + return size; +} + #endif -- cgit v1.2.3-55-g7522 From e67c904668d82ca4416cd91d37d9f5abcceef747 Mon Sep 17 00:00:00 2001 From: Tony Nguyen Date: Sat, 24 Aug 2019 04:36:48 +1000 Subject: memory: Access MemoryRegion with MemOp Convert memory_region_dispatch_{read|write} operand "unsigned size" into a "MemOp op". Signed-off-by: Tony Nguyen Reviewed-by: Richard Henderson Message-Id: <1dd82df5801866743f838f1d046475115a1d32da.1566466906.git.tony.nguyen@bt.com> Signed-off-by: Richard Henderson --- include/exec/memop.h | 20 ++++++++++++++------ include/exec/memory.h | 9 +++++---- memory.c | 7 +++++-- 3 files changed, 24 insertions(+), 12 deletions(-) (limited to 'include/exec') diff --git a/include/exec/memop.h b/include/exec/memop.h index dfd76a1604..0a610b75d9 100644 --- a/include/exec/memop.h +++ b/include/exec/memop.h @@ -12,6 +12,8 @@ #ifndef MEMOP_H #define MEMOP_H +#include "qemu/host-utils.h" + typedef enum MemOp { MO_8 = 0, MO_16 = 1, @@ -107,14 +109,20 @@ typedef enum MemOp { MO_SSIZE = MO_SIZE | MO_SIGN, } MemOp; +/* MemOp to size in bytes. */ +static inline unsigned memop_size(MemOp op) +{ + return 1 << (op & MO_SIZE); +} + /* Size in bytes to MemOp. */ -static inline unsigned size_memop(unsigned size) +static inline MemOp size_memop(unsigned size) { - /* - * FIXME: No-op to aid conversion of memory_region_dispatch_{read|write} - * "unsigned size" operand into a "MemOp op". - */ - return size; +#ifdef CONFIG_DEBUG_TCG + /* Power of 2 up to 8. */ + assert((size & (size - 1)) == 0 && size >= 1 && size <= 8); +#endif + return ctz32(size); } #endif diff --git a/include/exec/memory.h b/include/exec/memory.h index fddc2ff48a..192875b080 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -19,6 +19,7 @@ #include "exec/cpu-common.h" #include "exec/hwaddr.h" #include "exec/memattrs.h" +#include "exec/memop.h" #include "exec/ramlist.h" #include "qemu/bswap.h" #include "qemu/queue.h" @@ -1749,13 +1750,13 @@ void mtree_info(bool flatview, bool dispatch_tree, bool owner); * @mr: #MemoryRegion to access * @addr: address within that region * @pval: pointer to uint64_t which the data is written to - * @size: size of the access in bytes + * @op: size, sign, and endianness of the memory operation * @attrs: memory transaction attributes to use for the access */ MemTxResult memory_region_dispatch_read(MemoryRegion *mr, hwaddr addr, uint64_t *pval, - unsigned size, + MemOp op, MemTxAttrs attrs); /** * memory_region_dispatch_write: perform a write directly to the specified @@ -1764,13 +1765,13 @@ MemTxResult memory_region_dispatch_read(MemoryRegion *mr, * @mr: #MemoryRegion to access * @addr: address within that region * @data: data to write - * @size: size of the access in bytes + * @op: size, sign, and endianness of the memory operation * @attrs: memory transaction attributes to use for the access */ MemTxResult memory_region_dispatch_write(MemoryRegion *mr, hwaddr addr, uint64_t data, - unsigned size, + MemOp op, MemTxAttrs attrs); /** diff --git a/memory.c b/memory.c index 7fd93b1d42..3d87908784 100644 --- a/memory.c +++ b/memory.c @@ -1446,9 +1446,10 @@ static MemTxResult memory_region_dispatch_read1(MemoryRegion *mr, MemTxResult memory_region_dispatch_read(MemoryRegion *mr, hwaddr addr, uint64_t *pval, - unsigned size, + MemOp op, MemTxAttrs attrs) { + unsigned size = memop_size(op); MemTxResult r; if (!memory_region_access_valid(mr, addr, size, false, attrs)) { @@ -1490,9 +1491,11 @@ static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr, MemTxResult memory_region_dispatch_write(MemoryRegion *mr, hwaddr addr, uint64_t data, - unsigned size, + MemOp op, MemTxAttrs attrs) { + unsigned size = memop_size(op); + if (!memory_region_access_valid(mr, addr, size, true, attrs)) { unassigned_mem_write(mr, addr, data, size); return MEMTX_DECODE_ERROR; -- cgit v1.2.3-55-g7522 From d5d680cacc66ef7e3c02c81dc8f3a34eabce6dfe Mon Sep 17 00:00:00 2001 From: Tony Nguyen Date: Sat, 24 Aug 2019 04:36:52 +1000 Subject: memory: Access MemoryRegion with endianness Preparation for collapsing the two byte swaps adjust_endianness and handle_bswap into the former. Call memory_region_dispatch_{read|write} with endianness encoded into the "MemOp op" operand. This patch does not change any behaviour as memory_region_dispatch_{read|write} is yet to handle the endianness. Once it does handle endianness, callers with byte swaps can collapse them into adjust_endianness. Reviewed-by: Richard Henderson Signed-off-by: Tony Nguyen Message-Id: <8066ab3eb037c0388dfadfe53c5118429dd1de3a.1566466906.git.tony.nguyen@bt.com> Signed-off-by: Richard Henderson --- accel/tcg/cputlb.c | 8 ++++++-- exec.c | 13 +++++++++++-- hw/intc/armv7m_nvic.c | 15 ++++++++------- hw/s390x/s390-pci-inst.c | 6 ++++-- hw/vfio/pci-quirks.c | 5 +++-- hw/virtio/virtio-pci.c | 6 ++++-- include/exec/memory.h | 3 +++ memory.c | 18 ++++++++++++++++++ memory_ldst.inc.c | 24 ++++++++++++++++++------ 9 files changed, 75 insertions(+), 23 deletions(-) (limited to 'include/exec') diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 6c83878f73..f64c6b1c75 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -906,7 +906,8 @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, qemu_mutex_lock_iothread(); locked = true; } - r = memory_region_dispatch_read(mr, mr_offset, &val, size_memop(size), + r = memory_region_dispatch_read(mr, mr_offset, &val, + size_memop(size) | MO_TE, iotlbentry->attrs); if (r != MEMTX_OK) { hwaddr physaddr = mr_offset + @@ -947,7 +948,8 @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, qemu_mutex_lock_iothread(); locked = true; } - r = memory_region_dispatch_write(mr, mr_offset, val, size_memop(size), + r = memory_region_dispatch_write(mr, mr_offset, val, + size_memop(size) | MO_TE, iotlbentry->attrs); if (r != MEMTX_OK) { hwaddr physaddr = mr_offset + @@ -1305,6 +1307,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, } } + /* TODO: Merge bswap into io_readx -> memory_region_dispatch_read. */ res = io_readx(env, &env_tlb(env)->d[mmu_idx].iotlb[index], mmu_idx, addr, retaddr, access_type, size); return handle_bswap(res, size, big_endian); @@ -1553,6 +1556,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, } } + /* TODO: Merge bswap into io_writex -> memory_region_dispatch_write. */ io_writex(env, &env_tlb(env)->d[mmu_idx].iotlb[index], mmu_idx, handle_bswap(val, size, big_endian), addr, retaddr, size); diff --git a/exec.c b/exec.c index cc9697fe1b..e4652c0e75 100644 --- a/exec.c +++ b/exec.c @@ -3364,8 +3364,13 @@ static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr, /* XXX: could force current_cpu to NULL to avoid potential bugs */ val = ldn_p(buf, l); + /* + * TODO: Merge bswap from ldn_p into memory_region_dispatch_write + * by using ldn_he_p and dropping MO_TE to get a host-endian value. + */ result |= memory_region_dispatch_write(mr, addr1, val, - size_memop(l), attrs); + size_memop(l) | MO_TE, + attrs); } else { /* RAM case */ ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false); @@ -3426,8 +3431,12 @@ MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr, /* I/O case */ release_lock |= prepare_mmio_access(mr); l = memory_access_size(mr, l, addr1); + /* + * TODO: Merge bswap from stn_p into memory_region_dispatch_read + * by using stn_he_p and dropping MO_TE to get a host-endian value. + */ result |= memory_region_dispatch_read(mr, addr1, &val, - size_memop(l), attrs); + size_memop(l) | MO_TE, attrs); stn_p(buf, l, val); } else { /* RAM case */ diff --git a/hw/intc/armv7m_nvic.c b/hw/intc/armv7m_nvic.c index 7220940133..8e93e51e81 100644 --- a/hw/intc/armv7m_nvic.c +++ b/hw/intc/armv7m_nvic.c @@ -2349,8 +2349,8 @@ static MemTxResult nvic_sysreg_ns_write(void *opaque, hwaddr addr, if (attrs.secure) { /* S accesses to the alias act like NS accesses to the real region */ attrs.secure = 0; - return memory_region_dispatch_write(mr, addr, value, size_memop(size), - attrs); + return memory_region_dispatch_write(mr, addr, value, + size_memop(size) | MO_TE, attrs); } else { /* NS attrs are RAZ/WI for privileged, and BusFault for user */ if (attrs.user) { @@ -2369,8 +2369,8 @@ static MemTxResult nvic_sysreg_ns_read(void *opaque, hwaddr addr, if (attrs.secure) { /* S accesses to the alias act like NS accesses to the real region */ attrs.secure = 0; - return memory_region_dispatch_read(mr, addr, data, size_memop(size), - attrs); + return memory_region_dispatch_read(mr, addr, data, + size_memop(size) | MO_TE, attrs); } else { /* NS attrs are RAZ/WI for privileged, and BusFault for user */ if (attrs.user) { @@ -2396,8 +2396,8 @@ static MemTxResult nvic_systick_write(void *opaque, hwaddr addr, /* Direct the access to the correct systick */ mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->systick[attrs.secure]), 0); - return memory_region_dispatch_write(mr, addr, value, size_memop(size), - attrs); + return memory_region_dispatch_write(mr, addr, value, + size_memop(size) | MO_TE, attrs); } static MemTxResult nvic_systick_read(void *opaque, hwaddr addr, @@ -2409,7 +2409,8 @@ static MemTxResult nvic_systick_read(void *opaque, hwaddr addr, /* Direct the access to the correct systick */ mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->systick[attrs.secure]), 0); - return memory_region_dispatch_read(mr, addr, data, size_memop(size), attrs); + return memory_region_dispatch_read(mr, addr, data, size_memop(size) | MO_TE, + attrs); } static const MemoryRegionOps nvic_systick_ops = { diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c index 0e92a372ca..4b3bd4a804 100644 --- a/hw/s390x/s390-pci-inst.c +++ b/hw/s390x/s390-pci-inst.c @@ -373,7 +373,8 @@ static MemTxResult zpci_read_bar(S390PCIBusDevice *pbdev, uint8_t pcias, mr = pbdev->pdev->io_regions[pcias].memory; mr = s390_get_subregion(mr, offset, len); offset -= mr->addr; - return memory_region_dispatch_read(mr, offset, data, size_memop(len), + return memory_region_dispatch_read(mr, offset, data, + size_memop(len) | MO_BE, MEMTXATTRS_UNSPECIFIED); } @@ -472,7 +473,8 @@ static MemTxResult zpci_write_bar(S390PCIBusDevice *pbdev, uint8_t pcias, mr = pbdev->pdev->io_regions[pcias].memory; mr = s390_get_subregion(mr, offset, len); offset -= mr->addr; - return memory_region_dispatch_write(mr, offset, data, size_memop(len), + return memory_region_dispatch_write(mr, offset, data, + size_memop(len) | MO_BE, MEMTXATTRS_UNSPECIFIED); } diff --git a/hw/vfio/pci-quirks.c b/hw/vfio/pci-quirks.c index a4e1d2abb5..136f3a9ad6 100644 --- a/hw/vfio/pci-quirks.c +++ b/hw/vfio/pci-quirks.c @@ -1074,7 +1074,8 @@ static void vfio_rtl8168_quirk_address_write(void *opaque, hwaddr addr, /* Write to the proper guest MSI-X table instead */ memory_region_dispatch_write(&vdev->pdev.msix_table_mmio, - offset, val, size_memop(size), + offset, val, + size_memop(size) | MO_LE, MEMTXATTRS_UNSPECIFIED); } return; /* Do not write guest MSI-X data to hardware */ @@ -1105,7 +1106,7 @@ static uint64_t vfio_rtl8168_quirk_data_read(void *opaque, if (rtl->enabled && (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) { hwaddr offset = rtl->addr & 0xfff; memory_region_dispatch_read(&vdev->pdev.msix_table_mmio, offset, - &data, size_memop(size), + &data, size_memop(size) | MO_LE, MEMTXATTRS_UNSPECIFIED); trace_vfio_quirk_rtl8168_msix_read(vdev->vbasedev.name, offset, data); } diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 82c5e87a44..d89a85bb33 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -553,7 +553,8 @@ void virtio_address_space_write(VirtIOPCIProxy *proxy, hwaddr addr, /* As length is under guest control, handle illegal values. */ return; } - memory_region_dispatch_write(mr, addr, val, size_memop(len), + /* TODO: Merge bswap from cpu_to_leXX into memory_region_dispatch_write. */ + memory_region_dispatch_write(mr, addr, val, size_memop(len) | MO_LE, MEMTXATTRS_UNSPECIFIED); } @@ -577,7 +578,8 @@ virtio_address_space_read(VirtIOPCIProxy *proxy, hwaddr addr, /* Make sure caller aligned buf properly */ assert(!(((uintptr_t)buf) & (len - 1))); - memory_region_dispatch_read(mr, addr, &val, size_memop(len), + /* TODO: Merge bswap from leXX_to_cpu into memory_region_dispatch_read. */ + memory_region_dispatch_read(mr, addr, &val, size_memop(len) | MO_LE, MEMTXATTRS_UNSPECIFIED); switch (len) { case 1: diff --git a/include/exec/memory.h b/include/exec/memory.h index 192875b080..c4c86a6ff4 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -2211,6 +2211,9 @@ address_space_write_cached(MemoryRegionCache *cache, hwaddr addr, } } +/* enum device_endian to MemOp. */ +MemOp devend_memop(enum device_endian end); + #endif #endif diff --git a/memory.c b/memory.c index 3d87908784..d72143a18f 100644 --- a/memory.c +++ b/memory.c @@ -3285,3 +3285,21 @@ static void memory_register_types(void) } type_init(memory_register_types) + +MemOp devend_memop(enum device_endian end) +{ + static MemOp conv[] = { + [DEVICE_LITTLE_ENDIAN] = MO_LE, + [DEVICE_BIG_ENDIAN] = MO_BE, + [DEVICE_NATIVE_ENDIAN] = MO_TE, + [DEVICE_HOST_ENDIAN] = 0, + }; + switch (end) { + case DEVICE_LITTLE_ENDIAN: + case DEVICE_BIG_ENDIAN: + case DEVICE_NATIVE_ENDIAN: + return conv[end]; + default: + g_assert_not_reached(); + } +} diff --git a/memory_ldst.inc.c b/memory_ldst.inc.c index de658c40c4..809a7e8389 100644 --- a/memory_ldst.inc.c +++ b/memory_ldst.inc.c @@ -38,7 +38,9 @@ static inline uint32_t glue(address_space_ldl_internal, SUFFIX)(ARG1_DECL, release_lock |= prepare_mmio_access(mr); /* I/O case */ - r = memory_region_dispatch_read(mr, addr1, &val, MO_32, attrs); + /* TODO: Merge bswap32 into memory_region_dispatch_read. */ + r = memory_region_dispatch_read(mr, addr1, &val, + MO_32 | devend_memop(endian), attrs); #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap32(val); @@ -114,7 +116,9 @@ static inline uint64_t glue(address_space_ldq_internal, SUFFIX)(ARG1_DECL, release_lock |= prepare_mmio_access(mr); /* I/O case */ - r = memory_region_dispatch_read(mr, addr1, &val, MO_64, attrs); + /* TODO: Merge bswap64 into memory_region_dispatch_read. */ + r = memory_region_dispatch_read(mr, addr1, &val, + MO_64 | devend_memop(endian), attrs); #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap64(val); @@ -224,7 +228,9 @@ static inline uint32_t glue(address_space_lduw_internal, SUFFIX)(ARG1_DECL, release_lock |= prepare_mmio_access(mr); /* I/O case */ - r = memory_region_dispatch_read(mr, addr1, &val, MO_16, attrs); + /* TODO: Merge bswap16 into memory_region_dispatch_read. */ + r = memory_region_dispatch_read(mr, addr1, &val, + MO_16 | devend_memop(endian), attrs); #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap16(val); @@ -346,7 +352,9 @@ static inline void glue(address_space_stl_internal, SUFFIX)(ARG1_DECL, val = bswap32(val); } #endif - r = memory_region_dispatch_write(mr, addr1, val, MO_32, attrs); + /* TODO: Merge bswap32 into memory_region_dispatch_write. */ + r = memory_region_dispatch_write(mr, addr1, val, + MO_32 | devend_memop(endian), attrs); } else { /* RAM case */ ptr = qemu_map_ram_ptr(mr->ram_block, addr1); @@ -451,7 +459,9 @@ static inline void glue(address_space_stw_internal, SUFFIX)(ARG1_DECL, val = bswap16(val); } #endif - r = memory_region_dispatch_write(mr, addr1, val, MO_16, attrs); + /* TODO: Merge bswap16 into memory_region_dispatch_write. */ + r = memory_region_dispatch_write(mr, addr1, val, + MO_16 | devend_memop(endian), attrs); } else { /* RAM case */ ptr = qemu_map_ram_ptr(mr->ram_block, addr1); @@ -524,7 +534,9 @@ static void glue(address_space_stq_internal, SUFFIX)(ARG1_DECL, val = bswap64(val); } #endif - r = memory_region_dispatch_write(mr, addr1, val, MO_64, attrs); + /* TODO: Merge bswap64 into memory_region_dispatch_write. */ + r = memory_region_dispatch_write(mr, addr1, val, + MO_64 | devend_memop(endian), attrs); } else { /* RAM case */ ptr = qemu_map_ram_ptr(mr->ram_block, addr1); -- cgit v1.2.3-55-g7522 From be5c4787e9a6eed12fd765d9e890f7cc6cd63220 Mon Sep 17 00:00:00 2001 From: Tony Nguyen Date: Sat, 24 Aug 2019 04:36:53 +1000 Subject: cputlb: Replace size and endian operands for MemOp Preparation for collapsing the two byte swaps adjust_endianness and handle_bswap into the former. Signed-off-by: Tony Nguyen Reviewed-by: Richard Henderson Message-Id: <755b7104410956b743e1f1e9c34ab87db113360f.1566466906.git.tony.nguyen@bt.com> Signed-off-by: Richard Henderson --- accel/tcg/cputlb.c | 170 ++++++++++++++++++++++++--------------------------- include/exec/memop.h | 6 ++ 2 files changed, 87 insertions(+), 89 deletions(-) (limited to 'include/exec') diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index f64c6b1c75..5c12eef292 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -881,7 +881,7 @@ static void tlb_fill(CPUState *cpu, target_ulong addr, int size, static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, int mmu_idx, target_ulong addr, uintptr_t retaddr, - MMUAccessType access_type, int size) + MMUAccessType access_type, MemOp op) { CPUState *cpu = env_cpu(env); hwaddr mr_offset; @@ -906,15 +906,13 @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, qemu_mutex_lock_iothread(); locked = true; } - r = memory_region_dispatch_read(mr, mr_offset, &val, - size_memop(size) | MO_TE, - iotlbentry->attrs); + r = memory_region_dispatch_read(mr, mr_offset, &val, op, iotlbentry->attrs); if (r != MEMTX_OK) { hwaddr physaddr = mr_offset + section->offset_within_address_space - section->offset_within_region; - cpu_transaction_failed(cpu, physaddr, addr, size, access_type, + cpu_transaction_failed(cpu, physaddr, addr, memop_size(op), access_type, mmu_idx, iotlbentry->attrs, r, retaddr); } if (locked) { @@ -926,7 +924,7 @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, int mmu_idx, uint64_t val, target_ulong addr, - uintptr_t retaddr, int size) + uintptr_t retaddr, MemOp op) { CPUState *cpu = env_cpu(env); hwaddr mr_offset; @@ -948,16 +946,15 @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, qemu_mutex_lock_iothread(); locked = true; } - r = memory_region_dispatch_write(mr, mr_offset, val, - size_memop(size) | MO_TE, - iotlbentry->attrs); + r = memory_region_dispatch_write(mr, mr_offset, val, op, iotlbentry->attrs); if (r != MEMTX_OK) { hwaddr physaddr = mr_offset + section->offset_within_address_space - section->offset_within_region; - cpu_transaction_failed(cpu, physaddr, addr, size, MMU_DATA_STORE, - mmu_idx, iotlbentry->attrs, r, retaddr); + cpu_transaction_failed(cpu, physaddr, addr, memop_size(op), + MMU_DATA_STORE, mmu_idx, iotlbentry->attrs, r, + retaddr); } if (locked) { qemu_mutex_unlock_iothread(); @@ -1218,14 +1215,15 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, * access type. */ -static inline uint64_t handle_bswap(uint64_t val, int size, bool big_endian) +static inline uint64_t handle_bswap(uint64_t val, MemOp op) { - if ((big_endian && NEED_BE_BSWAP) || (!big_endian && NEED_LE_BSWAP)) { - switch (size) { - case 1: return val; - case 2: return bswap16(val); - case 4: return bswap32(val); - case 8: return bswap64(val); + if ((memop_big_endian(op) && NEED_BE_BSWAP) || + (!memop_big_endian(op) && NEED_LE_BSWAP)) { + switch (op & MO_SIZE) { + case MO_8: return val; + case MO_16: return bswap16(val); + case MO_32: return bswap32(val); + case MO_64: return bswap64(val); default: g_assert_not_reached(); } @@ -1248,7 +1246,7 @@ typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr, static inline uint64_t __attribute__((always_inline)) load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, - uintptr_t retaddr, size_t size, bool big_endian, bool code_read, + uintptr_t retaddr, MemOp op, bool code_read, FullLoadHelper *full_load) { uintptr_t mmu_idx = get_mmuidx(oi); @@ -1262,6 +1260,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, unsigned a_bits = get_alignment_bits(get_memop(oi)); void *haddr; uint64_t res; + size_t size = memop_size(op); /* Handle CPU specific unaligned behaviour */ if (addr & ((1 << a_bits) - 1)) { @@ -1309,8 +1308,8 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, /* TODO: Merge bswap into io_readx -> memory_region_dispatch_read. */ res = io_readx(env, &env_tlb(env)->d[mmu_idx].iotlb[index], - mmu_idx, addr, retaddr, access_type, size); - return handle_bswap(res, size, big_endian); + mmu_idx, addr, retaddr, access_type, op); + return handle_bswap(res, op); } /* Handle slow unaligned access (it spans two pages or IO). */ @@ -1327,7 +1326,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, r2 = full_load(env, addr2, oi, retaddr); shift = (addr & (size - 1)) * 8; - if (big_endian) { + if (memop_big_endian(op)) { /* Big-endian combine. */ res = (r1 << shift) | (r2 >> ((size * 8) - shift)); } else { @@ -1339,30 +1338,27 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, do_aligned_access: haddr = (void *)((uintptr_t)addr + entry->addend); - switch (size) { - case 1: + switch (op) { + case MO_UB: res = ldub_p(haddr); break; - case 2: - if (big_endian) { - res = lduw_be_p(haddr); - } else { - res = lduw_le_p(haddr); - } + case MO_BEUW: + res = lduw_be_p(haddr); break; - case 4: - if (big_endian) { - res = (uint32_t)ldl_be_p(haddr); - } else { - res = (uint32_t)ldl_le_p(haddr); - } + case MO_LEUW: + res = lduw_le_p(haddr); break; - case 8: - if (big_endian) { - res = ldq_be_p(haddr); - } else { - res = ldq_le_p(haddr); - } + case MO_BEUL: + res = (uint32_t)ldl_be_p(haddr); + break; + case MO_LEUL: + res = (uint32_t)ldl_le_p(haddr); + break; + case MO_BEQ: + res = ldq_be_p(haddr); + break; + case MO_LEQ: + res = ldq_le_p(haddr); break; default: g_assert_not_reached(); @@ -1384,8 +1380,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, static uint64_t full_ldub_mmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 1, false, false, - full_ldub_mmu); + return load_helper(env, addr, oi, retaddr, MO_UB, false, full_ldub_mmu); } tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr, @@ -1397,7 +1392,7 @@ tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr, static uint64_t full_le_lduw_mmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 2, false, false, + return load_helper(env, addr, oi, retaddr, MO_LEUW, false, full_le_lduw_mmu); } @@ -1410,7 +1405,7 @@ tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr, static uint64_t full_be_lduw_mmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 2, true, false, + return load_helper(env, addr, oi, retaddr, MO_BEUW, false, full_be_lduw_mmu); } @@ -1423,7 +1418,7 @@ tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr, static uint64_t full_le_ldul_mmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 4, false, false, + return load_helper(env, addr, oi, retaddr, MO_LEUL, false, full_le_ldul_mmu); } @@ -1436,7 +1431,7 @@ tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr, static uint64_t full_be_ldul_mmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 4, true, false, + return load_helper(env, addr, oi, retaddr, MO_BEUL, false, full_be_ldul_mmu); } @@ -1449,14 +1444,14 @@ tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr, uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 8, false, false, + return load_helper(env, addr, oi, retaddr, MO_LEQ, false, helper_le_ldq_mmu); } uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 8, true, false, + return load_helper(env, addr, oi, retaddr, MO_BEQ, false, helper_be_ldq_mmu); } @@ -1502,7 +1497,7 @@ tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr, static inline void __attribute__((always_inline)) store_helper(CPUArchState *env, target_ulong addr, uint64_t val, - TCGMemOpIdx oi, uintptr_t retaddr, size_t size, bool big_endian) + TCGMemOpIdx oi, uintptr_t retaddr, MemOp op) { uintptr_t mmu_idx = get_mmuidx(oi); uintptr_t index = tlb_index(env, mmu_idx, addr); @@ -1511,6 +1506,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, const size_t tlb_off = offsetof(CPUTLBEntry, addr_write); unsigned a_bits = get_alignment_bits(get_memop(oi)); void *haddr; + size_t size = memop_size(op); /* Handle CPU specific unaligned behaviour */ if (addr & ((1 << a_bits) - 1)) { @@ -1558,8 +1554,8 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, /* TODO: Merge bswap into io_writex -> memory_region_dispatch_write. */ io_writex(env, &env_tlb(env)->d[mmu_idx].iotlb[index], mmu_idx, - handle_bswap(val, size, big_endian), - addr, retaddr, size); + handle_bswap(val, op), + addr, retaddr, op); return; } @@ -1595,7 +1591,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, */ for (i = 0; i < size; ++i) { uint8_t val8; - if (big_endian) { + if (memop_big_endian(op)) { /* Big-endian extract. */ val8 = val >> (((size - 1) * 8) - (i * 8)); } else { @@ -1609,30 +1605,27 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, do_aligned_access: haddr = (void *)((uintptr_t)addr + entry->addend); - switch (size) { - case 1: + switch (op) { + case MO_UB: stb_p(haddr, val); break; - case 2: - if (big_endian) { - stw_be_p(haddr, val); - } else { - stw_le_p(haddr, val); - } + case MO_BEUW: + stw_be_p(haddr, val); break; - case 4: - if (big_endian) { - stl_be_p(haddr, val); - } else { - stl_le_p(haddr, val); - } + case MO_LEUW: + stw_le_p(haddr, val); break; - case 8: - if (big_endian) { - stq_be_p(haddr, val); - } else { - stq_le_p(haddr, val); - } + case MO_BEUL: + stl_be_p(haddr, val); + break; + case MO_LEUL: + stl_le_p(haddr, val); + break; + case MO_BEQ: + stq_be_p(haddr, val); + break; + case MO_LEQ: + stq_le_p(haddr, val); break; default: g_assert_not_reached(); @@ -1643,43 +1636,43 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val, TCGMemOpIdx oi, uintptr_t retaddr) { - store_helper(env, addr, val, oi, retaddr, 1, false); + store_helper(env, addr, val, oi, retaddr, MO_UB); } void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, TCGMemOpIdx oi, uintptr_t retaddr) { - store_helper(env, addr, val, oi, retaddr, 2, false); + store_helper(env, addr, val, oi, retaddr, MO_LEUW); } void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val, TCGMemOpIdx oi, uintptr_t retaddr) { - store_helper(env, addr, val, oi, retaddr, 2, true); + store_helper(env, addr, val, oi, retaddr, MO_BEUW); } void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, TCGMemOpIdx oi, uintptr_t retaddr) { - store_helper(env, addr, val, oi, retaddr, 4, false); + store_helper(env, addr, val, oi, retaddr, MO_LEUL); } void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val, TCGMemOpIdx oi, uintptr_t retaddr) { - store_helper(env, addr, val, oi, retaddr, 4, true); + store_helper(env, addr, val, oi, retaddr, MO_BEUL); } void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, TCGMemOpIdx oi, uintptr_t retaddr) { - store_helper(env, addr, val, oi, retaddr, 8, false); + store_helper(env, addr, val, oi, retaddr, MO_LEQ); } void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, TCGMemOpIdx oi, uintptr_t retaddr) { - store_helper(env, addr, val, oi, retaddr, 8, true); + store_helper(env, addr, val, oi, retaddr, MO_BEQ); } /* First set of helpers allows passing in of OI and RETADDR. This makes @@ -1744,8 +1737,7 @@ void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, static uint64_t full_ldub_cmmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 1, false, true, - full_ldub_cmmu); + return load_helper(env, addr, oi, retaddr, MO_8, true, full_ldub_cmmu); } uint8_t helper_ret_ldb_cmmu(CPUArchState *env, target_ulong addr, @@ -1757,7 +1749,7 @@ uint8_t helper_ret_ldb_cmmu(CPUArchState *env, target_ulong addr, static uint64_t full_le_lduw_cmmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 2, false, true, + return load_helper(env, addr, oi, retaddr, MO_LEUW, true, full_le_lduw_cmmu); } @@ -1770,7 +1762,7 @@ uint16_t helper_le_ldw_cmmu(CPUArchState *env, target_ulong addr, static uint64_t full_be_lduw_cmmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 2, true, true, + return load_helper(env, addr, oi, retaddr, MO_BEUW, true, full_be_lduw_cmmu); } @@ -1783,7 +1775,7 @@ uint16_t helper_be_ldw_cmmu(CPUArchState *env, target_ulong addr, static uint64_t full_le_ldul_cmmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 4, false, true, + return load_helper(env, addr, oi, retaddr, MO_LEUL, true, full_le_ldul_cmmu); } @@ -1796,7 +1788,7 @@ uint32_t helper_le_ldl_cmmu(CPUArchState *env, target_ulong addr, static uint64_t full_be_ldul_cmmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 4, true, true, + return load_helper(env, addr, oi, retaddr, MO_BEUL, true, full_be_ldul_cmmu); } @@ -1809,13 +1801,13 @@ uint32_t helper_be_ldl_cmmu(CPUArchState *env, target_ulong addr, uint64_t helper_le_ldq_cmmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 8, false, true, + return load_helper(env, addr, oi, retaddr, MO_LEQ, true, helper_le_ldq_cmmu); } uint64_t helper_be_ldq_cmmu(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, uintptr_t retaddr) { - return load_helper(env, addr, oi, retaddr, 8, true, true, + return load_helper(env, addr, oi, retaddr, MO_BEQ, true, helper_be_ldq_cmmu); } diff --git a/include/exec/memop.h b/include/exec/memop.h index 0a610b75d9..529d07b02d 100644 --- a/include/exec/memop.h +++ b/include/exec/memop.h @@ -125,4 +125,10 @@ static inline MemOp size_memop(unsigned size) return ctz32(size); } +/* Big endianness from MemOp. */ +static inline bool memop_big_endian(MemOp op) +{ + return (op & MO_BSWAP) == MO_BE; +} + #endif -- cgit v1.2.3-55-g7522 From a26fc6f5152b47f1d7ed928f9c9d462d01ff1624 Mon Sep 17 00:00:00 2001 From: Tony Nguyen Date: Sat, 24 Aug 2019 04:36:56 +1000 Subject: cputlb: Byte swap memory transaction attribute Notice new attribute, byte swap, and force the transaction through the memory slow path. Required by architectures that can invert endianness of memory transaction, e.g. SPARC64 has the Invert Endian TTE bit. Suggested-by: Richard Henderson Signed-off-by: Tony Nguyen Reviewed-by: Richard Henderson Message-Id: <2a10a1f1c00a894af1212c8f68ef09c2966023c1.1566466906.git.tony.nguyen@bt.com> Signed-off-by: Richard Henderson --- accel/tcg/cputlb.c | 12 ++++++++++++ include/exec/memattrs.h | 2 ++ 2 files changed, 14 insertions(+) (limited to 'include/exec') diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 3c9e634d99..d9787cc893 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -738,6 +738,10 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, */ address |= TLB_RECHECK; } + if (attrs.byte_swap) { + /* Force the access through the I/O slow path. */ + address |= TLB_MMIO; + } if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) { /* IO memory case */ @@ -891,6 +895,10 @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry, bool locked = false; MemTxResult r; + if (iotlbentry->attrs.byte_swap) { + op ^= MO_BSWAP; + } + section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs); mr = section->mr; mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr; @@ -933,6 +941,10 @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, bool locked = false; MemTxResult r; + if (iotlbentry->attrs.byte_swap) { + op ^= MO_BSWAP; + } + section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs); mr = section->mr; mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr; diff --git a/include/exec/memattrs.h b/include/exec/memattrs.h index d4a3477d71..95f2d20d55 100644 --- a/include/exec/memattrs.h +++ b/include/exec/memattrs.h @@ -37,6 +37,8 @@ typedef struct MemTxAttrs { unsigned int user:1; /* Requester ID (for MSI for example) */ unsigned int requester_id:16; + /* Invert endianness for this page */ + unsigned int byte_swap:1; /* * The following are target-specific page-table bits. These are not * related to actual memory transactions at all. However, this structure -- cgit v1.2.3-55-g7522 From 30d7e098d5c38644359820317fcf72e3e129ec53 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 23 Aug 2019 15:12:32 -0700 Subject: cputlb: Fold TLB_RECHECK into TLB_INVALID_MASK We had two different mechanisms to force a recheck of the tlb. Before TLB_RECHECK was introduced, we had a PAGE_WRITE_INV bit that would immediate set TLB_INVALID_MASK, which automatically means that a second check of the tlb entry fails. We can use the same mechanism to handle small pages. Conserve TLB_* bits by removing TLB_RECHECK. Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- accel/tcg/cputlb.c | 86 ++++++++++++++------------------------------------ include/exec/cpu-all.h | 5 +-- 2 files changed, 24 insertions(+), 67 deletions(-) (limited to 'include/exec') diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index d9787cc893..c9576bebcf 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -732,11 +732,8 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, address = vaddr_page; if (size < TARGET_PAGE_SIZE) { - /* - * Slow-path the TLB entries; we will repeat the MMU check and TLB - * fill on every access. - */ - address |= TLB_RECHECK; + /* Repeat the MMU check and TLB fill on every access. */ + address |= TLB_INVALID_MASK; } if (attrs.byte_swap) { /* Force the access through the I/O slow path. */ @@ -1026,10 +1023,15 @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \ (ADDR) & TARGET_PAGE_MASK) -/* NOTE: this function can trigger an exception */ -/* NOTE2: the returned address is not exactly the physical address: it - * is actually a ram_addr_t (in system mode; the user mode emulation - * version of this function returns a guest virtual address). +/* + * Return a ram_addr_t for the virtual address for execution. + * + * Return -1 if we can't translate and execute from an entire page + * of RAM. This will force us to execute by loading and translating + * one insn at a time, without caching. + * + * NOTE: This function will trigger an exception if the page is + * not executable. */ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) { @@ -1043,19 +1045,20 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) tlb_fill(env_cpu(env), addr, 0, MMU_INST_FETCH, mmu_idx, 0); index = tlb_index(env, mmu_idx, addr); entry = tlb_entry(env, mmu_idx, addr); + + if (unlikely(entry->addr_code & TLB_INVALID_MASK)) { + /* + * The MMU protection covers a smaller range than a target + * page, so we must redo the MMU check for every insn. + */ + return -1; + } } assert(tlb_hit(entry->addr_code, addr)); } - if (unlikely(entry->addr_code & (TLB_RECHECK | TLB_MMIO))) { - /* - * Return -1 if we can't translate and execute from an entire - * page of RAM here, which will cause us to execute by loading - * and translating one insn at a time, without caching: - * - TLB_RECHECK: means the MMU protection covers a smaller range - * than a target page, so we must redo the MMU check every insn - * - TLB_MMIO: region is not backed by RAM - */ + if (unlikely(entry->addr_code & TLB_MMIO)) { + /* The region is not backed by RAM. */ return -1; } @@ -1180,7 +1183,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, } /* Notice an IO access or a needs-MMU-lookup access */ - if (unlikely(tlb_addr & (TLB_MMIO | TLB_RECHECK))) { + if (unlikely(tlb_addr & TLB_MMIO)) { /* There's really nothing that can be done to support this apart from stop-the-world. */ goto stop_the_world; @@ -1258,6 +1261,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, entry = tlb_entry(env, mmu_idx, addr); } tlb_addr = code_read ? entry->addr_code : entry->addr_read; + tlb_addr &= ~TLB_INVALID_MASK; } /* Handle an IO access. */ @@ -1265,27 +1269,6 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, if ((addr & (size - 1)) != 0) { goto do_unaligned_access; } - - if (tlb_addr & TLB_RECHECK) { - /* - * This is a TLB_RECHECK access, where the MMU protection - * covers a smaller range than a target page, and we must - * repeat the MMU check here. This tlb_fill() call might - * longjump out if this access should cause a guest exception. - */ - tlb_fill(env_cpu(env), addr, size, - access_type, mmu_idx, retaddr); - index = tlb_index(env, mmu_idx, addr); - entry = tlb_entry(env, mmu_idx, addr); - - tlb_addr = code_read ? entry->addr_code : entry->addr_read; - tlb_addr &= ~TLB_RECHECK; - if (!(tlb_addr & ~TARGET_PAGE_MASK)) { - /* RAM access */ - goto do_aligned_access; - } - } - return io_readx(env, &env_tlb(env)->d[mmu_idx].iotlb[index], mmu_idx, addr, retaddr, access_type, op); } @@ -1314,7 +1297,6 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, return res & MAKE_64BIT_MASK(0, size * 8); } - do_aligned_access: haddr = (void *)((uintptr_t)addr + entry->addend); switch (op) { case MO_UB: @@ -1509,27 +1491,6 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, if ((addr & (size - 1)) != 0) { goto do_unaligned_access; } - - if (tlb_addr & TLB_RECHECK) { - /* - * This is a TLB_RECHECK access, where the MMU protection - * covers a smaller range than a target page, and we must - * repeat the MMU check here. This tlb_fill() call might - * longjump out if this access should cause a guest exception. - */ - tlb_fill(env_cpu(env), addr, size, MMU_DATA_STORE, - mmu_idx, retaddr); - index = tlb_index(env, mmu_idx, addr); - entry = tlb_entry(env, mmu_idx, addr); - - tlb_addr = tlb_addr_write(entry); - tlb_addr &= ~TLB_RECHECK; - if (!(tlb_addr & ~TARGET_PAGE_MASK)) { - /* RAM access */ - goto do_aligned_access; - } - } - io_writex(env, &env_tlb(env)->d[mmu_idx].iotlb[index], mmu_idx, val, addr, retaddr, op); return; @@ -1579,7 +1540,6 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, return; } - do_aligned_access: haddr = (void *)((uintptr_t)addr + entry->addend); switch (op) { case MO_UB: diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index 8323094648..8d07ae23a5 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -329,14 +329,11 @@ CPUArchState *cpu_copy(CPUArchState *env); #define TLB_NOTDIRTY (1 << (TARGET_PAGE_BITS - 2)) /* Set if TLB entry is an IO callback. */ #define TLB_MMIO (1 << (TARGET_PAGE_BITS - 3)) -/* Set if TLB entry must have MMU lookup repeated for every access */ -#define TLB_RECHECK (1 << (TARGET_PAGE_BITS - 4)) /* Use this mask to check interception with an alignment mask * in a TCG backend. */ -#define TLB_FLAGS_MASK (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO \ - | TLB_RECHECK) +#define TLB_FLAGS_MASK (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO) /** * tlb_hit_page: return true if page aligned @addr is a hit against the -- cgit v1.2.3-55-g7522 From 50b107c5d617eaf93301cef20221312e7a986701 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 24 Aug 2019 09:51:09 -0700 Subject: cputlb: Handle watchpoints via TLB_WATCHPOINT The raising of exceptions from check_watchpoint, buried inside of the I/O subsystem, is fundamentally broken. We do not have the helper return address with which we can unwind guest state. Replace PHYS_SECTION_WATCH and io_mem_watch with TLB_WATCHPOINT. Move the call to cpu_check_watchpoint into the cputlb helpers where we do have the helper return address. This allows watchpoints on RAM to bypass the full i/o access path. Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: David Hildenbrand Signed-off-by: Richard Henderson --- accel/tcg/cputlb.c | 89 +++++++++++++++++++++++++++++++++----- exec.c | 114 +++---------------------------------------------- include/exec/cpu-all.h | 5 ++- 3 files changed, 90 insertions(+), 118 deletions(-) (limited to 'include/exec') diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index d0f8db33a2..9a9a626938 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -710,6 +710,7 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, hwaddr iotlb, xlat, sz, paddr_page; target_ulong vaddr_page; int asidx = cpu_asidx_from_attrs(cpu, attrs); + int wp_flags; assert_cpu_is_self(cpu); @@ -752,6 +753,8 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, code_address = address; iotlb = memory_region_section_get_iotlb(cpu, section, vaddr_page, paddr_page, xlat, prot, &address); + wp_flags = cpu_watchpoint_address_matches(cpu, vaddr_page, + TARGET_PAGE_SIZE); index = tlb_index(env, mmu_idx, vaddr_page); te = tlb_entry(env, mmu_idx, vaddr_page); @@ -805,6 +808,9 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, tn.addend = addend - vaddr_page; if (prot & PAGE_READ) { tn.addr_read = address; + if (wp_flags & BP_MEM_READ) { + tn.addr_read |= TLB_WATCHPOINT; + } } else { tn.addr_read = -1; } @@ -831,6 +837,9 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, if (prot & PAGE_WRITE_INV) { tn.addr_write |= TLB_INVALID_MASK; } + if (wp_flags & BP_MEM_WRITE) { + tn.addr_write |= TLB_WATCHPOINT; + } } copy_tlb_helper_locked(te, &tn); @@ -1264,13 +1273,33 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, tlb_addr &= ~TLB_INVALID_MASK; } - /* Handle an IO access. */ + /* Handle anything that isn't just a straight memory access. */ if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) { + CPUIOTLBEntry *iotlbentry; + + /* For anything that is unaligned, recurse through full_load. */ if ((addr & (size - 1)) != 0) { goto do_unaligned_access; } - return io_readx(env, &env_tlb(env)->d[mmu_idx].iotlb[index], - mmu_idx, addr, retaddr, access_type, op); + + iotlbentry = &env_tlb(env)->d[mmu_idx].iotlb[index]; + + /* Handle watchpoints. */ + if (unlikely(tlb_addr & TLB_WATCHPOINT)) { + /* On watchpoint hit, this will longjmp out. */ + cpu_check_watchpoint(env_cpu(env), addr, size, + iotlbentry->attrs, BP_MEM_READ, retaddr); + + /* The backing page may or may not require I/O. */ + tlb_addr &= ~TLB_WATCHPOINT; + if ((tlb_addr & ~TARGET_PAGE_MASK) == 0) { + goto do_aligned_access; + } + } + + /* Handle I/O access. */ + return io_readx(env, iotlbentry, mmu_idx, addr, + retaddr, access_type, op); } /* Handle slow unaligned access (it spans two pages or IO). */ @@ -1297,6 +1326,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi, return res & MAKE_64BIT_MASK(0, size * 8); } + do_aligned_access: haddr = (void *)((uintptr_t)addr + entry->addend); switch (op) { case MO_UB: @@ -1486,13 +1516,32 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK; } - /* Handle an IO access. */ + /* Handle anything that isn't just a straight memory access. */ if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) { + CPUIOTLBEntry *iotlbentry; + + /* For anything that is unaligned, recurse through byte stores. */ if ((addr & (size - 1)) != 0) { goto do_unaligned_access; } - io_writex(env, &env_tlb(env)->d[mmu_idx].iotlb[index], mmu_idx, - val, addr, retaddr, op); + + iotlbentry = &env_tlb(env)->d[mmu_idx].iotlb[index]; + + /* Handle watchpoints. */ + if (unlikely(tlb_addr & TLB_WATCHPOINT)) { + /* On watchpoint hit, this will longjmp out. */ + cpu_check_watchpoint(env_cpu(env), addr, size, + iotlbentry->attrs, BP_MEM_WRITE, retaddr); + + /* The backing page may or may not require I/O. */ + tlb_addr &= ~TLB_WATCHPOINT; + if ((tlb_addr & ~TARGET_PAGE_MASK) == 0) { + goto do_aligned_access; + } + } + + /* Handle I/O access. */ + io_writex(env, iotlbentry, mmu_idx, val, addr, retaddr, op); return; } @@ -1517,10 +1566,29 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, index2 = tlb_index(env, mmu_idx, page2); entry2 = tlb_entry(env, mmu_idx, page2); tlb_addr2 = tlb_addr_write(entry2); - if (!tlb_hit_page(tlb_addr2, page2) - && !victim_tlb_hit(env, mmu_idx, index2, tlb_off, page2)) { - tlb_fill(env_cpu(env), page2, size2, MMU_DATA_STORE, - mmu_idx, retaddr); + if (!tlb_hit_page(tlb_addr2, page2)) { + if (!victim_tlb_hit(env, mmu_idx, index2, tlb_off, page2)) { + tlb_fill(env_cpu(env), page2, size2, MMU_DATA_STORE, + mmu_idx, retaddr); + index2 = tlb_index(env, mmu_idx, page2); + entry2 = tlb_entry(env, mmu_idx, page2); + } + tlb_addr2 = tlb_addr_write(entry2); + } + + /* + * Handle watchpoints. Since this may trap, all checks + * must happen before any store. + */ + if (unlikely(tlb_addr & TLB_WATCHPOINT)) { + cpu_check_watchpoint(env_cpu(env), addr, size - size2, + env_tlb(env)->d[mmu_idx].iotlb[index].attrs, + BP_MEM_WRITE, retaddr); + } + if (unlikely(tlb_addr2 & TLB_WATCHPOINT)) { + cpu_check_watchpoint(env_cpu(env), page2, size2, + env_tlb(env)->d[mmu_idx].iotlb[index2].attrs, + BP_MEM_WRITE, retaddr); } /* @@ -1542,6 +1610,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val, return; } + do_aligned_access: haddr = (void *)((uintptr_t)addr + entry->addend); switch (op) { case MO_UB: diff --git a/exec.c b/exec.c index 8575ce51ad..ad0f4a598f 100644 --- a/exec.c +++ b/exec.c @@ -193,15 +193,12 @@ typedef struct subpage_t { #define PHYS_SECTION_UNASSIGNED 0 #define PHYS_SECTION_NOTDIRTY 1 #define PHYS_SECTION_ROM 2 -#define PHYS_SECTION_WATCH 3 static void io_mem_init(void); static void memory_map_init(void); static void tcg_log_global_after_sync(MemoryListener *listener); static void tcg_commit(MemoryListener *listener); -static MemoryRegion io_mem_watch; - /** * CPUAddressSpace: all the information a CPU needs about an AddressSpace * @cpu: the CPU whose AddressSpace this is @@ -1472,7 +1469,6 @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu, target_ulong *address) { hwaddr iotlb; - int flags, match; if (memory_region_is_ram(section->mr)) { /* Normal RAM. */ @@ -1490,19 +1486,6 @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu, iotlb += xlat; } - /* Avoid trapping reads of pages with a write breakpoint. */ - match = (prot & PAGE_READ ? BP_MEM_READ : 0) - | (prot & PAGE_WRITE ? BP_MEM_WRITE : 0); - flags = cpu_watchpoint_address_matches(cpu, vaddr, TARGET_PAGE_SIZE); - if (flags & match) { - /* - * Make accesses to pages with watchpoints go via the - * watchpoint trap routines. - */ - iotlb = PHYS_SECTION_WATCH + paddr; - *address |= TLB_MMIO; - } - return iotlb; } #endif /* defined(CONFIG_USER_ONLY) */ @@ -2810,10 +2793,14 @@ void cpu_check_watchpoint(CPUState *cpu, vaddr addr, vaddr len, assert(tcg_enabled()); if (cpu->watchpoint_hit) { - /* We re-entered the check after replacing the TB. Now raise - * the debug interrupt so that is will trigger after the - * current instruction. */ + /* + * We re-entered the check after replacing the TB. + * Now raise the debug interrupt so that it will + * trigger after the current instruction. + */ + qemu_mutex_lock_iothread(); cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG); + qemu_mutex_unlock_iothread(); return; } @@ -2858,88 +2845,6 @@ void cpu_check_watchpoint(CPUState *cpu, vaddr addr, vaddr len, } } -static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags) -{ - CPUState *cpu = current_cpu; - vaddr addr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset; - - cpu_check_watchpoint(cpu, addr, len, attrs, flags, 0); -} - -/* Watchpoint access routines. Watchpoints are inserted using TLB tricks, - so these check for a hit then pass through to the normal out-of-line - phys routines. */ -static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata, - unsigned size, MemTxAttrs attrs) -{ - MemTxResult res; - uint64_t data; - int asidx = cpu_asidx_from_attrs(current_cpu, attrs); - AddressSpace *as = current_cpu->cpu_ases[asidx].as; - - check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ); - switch (size) { - case 1: - data = address_space_ldub(as, addr, attrs, &res); - break; - case 2: - data = address_space_lduw(as, addr, attrs, &res); - break; - case 4: - data = address_space_ldl(as, addr, attrs, &res); - break; - case 8: - data = address_space_ldq(as, addr, attrs, &res); - break; - default: abort(); - } - *pdata = data; - return res; -} - -static MemTxResult watch_mem_write(void *opaque, hwaddr addr, - uint64_t val, unsigned size, - MemTxAttrs attrs) -{ - MemTxResult res; - int asidx = cpu_asidx_from_attrs(current_cpu, attrs); - AddressSpace *as = current_cpu->cpu_ases[asidx].as; - - check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE); - switch (size) { - case 1: - address_space_stb(as, addr, val, attrs, &res); - break; - case 2: - address_space_stw(as, addr, val, attrs, &res); - break; - case 4: - address_space_stl(as, addr, val, attrs, &res); - break; - case 8: - address_space_stq(as, addr, val, attrs, &res); - break; - default: abort(); - } - return res; -} - -static const MemoryRegionOps watch_mem_ops = { - .read_with_attrs = watch_mem_read, - .write_with_attrs = watch_mem_write, - .endianness = DEVICE_NATIVE_ENDIAN, - .valid = { - .min_access_size = 1, - .max_access_size = 8, - .unaligned = false, - }, - .impl = { - .min_access_size = 1, - .max_access_size = 8, - .unaligned = false, - }, -}; - static MemTxResult flatview_read(FlatView *fv, hwaddr addr, MemTxAttrs attrs, uint8_t *buf, hwaddr len); static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs, @@ -3115,9 +3020,6 @@ static void io_mem_init(void) memory_region_init_io(&io_mem_notdirty, NULL, ¬dirty_mem_ops, NULL, NULL, UINT64_MAX); memory_region_clear_global_locking(&io_mem_notdirty); - - memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL, - NULL, UINT64_MAX); } AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv) @@ -3131,8 +3033,6 @@ AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv) assert(n == PHYS_SECTION_NOTDIRTY); n = dummy_section(&d->map, fv, &io_mem_rom); assert(n == PHYS_SECTION_ROM); - n = dummy_section(&d->map, fv, &io_mem_watch); - assert(n == PHYS_SECTION_WATCH); d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 }; diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index 8d07ae23a5..d2d443c4f9 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -329,11 +329,14 @@ CPUArchState *cpu_copy(CPUArchState *env); #define TLB_NOTDIRTY (1 << (TARGET_PAGE_BITS - 2)) /* Set if TLB entry is an IO callback. */ #define TLB_MMIO (1 << (TARGET_PAGE_BITS - 3)) +/* Set if TLB entry contains a watchpoint. */ +#define TLB_WATCHPOINT (1 << (TARGET_PAGE_BITS - 4)) /* Use this mask to check interception with an alignment mask * in a TCG backend. */ -#define TLB_FLAGS_MASK (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO) +#define TLB_FLAGS_MASK \ + (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO | TLB_WATCHPOINT) /** * tlb_hit_page: return true if page aligned @addr is a hit against the -- cgit v1.2.3-55-g7522 From 59e96ac6cb13951dd09afc70622858089abf3384 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 26 Aug 2019 09:51:08 +0200 Subject: tcg: Factor out CONFIG_USER_ONLY probe_write() from s390x code Factor it out into common code. Similar to the !CONFIG_USER_ONLY variant, let's not allow to cross page boundaries. Signed-off-by: David Hildenbrand Reviewed-by: Richard Henderson Message-Id: <20190826075112.25637-4-david@redhat.com> [rth: Move cpu & cc variables inside if block.] Signed-off-by: Richard Henderson --- accel/tcg/user-exec.c | 14 ++++++++++++++ include/exec/exec-all.h | 4 ++-- target/s390x/mem_helper.c | 7 ------- 3 files changed, 16 insertions(+), 9 deletions(-) (limited to 'include/exec') diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 897d1571c4..86e6827201 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -188,6 +188,20 @@ static inline int handle_cpu_signal(uintptr_t pc, siginfo_t *info, g_assert_not_reached(); } +void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, + uintptr_t retaddr) +{ + if (!guest_addr_valid(addr) || + page_check_range(addr, size, PAGE_WRITE) < 0) { + CPUState *cpu = env_cpu(env); + CPUClass *cc = CPU_GET_CLASS(cpu); + + cc->tlb_fill(cpu, addr, size, MMU_DATA_STORE, MMU_USER_IDX, false, + retaddr); + g_assert_not_reached(); + } +} + #if defined(__i386__) #if defined(__NetBSD__) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 135aeaab0d..cbcc85add3 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -260,8 +260,6 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, void tlb_set_page(CPUState *cpu, target_ulong vaddr, hwaddr paddr, int prot, int mmu_idx, target_ulong size); -void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, - uintptr_t retaddr); #else static inline void tlb_init(CPUState *cpu) { @@ -312,6 +310,8 @@ static inline void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu, { } #endif +void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, + uintptr_t retaddr); #define CODE_GEN_ALIGN 16 /* must be >= of the size of a icache line */ diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c index 4b43440e89..fdff60ce5d 100644 --- a/target/s390x/mem_helper.c +++ b/target/s390x/mem_helper.c @@ -2615,12 +2615,6 @@ uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) void probe_write_access(CPUS390XState *env, uint64_t addr, uint64_t len, uintptr_t ra) { -#ifdef CONFIG_USER_ONLY - if (!guest_addr_valid(addr) || !guest_addr_valid(addr + len - 1) || - page_check_range(addr, len, PAGE_WRITE) < 0) { - s390_program_interrupt(env, PGM_ADDRESSING, ILEN_AUTO, ra); - } -#else /* test the actual access, not just any access to the page due to LAP */ while (len) { const uint64_t pagelen = -(addr | TARGET_PAGE_MASK); @@ -2630,7 +2624,6 @@ void probe_write_access(CPUS390XState *env, uint64_t addr, uint64_t len, addr = wrap_address(env, addr + curlen); len -= curlen; } -#endif } void HELPER(probe_write_access)(CPUS390XState *env, uint64_t addr, uint64_t len) -- cgit v1.2.3-55-g7522 From fef39ccd567032d3ad520ed80f3576068e6eb2e3 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Fri, 30 Aug 2019 12:09:58 +0200 Subject: tcg: Make probe_write() return a pointer to the host page ... similar to tlb_vaddr_to_host(); however, allow access to the host page except when TLB_NOTDIRTY or TLB_MMIO is set. Reviewed-by: Richard Henderson Signed-off-by: David Hildenbrand Message-Id: <20190830100959.26615-2-david@redhat.com> Signed-off-by: Richard Henderson --- accel/tcg/cputlb.c | 21 ++++++++++++++++----- accel/tcg/user-exec.c | 6 ++++-- include/exec/exec-all.h | 4 ++-- 3 files changed, 22 insertions(+), 9 deletions(-) (limited to 'include/exec') diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 707adf7631..cb969d8372 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -1078,11 +1078,11 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) /* Probe for whether the specified guest write access is permitted. * If it is not permitted then an exception will be taken in the same * way as if this were a real write access (and we will not return). - * Otherwise the function will return, and there will be a valid - * entry in the TLB for this access. + * If the size is 0 or the page requires I/O access, returns NULL; otherwise, + * returns the address of the host page similar to tlb_vaddr_to_host(). */ -void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, - uintptr_t retaddr) +void *probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, + uintptr_t retaddr) { uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); @@ -1101,12 +1101,23 @@ void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, tlb_addr = tlb_addr_write(entry); } + if (!size) { + return NULL; + } + /* Handle watchpoints. */ - if ((tlb_addr & TLB_WATCHPOINT) && size > 0) { + if (tlb_addr & TLB_WATCHPOINT) { cpu_check_watchpoint(env_cpu(env), addr, size, env_tlb(env)->d[mmu_idx].iotlb[index].attrs, BP_MEM_WRITE, retaddr); } + + if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO)) { + /* I/O access */ + return NULL; + } + + return (void *)((uintptr_t)addr + entry->addend); } void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 625c33f893..5720bf8056 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -188,8 +188,8 @@ static inline int handle_cpu_signal(uintptr_t pc, siginfo_t *info, g_assert_not_reached(); } -void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, - uintptr_t retaddr) +void *probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, + uintptr_t retaddr) { g_assert(-(addr | TARGET_PAGE_MASK) >= size); @@ -202,6 +202,8 @@ void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, retaddr); g_assert_not_reached(); } + + return size ? g2h(addr) : NULL; } #if defined(__i386__) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index cbcc85add3..a7893ed16b 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -310,8 +310,8 @@ static inline void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu, { } #endif -void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, - uintptr_t retaddr); +void *probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, + uintptr_t retaddr); #define CODE_GEN_ALIGN 16 /* must be >= of the size of a icache line */ -- cgit v1.2.3-55-g7522 From c25c283df0f08582df29f1d5d7be1516b851532d Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Fri, 30 Aug 2019 12:09:59 +0200 Subject: tcg: Factor out probe_write() logic into probe_access() Let's also allow to probe other access types. Reviewed-by: Richard Henderson Signed-off-by: David Hildenbrand Message-Id: <20190830100959.26615-3-david@redhat.com> Signed-off-by: Richard Henderson --- accel/tcg/cputlb.c | 43 ++++++++++++++++++++++++++++++++----------- accel/tcg/user-exec.c | 26 ++++++++++++++++++++------ include/exec/exec-all.h | 10 ++++++++-- 3 files changed, 60 insertions(+), 19 deletions(-) (limited to 'include/exec') diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index cb969d8372..abae79650c 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -1075,30 +1075,51 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr) return qemu_ram_addr_from_host_nofail(p); } -/* Probe for whether the specified guest write access is permitted. - * If it is not permitted then an exception will be taken in the same - * way as if this were a real write access (and we will not return). +/* + * Probe for whether the specified guest access is permitted. If it is not + * permitted then an exception will be taken in the same way as if this + * were a real access (and we will not return). * If the size is 0 or the page requires I/O access, returns NULL; otherwise, * returns the address of the host page similar to tlb_vaddr_to_host(). */ -void *probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, - uintptr_t retaddr) +void *probe_access(CPUArchState *env, target_ulong addr, int size, + MMUAccessType access_type, int mmu_idx, uintptr_t retaddr) { uintptr_t index = tlb_index(env, mmu_idx, addr); CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); - target_ulong tlb_addr = tlb_addr_write(entry); + target_ulong tlb_addr; + size_t elt_ofs; + int wp_access; g_assert(-(addr | TARGET_PAGE_MASK) >= size); + switch (access_type) { + case MMU_DATA_LOAD: + elt_ofs = offsetof(CPUTLBEntry, addr_read); + wp_access = BP_MEM_READ; + break; + case MMU_DATA_STORE: + elt_ofs = offsetof(CPUTLBEntry, addr_write); + wp_access = BP_MEM_WRITE; + break; + case MMU_INST_FETCH: + elt_ofs = offsetof(CPUTLBEntry, addr_code); + wp_access = BP_MEM_READ; + break; + default: + g_assert_not_reached(); + } + tlb_addr = tlb_read_ofs(entry, elt_ofs); + if (unlikely(!tlb_hit(tlb_addr, addr))) { - if (!VICTIM_TLB_HIT(addr_write, addr)) { - tlb_fill(env_cpu(env), addr, size, MMU_DATA_STORE, - mmu_idx, retaddr); + if (!victim_tlb_hit(env, mmu_idx, index, elt_ofs, + addr & TARGET_PAGE_MASK)) { + tlb_fill(env_cpu(env), addr, size, access_type, mmu_idx, retaddr); /* TLB resize via tlb_fill may have moved the entry. */ index = tlb_index(env, mmu_idx, addr); entry = tlb_entry(env, mmu_idx, addr); } - tlb_addr = tlb_addr_write(entry); + tlb_addr = tlb_read_ofs(entry, elt_ofs); } if (!size) { @@ -1109,7 +1130,7 @@ void *probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, if (tlb_addr & TLB_WATCHPOINT) { cpu_check_watchpoint(env_cpu(env), addr, size, env_tlb(env)->d[mmu_idx].iotlb[index].attrs, - BP_MEM_WRITE, retaddr); + wp_access, retaddr); } if (tlb_addr & (TLB_NOTDIRTY | TLB_MMIO)) { diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 5720bf8056..71c4bf6477 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -188,17 +188,31 @@ static inline int handle_cpu_signal(uintptr_t pc, siginfo_t *info, g_assert_not_reached(); } -void *probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, - uintptr_t retaddr) +void *probe_access(CPUArchState *env, target_ulong addr, int size, + MMUAccessType access_type, int mmu_idx, uintptr_t retaddr) { + int flags; + g_assert(-(addr | TARGET_PAGE_MASK) >= size); - if (!guest_addr_valid(addr) || - page_check_range(addr, size, PAGE_WRITE) < 0) { + switch (access_type) { + case MMU_DATA_STORE: + flags = PAGE_WRITE; + break; + case MMU_DATA_LOAD: + flags = PAGE_READ; + break; + case MMU_INST_FETCH: + flags = PAGE_EXEC; + break; + default: + g_assert_not_reached(); + } + + if (!guest_addr_valid(addr) || page_check_range(addr, size, flags) < 0) { CPUState *cpu = env_cpu(env); CPUClass *cc = CPU_GET_CLASS(cpu); - - cc->tlb_fill(cpu, addr, size, MMU_DATA_STORE, MMU_USER_IDX, false, + cc->tlb_fill(cpu, addr, size, access_type, MMU_USER_IDX, false, retaddr); g_assert_not_reached(); } diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index a7893ed16b..81b02eb2fe 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -310,8 +310,14 @@ static inline void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu, { } #endif -void *probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, - uintptr_t retaddr); +void *probe_access(CPUArchState *env, target_ulong addr, int size, + MMUAccessType access_type, int mmu_idx, uintptr_t retaddr); + +static inline void *probe_write(CPUArchState *env, target_ulong addr, int size, + int mmu_idx, uintptr_t retaddr) +{ + return probe_access(env, addr, size, MMU_DATA_STORE, mmu_idx, retaddr); +} #define CODE_GEN_ALIGN 16 /* must be >= of the size of a icache line */ -- cgit v1.2.3-55-g7522