From 785a507ec78bbda1c346f3d3593e5a58b62e73ef Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 5 Mar 2018 00:31:20 +0100 Subject: memory: inline some performance-sensitive accessors These accessors are called from inlined functions, and the call sequence is much more expensive than just inlining the access. Move the struct declaration to memory-internal.h so that exec.c and memory.c can both use an inline function. Reviewed-by: Alexey Kardashevskiy Signed-off-by: Paolo Bonzini --- include/exec/memory-internal.h | 13 +++++++++---- include/exec/memory.h | 22 +++++++++++++++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) (limited to 'include/exec') diff --git a/include/exec/memory-internal.h b/include/exec/memory-internal.h index 4162474fd5..6a5ee42d36 100644 --- a/include/exec/memory-internal.h +++ b/include/exec/memory-internal.h @@ -21,7 +21,15 @@ #define MEMORY_INTERNAL_H #ifndef CONFIG_USER_ONLY -typedef struct AddressSpaceDispatch AddressSpaceDispatch; +static inline AddressSpaceDispatch *flatview_to_dispatch(FlatView *fv) +{ + return fv->dispatch; +} + +static inline AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as) +{ + return flatview_to_dispatch(address_space_to_flatview(as)); +} extern const MemoryRegionOps unassigned_mem_ops; @@ -31,9 +39,6 @@ bool memory_region_access_valid(MemoryRegion *mr, hwaddr addr, void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section); AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv); void address_space_dispatch_compact(AddressSpaceDispatch *d); - -AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as); -AddressSpaceDispatch *flatview_to_dispatch(FlatView *fv); void address_space_dispatch_free(AddressSpaceDispatch *d); void mtree_print_dispatch(fprintf_function mon, void *f, diff --git a/include/exec/memory.h b/include/exec/memory.h index 15e81113ba..4b65b566cb 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -326,7 +326,27 @@ struct AddressSpace { QTAILQ_ENTRY(AddressSpace) address_spaces_link; }; -FlatView *address_space_to_flatview(AddressSpace *as); +typedef struct AddressSpaceDispatch AddressSpaceDispatch; +typedef struct FlatRange FlatRange; + +/* Flattened global view of current active memory hierarchy. Kept in sorted + * order. + */ +struct FlatView { + struct rcu_head rcu; + unsigned ref; + FlatRange *ranges; + unsigned nr; + unsigned nr_allocated; + struct AddressSpaceDispatch *dispatch; + MemoryRegion *root; +}; + +static inline FlatView *address_space_to_flatview(AddressSpace *as) +{ + return atomic_rcu_read(&as->current_map); +} + /** * MemoryRegionSection: describes a fragment of a #MemoryRegion -- cgit v1.2.3-55-g7522 From b2a44fcad74f1cc7a6786d38eba7db12ab2352ba Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 5 Mar 2018 00:19:49 +0100 Subject: address_space_read: address_space_to_flatview needs RCU lock address_space_read is calling address_space_to_flatview but it can be called outside the RCU lock. To fix it, push the rcu_read_lock/unlock pair up from flatview_read_full to address_space_read's constant size fast path and address_space_read_full. Reviewed-by: Alexey Kardashevskiy Signed-off-by: Paolo Bonzini --- exec.c | 38 +++++++++++++++++++++++++------------- include/exec/memory.h | 25 ++++++++++--------------- 2 files changed, 35 insertions(+), 28 deletions(-) (limited to 'include/exec') diff --git a/exec.c b/exec.c index ec91162177..e4552ee228 100644 --- a/exec.c +++ b/exec.c @@ -2616,6 +2616,8 @@ static const MemoryRegionOps watch_mem_ops = { }, }; +static MemTxResult flatview_read(FlatView *fv, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, int len); static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs, const uint8_t *buf, int len); static bool flatview_access_valid(FlatView *fv, hwaddr addr, int len, @@ -3165,24 +3167,18 @@ MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr, return result; } -MemTxResult flatview_read_full(FlatView *fv, hwaddr addr, - MemTxAttrs attrs, uint8_t *buf, int len) +/* Called from RCU critical section. */ +static MemTxResult flatview_read(FlatView *fv, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, int len) { hwaddr l; hwaddr addr1; MemoryRegion *mr; - MemTxResult result = MEMTX_OK; - - if (len > 0) { - rcu_read_lock(); - l = len; - mr = flatview_translate(fv, addr, &addr1, &l, false); - result = flatview_read_continue(fv, addr, attrs, buf, len, - addr1, l, mr); - rcu_read_unlock(); - } - return result; + l = len; + mr = flatview_translate(fv, addr, &addr1, &l, false); + return flatview_read_continue(fv, addr, attrs, buf, len, + addr1, l, mr); } static MemTxResult flatview_rw(FlatView *fv, hwaddr addr, MemTxAttrs attrs, @@ -3203,6 +3199,22 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, addr, attrs, buf, len, is_write); } +MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, int len) +{ + MemTxResult result = MEMTX_OK; + FlatView *fv; + + if (len > 0) { + rcu_read_lock(); + fv = address_space_to_flatview(as); + result = flatview_read(fv, addr, attrs, buf, len); + rcu_read_unlock(); + } + + return result; +} + MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, const uint8_t *buf, int len) diff --git a/include/exec/memory.h b/include/exec/memory.h index 4b65b566cb..31eae0a640 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -1917,13 +1917,12 @@ void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len, /* Internal functions, part of the implementation of address_space_read. */ +MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, int len); MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr, MemTxAttrs attrs, uint8_t *buf, int len, hwaddr addr1, hwaddr l, MemoryRegion *mr); - -MemTxResult flatview_read_full(FlatView *fv, hwaddr addr, - MemTxAttrs attrs, uint8_t *buf, int len); void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr); static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write) @@ -1942,25 +1941,28 @@ static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write) * * Return a MemTxResult indicating whether the operation succeeded * or failed (eg unassigned memory, device rejected the transaction, - * IOMMU fault). + * IOMMU fault). Called within RCU critical section. * - * @fv: #FlatView to be accessed + * @as: #AddressSpace to be accessed * @addr: address within that address space * @attrs: memory transaction attributes * @buf: buffer with the data transferred */ static inline __attribute__((__always_inline__)) -MemTxResult flatview_read(FlatView *fv, hwaddr addr, MemTxAttrs attrs, - uint8_t *buf, int len) +MemTxResult address_space_read(AddressSpace *as, hwaddr addr, + MemTxAttrs attrs, uint8_t *buf, + int len) { MemTxResult result = MEMTX_OK; hwaddr l, addr1; void *ptr; MemoryRegion *mr; + FlatView *fv; if (__builtin_constant_p(len)) { if (len) { rcu_read_lock(); + fv = address_space_to_flatview(as); l = len; mr = flatview_translate(fv, addr, &addr1, &l, false); if (len == l && memory_access_is_direct(mr, false)) { @@ -1973,18 +1975,11 @@ MemTxResult flatview_read(FlatView *fv, hwaddr addr, MemTxAttrs attrs, rcu_read_unlock(); } } else { - result = flatview_read_full(fv, addr, attrs, buf, len); + result = address_space_read_full(as, addr, attrs, buf, len); } return result; } -static inline MemTxResult address_space_read(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, uint8_t *buf, - int len) -{ - return flatview_read(address_space_to_flatview(as), addr, attrs, buf, len); -} - /** * address_space_read_cached: read from a cached RAM region * -- cgit v1.2.3-55-g7522