From db94604b20278c1dc227a04e4c564d80230e6c3f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 21 May 2015 15:12:29 +0200 Subject: exec: optimize phys_page_set_level phys_page_set_level is writing zeroes to a struct that has just been filled in by phys_map_node_alloc. Instead, tell phys_map_node_alloc whether to fill in the page "as a leaf" or "as a non-leaf". memcpy is faster than struct assignment, which copies each bitfield individually. A compiler bug (https://gcc.gnu.org/PR66391), and small memcpys like this one are special-cased anyway, and optimized to a register move, so just use the memcpy. This cuts the cost of phys_page_set_level from 25% to 5% when booting qboot. Reviewed-by: Stefan Hajnoczi Signed-off-by: Paolo Bonzini --- exec.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index e19ab22cd6..fc8d05d50a 100644 --- a/exec.c +++ b/exec.c @@ -173,17 +173,22 @@ static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes) } } -static uint32_t phys_map_node_alloc(PhysPageMap *map) +static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf) { unsigned i; uint32_t ret; + PhysPageEntry e; + PhysPageEntry *p; ret = map->nodes_nb++; + p = map->nodes[ret]; assert(ret != PHYS_MAP_NODE_NIL); assert(ret != map->nodes_nb_alloc); + + e.skip = leaf ? 0 : 1; + e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL; for (i = 0; i < P_L2_SIZE; ++i) { - map->nodes[ret][i].skip = 1; - map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL; + memcpy(&p[i], &e, sizeof(e)); } return ret; } @@ -193,21 +198,12 @@ static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp, int level) { PhysPageEntry *p; - int i; hwaddr step = (hwaddr)1 << (level * P_L2_BITS); if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) { - lp->ptr = phys_map_node_alloc(map); - p = map->nodes[lp->ptr]; - if (level == 0) { - for (i = 0; i < P_L2_SIZE; i++) { - p[i].skip = 0; - p[i].ptr = PHYS_SECTION_UNASSIGNED; - } - } - } else { - p = map->nodes[lp->ptr]; + lp->ptr = phys_map_node_alloc(map, level == 0); } + p = map->nodes[lp->ptr]; lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)]; while (*nb && lp < &p[P_L2_SIZE]) { -- cgit v1.2.3-55-g7522 From 49dfcec40349245ad365964468b67e132c3cedc7 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 Mar 2015 11:35:19 +0100 Subject: ram_addr: tweaks to xen_modified_memory Invoke xen_modified_memory from cpu_physical_memory_set_dirty_range_nocode; it is akin to DIRTY_MEMORY_MIGRATION, so set it together with that bitmap. The remaining call from invalidate_and_set_dirty's "else" branch will go away soon. Second, fix the second argument to the function in the cpu_physical_memory_set_dirty_lebitmap call site. That function is only used by KVM, but it is better to be clean anyway. Acked-by: Stefano Stabellini Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 3 ++- include/exec/ram_addr.h | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index fc8d05d50a..6f6dbc8e60 100644 --- a/exec.c +++ b/exec.c @@ -2281,8 +2281,9 @@ static void invalidate_and_set_dirty(hwaddr addr, if (cpu_physical_memory_range_includes_clean(addr, length)) { tb_invalidate_phys_range(addr, addr + length, 0); cpu_physical_memory_set_dirty_range_nocode(addr, length); + } else { + xen_modified_memory(addr, length); } - xen_modified_memory(addr, length); } static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr) diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index ff558a4734..7f6e928a7f 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -112,6 +112,7 @@ static inline void cpu_physical_memory_set_dirty_range_nocode(ram_addr_t start, page = start >> TARGET_PAGE_BITS; bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); + xen_modified_memory(start, length); } static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, @@ -155,7 +156,7 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_list.dirty_memory[DIRTY_MEMORY_CODE][page + k] |= temp; } } - xen_modified_memory(start, pages); + xen_modified_memory(start, pages << TARGET_PAGE_BITS); } else { /* * bitmap-traveling is faster than memory-traveling (for addr...) -- cgit v1.2.3-55-g7522 From 845b6214a309fa58a4405050bf8313e19fde5c91 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 Mar 2015 11:45:53 +0100 Subject: exec: use memory_region_get_dirty_log_mask to optimize dirty tracking The memory API can now return the exact set of bitmaps that have to be tracked. Use it instead of the in_migration variable. In the next patches, we will also use it to set only DIRTY_MEMORY_VGA or DIRTY_MEMORY_MIGRATION if necessary. This can make a difference for dataplane, especially after the dirty bitmap is changed to use more expensive atomic operations. Of some interest is the change to stl_phys_notdirty. When migration was introduced, stl_phys_notdirty was changed to effectively behave as stl_phys during migration. In fact, if one looks at the function as it was in the beginning (commit 8df1cd0, physical memory access functions, 2005-01-28), at the time the dirty bitmap was the equivalent of DIRTY_MEMORY_CODE nowadays; hence, the function simply should not touch the dirty code bits. This patch changes it to do the intended thing. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 59 +++++++++++++++++++---------------------------------------- 1 file changed, 19 insertions(+), 40 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index 6f6dbc8e60..950fea410e 100644 --- a/exec.c +++ b/exec.c @@ -59,8 +59,6 @@ //#define DEBUG_SUBPAGE #if !defined(CONFIG_USER_ONLY) -static bool in_migration; - /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes * are protected by the ramlist lock. */ @@ -866,11 +864,6 @@ void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length, } } -static void cpu_physical_memory_set_dirty_tracking(bool enable) -{ - in_migration = enable; -} - /* Called from RCU critical section */ hwaddr memory_region_section_get_iotlb(CPUState *cpu, MemoryRegionSection *section, @@ -2161,22 +2154,6 @@ static void tcg_commit(MemoryListener *listener) } } -static void core_log_global_start(MemoryListener *listener) -{ - cpu_physical_memory_set_dirty_tracking(true); -} - -static void core_log_global_stop(MemoryListener *listener) -{ - cpu_physical_memory_set_dirty_tracking(false); -} - -static MemoryListener core_memory_listener = { - .log_global_start = core_log_global_start, - .log_global_stop = core_log_global_stop, - .priority = 1, -}; - void address_space_init_dispatch(AddressSpace *as) { as->dispatch = NULL; @@ -2216,8 +2193,6 @@ static void memory_map_init(void) memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io", 65536); address_space_init(&address_space_io, system_io, "I/O"); - - memory_listener_register(&core_memory_listener, &address_space_memory); } MemoryRegion *get_system_memory(void) @@ -2275,12 +2250,18 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr, #else -static void invalidate_and_set_dirty(hwaddr addr, +static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, hwaddr length) { if (cpu_physical_memory_range_includes_clean(addr, length)) { - tb_invalidate_phys_range(addr, addr + length, 0); - cpu_physical_memory_set_dirty_range_nocode(addr, length); + uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr); + if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { + tb_invalidate_phys_range(addr, addr + length, 0); + dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); + } + if (dirty_log_mask) { + cpu_physical_memory_set_dirty_range_nocode(addr, length); + } } else { xen_modified_memory(addr, length); } @@ -2368,7 +2349,7 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, /* RAM case */ ptr = qemu_get_ram_ptr(addr1); memcpy(ptr, buf, l); - invalidate_and_set_dirty(addr1, l); + invalidate_and_set_dirty(mr, addr1, l); } } else { if (!memory_access_is_direct(mr, is_write)) { @@ -2465,7 +2446,7 @@ static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as, switch (type) { case WRITE_DATA: memcpy(ptr, buf, l); - invalidate_and_set_dirty(addr1, l); + invalidate_and_set_dirty(mr, addr1, l); break; case FLUSH_CACHE: flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l); @@ -2690,7 +2671,7 @@ void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len, mr = qemu_ram_addr_from_host(buffer, &addr1); assert(mr != NULL); if (is_write) { - invalidate_and_set_dirty(addr1, access_len); + invalidate_and_set_dirty(mr, addr1, access_len); } if (xen_enabled()) { xen_invalidate_map_cache_entry(buffer); @@ -3019,6 +3000,7 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val, hwaddr l = 4; hwaddr addr1; MemTxResult r; + uint8_t dirty_log_mask; rcu_read_lock(); mr = address_space_translate(as, addr, &addr1, &l, @@ -3030,13 +3012,10 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val, ptr = qemu_get_ram_ptr(addr1); stl_p(ptr, val); - if (unlikely(in_migration)) { - if (cpu_physical_memory_is_clean(addr1)) { - /* invalidate code */ - tb_invalidate_phys_page_range(addr1, addr1 + 4, 0); - /* set dirty bit */ - cpu_physical_memory_set_dirty_range_nocode(addr1, 4); - } + dirty_log_mask = memory_region_get_dirty_log_mask(mr); + dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); + if (dirty_log_mask) { + cpu_physical_memory_set_dirty_range_nocode(addr1, 4); } r = MEMTX_OK; } @@ -3093,7 +3072,7 @@ static inline void address_space_stl_internal(AddressSpace *as, stl_p(ptr, val); break; } - invalidate_and_set_dirty(addr1, 4); + invalidate_and_set_dirty(mr, addr1, 4); r = MEMTX_OK; } if (result) { @@ -3197,7 +3176,7 @@ static inline void address_space_stw_internal(AddressSpace *as, stw_p(ptr, val); break; } - invalidate_and_set_dirty(addr1, 2); + invalidate_and_set_dirty(mr, addr1, 2); r = MEMTX_OK; } if (result) { -- cgit v1.2.3-55-g7522 From 358653391b0c0beaa0e3f9e28304e1918cd223b3 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 22 Apr 2015 14:20:35 +0200 Subject: translate-all: remove unnecessary argument to tb_invalidate_phys_range The is_cpu_write_access argument is always 0, remove it. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 2 +- linux-user/mmap.c | 6 +++--- translate-all.c | 5 ++--- translate-all.h | 3 +-- 4 files changed, 7 insertions(+), 9 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index 950fea410e..650cfa8cb8 100644 --- a/exec.c +++ b/exec.c @@ -2256,7 +2256,7 @@ static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, if (cpu_physical_memory_range_includes_clean(addr, length)) { uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr); if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { - tb_invalidate_phys_range(addr, addr + length, 0); + tb_invalidate_phys_range(addr, addr + length); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); } if (dirty_log_mask) { diff --git a/linux-user/mmap.c b/linux-user/mmap.c index 959ff4d003..78e1b2df43 100644 --- a/linux-user/mmap.c +++ b/linux-user/mmap.c @@ -575,7 +575,7 @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int prot, page_dump(stdout); printf("\n"); #endif - tb_invalidate_phys_range(start, start + len, 0); + tb_invalidate_phys_range(start, start + len); mmap_unlock(); return start; fail: @@ -680,7 +680,7 @@ int target_munmap(abi_ulong start, abi_ulong len) if (ret == 0) { page_set_flags(start, start + len, 0); - tb_invalidate_phys_range(start, start + len, 0); + tb_invalidate_phys_range(start, start + len); } mmap_unlock(); return ret; @@ -759,7 +759,7 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size, page_set_flags(old_addr, old_addr + old_size, 0); page_set_flags(new_addr, new_addr + new_size, prot | PAGE_VALID); } - tb_invalidate_phys_range(new_addr, new_addr + new_size, 0); + tb_invalidate_phys_range(new_addr, new_addr + new_size); mmap_unlock(); return new_addr; } diff --git a/translate-all.c b/translate-all.c index 536008f52d..d118c6c2af 100644 --- a/translate-all.c +++ b/translate-all.c @@ -1042,11 +1042,10 @@ TranslationBlock *tb_gen_code(CPUState *cpu, * access: the virtual CPU will exit the current TB if code is modified inside * this TB. */ -void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end, - int is_cpu_write_access) +void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) { while (start < end) { - tb_invalidate_phys_page_range(start, end, is_cpu_write_access); + tb_invalidate_phys_page_range(start, end, 0); start &= TARGET_PAGE_MASK; start += TARGET_PAGE_SIZE; } diff --git a/translate-all.h b/translate-all.h index f1a40a53c2..038464005f 100644 --- a/translate-all.h +++ b/translate-all.h @@ -23,8 +23,7 @@ void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len); void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, int is_cpu_write_access); -void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end, - int is_cpu_write_access); +void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end); void tb_check_watchpoint(CPUState *cpu); #ifdef CONFIG_USER_ONLY -- cgit v1.2.3-55-g7522 From 58d2707e8713ef17b89b8b4c9ce586c76655a385 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 Mar 2015 11:56:01 +0100 Subject: exec: pass client mask to cpu_physical_memory_set_dirty_range This cuts in half the cost of bitmap operations (which will become more expensive when made atomic) during migration on non-VRAM regions. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 20 +++++++++++--------- include/exec/ram_addr.h | 33 ++++++++++++++++----------------- memory.c | 3 ++- 3 files changed, 29 insertions(+), 27 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index 650cfa8cb8..fe137bd725 100644 --- a/exec.c +++ b/exec.c @@ -1351,7 +1351,8 @@ int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp) cpu_physical_memory_clear_dirty_range(block->offset, block->used_length); block->used_length = newsize; - cpu_physical_memory_set_dirty_range(block->offset, block->used_length); + cpu_physical_memory_set_dirty_range(block->offset, block->used_length, + DIRTY_CLIENTS_ALL); memory_region_set_size(block->mr, newsize); if (block->resized) { block->resized(block->idstr, newsize, block->host); @@ -1425,7 +1426,8 @@ static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp) } } cpu_physical_memory_set_dirty_range(new_block->offset, - new_block->used_length); + new_block->used_length, + DIRTY_CLIENTS_ALL); if (new_block->host) { qemu_ram_setup_dump(new_block->host, new_block->max_length); @@ -1813,7 +1815,11 @@ static void notdirty_mem_write(void *opaque, hwaddr ram_addr, default: abort(); } - cpu_physical_memory_set_dirty_range_nocode(ram_addr, size); + /* Set both VGA and migration bits for simplicity and to remove + * the notdirty callback faster. + */ + cpu_physical_memory_set_dirty_range(ram_addr, size, + DIRTY_CLIENTS_NOCODE); /* we remove the notdirty callback only if the code has been flushed */ if (!cpu_physical_memory_is_clean(ram_addr)) { @@ -2259,9 +2265,7 @@ static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, tb_invalidate_phys_range(addr, addr + length); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); } - if (dirty_log_mask) { - cpu_physical_memory_set_dirty_range_nocode(addr, length); - } + cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); } else { xen_modified_memory(addr, length); } @@ -3014,9 +3018,7 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val, dirty_log_mask = memory_region_get_dirty_log_mask(mr); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); - if (dirty_log_mask) { - cpu_physical_memory_set_dirty_range_nocode(addr1, 4); - } + cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask); r = MEMTX_OK; } if (result) { diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 7f6e928a7f..5bbc7bb41d 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -41,6 +41,9 @@ void qemu_ram_free_from_ptr(ram_addr_t addr); int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp); +#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) +#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) + static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, ram_addr_t length, unsigned client) @@ -103,28 +106,23 @@ static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, set_bit(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]); } -static inline void cpu_physical_memory_set_dirty_range_nocode(ram_addr_t start, - ram_addr_t length) -{ - unsigned long end, page; - - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); - xen_modified_memory(start, length); -} - static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, - ram_addr_t length) + ram_addr_t length, + uint8_t mask) { unsigned long end, page; end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; page = start >> TARGET_PAGE_BITS; - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_CODE], page, end - page); + if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { + bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { + bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { + bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_CODE], page, end - page); + } xen_modified_memory(start, length); } @@ -172,7 +170,8 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, addr = page_number * TARGET_PAGE_SIZE; ram_addr = start + addr; cpu_physical_memory_set_dirty_range(ram_addr, - TARGET_PAGE_SIZE * hpratio); + TARGET_PAGE_SIZE * hpratio, + DIRTY_CLIENTS_ALL); } while (c != 0); } } diff --git a/memory.c b/memory.c index b7ca987c07..418cac7ce1 100644 --- a/memory.c +++ b/memory.c @@ -1461,7 +1461,8 @@ void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, hwaddr size) { assert(mr->terminates); - cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size); + cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size, + memory_region_get_dirty_log_mask(mr)); } bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr, -- cgit v1.2.3-55-g7522 From e87f7778b64d4a6a78e16c288c7fdc6c15317d5f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 25 Mar 2015 15:21:39 +0100 Subject: exec: only check relevant bitmaps for cleanliness Most of the time, not all bitmaps have to be marked as dirty; do not do anything if the interesting ones are already dirty. Previously, any clean bitmap would have cause all the bitmaps to be marked dirty. In fact, unless running TCG most of the time bitmap operations need not be done at all, because memory_region_is_logging returns zero. In this case, skip the call to cpu_physical_memory_range_includes_clean altogether as well. With this patch, cpu_physical_memory_set_dirty_range is called unconditionally, so there need not be anymore a separate call to xen_modified_memory. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 22 +++++++++++++--------- include/exec/ram_addr.h | 25 ++++++++++++++++++------- 2 files changed, 31 insertions(+), 16 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index fe137bd725..162c57908e 100644 --- a/exec.c +++ b/exec.c @@ -2259,16 +2259,20 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr, static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, hwaddr length) { - if (cpu_physical_memory_range_includes_clean(addr, length)) { - uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr); - if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { - tb_invalidate_phys_range(addr, addr + length); - dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); - } - cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); - } else { - xen_modified_memory(addr, length); + uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr); + /* No early return if dirty_log_mask is or becomes 0, because + * cpu_physical_memory_set_dirty_range will still call + * xen_modified_memory. + */ + if (dirty_log_mask) { + dirty_log_mask = + cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask); + } + if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { + tb_invalidate_phys_range(addr, addr + length); + dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); } + cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); } static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr) diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index c221bd7dd0..3e42b4f602 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -89,14 +89,25 @@ static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) return !(vga && code && migration); } -static inline bool cpu_physical_memory_range_includes_clean(ram_addr_t start, - ram_addr_t length) +static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, + ram_addr_t length, + uint8_t mask) { - bool vga = !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA); - bool code = !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE); - bool migration = - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION); - return vga || code || migration; + uint8_t ret = 0; + + if (mask & (1 << DIRTY_MEMORY_VGA) && + !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { + ret |= (1 << DIRTY_MEMORY_VGA); + } + if (mask & (1 << DIRTY_MEMORY_CODE) && + !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { + ret |= (1 << DIRTY_MEMORY_CODE); + } + if (mask & (1 << DIRTY_MEMORY_MIGRATION) && + !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { + ret |= (1 << DIRTY_MEMORY_MIGRATION); + } + return ret; } static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, -- cgit v1.2.3-55-g7522 From 03eebc9e3246b9b3f5925aa41f7dfd7c1e467875 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Tue, 2 Dec 2014 11:23:18 +0000 Subject: memory: replace cpu_physical_memory_reset_dirty() with test-and-clear The cpu_physical_memory_reset_dirty() function is sometimes used together with cpu_physical_memory_get_dirty(). This is not atomic since two separate accesses to the dirty memory bitmap are made. Turn cpu_physical_memory_reset_dirty() and cpu_physical_memory_clear_dirty_range_type() into the atomic cpu_physical_memory_test_and_clear_dirty(). Signed-off-by: Stefan Hajnoczi Message-Id: <1417519399-3166-6-git-send-email-stefanha@redhat.com> Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- cputlb.c | 4 ++-- exec.c | 23 +++++++++++++++++------ include/exec/ram_addr.h | 33 ++++++++++----------------------- memory.c | 11 ++++------- 4 files changed, 33 insertions(+), 38 deletions(-) (limited to 'exec.c') diff --git a/cputlb.c b/cputlb.c index dd1203b938..a50608676c 100644 --- a/cputlb.c +++ b/cputlb.c @@ -125,8 +125,8 @@ void tlb_flush_page(CPUState *cpu, target_ulong addr) can be detected */ void tlb_protect_code(ram_addr_t ram_addr) { - cpu_physical_memory_reset_dirty(ram_addr, TARGET_PAGE_SIZE, - DIRTY_MEMORY_CODE); + cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE, + DIRTY_MEMORY_CODE); } /* update the TLB so that writes in physical page 'phys_addr' are no longer diff --git a/exec.c b/exec.c index 162c57908e..487583b1bd 100644 --- a/exec.c +++ b/exec.c @@ -852,16 +852,27 @@ static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length) } /* Note: start and end must be within the same ram block. */ -void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length, - unsigned client) +bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, + ram_addr_t length, + unsigned client) { - if (length == 0) - return; - cpu_physical_memory_clear_dirty_range_type(start, length, client); + unsigned long end, page; + bool dirty; - if (tcg_enabled()) { + if (length == 0) { + return false; + } + + end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; + page = start >> TARGET_PAGE_BITS; + dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client], + page, end - page); + + if (dirty && tcg_enabled()) { tlb_reset_dirty_range_all(start, length); } + + return dirty; } /* Called from RCU critical section */ diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 63db371850..a2beea7925 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -194,30 +194,19 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, } #endif /* not _WIN32 */ -static inline void cpu_physical_memory_clear_dirty_range_type(ram_addr_t start, - ram_addr_t length, - unsigned client) -{ - unsigned long end, page; - - assert(client < DIRTY_MEMORY_NUM); - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - bitmap_clear(ram_list.dirty_memory[client], page, end - page); -} +bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, + ram_addr_t length, + unsigned client); static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, ram_addr_t length) { - cpu_physical_memory_clear_dirty_range_type(start, length, DIRTY_MEMORY_MIGRATION); - cpu_physical_memory_clear_dirty_range_type(start, length, DIRTY_MEMORY_VGA); - cpu_physical_memory_clear_dirty_range_type(start, length, DIRTY_MEMORY_CODE); + cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); + cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); + cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); } -void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length, - unsigned client); - static inline uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest, ram_addr_t start, @@ -245,16 +234,14 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest, } } else { for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { - if (cpu_physical_memory_get_dirty(start + addr, - TARGET_PAGE_SIZE, - DIRTY_MEMORY_MIGRATION)) { + if (cpu_physical_memory_test_and_clear_dirty( + start + addr, + TARGET_PAGE_SIZE, + DIRTY_MEMORY_MIGRATION)) { long k = (start + addr) >> TARGET_PAGE_BITS; if (!test_and_set_bit(k, dest)) { num_dirty++; } - cpu_physical_memory_reset_dirty(start + addr, - TARGET_PAGE_SIZE, - DIRTY_MEMORY_MIGRATION); } } } diff --git a/memory.c b/memory.c index 418cac7ce1..0c5d32807e 100644 --- a/memory.c +++ b/memory.c @@ -1468,13 +1468,9 @@ void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr, hwaddr size, unsigned client) { - bool ret; assert(mr->terminates); - ret = cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, client); - if (ret) { - cpu_physical_memory_reset_dirty(mr->ram_addr + addr, size, client); - } - return ret; + return cpu_physical_memory_test_and_clear_dirty(mr->ram_addr + addr, + size, client); } @@ -1518,7 +1514,8 @@ void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr, hwaddr size, unsigned client) { assert(mr->terminates); - cpu_physical_memory_reset_dirty(mr->ram_addr + addr, size, client); + cpu_physical_memory_test_and_clear_dirty(mr->ram_addr + addr, size, + client); } int memory_region_get_fd(MemoryRegion *mr) -- cgit v1.2.3-55-g7522