From 49dfcec40349245ad365964468b67e132c3cedc7 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 Mar 2015 11:35:19 +0100 Subject: ram_addr: tweaks to xen_modified_memory Invoke xen_modified_memory from cpu_physical_memory_set_dirty_range_nocode; it is akin to DIRTY_MEMORY_MIGRATION, so set it together with that bitmap. The remaining call from invalidate_and_set_dirty's "else" branch will go away soon. Second, fix the second argument to the function in the cpu_physical_memory_set_dirty_lebitmap call site. That function is only used by KVM, but it is better to be clean anyway. Acked-by: Stefano Stabellini Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- include/exec/ram_addr.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/exec/ram_addr.h') diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index ff558a4734..7f6e928a7f 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -112,6 +112,7 @@ static inline void cpu_physical_memory_set_dirty_range_nocode(ram_addr_t start, page = start >> TARGET_PAGE_BITS; bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); + xen_modified_memory(start, length); } static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, @@ -155,7 +156,7 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_list.dirty_memory[DIRTY_MEMORY_CODE][page + k] |= temp; } } - xen_modified_memory(start, pages); + xen_modified_memory(start, pages << TARGET_PAGE_BITS); } else { /* * bitmap-traveling is faster than memory-traveling (for addr...) -- cgit v1.2.3 From 58d2707e8713ef17b89b8b4c9ce586c76655a385 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 Mar 2015 11:56:01 +0100 Subject: exec: pass client mask to cpu_physical_memory_set_dirty_range This cuts in half the cost of bitmap operations (which will become more expensive when made atomic) during migration on non-VRAM regions. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- include/exec/ram_addr.h | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) (limited to 'include/exec/ram_addr.h') diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 7f6e928a7f..5bbc7bb41d 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -41,6 +41,9 @@ void qemu_ram_free_from_ptr(ram_addr_t addr); int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp); +#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) +#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) + static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, ram_addr_t length, unsigned client) @@ -103,28 +106,23 @@ static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, set_bit(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]); } -static inline void cpu_physical_memory_set_dirty_range_nocode(ram_addr_t start, - ram_addr_t length) -{ - unsigned long end, page; - - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); - xen_modified_memory(start, length); -} - static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, - ram_addr_t length) + ram_addr_t length, + uint8_t mask) { unsigned long end, page; end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; page = start >> TARGET_PAGE_BITS; - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_CODE], page, end - page); + if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { + bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { + bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { + bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_CODE], page, end - page); + } xen_modified_memory(start, length); } @@ -172,7 +170,8 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, addr = page_number * TARGET_PAGE_SIZE; ram_addr = start + addr; cpu_physical_memory_set_dirty_range(ram_addr, - TARGET_PAGE_SIZE * hpratio); + TARGET_PAGE_SIZE * hpratio, + DIRTY_CLIENTS_ALL); } while (c != 0); } } -- cgit v1.2.3 From 72b47e79cef36ed6ffc718f10e21001d7ec2a66f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 22 Apr 2015 13:48:25 +0200 Subject: exec: invert return value of cpu_physical_memory_get_clean, rename While it is obvious that cpu_physical_memory_get_dirty returns true even if a single page is dirty, the same is not true for cpu_physical_memory_get_clean; one would expect that it returns true only if all the pages are clean, but it actually looks for even one clean page. (By contrast, the caller of that function, cpu_physical_memory_range_includes_clean, has a good name). To clarify, rename the function to cpu_physical_memory_all_dirty and return true if _all_ the pages are dirty. This is the opposite of the previous meaning, because "all are 1" is the same as "not (any is 0)", so we have to modify cpu_physical_memory_range_includes_clean as well. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- include/exec/ram_addr.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'include/exec/ram_addr.h') diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 5bbc7bb41d..c221bd7dd0 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -59,7 +59,7 @@ static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, return next < end; } -static inline bool cpu_physical_memory_get_clean(ram_addr_t start, +static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, ram_addr_t length, unsigned client) { @@ -71,7 +71,7 @@ static inline bool cpu_physical_memory_get_clean(ram_addr_t start, page = start >> TARGET_PAGE_BITS; next = find_next_zero_bit(ram_list.dirty_memory[client], end, page); - return next < end; + return next >= end; } static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, @@ -92,10 +92,10 @@ static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) static inline bool cpu_physical_memory_range_includes_clean(ram_addr_t start, ram_addr_t length) { - bool vga = cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_VGA); - bool code = cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_CODE); + bool vga = !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA); + bool code = !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE); bool migration = - cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_MIGRATION); + !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION); return vga || code || migration; } -- cgit v1.2.3 From e87f7778b64d4a6a78e16c288c7fdc6c15317d5f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 25 Mar 2015 15:21:39 +0100 Subject: exec: only check relevant bitmaps for cleanliness Most of the time, not all bitmaps have to be marked as dirty; do not do anything if the interesting ones are already dirty. Previously, any clean bitmap would have cause all the bitmaps to be marked dirty. In fact, unless running TCG most of the time bitmap operations need not be done at all, because memory_region_is_logging returns zero. In this case, skip the call to cpu_physical_memory_range_includes_clean altogether as well. With this patch, cpu_physical_memory_set_dirty_range is called unconditionally, so there need not be anymore a separate call to xen_modified_memory. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- include/exec/ram_addr.h | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'include/exec/ram_addr.h') diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index c221bd7dd0..3e42b4f602 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -89,14 +89,25 @@ static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) return !(vga && code && migration); } -static inline bool cpu_physical_memory_range_includes_clean(ram_addr_t start, - ram_addr_t length) +static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, + ram_addr_t length, + uint8_t mask) { - bool vga = !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA); - bool code = !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE); - bool migration = - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION); - return vga || code || migration; + uint8_t ret = 0; + + if (mask & (1 << DIRTY_MEMORY_VGA) && + !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { + ret |= (1 << DIRTY_MEMORY_VGA); + } + if (mask & (1 << DIRTY_MEMORY_CODE) && + !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { + ret |= (1 << DIRTY_MEMORY_CODE); + } + if (mask & (1 << DIRTY_MEMORY_MIGRATION) && + !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { + ret |= (1 << DIRTY_MEMORY_MIGRATION); + } + return ret; } static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, -- cgit v1.2.3 From 9460dee4b2258e3990906fb34099481c8334c267 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 Mar 2015 11:41:32 +0100 Subject: memory: do not touch code dirty bitmap unless TCG is enabled cpu_physical_memory_set_dirty_lebitmap unconditionally syncs the DIRTY_MEMORY_CODE bitmap. This however is unused unless TCG is enabled. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- include/exec/ram_addr.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'include/exec/ram_addr.h') diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 3e42b4f602..98110108b7 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -162,11 +162,14 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION][page + k] |= temp; ram_list.dirty_memory[DIRTY_MEMORY_VGA][page + k] |= temp; - ram_list.dirty_memory[DIRTY_MEMORY_CODE][page + k] |= temp; + if (tcg_enabled()) { + ram_list.dirty_memory[DIRTY_MEMORY_CODE][page + k] |= temp; + } } } xen_modified_memory(start, pages << TARGET_PAGE_BITS); } else { + uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; /* * bitmap-traveling is faster than memory-traveling (for addr...) * especially when most of the memory is not dirty. @@ -181,8 +184,7 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, addr = page_number * TARGET_PAGE_SIZE; ram_addr = start + addr; cpu_physical_memory_set_dirty_range(ram_addr, - TARGET_PAGE_SIZE * hpratio, - DIRTY_CLIENTS_ALL); + TARGET_PAGE_SIZE * hpratio, clients); } while (c != 0); } } -- cgit v1.2.3 From d114875b9a1c21162f69a12d72f69a22e7bab376 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Tue, 2 Dec 2014 11:23:16 +0000 Subject: memory: use atomic ops for setting dirty memory bits Use set_bit_atomic() and bitmap_set_atomic() so that multiple threads can dirty memory without race conditions. Signed-off-by: Stefan Hajnoczi Message-Id: <1417519399-3166-4-git-send-email-stefanha@redhat.com> Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- include/exec/ram_addr.h | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'include/exec/ram_addr.h') diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 98110108b7..9f73076044 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -114,7 +114,7 @@ static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) { assert(client < DIRTY_MEMORY_NUM); - set_bit(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]); + set_bit_atomic(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]); } static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, @@ -122,17 +122,18 @@ static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, uint8_t mask) { unsigned long end, page; + unsigned long **d = ram_list.dirty_memory; end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; page = start >> TARGET_PAGE_BITS; if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page); + bitmap_set_atomic(d[DIRTY_MEMORY_MIGRATION], page, end - page); } if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page); + bitmap_set_atomic(d[DIRTY_MEMORY_VGA], page, end - page); } if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { - bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_CODE], page, end - page); + bitmap_set_atomic(d[DIRTY_MEMORY_CODE], page, end - page); } xen_modified_memory(start, length); } @@ -159,11 +160,12 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, for (k = 0; k < nr; k++) { if (bitmap[k]) { unsigned long temp = leul_to_cpu(bitmap[k]); + unsigned long **d = ram_list.dirty_memory; - ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION][page + k] |= temp; - ram_list.dirty_memory[DIRTY_MEMORY_VGA][page + k] |= temp; + atomic_or(&d[DIRTY_MEMORY_MIGRATION][page + k], temp); + atomic_or(&d[DIRTY_MEMORY_VGA][page + k], temp); if (tcg_enabled()) { - ram_list.dirty_memory[DIRTY_MEMORY_CODE][page + k] |= temp; + atomic_or(&d[DIRTY_MEMORY_CODE][page + k], temp); } } } -- cgit v1.2.3 From 20015f72bda7d2f356c43580a5542a659afedf83 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Tue, 2 Dec 2014 11:23:17 +0000 Subject: migration: move dirty bitmap sync to ram_addr.h The dirty memory bitmap is managed by ram_addr.h and copied to migration_bitmap[] periodically during live migration. Move the code to sync the bitmap to ram_addr.h where related code lives. Signed-off-by: Stefan Hajnoczi Message-Id: <1417519399-3166-5-git-send-email-stefanha@redhat.com> Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- include/exec/ram_addr.h | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) (limited to 'include/exec/ram_addr.h') diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 9f73076044..63db371850 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -218,5 +218,49 @@ static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length, unsigned client); +static inline +uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest, + ram_addr_t start, + ram_addr_t length) +{ + ram_addr_t addr; + unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); + uint64_t num_dirty = 0; + + /* start address is aligned at the start of a word? */ + if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) { + int k; + int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); + unsigned long *src = ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]; + + for (k = page; k < page + nr; k++) { + if (src[k]) { + unsigned long new_dirty; + new_dirty = ~dest[k]; + dest[k] |= src[k]; + new_dirty &= src[k]; + num_dirty += ctpopl(new_dirty); + src[k] = 0; + } + } + } else { + for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { + if (cpu_physical_memory_get_dirty(start + addr, + TARGET_PAGE_SIZE, + DIRTY_MEMORY_MIGRATION)) { + long k = (start + addr) >> TARGET_PAGE_BITS; + if (!test_and_set_bit(k, dest)) { + num_dirty++; + } + cpu_physical_memory_reset_dirty(start + addr, + TARGET_PAGE_SIZE, + DIRTY_MEMORY_MIGRATION); + } + } + } + + return num_dirty; +} + #endif #endif -- cgit v1.2.3 From 03eebc9e3246b9b3f5925aa41f7dfd7c1e467875 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Tue, 2 Dec 2014 11:23:18 +0000 Subject: memory: replace cpu_physical_memory_reset_dirty() with test-and-clear The cpu_physical_memory_reset_dirty() function is sometimes used together with cpu_physical_memory_get_dirty(). This is not atomic since two separate accesses to the dirty memory bitmap are made. Turn cpu_physical_memory_reset_dirty() and cpu_physical_memory_clear_dirty_range_type() into the atomic cpu_physical_memory_test_and_clear_dirty(). Signed-off-by: Stefan Hajnoczi Message-Id: <1417519399-3166-6-git-send-email-stefanha@redhat.com> Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- include/exec/ram_addr.h | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) (limited to 'include/exec/ram_addr.h') diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index 63db371850..a2beea7925 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -194,30 +194,19 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, } #endif /* not _WIN32 */ -static inline void cpu_physical_memory_clear_dirty_range_type(ram_addr_t start, - ram_addr_t length, - unsigned client) -{ - unsigned long end, page; - - assert(client < DIRTY_MEMORY_NUM); - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - bitmap_clear(ram_list.dirty_memory[client], page, end - page); -} +bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, + ram_addr_t length, + unsigned client); static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, ram_addr_t length) { - cpu_physical_memory_clear_dirty_range_type(start, length, DIRTY_MEMORY_MIGRATION); - cpu_physical_memory_clear_dirty_range_type(start, length, DIRTY_MEMORY_VGA); - cpu_physical_memory_clear_dirty_range_type(start, length, DIRTY_MEMORY_CODE); + cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); + cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); + cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); } -void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length, - unsigned client); - static inline uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest, ram_addr_t start, @@ -245,16 +234,14 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest, } } else { for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { - if (cpu_physical_memory_get_dirty(start + addr, - TARGET_PAGE_SIZE, - DIRTY_MEMORY_MIGRATION)) { + if (cpu_physical_memory_test_and_clear_dirty( + start + addr, + TARGET_PAGE_SIZE, + DIRTY_MEMORY_MIGRATION)) { long k = (start + addr) >> TARGET_PAGE_BITS; if (!test_and_set_bit(k, dest)) { num_dirty++; } - cpu_physical_memory_reset_dirty(start + addr, - TARGET_PAGE_SIZE, - DIRTY_MEMORY_MIGRATION); } } } -- cgit v1.2.3 From 5f2cb94688bd0b2c88e0fc1ac3c4582965b7b106 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Tue, 2 Dec 2014 11:23:19 +0000 Subject: memory: make cpu_physical_memory_sync_dirty_bitmap() fully atomic The fast path of cpu_physical_memory_sync_dirty_bitmap() directly manipulates the dirty bitmap. Use atomic_xchg() to make the test-and-clear atomic. Signed-off-by: Stefan Hajnoczi Message-Id: <1417519399-3166-7-git-send-email-stefanha@redhat.com> [Only do xchg on nonzero words. - Paolo] Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- include/exec/ram_addr.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/exec/ram_addr.h') diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index a2beea7925..c113f21140 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -224,12 +224,12 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest, for (k = page; k < page + nr; k++) { if (src[k]) { + unsigned long bits = atomic_xchg(&src[k], 0); unsigned long new_dirty; new_dirty = ~dest[k]; - dest[k] |= src[k]; - new_dirty &= src[k]; + dest[k] |= bits; + new_dirty &= bits; num_dirty += ctpopl(new_dirty); - src[k] = 0; } } } else { -- cgit v1.2.3