From 395cb450095408e0f7b57d69c22a14120f47dc52 Mon Sep 17 00:00:00 2001 From: Bihong Yu Date: Tue, 20 Oct 2020 11:10:43 +0800 Subject: migration: Add spaces around operator Signed-off-by: Bihong Yu Reviewed-by: Chuan Zheng Reviewed-by: Dr. David Alan Gilbert Message-Id: <1603163448-27122-4-git-send-email-yubihong@huawei.com> Reviewed-by: Juan Quintela Signed-off-by: Dr. David Alan Gilbert --- migration/migration.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'migration/migration.c') diff --git a/migration/migration.c b/migration/migration.c index 0575ecb379..e050f5726b 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -2478,8 +2478,8 @@ static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname, * Since we currently insist on matching page sizes, just sanity check * we're being asked for whole host pages. */ - if (start & (our_host_ps-1) || - (len & (our_host_ps-1))) { + if (start & (our_host_ps - 1) || + (len & (our_host_ps - 1))) { error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT " len: %zd", __func__, start, len); mark_source_rp_bad(ms); -- cgit v1.2.3 From 7a267fc49b5a06cb395fe437996d5df517783ff8 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 21 Oct 2020 17:27:17 -0400 Subject: migration: Introduce migrate_send_rp_message_req_pages() This is another layer wrapper for sending a page request to the source VM. The new migrate_send_rp_message_req_pages() will be used elsewhere in coming patches. Reviewed-by: Dr. David Alan Gilbert Signed-off-by: Peter Xu Message-Id: <20201021212721.440373-3-peterx@redhat.com> Signed-off-by: Dr. David Alan Gilbert --- migration/migration.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'migration/migration.c') diff --git a/migration/migration.c b/migration/migration.c index e050f5726b..ed122ff8d9 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -316,8 +316,8 @@ error: * Start: Address offset within the RB * Len: Length in bytes required - must be a multiple of pagesize */ -int migrate_send_rp_req_pages(MigrationIncomingState *mis, RAMBlock *rb, - ram_addr_t start) +int migrate_send_rp_message_req_pages(MigrationIncomingState *mis, + RAMBlock *rb, ram_addr_t start) { uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */ size_t msglen = 12; /* start + len */ @@ -353,6 +353,12 @@ int migrate_send_rp_req_pages(MigrationIncomingState *mis, RAMBlock *rb, return migrate_send_rp_message(mis, msg_type, msglen, bufc); } +int migrate_send_rp_req_pages(MigrationIncomingState *mis, + RAMBlock *rb, ram_addr_t start) +{ + return migrate_send_rp_message_req_pages(mis, rb, start); +} + static bool migration_colo_enabled; bool migration_incoming_colo_enabled(void) { -- cgit v1.2.3 From 8f8bfffcf1b486cee9a3bc79bb9b174682b06e22 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 21 Oct 2020 17:27:18 -0400 Subject: migration: Maintain postcopy faulted addresses Maintain a list of faulted addresses on the destination host for which we're waiting on. This is implemented using a GTree rather than a real list to make sure even there're plenty of vCPUs/threads that are faulting, the lookup will still be fast with O(log(N)) (because we'll do that after placing each page). It should bring a slight overhead, but ideally that shouldn't be a big problem simply because in most cases the requested page list will be short. Actually we did similar things for postcopy blocktime measurements. This patch didn't use that simply because: (1) blocktime measurement is towards vcpu threads only, but here we need to record all faulted addresses, including main thread and external thread (like, DPDK via vhost-user). (2) blocktime measurement will require UFFD_FEATURE_THREAD_ID, but here we don't want to add that extra dependency on the kernel version since not necessary. E.g., we don't need to know which thread faulted on which page, we also don't care about multiple threads faulting on the same page. But we only care about what addresses are faulted so waiting for a page copying from src. (3) blocktime measurement is not enabled by default. However we need this by default especially for postcopy recover. Another thing to mention is that this patch introduced a new mutex to serialize the receivedmap and the page_requested tree, however that serialization does not cover other procedures like UFFDIO_COPY. Signed-off-by: Peter Xu Message-Id: <20201021212721.440373-4-peterx@redhat.com> Reviewed-by: Dr. David Alan Gilbert Signed-off-by: Dr. David Alan Gilbert --- migration/migration.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) (limited to 'migration/migration.c') diff --git a/migration/migration.c b/migration/migration.c index ed122ff8d9..5421042d4a 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -143,6 +143,13 @@ static int migration_maybe_pause(MigrationState *s, int new_state); static void migrate_fd_cancel(MigrationState *s); +static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp) +{ + uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp; + + return (a > b) - (a < b); +} + void migration_object_init(void) { MachineState *ms = MACHINE(qdev_get_machine()); @@ -165,6 +172,8 @@ void migration_object_init(void) qemu_event_init(¤t_incoming->main_thread_load_event, false); qemu_sem_init(¤t_incoming->postcopy_pause_sem_dst, 0); qemu_sem_init(¤t_incoming->postcopy_pause_sem_fault, 0); + qemu_mutex_init(¤t_incoming->page_request_mutex); + current_incoming->page_requested = g_tree_new(page_request_addr_cmp); if (!migration_object_check(current_migration, &err)) { error_report_err(err); @@ -240,6 +249,11 @@ void migration_incoming_state_destroy(void) qemu_event_reset(&mis->main_thread_load_event); + if (mis->page_requested) { + g_tree_destroy(mis->page_requested); + mis->page_requested = NULL; + } + if (mis->socket_address_list) { qapi_free_SocketAddressList(mis->socket_address_list); mis->socket_address_list = NULL; @@ -354,8 +368,33 @@ int migrate_send_rp_message_req_pages(MigrationIncomingState *mis, } int migrate_send_rp_req_pages(MigrationIncomingState *mis, - RAMBlock *rb, ram_addr_t start) + RAMBlock *rb, ram_addr_t start, uint64_t haddr) { + void *aligned = (void *)(uintptr_t)(haddr & (-qemu_ram_pagesize(rb))); + bool received; + + WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) { + received = ramblock_recv_bitmap_test_byte_offset(rb, start); + if (!received && !g_tree_lookup(mis->page_requested, aligned)) { + /* + * The page has not been received, and it's not yet in the page + * request list. Queue it. Set the value of element to 1, so that + * things like g_tree_lookup() will return TRUE (1) when found. + */ + g_tree_insert(mis->page_requested, aligned, (gpointer)1); + mis->page_requested_count++; + trace_postcopy_page_req_add(aligned, mis->page_requested_count); + } + } + + /* + * If the page is there, skip sending the message. We don't even need the + * lock because as long as the page arrived, it'll be there forever. + */ + if (received) { + return 0; + } + return migrate_send_rp_message_req_pages(mis, rb, start); } -- cgit v1.2.3 From d246ea5039fd6d5344becd0943fcbb7f8e6bbfe7 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 21 Oct 2020 17:27:20 -0400 Subject: migration/postcopy: Release fd before going into 'postcopy-pause' Logically below race could trigger with the old code: test program migration thread ------------ ---------------- wait_until('postcopy-pause') postcopy_pause() set_state('postcopy-pause') do_postcopy_recover() arm s->to_dst_file with new fd release s->to_dst_file [1] Here [1] could have released the just-installed recoverying channel. Then the migration could hang without really resuming. Instead, it should be very safe to release the fd before setting the state into 'postcopy-pause', because there's no reason for any other thread to touch it during 'postcopy-active'. Dave reported a very rare postcopy recovery hang that the migration-test program waited for the migration to complete in migrate_postcopy_complete(). We do suspect it's the same thing that we're gonna fix here. Hard to tell. However since we've noticed this, fix this irrelevant of the hang report. Cc: Dr. David Alan Gilbert Cc: Juan Quintela Reviewed-by: Dr. David Alan Gilbert Signed-off-by: Peter Xu Message-Id: <20201021212721.440373-6-peterx@redhat.com> Signed-off-by: Dr. David Alan Gilbert --- migration/migration.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'migration/migration.c') diff --git a/migration/migration.c b/migration/migration.c index 5421042d4a..627f194b8b 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -3178,9 +3178,6 @@ static MigThrError postcopy_pause(MigrationState *s) while (true) { QEMUFile *file; - migrate_set_state(&s->state, s->state, - MIGRATION_STATUS_POSTCOPY_PAUSED); - /* Current channel is possibly broken. Release it. */ assert(s->to_dst_file); qemu_mutex_lock(&s->qemu_file_lock); @@ -3191,6 +3188,9 @@ static MigThrError postcopy_pause(MigrationState *s) qemu_file_shutdown(file); qemu_fclose(file); + migrate_set_state(&s->state, s->state, + MIGRATION_STATUS_POSTCOPY_PAUSED); + error_report("Detected IO failure for postcopy. " "Migration paused."); -- cgit v1.2.3