From 35a6ed4f716c1027c04d12a3c499b2e9d6b62c24 Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:42:52 +0800 Subject: migration: Introduce capability 'x-colo' to migration We add helper function colo_supported() to indicate whether colo is supported or not, with which we use to control whether or not showing 'x-colo' string to users, they can use qmp command 'query-migrate-capabilities' or hmp command 'info migrate_capabilities' to learn if colo is supported. The default value for COLO (COarse-Grain LOck Stepping) is disabled. Cc: Juan Quintela Cc: Amit Shah Cc: Eric Blake Cc: Markus Armbruster Signed-off-by: zhanghailiang Signed-off-by: Li Zhijian Signed-off-by: Gonglei Reviewed-by: Eric Blake Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- include/migration/colo.h | 20 ++++++++++++++++++++ include/migration/migration.h | 1 + 2 files changed, 21 insertions(+) create mode 100644 include/migration/colo.h (limited to 'include') diff --git a/include/migration/colo.h b/include/migration/colo.h new file mode 100644 index 0000000000..59a632a418 --- /dev/null +++ b/include/migration/colo.h @@ -0,0 +1,20 @@ +/* + * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO) + * (a.k.a. Fault Tolerance or Continuous Replication) + * + * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. + * Copyright (c) 2016 FUJITSU LIMITED + * Copyright (c) 2016 Intel Corporation + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + */ + +#ifndef QEMU_COLO_H +#define QEMU_COLO_H + +#include "qemu-common.h" + +bool colo_supported(void); + +#endif diff --git a/include/migration/migration.h b/include/migration/migration.h index 2791b90c00..34f442b2b3 100644 --- a/include/migration/migration.h +++ b/include/migration/migration.h @@ -298,6 +298,7 @@ int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen); int migrate_use_xbzrle(void); int64_t migrate_xbzrle_cache_size(void); +bool migrate_colo_enabled(void); int64_t xbzrle_cache_resize(int64_t new_size); -- cgit v1.2.3-55-g7522 From 5821ebf93b1da9f74dc04c20e2923aadfaf803df Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:42:53 +0800 Subject: COLO: migrate COLO related info to secondary node We can determine whether or not VM in destination should go into COLO mode by referring to the info that was migrated. We skip this section if COLO is not enabled (i.e. migrate_set_capability colo off), so that, It doesn't break compatibility with migration no matter whether users configure the --enable-colo/disable-colo on the source/destination side or not; Signed-off-by: zhanghailiang Signed-off-by: Li Zhijian Signed-off-by: Gonglei Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- include/migration/colo.h | 2 ++ migration/Makefile.objs | 1 + migration/colo-comm.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ vl.c | 3 +++ 4 files changed, 57 insertions(+) create mode 100644 migration/colo-comm.c (limited to 'include') diff --git a/include/migration/colo.h b/include/migration/colo.h index 59a632a418..1c899a0bc4 100644 --- a/include/migration/colo.h +++ b/include/migration/colo.h @@ -14,7 +14,9 @@ #define QEMU_COLO_H #include "qemu-common.h" +#include "migration/migration.h" bool colo_supported(void); +void colo_info_init(void); #endif diff --git a/migration/Makefile.objs b/migration/Makefile.objs index cff96f0a06..4bbe9ab313 100644 --- a/migration/Makefile.objs +++ b/migration/Makefile.objs @@ -1,6 +1,7 @@ common-obj-y += migration.o socket.o fd.o exec.o common-obj-y += tls.o common-obj-$(CONFIG_COLO) += colo.o +common-obj-y += colo-comm.o common-obj-y += vmstate.o common-obj-y += qemu-file.o common-obj-y += qemu-file-channel.o diff --git a/migration/colo-comm.c b/migration/colo-comm.c new file mode 100644 index 0000000000..a2d5185a8a --- /dev/null +++ b/migration/colo-comm.c @@ -0,0 +1,51 @@ +/* + * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO) + * (a.k.a. Fault Tolerance or Continuous Replication) + * + * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. + * Copyright (c) 2016 FUJITSU LIMITED + * Copyright (c) 2016 Intel Corporation + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + * + */ + +#include "qemu/osdep.h" +#include +#include "trace.h" + +typedef struct { + bool colo_requested; +} COLOInfo; + +static COLOInfo colo_info; + +static void colo_info_pre_save(void *opaque) +{ + COLOInfo *s = opaque; + + s->colo_requested = migrate_colo_enabled(); +} + +static bool colo_info_need(void *opaque) +{ + return migrate_colo_enabled(); +} + +static const VMStateDescription colo_state = { + .name = "COLOState", + .version_id = 1, + .minimum_version_id = 1, + .pre_save = colo_info_pre_save, + .needed = colo_info_need, + .fields = (VMStateField[]) { + VMSTATE_BOOL(colo_requested, COLOInfo), + VMSTATE_END_OF_LIST() + }, +}; + +void colo_info_init(void) +{ + vmstate_register(NULL, 0, &colo_state, &colo_info); +} diff --git a/vl.c b/vl.c index 74dfe4eef9..a1cee4f3f4 100644 --- a/vl.c +++ b/vl.c @@ -90,6 +90,7 @@ int main(int argc, char **argv) #include "audio/audio.h" #include "migration/migration.h" #include "sysemu/cpus.h" +#include "migration/colo.h" #include "sysemu/kvm.h" #include "qapi/qmp/qjson.h" #include "qemu/option.h" @@ -4426,6 +4427,8 @@ int main(int argc, char **argv, char **envp) #endif } + colo_info_init(); + if (net_init_clients() < 0) { exit(1); } -- cgit v1.2.3-55-g7522 From 0b827d5e7291193887d22d058bc20c12b423047c Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:42:54 +0800 Subject: migration: Enter into COLO mode after migration if COLO is enabled Add a new migration state: MIGRATION_STATUS_COLO. Migration source side enters this state after the first live migration successfully finished if COLO is enabled by command 'migrate_set_capability x-colo on'. We reuse migration thread, so the process of checkpointing will be handled in migration thread. Signed-off-by: zhanghailiang Signed-off-by: Li Zhijian Signed-off-by: Gonglei Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- include/migration/colo.h | 3 +++ migration/colo.c | 29 +++++++++++++++++++++++++++++ migration/migration.c | 38 +++++++++++++++++++++++++++++++++----- migration/trace-events | 3 +++ qapi-schema.json | 4 +++- stubs/migration-colo.c | 9 +++++++++ 6 files changed, 80 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/include/migration/colo.h b/include/migration/colo.h index 1c899a0bc4..bf84b99cc5 100644 --- a/include/migration/colo.h +++ b/include/migration/colo.h @@ -19,4 +19,7 @@ bool colo_supported(void); void colo_info_init(void); +void migrate_start_colo_process(MigrationState *s); +bool migration_in_colo_state(void); + #endif diff --git a/migration/colo.c b/migration/colo.c index d21505794e..f480431972 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -11,9 +11,38 @@ */ #include "qemu/osdep.h" +#include "sysemu/sysemu.h" #include "migration/colo.h" +#include "trace.h" bool colo_supported(void) { return false; } + +bool migration_in_colo_state(void) +{ + MigrationState *s = migrate_get_current(); + + return (s->state == MIGRATION_STATUS_COLO); +} + +static void colo_process_checkpoint(MigrationState *s) +{ + qemu_mutex_lock_iothread(); + vm_start(); + qemu_mutex_unlock_iothread(); + trace_colo_vm_state_change("stop", "run"); + + /* TODO: COLO checkpoint savevm loop */ + +} + +void migrate_start_colo_process(MigrationState *s) +{ + qemu_mutex_unlock_iothread(); + migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_COLO); + colo_process_checkpoint(s); + qemu_mutex_lock_iothread(); +} diff --git a/migration/migration.c b/migration/migration.c index c7869e5779..aff3eb460d 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -695,6 +695,10 @@ MigrationInfo *qmp_query_migrate(Error **errp) get_xbzrle_cache_stats(info); break; + case MIGRATION_STATUS_COLO: + info->has_status = true; + /* TODO: display COLO specific information (checkpoint info etc.) */ + break; case MIGRATION_STATUS_COMPLETED: get_xbzrle_cache_stats(info); @@ -1113,7 +1117,8 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk, params.shared = has_inc && inc; if (migration_is_setup_or_active(s->state) || - s->state == MIGRATION_STATUS_CANCELLING) { + s->state == MIGRATION_STATUS_CANCELLING || + s->state == MIGRATION_STATUS_COLO) { error_setg(errp, QERR_MIGRATION_ACTIVE); return; } @@ -1661,7 +1666,11 @@ static void migration_completion(MigrationState *s, int current_active_state, if (!ret) { ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE); - if (ret >= 0) { + /* + * Don't mark the image with BDRV_O_INACTIVE flag if + * we will go into COLO stage later. + */ + if (ret >= 0 && !migrate_colo_enabled()) { ret = bdrv_inactivate_all(); } if (ret >= 0) { @@ -1703,8 +1712,11 @@ static void migration_completion(MigrationState *s, int current_active_state, goto fail_invalidate; } - migrate_set_state(&s->state, current_active_state, - MIGRATION_STATUS_COMPLETED); + if (!migrate_colo_enabled()) { + migrate_set_state(&s->state, current_active_state, + MIGRATION_STATUS_COMPLETED); + } + return; fail_invalidate: @@ -1749,6 +1761,7 @@ static void *migration_thread(void *opaque) bool entered_postcopy = false; /* The active state we expect to be in; ACTIVE or POSTCOPY_ACTIVE */ enum MigrationStatus current_active_state = MIGRATION_STATUS_ACTIVE; + bool enable_colo = migrate_colo_enabled(); rcu_register_thread(); @@ -1857,7 +1870,13 @@ static void *migration_thread(void *opaque) end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); qemu_mutex_lock_iothread(); - qemu_savevm_state_cleanup(); + /* + * The resource has been allocated by migration will be reused in COLO + * process, so don't release them. + */ + if (!enable_colo) { + qemu_savevm_state_cleanup(); + } if (s->state == MIGRATION_STATUS_COMPLETED) { uint64_t transferred_bytes = qemu_ftell(s->to_dst_file); s->total_time = end_time - s->total_time; @@ -1870,6 +1889,15 @@ static void *migration_thread(void *opaque) } runstate_set(RUN_STATE_POSTMIGRATE); } else { + if (s->state == MIGRATION_STATUS_ACTIVE && enable_colo) { + migrate_start_colo_process(s); + qemu_savevm_state_cleanup(); + /* + * Fixme: we will run VM in COLO no matter its old running state. + * After exited COLO, we will keep running. + */ + old_vm_running = true; + } if (old_vm_running && !entered_postcopy) { vm_start(); } else { diff --git a/migration/trace-events b/migration/trace-events index dfee75abf4..a29e5a09ea 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -207,3 +207,6 @@ migration_tls_outgoing_handshake_complete(void) "" migration_tls_incoming_handshake_start(void) "" migration_tls_incoming_handshake_error(const char *err) "err=%s" migration_tls_incoming_handshake_complete(void) "" + +# migration/colo.c +colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'" diff --git a/qapi-schema.json b/qapi-schema.json index 0fb4d7ed3d..8b212152a3 100644 --- a/qapi-schema.json +++ b/qapi-schema.json @@ -459,12 +459,14 @@ # # @failed: some error occurred during migration process. # +# @colo: VM is in the process of fault tolerance. (since 2.8) +# # Since: 2.3 # ## { 'enum': 'MigrationStatus', 'data': [ 'none', 'setup', 'cancelling', 'cancelled', - 'active', 'postcopy-active', 'completed', 'failed' ] } + 'active', 'postcopy-active', 'completed', 'failed', 'colo' ] } ## # @MigrationInfo diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c index d21505794e..0c8eef4f38 100644 --- a/stubs/migration-colo.c +++ b/stubs/migration-colo.c @@ -17,3 +17,12 @@ bool colo_supported(void) { return false; } + +bool migration_in_colo_state(void) +{ + return false; +} + +void migrate_start_colo_process(MigrationState *s) +{ +} -- cgit v1.2.3-55-g7522 From 25d0c16f625feb3b6b9bf8079388cdd314e63916 Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:42:55 +0800 Subject: migration: Switch to COLO process after finishing loadvm Switch from normal migration loadvm process into COLO checkpoint process if COLO mode is enabled. We add three new members to struct MigrationIncomingState, 'have_colo_incoming_thread' and 'colo_incoming_thread' record the COLO related thread for secondary VM, 'migration_incoming_co' records the original migration incoming coroutine. Signed-off-by: zhanghailiang Signed-off-by: Li Zhijian Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- include/migration/colo.h | 7 +++++++ include/migration/migration.h | 7 +++++++ migration/colo-comm.c | 10 ++++++++++ migration/colo.c | 21 +++++++++++++++++++++ migration/migration.c | 12 ++++++++++++ stubs/migration-colo.c | 10 ++++++++++ 6 files changed, 67 insertions(+) (limited to 'include') diff --git a/include/migration/colo.h b/include/migration/colo.h index bf84b99cc5..b40676c748 100644 --- a/include/migration/colo.h +++ b/include/migration/colo.h @@ -15,6 +15,8 @@ #include "qemu-common.h" #include "migration/migration.h" +#include "qemu/coroutine_int.h" +#include "qemu/thread.h" bool colo_supported(void); void colo_info_init(void); @@ -22,4 +24,9 @@ void colo_info_init(void); void migrate_start_colo_process(MigrationState *s); bool migration_in_colo_state(void); +/* loadvm */ +bool migration_incoming_enable_colo(void); +void migration_incoming_exit_colo(void); +void *colo_process_incoming_thread(void *opaque); +bool migration_incoming_in_colo_state(void); #endif diff --git a/include/migration/migration.h b/include/migration/migration.h index 34f442b2b3..c309d23370 100644 --- a/include/migration/migration.h +++ b/include/migration/migration.h @@ -21,6 +21,7 @@ #include "migration/vmstate.h" #include "qapi-types.h" #include "exec/cpu-common.h" +#include "qemu/coroutine_int.h" #define QEMU_VM_FILE_MAGIC 0x5145564d #define QEMU_VM_FILE_VERSION_COMPAT 0x00000002 @@ -107,6 +108,12 @@ struct MigrationIncomingState { QEMUBH *bh; int state; + + bool have_colo_incoming_thread; + QemuThread colo_incoming_thread; + /* The coroutine we should enter (back) after failover */ + Coroutine *migration_incoming_co; + /* See savevm.c */ LoadStateEntry_Head loadvm_handlers; }; diff --git a/migration/colo-comm.c b/migration/colo-comm.c index a2d5185a8a..bf44f76440 100644 --- a/migration/colo-comm.c +++ b/migration/colo-comm.c @@ -49,3 +49,13 @@ void colo_info_init(void) { vmstate_register(NULL, 0, &colo_state, &colo_info); } + +bool migration_incoming_enable_colo(void) +{ + return colo_info.colo_requested; +} + +void migration_incoming_exit_colo(void) +{ + colo_info.colo_requested = false; +} diff --git a/migration/colo.c b/migration/colo.c index f480431972..550cc5f122 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -27,6 +27,13 @@ bool migration_in_colo_state(void) return (s->state == MIGRATION_STATUS_COLO); } +bool migration_incoming_in_colo_state(void) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + + return mis && (mis->state == MIGRATION_STATUS_COLO); +} + static void colo_process_checkpoint(MigrationState *s) { qemu_mutex_lock_iothread(); @@ -46,3 +53,17 @@ void migrate_start_colo_process(MigrationState *s) colo_process_checkpoint(s); qemu_mutex_lock_iothread(); } + +void *colo_process_incoming_thread(void *opaque) +{ + MigrationIncomingState *mis = opaque; + + migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_COLO); + + /* TODO: COLO checkpoint restore loop */ + + migration_incoming_exit_colo(); + + return NULL; +} diff --git a/migration/migration.c b/migration/migration.c index aff3eb460d..d8421b5085 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -407,6 +407,18 @@ static void process_incoming_migration_co(void *opaque) /* Else if something went wrong then just fall out of the normal exit */ } + /* we get COLO info, and know if we are in COLO mode */ + if (!ret && migration_incoming_enable_colo()) { + mis->migration_incoming_co = qemu_coroutine_self(); + qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming", + colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE); + mis->have_colo_incoming_thread = true; + qemu_coroutine_yield(); + + /* Wait checkpoint incoming thread exit before free resource */ + qemu_thread_join(&mis->colo_incoming_thread); + } + qemu_fclose(f); free_xbzrle_decoded_buf(); diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c index 0c8eef4f38..7b723954d5 100644 --- a/stubs/migration-colo.c +++ b/stubs/migration-colo.c @@ -23,6 +23,16 @@ bool migration_in_colo_state(void) return false; } +bool migration_incoming_in_colo_state(void) +{ + return false; +} + void migrate_start_colo_process(MigrationState *s) { } + +void *colo_process_incoming_thread(void *opaque) +{ + return NULL; +} -- cgit v1.2.3-55-g7522 From d89e666e0666a0023e4aa6b6b4c4d25d049c5215 Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:43:03 +0800 Subject: COLO: Add 'x-colo-lost-heartbeat' command to trigger failover We leave users to choose whatever heartbeat solution they want, if the heartbeat is lost, or other errors they detect, they can use experimental command 'x_colo_lost_heartbeat' to tell COLO to do failover, COLO will do operations accordingly. For example, if the command is sent to the Primary side, the Primary side will exit COLO mode, does cleanup work, and then, PVM will take over the service work. If sent to the Secondary side, the Secondary side will run failover work, then takes over PVM's service work. Cc: Luiz Capitulino Cc: Eric Blake Cc: Markus Armbruster Signed-off-by: zhanghailiang Signed-off-by: Li Zhijian Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- docs/qmp-commands.txt | 10 ++++++++++ hmp-commands.hx | 15 +++++++++++++++ hmp.c | 8 ++++++++ hmp.h | 1 + include/migration/colo.h | 3 +++ include/migration/failover.h | 20 ++++++++++++++++++++ migration/Makefile.objs | 2 +- migration/colo-comm.c | 11 +++++++++++ migration/colo-failover.c | 42 ++++++++++++++++++++++++++++++++++++++++++ migration/colo.c | 1 + qapi-schema.json | 29 +++++++++++++++++++++++++++++ stubs/migration-colo.c | 8 ++++++++ 12 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 include/migration/failover.h create mode 100644 migration/colo-failover.c (limited to 'include') diff --git a/docs/qmp-commands.txt b/docs/qmp-commands.txt index 2e42929b6e..a4732a570c 100644 --- a/docs/qmp-commands.txt +++ b/docs/qmp-commands.txt @@ -554,6 +554,16 @@ Example: -> { "execute": "migrate_set_downtime", "arguments": { "value": 0.1 } } <- { "return": {} } +x-colo-lost-heartbeat +-------------------- + +Tell COLO that heartbeat is lost, a failover or takeover is needed. + +Example: + +-> { "execute": "x-colo-lost-heartbeat" } +<- { "return": {} } + client_migrate_info ------------------- diff --git a/hmp-commands.hx b/hmp-commands.hx index 06bef470b9..88192817b2 100644 --- a/hmp-commands.hx +++ b/hmp-commands.hx @@ -1037,6 +1037,21 @@ STEXI @findex migrate_start_postcopy Switch in-progress migration to postcopy mode. Ignored after the end of migration (or once already in postcopy). +ETEXI + + { + .name = "x_colo_lost_heartbeat", + .args_type = "", + .params = "", + .help = "Tell COLO that heartbeat is lost,\n\t\t\t" + "a failover or takeover is needed.", + .cmd = hmp_x_colo_lost_heartbeat, + }, + +STEXI +@item x_colo_lost_heartbeat +@findex x_colo_lost_heartbeat +Tell COLO that heartbeat is lost, a failover or takeover is needed. ETEXI { diff --git a/hmp.c b/hmp.c index f2b831a4a3..00af4230bf 100644 --- a/hmp.c +++ b/hmp.c @@ -1451,6 +1451,14 @@ void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict) hmp_handle_error(mon, &err); } +void hmp_x_colo_lost_heartbeat(Monitor *mon, const QDict *qdict) +{ + Error *err = NULL; + + qmp_x_colo_lost_heartbeat(&err); + hmp_handle_error(mon, &err); +} + void hmp_set_password(Monitor *mon, const QDict *qdict) { const char *protocol = qdict_get_str(qdict, "protocol"); diff --git a/hmp.h b/hmp.h index 184769c13f..05daf7cd5c 100644 --- a/hmp.h +++ b/hmp.h @@ -72,6 +72,7 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict); void hmp_migrate_set_cache_size(Monitor *mon, const QDict *qdict); void hmp_client_migrate_info(Monitor *mon, const QDict *qdict); void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict); +void hmp_x_colo_lost_heartbeat(Monitor *mon, const QDict *qdict); void hmp_set_password(Monitor *mon, const QDict *qdict); void hmp_expire_password(Monitor *mon, const QDict *qdict); void hmp_eject(Monitor *mon, const QDict *qdict); diff --git a/include/migration/colo.h b/include/migration/colo.h index b40676c748..e9ac2c343a 100644 --- a/include/migration/colo.h +++ b/include/migration/colo.h @@ -17,6 +17,7 @@ #include "migration/migration.h" #include "qemu/coroutine_int.h" #include "qemu/thread.h" +#include "qemu/main-loop.h" bool colo_supported(void); void colo_info_init(void); @@ -29,4 +30,6 @@ bool migration_incoming_enable_colo(void); void migration_incoming_exit_colo(void); void *colo_process_incoming_thread(void *opaque); bool migration_incoming_in_colo_state(void); + +COLOMode get_colo_mode(void); #endif diff --git a/include/migration/failover.h b/include/migration/failover.h new file mode 100644 index 0000000000..3274735068 --- /dev/null +++ b/include/migration/failover.h @@ -0,0 +1,20 @@ +/* + * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO) + * (a.k.a. Fault Tolerance or Continuous Replication) + * + * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD. + * Copyright (c) 2016 FUJITSU LIMITED + * Copyright (c) 2016 Intel Corporation + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + */ + +#ifndef QEMU_FAILOVER_H +#define QEMU_FAILOVER_H + +#include "qemu-common.h" + +void failover_request_active(Error **errp); + +#endif diff --git a/migration/Makefile.objs b/migration/Makefile.objs index 4bbe9ab313..3f3e237142 100644 --- a/migration/Makefile.objs +++ b/migration/Makefile.objs @@ -1,7 +1,7 @@ common-obj-y += migration.o socket.o fd.o exec.o common-obj-y += tls.o -common-obj-$(CONFIG_COLO) += colo.o common-obj-y += colo-comm.o +common-obj-$(CONFIG_COLO) += colo.o colo-failover.o common-obj-y += vmstate.o common-obj-y += qemu-file.o common-obj-y += qemu-file-channel.o diff --git a/migration/colo-comm.c b/migration/colo-comm.c index bf44f76440..20b60ec384 100644 --- a/migration/colo-comm.c +++ b/migration/colo-comm.c @@ -21,6 +21,17 @@ typedef struct { static COLOInfo colo_info; +COLOMode get_colo_mode(void) +{ + if (migration_in_colo_state()) { + return COLO_MODE_PRIMARY; + } else if (migration_incoming_in_colo_state()) { + return COLO_MODE_SECONDARY; + } else { + return COLO_MODE_UNKNOWN; + } +} + static void colo_info_pre_save(void *opaque) { COLOInfo *s = opaque; diff --git a/migration/colo-failover.c b/migration/colo-failover.c new file mode 100644 index 0000000000..e31fc10c82 --- /dev/null +++ b/migration/colo-failover.c @@ -0,0 +1,42 @@ +/* + * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO) + * (a.k.a. Fault Tolerance or Continuous Replication) + * + * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. + * Copyright (c) 2016 FUJITSU LIMITED + * Copyright (c) 2016 Intel Corporation + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "migration/colo.h" +#include "migration/failover.h" +#include "qmp-commands.h" +#include "qapi/qmp/qerror.h" + +static QEMUBH *failover_bh; + +static void colo_failover_bh(void *opaque) +{ + qemu_bh_delete(failover_bh); + failover_bh = NULL; + /* TODO: Do failover work */ +} + +void failover_request_active(Error **errp) +{ + failover_bh = qemu_bh_new(colo_failover_bh, NULL); + qemu_bh_schedule(failover_bh); +} + +void qmp_x_colo_lost_heartbeat(Error **errp) +{ + if (get_colo_mode() == COLO_MODE_UNKNOWN) { + error_setg(errp, QERR_FEATURE_DISABLED, "colo"); + return; + } + + failover_request_active(errp); +} diff --git a/migration/colo.c b/migration/colo.c index a35cc59731..12fa0b4111 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -18,6 +18,7 @@ #include "trace.h" #include "qemu/error-report.h" #include "qapi/error.h" +#include "migration/failover.h" #define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024) diff --git a/qapi-schema.json b/qapi-schema.json index 05fed9a2c0..a18484115f 100644 --- a/qapi-schema.json +++ b/qapi-schema.json @@ -823,6 +823,35 @@ 'vmstate-send', 'vmstate-size', 'vmstate-received', 'vmstate-loaded' ] } +## +# @COLOMode +# +# The colo mode +# +# @unknown: unknown mode +# +# @primary: master side +# +# @secondary: slave side +# +# Since: 2.8 +## +{ 'enum': 'COLOMode', + 'data': [ 'unknown', 'primary', 'secondary'] } + +## +# @x-colo-lost-heartbeat +# +# Tell qemu that heartbeat is lost, request it to do takeover procedures. +# If this command is sent to the PVM, the Primary side will exit COLO mode. +# If sent to the Secondary, the Secondary side will run failover work, +# then takes over server operation to become the service VM. +# +# Since: 2.8 +## +{ 'command': 'x-colo-lost-heartbeat' } + +## # @MouseInfo: # # Information about a mouse device. diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c index 7b723954d5..7811764c4b 100644 --- a/stubs/migration-colo.c +++ b/stubs/migration-colo.c @@ -12,6 +12,7 @@ #include "qemu/osdep.h" #include "migration/colo.h" +#include "qmp-commands.h" bool colo_supported(void) { @@ -36,3 +37,10 @@ void *colo_process_incoming_thread(void *opaque) { return NULL; } + +void qmp_x_colo_lost_heartbeat(Error **errp) +{ + error_setg(errp, "COLO is not supported, please rerun configure" + " with --enable-colo option in order to support" + " COLO feature"); +} -- cgit v1.2.3-55-g7522 From aef060850bd0e35aa7128e0ae3cef9d62c328314 Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:43:04 +0800 Subject: COLO: Introduce state to record failover process When handling failover, COLO processes differently according to the different stage of failover process, here we introduce a global atomic variable to record the status of failover. We add four failover status to indicate the different stage of failover process. You should use the helpers to get and set the value. Signed-off-by: zhanghailiang Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- include/migration/failover.h | 5 +++++ migration/colo-failover.c | 41 +++++++++++++++++++++++++++++++++++++++++ migration/colo.c | 4 ++++ migration/trace-events | 1 + qapi-schema.json | 18 ++++++++++++++++++ 5 files changed, 69 insertions(+) (limited to 'include') diff --git a/include/migration/failover.h b/include/migration/failover.h index 3274735068..7e0f36a274 100644 --- a/include/migration/failover.h +++ b/include/migration/failover.h @@ -14,7 +14,12 @@ #define QEMU_FAILOVER_H #include "qemu-common.h" +#include "qapi-types.h" +void failover_init_state(void); +FailoverStatus failover_set_state(FailoverStatus old_state, + FailoverStatus new_state); +FailoverStatus failover_get_state(void); void failover_request_active(Error **errp); #endif diff --git a/migration/colo-failover.c b/migration/colo-failover.c index e31fc10c82..6cca039eb9 100644 --- a/migration/colo-failover.c +++ b/migration/colo-failover.c @@ -15,22 +15,63 @@ #include "migration/failover.h" #include "qmp-commands.h" #include "qapi/qmp/qerror.h" +#include "qemu/error-report.h" +#include "trace.h" static QEMUBH *failover_bh; +static FailoverStatus failover_state; static void colo_failover_bh(void *opaque) { + int old_state; + qemu_bh_delete(failover_bh); failover_bh = NULL; + + old_state = failover_set_state(FAILOVER_STATUS_REQUIRE, + FAILOVER_STATUS_ACTIVE); + if (old_state != FAILOVER_STATUS_REQUIRE) { + error_report("Unknown error for failover, old_state = %s", + FailoverStatus_lookup[old_state]); + return; + } + /* TODO: Do failover work */ } void failover_request_active(Error **errp) { + if (failover_set_state(FAILOVER_STATUS_NONE, + FAILOVER_STATUS_REQUIRE) != FAILOVER_STATUS_NONE) { + error_setg(errp, "COLO failover is already actived"); + return; + } failover_bh = qemu_bh_new(colo_failover_bh, NULL); qemu_bh_schedule(failover_bh); } +void failover_init_state(void) +{ + failover_state = FAILOVER_STATUS_NONE; +} + +FailoverStatus failover_set_state(FailoverStatus old_state, + FailoverStatus new_state) +{ + FailoverStatus old; + + old = atomic_cmpxchg(&failover_state, old_state, new_state); + if (old == old_state) { + trace_colo_failover_set_state(FailoverStatus_lookup[new_state]); + } + return old; +} + +FailoverStatus failover_get_state(void) +{ + return atomic_read(&failover_state); +} + void qmp_x_colo_lost_heartbeat(Error **errp) { if (get_colo_mode() == COLO_MODE_UNKNOWN) { diff --git a/migration/colo.c b/migration/colo.c index 12fa0b4111..6b32c9183c 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -232,6 +232,8 @@ static void colo_process_checkpoint(MigrationState *s) Error *local_err = NULL; int ret; + failover_init_state(); + s->rp_state.from_dst_file = qemu_file_get_return_path(s->to_dst_file); if (!s->rp_state.from_dst_file) { error_report("Open QEMUFile from_dst_file failed"); @@ -332,6 +334,8 @@ void *colo_process_incoming_thread(void *opaque) migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, MIGRATION_STATUS_COLO); + failover_init_state(); + mis->to_src_file = qemu_file_get_return_path(mis->from_src_file); if (!mis->to_src_file) { error_report("COLO incoming thread: Open QEMUFile to_src_file failed"); diff --git a/migration/trace-events b/migration/trace-events index f374c8cf27..94134f700b 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -212,3 +212,4 @@ migration_tls_incoming_handshake_complete(void) "" colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'" colo_send_message(const char *msg) "Send '%s' message" colo_receive_message(const char *msg) "Receive '%s' message" +colo_failover_set_state(const char *new_state) "new state %s" diff --git a/qapi-schema.json b/qapi-schema.json index a18484115f..8a7b527091 100644 --- a/qapi-schema.json +++ b/qapi-schema.json @@ -839,6 +839,24 @@ { 'enum': 'COLOMode', 'data': [ 'unknown', 'primary', 'secondary'] } +## +# @FailoverStatus +# +# An enumeration of COLO failover status +# +# @none: no failover has ever happened +# +# @require: got failover requirement but not handled +# +# @active: in the process of doing failover +# +# @completed: finish the process of failover +# +# Since: 2.8 +## +{ 'enum': 'FailoverStatus', + 'data': [ 'none', 'require', 'active', 'completed'] } + ## # @x-colo-lost-heartbeat # -- cgit v1.2.3-55-g7522 From b3f7f0c5e6449be7275f1762bccbfa2177395a3b Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:43:05 +0800 Subject: COLO: Implement the process of failover for primary VM For primary side, if COLO gets failover request from users. To be exact, gets 'x_colo_lost_heartbeat' command. COLO thread will exit the loop while the failover BH does the cleanup work and resumes VM. Signed-off-by: zhanghailiang Signed-off-by: Li Zhijian Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- include/migration/colo.h | 3 +++ include/migration/failover.h | 1 + migration/colo-failover.c | 2 +- migration/colo.c | 50 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/migration/colo.h b/include/migration/colo.h index e9ac2c343a..e32eef4763 100644 --- a/include/migration/colo.h +++ b/include/migration/colo.h @@ -32,4 +32,7 @@ void *colo_process_incoming_thread(void *opaque); bool migration_incoming_in_colo_state(void); COLOMode get_colo_mode(void); + +/* failover */ +void colo_do_failover(MigrationState *s); #endif diff --git a/include/migration/failover.h b/include/migration/failover.h index 7e0f36a274..ad91ef2381 100644 --- a/include/migration/failover.h +++ b/include/migration/failover.h @@ -21,5 +21,6 @@ FailoverStatus failover_set_state(FailoverStatus old_state, FailoverStatus new_state); FailoverStatus failover_get_state(void); void failover_request_active(Error **errp); +bool failover_request_is_active(void); #endif diff --git a/migration/colo-failover.c b/migration/colo-failover.c index 6cca039eb9..cc229f5ab1 100644 --- a/migration/colo-failover.c +++ b/migration/colo-failover.c @@ -36,7 +36,7 @@ static void colo_failover_bh(void *opaque) return; } - /* TODO: Do failover work */ + colo_do_failover(NULL); } void failover_request_active(Error **errp) diff --git a/migration/colo.c b/migration/colo.c index 6b32c9183c..04f7c55a84 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -41,6 +41,40 @@ bool migration_incoming_in_colo_state(void) return mis && (mis->state == MIGRATION_STATUS_COLO); } +static bool colo_runstate_is_stopped(void) +{ + return runstate_check(RUN_STATE_COLO) || !runstate_is_running(); +} + +static void primary_vm_do_failover(void) +{ + MigrationState *s = migrate_get_current(); + int old_state; + + migrate_set_state(&s->state, MIGRATION_STATUS_COLO, + MIGRATION_STATUS_COMPLETED); + + old_state = failover_set_state(FAILOVER_STATUS_ACTIVE, + FAILOVER_STATUS_COMPLETED); + if (old_state != FAILOVER_STATUS_ACTIVE) { + error_report("Incorrect state (%s) while doing failover for Primary VM", + FailoverStatus_lookup[old_state]); + return; + } +} + +void colo_do_failover(MigrationState *s) +{ + /* Make sure VM stopped while failover happened. */ + if (!colo_runstate_is_stopped()) { + vm_stop_force_state(RUN_STATE_COLO); + } + + if (get_colo_mode() == COLO_MODE_PRIMARY) { + primary_vm_do_failover(); + } +} + static void colo_send_message(QEMUFile *f, COLOMessage msg, Error **errp) { @@ -162,9 +196,20 @@ static int colo_do_checkpoint_transaction(MigrationState *s, bioc->usage = 0; qemu_mutex_lock_iothread(); + if (failover_get_state() != FAILOVER_STATUS_NONE) { + qemu_mutex_unlock_iothread(); + goto out; + } vm_stop_force_state(RUN_STATE_COLO); qemu_mutex_unlock_iothread(); trace_colo_vm_state_change("run", "stop"); + /* + * Failover request bh could be called after vm_stop_force_state(), + * So we need check failover_request_is_active() again. + */ + if (failover_get_state() != FAILOVER_STATUS_NONE) { + goto out; + } /* Disable block migration */ s->params.blk = 0; @@ -259,6 +304,11 @@ static void colo_process_checkpoint(MigrationState *s) trace_colo_vm_state_change("stop", "run"); while (s->state == MIGRATION_STATUS_COLO) { + if (failover_get_state() != FAILOVER_STATUS_NONE) { + error_report("failover request"); + goto out; + } + current_time = qemu_clock_get_ms(QEMU_CLOCK_HOST); if (current_time - checkpoint_time < s->parameters.x_checkpoint_delay) { -- cgit v1.2.3-55-g7522