From 35a6ed4f716c1027c04d12a3c499b2e9d6b62c24 Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:42:52 +0800 Subject: migration: Introduce capability 'x-colo' to migration We add helper function colo_supported() to indicate whether colo is supported or not, with which we use to control whether or not showing 'x-colo' string to users, they can use qmp command 'query-migrate-capabilities' or hmp command 'info migrate_capabilities' to learn if colo is supported. The default value for COLO (COarse-Grain LOck Stepping) is disabled. Cc: Juan Quintela Cc: Amit Shah Cc: Eric Blake Cc: Markus Armbruster Signed-off-by: zhanghailiang Signed-off-by: Li Zhijian Signed-off-by: Gonglei Reviewed-by: Eric Blake Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- docs/qmp-commands.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'docs') diff --git a/docs/qmp-commands.txt b/docs/qmp-commands.txt index 284576d795..9bdbbfe42f 100644 --- a/docs/qmp-commands.txt +++ b/docs/qmp-commands.txt @@ -2861,6 +2861,7 @@ Enable/Disable migration capabilities - "compress": use multiple compression threads to accelerate live migration - "events": generate events for each migration state change - "postcopy-ram": postcopy mode for live migration +- "x-colo": COarse-Grain LOck Stepping (COLO) for Non-stop Service Arguments: @@ -2882,6 +2883,7 @@ Query current migration capabilities - "compress": Multiple compression threads state (json-bool) - "events": Migration state change event state (json-bool) - "postcopy-ram": postcopy ram state (json-bool) + - "x-colo": COarse-Grain LOck Stepping for Non-stop Service (json-bool) Arguments: @@ -2895,7 +2897,8 @@ Example: {"state": false, "capability": "zero-blocks"}, {"state": false, "capability": "compress"}, {"state": true, "capability": "events"}, - {"state": false, "capability": "postcopy-ram"} + {"state": false, "capability": "postcopy-ram"}, + {"state": false, "capability": "x-colo"} ]} migrate-set-parameters -- cgit v1.2.3-55-g7522 From 68b5359187c3d4164cc546dcdd5ba3e37f0ffb55 Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:43:01 +0800 Subject: COLO: Add checkpoint-delay parameter for migrate-set-parameters Add checkpoint-delay parameter for migrate-set-parameters, so that we can control the checkpoint frequency when COLO is in periodic mode. Cc: Luiz Capitulino Cc: Eric Blake Cc: Markus Armbruster Signed-off-by: zhanghailiang Signed-off-by: Li Zhijian Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Eric Blake Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- docs/qmp-commands.txt | 2 ++ hmp.c | 8 ++++++++ migration/migration.c | 16 ++++++++++++++++ qapi-schema.json | 12 ++++++++++-- 4 files changed, 36 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/qmp-commands.txt b/docs/qmp-commands.txt index 9bdbbfe42f..2e42929b6e 100644 --- a/docs/qmp-commands.txt +++ b/docs/qmp-commands.txt @@ -2916,6 +2916,8 @@ Set migration parameters - "max-bandwidth": set maximum speed for migrations (in bytes/sec) (json-int) - "downtime-limit": set maximum tolerated downtime (in milliseconds) for migrations (json-int) +- "x-checkpoint-delay": set the delay time for periodic checkpoint (json-int) + Arguments: Example: diff --git a/hmp.c b/hmp.c index 3d602594a2..f2b831a4a3 100644 --- a/hmp.c +++ b/hmp.c @@ -318,6 +318,9 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict) monitor_printf(mon, " %s: %" PRId64 " milliseconds", MigrationParameter_lookup[MIGRATION_PARAMETER_DOWNTIME_LIMIT], params->downtime_limit); + monitor_printf(mon, " %s: %" PRId64, + MigrationParameter_lookup[MIGRATION_PARAMETER_X_CHECKPOINT_DELAY], + params->x_checkpoint_delay); monitor_printf(mon, "\n"); } @@ -1386,6 +1389,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict) p.has_downtime_limit = true; use_int_value = true; break; + case MIGRATION_PARAMETER_X_CHECKPOINT_DELAY: + p.has_x_checkpoint_delay = true; + use_int_value = true; + break; } if (use_int_value) { @@ -1402,6 +1409,7 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict) p.cpu_throttle_initial = valueint; p.cpu_throttle_increment = valueint; p.downtime_limit = valueint; + p.x_checkpoint_delay = valueint; } qmp_migrate_set_parameters(&p, &err); diff --git a/migration/migration.c b/migration/migration.c index d8421b5085..d87065a270 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -63,6 +63,11 @@ /* Migration XBZRLE default cache size */ #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024) +/* The delay time (in ms) between two COLO checkpoints + * Note: Please change this default value to 10000 when we support hybrid mode. + */ +#define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY 200 + static NotifierList migration_state_notifiers = NOTIFIER_LIST_INITIALIZER(migration_state_notifiers); @@ -95,6 +100,7 @@ MigrationState *migrate_get_current(void) .cpu_throttle_increment = DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT, .max_bandwidth = MAX_THROTTLE, .downtime_limit = DEFAULT_MIGRATE_SET_DOWNTIME, + .x_checkpoint_delay = DEFAULT_MIGRATE_X_CHECKPOINT_DELAY, }, }; @@ -587,6 +593,7 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp) params->max_bandwidth = s->parameters.max_bandwidth; params->has_downtime_limit = true; params->downtime_limit = s->parameters.downtime_limit; + params->x_checkpoint_delay = s->parameters.x_checkpoint_delay; return params; } @@ -845,6 +852,11 @@ void qmp_migrate_set_parameters(MigrationParameters *params, Error **errp) "an integer in the range of 0 to 2000000 milliseconds"); return; } + if (params->has_x_checkpoint_delay && (params->x_checkpoint_delay < 0)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "x_checkpoint_delay", + "is invalid, it should be positive"); + } if (params->has_compress_level) { s->parameters.compress_level = params->compress_level; @@ -879,6 +891,10 @@ void qmp_migrate_set_parameters(MigrationParameters *params, Error **errp) if (params->has_downtime_limit) { s->parameters.downtime_limit = params->downtime_limit; } + + if (params->has_x_checkpoint_delay) { + s->parameters.x_checkpoint_delay = params->x_checkpoint_delay; + } } diff --git a/qapi-schema.json b/qapi-schema.json index ee58fe61cd..05fed9a2c0 100644 --- a/qapi-schema.json +++ b/qapi-schema.json @@ -674,19 +674,24 @@ # @downtime-limit: set maximum tolerated downtime for migration. maximum # downtime in milliseconds (Since 2.8) # +# @x-checkpoint-delay: The delay time (in ms) between two COLO checkpoints in +# periodic mode. (Since 2.8) +# # Since: 2.4 ## { 'enum': 'MigrationParameter', 'data': ['compress-level', 'compress-threads', 'decompress-threads', 'cpu-throttle-initial', 'cpu-throttle-increment', 'tls-creds', 'tls-hostname', 'max-bandwidth', - 'downtime-limit'] } + 'downtime-limit', 'x-checkpoint-delay' ] } # # @migrate-set-parameters # # Set various migration parameters. See MigrationParameters for details. # +# @x-checkpoint-delay: the delay time between two checkpoints. (Since 2.8) +# # Since: 2.4 ## { 'command': 'migrate-set-parameters', 'boxed': true, @@ -735,6 +740,8 @@ # @downtime-limit: set maximum tolerated downtime for migration. maximum # downtime in milliseconds (Since 2.8) # +# @x-checkpoint-delay: the delay time between two COLO checkpoints. (Since 2.8) +# # Since: 2.4 ## { 'struct': 'MigrationParameters', @@ -746,7 +753,8 @@ '*tls-creds': 'str', '*tls-hostname': 'str', '*max-bandwidth': 'int', - '*downtime-limit': 'int'} } + '*downtime-limit': 'int', + '*x-checkpoint-delay': 'int'} } ## # @query-migrate-parameters -- cgit v1.2.3-55-g7522 From d89e666e0666a0023e4aa6b6b4c4d25d049c5215 Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:43:03 +0800 Subject: COLO: Add 'x-colo-lost-heartbeat' command to trigger failover We leave users to choose whatever heartbeat solution they want, if the heartbeat is lost, or other errors they detect, they can use experimental command 'x_colo_lost_heartbeat' to tell COLO to do failover, COLO will do operations accordingly. For example, if the command is sent to the Primary side, the Primary side will exit COLO mode, does cleanup work, and then, PVM will take over the service work. If sent to the Secondary side, the Secondary side will run failover work, then takes over PVM's service work. Cc: Luiz Capitulino Cc: Eric Blake Cc: Markus Armbruster Signed-off-by: zhanghailiang Signed-off-by: Li Zhijian Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- docs/qmp-commands.txt | 10 ++++++++++ hmp-commands.hx | 15 +++++++++++++++ hmp.c | 8 ++++++++ hmp.h | 1 + include/migration/colo.h | 3 +++ include/migration/failover.h | 20 ++++++++++++++++++++ migration/Makefile.objs | 2 +- migration/colo-comm.c | 11 +++++++++++ migration/colo-failover.c | 42 ++++++++++++++++++++++++++++++++++++++++++ migration/colo.c | 1 + qapi-schema.json | 29 +++++++++++++++++++++++++++++ stubs/migration-colo.c | 8 ++++++++ 12 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 include/migration/failover.h create mode 100644 migration/colo-failover.c (limited to 'docs') diff --git a/docs/qmp-commands.txt b/docs/qmp-commands.txt index 2e42929b6e..a4732a570c 100644 --- a/docs/qmp-commands.txt +++ b/docs/qmp-commands.txt @@ -554,6 +554,16 @@ Example: -> { "execute": "migrate_set_downtime", "arguments": { "value": 0.1 } } <- { "return": {} } +x-colo-lost-heartbeat +-------------------- + +Tell COLO that heartbeat is lost, a failover or takeover is needed. + +Example: + +-> { "execute": "x-colo-lost-heartbeat" } +<- { "return": {} } + client_migrate_info ------------------- diff --git a/hmp-commands.hx b/hmp-commands.hx index 06bef470b9..88192817b2 100644 --- a/hmp-commands.hx +++ b/hmp-commands.hx @@ -1037,6 +1037,21 @@ STEXI @findex migrate_start_postcopy Switch in-progress migration to postcopy mode. Ignored after the end of migration (or once already in postcopy). +ETEXI + + { + .name = "x_colo_lost_heartbeat", + .args_type = "", + .params = "", + .help = "Tell COLO that heartbeat is lost,\n\t\t\t" + "a failover or takeover is needed.", + .cmd = hmp_x_colo_lost_heartbeat, + }, + +STEXI +@item x_colo_lost_heartbeat +@findex x_colo_lost_heartbeat +Tell COLO that heartbeat is lost, a failover or takeover is needed. ETEXI { diff --git a/hmp.c b/hmp.c index f2b831a4a3..00af4230bf 100644 --- a/hmp.c +++ b/hmp.c @@ -1451,6 +1451,14 @@ void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict) hmp_handle_error(mon, &err); } +void hmp_x_colo_lost_heartbeat(Monitor *mon, const QDict *qdict) +{ + Error *err = NULL; + + qmp_x_colo_lost_heartbeat(&err); + hmp_handle_error(mon, &err); +} + void hmp_set_password(Monitor *mon, const QDict *qdict) { const char *protocol = qdict_get_str(qdict, "protocol"); diff --git a/hmp.h b/hmp.h index 184769c13f..05daf7cd5c 100644 --- a/hmp.h +++ b/hmp.h @@ -72,6 +72,7 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict); void hmp_migrate_set_cache_size(Monitor *mon, const QDict *qdict); void hmp_client_migrate_info(Monitor *mon, const QDict *qdict); void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict); +void hmp_x_colo_lost_heartbeat(Monitor *mon, const QDict *qdict); void hmp_set_password(Monitor *mon, const QDict *qdict); void hmp_expire_password(Monitor *mon, const QDict *qdict); void hmp_eject(Monitor *mon, const QDict *qdict); diff --git a/include/migration/colo.h b/include/migration/colo.h index b40676c748..e9ac2c343a 100644 --- a/include/migration/colo.h +++ b/include/migration/colo.h @@ -17,6 +17,7 @@ #include "migration/migration.h" #include "qemu/coroutine_int.h" #include "qemu/thread.h" +#include "qemu/main-loop.h" bool colo_supported(void); void colo_info_init(void); @@ -29,4 +30,6 @@ bool migration_incoming_enable_colo(void); void migration_incoming_exit_colo(void); void *colo_process_incoming_thread(void *opaque); bool migration_incoming_in_colo_state(void); + +COLOMode get_colo_mode(void); #endif diff --git a/include/migration/failover.h b/include/migration/failover.h new file mode 100644 index 0000000000..3274735068 --- /dev/null +++ b/include/migration/failover.h @@ -0,0 +1,20 @@ +/* + * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO) + * (a.k.a. Fault Tolerance or Continuous Replication) + * + * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD. + * Copyright (c) 2016 FUJITSU LIMITED + * Copyright (c) 2016 Intel Corporation + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + */ + +#ifndef QEMU_FAILOVER_H +#define QEMU_FAILOVER_H + +#include "qemu-common.h" + +void failover_request_active(Error **errp); + +#endif diff --git a/migration/Makefile.objs b/migration/Makefile.objs index 4bbe9ab313..3f3e237142 100644 --- a/migration/Makefile.objs +++ b/migration/Makefile.objs @@ -1,7 +1,7 @@ common-obj-y += migration.o socket.o fd.o exec.o common-obj-y += tls.o -common-obj-$(CONFIG_COLO) += colo.o common-obj-y += colo-comm.o +common-obj-$(CONFIG_COLO) += colo.o colo-failover.o common-obj-y += vmstate.o common-obj-y += qemu-file.o common-obj-y += qemu-file-channel.o diff --git a/migration/colo-comm.c b/migration/colo-comm.c index bf44f76440..20b60ec384 100644 --- a/migration/colo-comm.c +++ b/migration/colo-comm.c @@ -21,6 +21,17 @@ typedef struct { static COLOInfo colo_info; +COLOMode get_colo_mode(void) +{ + if (migration_in_colo_state()) { + return COLO_MODE_PRIMARY; + } else if (migration_incoming_in_colo_state()) { + return COLO_MODE_SECONDARY; + } else { + return COLO_MODE_UNKNOWN; + } +} + static void colo_info_pre_save(void *opaque) { COLOInfo *s = opaque; diff --git a/migration/colo-failover.c b/migration/colo-failover.c new file mode 100644 index 0000000000..e31fc10c82 --- /dev/null +++ b/migration/colo-failover.c @@ -0,0 +1,42 @@ +/* + * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO) + * (a.k.a. Fault Tolerance or Continuous Replication) + * + * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. + * Copyright (c) 2016 FUJITSU LIMITED + * Copyright (c) 2016 Intel Corporation + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "migration/colo.h" +#include "migration/failover.h" +#include "qmp-commands.h" +#include "qapi/qmp/qerror.h" + +static QEMUBH *failover_bh; + +static void colo_failover_bh(void *opaque) +{ + qemu_bh_delete(failover_bh); + failover_bh = NULL; + /* TODO: Do failover work */ +} + +void failover_request_active(Error **errp) +{ + failover_bh = qemu_bh_new(colo_failover_bh, NULL); + qemu_bh_schedule(failover_bh); +} + +void qmp_x_colo_lost_heartbeat(Error **errp) +{ + if (get_colo_mode() == COLO_MODE_UNKNOWN) { + error_setg(errp, QERR_FEATURE_DISABLED, "colo"); + return; + } + + failover_request_active(errp); +} diff --git a/migration/colo.c b/migration/colo.c index a35cc59731..12fa0b4111 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -18,6 +18,7 @@ #include "trace.h" #include "qemu/error-report.h" #include "qapi/error.h" +#include "migration/failover.h" #define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024) diff --git a/qapi-schema.json b/qapi-schema.json index 05fed9a2c0..a18484115f 100644 --- a/qapi-schema.json +++ b/qapi-schema.json @@ -823,6 +823,35 @@ 'vmstate-send', 'vmstate-size', 'vmstate-received', 'vmstate-loaded' ] } +## +# @COLOMode +# +# The colo mode +# +# @unknown: unknown mode +# +# @primary: master side +# +# @secondary: slave side +# +# Since: 2.8 +## +{ 'enum': 'COLOMode', + 'data': [ 'unknown', 'primary', 'secondary'] } + +## +# @x-colo-lost-heartbeat +# +# Tell qemu that heartbeat is lost, request it to do takeover procedures. +# If this command is sent to the PVM, the Primary side will exit COLO mode. +# If sent to the Secondary, the Secondary side will run failover work, +# then takes over server operation to become the service VM. +# +# Since: 2.8 +## +{ 'command': 'x-colo-lost-heartbeat' } + +## # @MouseInfo: # # Information about a mouse device. diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c index 7b723954d5..7811764c4b 100644 --- a/stubs/migration-colo.c +++ b/stubs/migration-colo.c @@ -12,6 +12,7 @@ #include "qemu/osdep.h" #include "migration/colo.h" +#include "qmp-commands.h" bool colo_supported(void) { @@ -36,3 +37,10 @@ void *colo_process_incoming_thread(void *opaque) { return NULL; } + +void qmp_x_colo_lost_heartbeat(Error **errp) +{ + error_setg(errp, "COLO is not supported, please rerun configure" + " with --enable-colo option in order to support" + " COLO feature"); +} -- cgit v1.2.3-55-g7522 From e59887d8c9ebc132d14567f6552bb017e5bc724d Mon Sep 17 00:00:00 2001 From: zhanghailiang Date: Thu, 27 Oct 2016 14:43:07 +0800 Subject: docs: Add documentation for COLO feature Introduce the design of COLO, and how to test it. Signed-off-by: zhanghailiang Reviewed-by: Amit Shah Signed-off-by: Amit Shah --- docs/COLO-FT.txt | 189 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 docs/COLO-FT.txt (limited to 'docs') diff --git a/docs/COLO-FT.txt b/docs/COLO-FT.txt new file mode 100644 index 0000000000..628293824c --- /dev/null +++ b/docs/COLO-FT.txt @@ -0,0 +1,189 @@ +COarse-grained LOck-stepping Virtual Machines for Non-stop Service +---------------------------------------- +Copyright (c) 2016 Intel Corporation +Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. +Copyright (c) 2016 Fujitsu, Corp. + +This work is licensed under the terms of the GNU GPL, version 2 or later. +See the COPYING file in the top-level directory. + +This document gives an overview of COLO's design and how to use it. + +== Background == +Virtual machine (VM) replication is a well known technique for providing +application-agnostic software-implemented hardware fault tolerance, +also known as "non-stop service". + +COLO (COarse-grained LOck-stepping) is a high availability solution. +Both primary VM (PVM) and secondary VM (SVM) run in parallel. They receive the +same request from client, and generate response in parallel too. +If the response packets from PVM and SVM are identical, they are released +immediately. Otherwise, a VM checkpoint (on demand) is conducted. + +== Architecture == + +The architecture of COLO is shown in the diagram below. +It consists of a pair of networked physical nodes: +The primary node running the PVM, and the secondary node running the SVM +to maintain a valid replica of the PVM. +PVM and SVM execute in parallel and generate output of response packets for +client requests according to the application semantics. + +The incoming packets from the client or external network are received by the +primary node, and then forwarded to the secondary node, so that both the PVM +and the SVM are stimulated with the same requests. + +COLO receives the outbound packets from both the PVM and SVM and compares them +before allowing the output to be sent to clients. + +The SVM is qualified as a valid replica of the PVM, as long as it generates +identical responses to all client requests. Once the differences in the outputs +are detected between the PVM and SVM, COLO withholds transmission of the +outbound packets until it has successfully synchronized the PVM state to the SVM. + + Primary Node Secondary Node + +------------+ +-----------------------+ +------------------------+ +------------+ + | | | HeartBeat |<----->| HeartBeat | | | + | Primary VM | +-----------|-----------+ +-----------|------------+ |Secondary VM| + | | | | | | + | | +-----------|-----------+ +-----------|------------+ | | + | | |QEMU +---v----+ | |QEMU +----v---+ | | | + | | | |Failover| | | |Failover| | | | + | | | +--------+ | | +--------+ | | | + | | | +---------------+ | | +---------------+ | | | + | | | | VM Checkpoint |-------------->| VM Checkpoint | | | | + | | | +---------------+ | | +---------------+ | | | + | | | | | | | | + |Requests<---------------------------^------------------------------------------>Requests| + |Responses----------------------\ /--|--------------\ /------------------------Responses| + | | | | | | | | | | | | | + | | | +-----------+ | | | | | | | +------------+ | | | + | | | | COLO disk | | | | | | | | | COLO disk | | | | + | | | | Manager |-|-|--|--------------|--|->| Manager | | | | + | | | +|----------+ | | | | | | | +-----------|+ | | | + | | | | | | | | | | | | | | | + +------------+ +--|------------|-|--|--+ +---|--|--------------|--+ +------------+ + | | | | | | | + +-------------+ | +----------v-v--|--+ +---|--v-----------+ | +-------------+ + | VM Monitor | | | COLO Proxy | | COLO Proxy | | | VM Monitor | + | | | |(compare packet) | | (adjust sequence)| | | | + +-------------+ | +----------|----^--+ +------------------+ | +-------------+ + | | | | + +------------------|------------|----|--+ +---------------------|------------------+ + | Kernel | | | | | Kernel | | + +------------------|------------|----|--+ +---------------------|------------------+ + | | | | + +--------------v+ +--------v----|--+ +------------------+ +v-------------+ + | Storage | |External Network| | External Network | | Storage | + +---------------+ +----------------+ +------------------+ +--------------+ + +== Components introduction == + +You can see there are several components in COLO's diagram of architecture. +Their functions are described below. + +HeartBeat: +Runs on both the primary and secondary nodes, to periodically check platform +availability. When the primary node suffers a hardware fail-stop failure, +the heartbeat stops responding, the secondary node will trigger a failover +as soon as it determines the absence. + +COLO disk Manager: +When primary VM writes data into image, the colo disk manger captures this data +and sends it to secondary VM's which makes sure the context of secondary VM's +image is consistent with the context of primary VM 's image. +For more details, please refer to docs/block-replication.txt. + +Checkpoint/Failover Controller: +Modifications of save/restore flow to realize continuous migration, +to make sure the state of VM in Secondary side is always consistent with VM in +Primary side. + +COLO Proxy: +Delivers packets to Primary and Seconday, and then compare the responses from +both side. Then decide whether to start a checkpoint according to some rules. +Please refer to docs/colo-proxy.txt for more informations. + +Note: +HeartBeat has not been implemented yet, so you need to trigger failover process +by using 'x-colo-lost-heartbeat' command. + +== Test procedure == +1. Startup qemu +Primary: +# qemu-kvm -enable-kvm -m 2048 -smp 2 -qmp stdio -vnc :7 -name primary \ + -device piix3-usb-uhci \ + -device usb-tablet -netdev tap,id=hn0,vhost=off \ + -device virtio-net-pci,id=net-pci0,netdev=hn0 \ + -drive if=virtio,id=primary-disk0,driver=quorum,read-pattern=fifo,vote-threshold=1,\ + children.0.file.filename=1.raw,\ + children.0.driver=raw -S +Secondary: +# qemu-kvm -enable-kvm -m 2048 -smp 2 -qmp stdio -vnc :7 -name secondary \ + -device piix3-usb-uhci \ + -device usb-tablet -netdev tap,id=hn0,vhost=off \ + -device virtio-net-pci,id=net-pci0,netdev=hn0 \ + -drive if=none,id=secondary-disk0,file.filename=1.raw,driver=raw,node-name=node0 \ + -drive if=virtio,id=active-disk0,driver=replication,mode=secondary,\ + file.driver=qcow2,top-id=active-disk0,\ + file.file.filename=/mnt/ramfs/active_disk.img,\ + file.backing.driver=qcow2,\ + file.backing.file.filename=/mnt/ramfs/hidden_disk.img,\ + file.backing.backing=secondary-disk0 \ + -incoming tcp:0:8888 + +2. On Secondary VM's QEMU monitor, issue command +{'execute':'qmp_capabilities'} +{ 'execute': 'nbd-server-start', + 'arguments': {'addr': {'type': 'inet', 'data': {'host': 'xx.xx.xx.xx', 'port': '8889'} } } +} +{'execute': 'nbd-server-add', 'arguments': {'device': 'secondeary-disk0', 'writable': true } } + +Note: + a. The qmp command nbd-server-start and nbd-server-add must be run + before running the qmp command migrate on primary QEMU + b. Active disk, hidden disk and nbd target's length should be the + same. + c. It is better to put active disk and hidden disk in ramdisk. + +3. On Primary VM's QEMU monitor, issue command: +{'execute':'qmp_capabilities'} +{ 'execute': 'human-monitor-command', + 'arguments': {'command-line': 'drive_add -n buddy driver=replication,mode=primary,file.driver=nbd,file.host=xx.xx.xx.xx,file.port=8889,file.export=secondary-disk0,node-name=nbd_client0'}} +{ 'execute':'x-blockdev-change', 'arguments':{'parent': 'primary-disk0', 'node': 'nbd_client0' } } +{ 'execute': 'migrate-set-capabilities', + 'arguments': {'capabilities': [ {'capability': 'x-colo', 'state': true } ] } } +{ 'execute': 'migrate', 'arguments': {'uri': 'tcp:xx.xx.xx.xx:8888' } } + + Note: + a. There should be only one NBD Client for each primary disk. + b. xx.xx.xx.xx is the secondary physical machine's hostname or IP + c. The qmp command line must be run after running qmp command line in + secondary qemu. + +4. After the above steps, you will see, whenever you make changes to PVM, SVM will be synced. +You can issue command '{ "execute": "migrate-set-parameters" , "arguments":{ "x-checkpoint-delay": 2000 } }' +to change the checkpoint period time + +5. Failover test +You can kill Primary VM and run 'x_colo_lost_heartbeat' in Secondary VM's +monitor at the same time, then SVM will failover and client will not detect this +change. + +Before issuing '{ "execute": "x-colo-lost-heartbeat" }' command, we have to +issue block related command to stop block replication. +Primary: + Remove the nbd child from the quorum: + { 'execute': 'x-blockdev-change', 'arguments': {'parent': 'colo-disk0', 'child': 'children.1'}} + { 'execute': 'human-monitor-command','arguments': {'command-line': 'drive_del blk-buddy0'}} + Note: there is no qmp command to remove the blockdev now + +Secondary: + The primary host is down, so we should do the following thing: + { 'execute': 'nbd-server-stop' } + +== TODO == +1. Support continuous VM replication. +2. Support shared storage. +3. Develop the heartbeat part. +4. Reduce checkpoint VM’s downtime while doing checkpoint. -- cgit v1.2.3-55-g7522