From b0acf414158ed72f3f2ac7550839af74b0530c8e Mon Sep 17 00:00:00 2001

From: Nigel Croxon <ncroxon@redhat.com>

Date: Wed, 31 Jul 2013 15:12:19 +0200

Subject: Force auto-convegence of live migration

RH-Author: Nigel Croxon <ncroxon@redhat.com>

Message-id: <1375283539-18714-4-git-send-email-ncroxon@redhat.com>

Patchwork-id: 52873

O-Subject: [RHEL7 PATCH 3/3] Force auto-convegence of live migration

Bugzilla: 985958

RH-Acked-by: Orit Wasserman <owasserm@redhat.com>

RH-Acked-by: Paolo Bonzini <pbonzini@redhat.com>

RH-Acked-by: Laszlo Ersek <lersek@redhat.com>

Bugzilla: 985958 - Throttle-down guest to help with live migration convergence (backport to RHEL7.0)

https://bugzilla.redhat.com/show_bug.cgi?id=985958

Backported from the following upstream commit:

commit 7ca1dfad952d8a8655b32e78623edcc38a51b14a

Author: Chegu Vinod <chegu_vinod@hp.com>

Date:   Mon Jun 24 03:47:39 2013 -0600

    Force auto-convegence of live migration

    If a user chooses to turn on the auto-converge migration capability

    these changes detect the lack of convergence and throttle down the

    guest. i.e. force the VCPUs out of the guest for some duration

    and let the migration thread catchup and help converge.

    Verified the convergence using the following :

     - Java Warehouse workload running on a 20VCPU/256G guest(~80% busy)

     - OLTP like workload running on a 80VCPU/512G guest (~80% busy)

    Sample results with Java warehouse workload : (migrate speed set to 20Gb and

    migrate downtime set to 4seconds).

     (qemu) info migrate

     capabilities: xbzrle: off auto-converge: off  <----

     Migration status: active

     total time: 1487503 milliseconds

     expected downtime: 519 milliseconds

     transferred ram: 383749347 kbytes

     remaining ram: 2753372 kbytes

     total ram: 268444224 kbytes

     duplicate: 65461532 pages

     skipped: 64901568 pages

     normal: 95750218 pages

     normal bytes: 383000872 kbytes

     dirty pages rate: 67551 pages

     ---

     (qemu) info migrate

     capabilities: xbzrle: off auto-converge: on   <----

     Migration status: completed

     total time: 241161 milliseconds

     downtime: 6373 milliseconds

     transferred ram: 28235307 kbytes

     remaining ram: 0 kbytes

     total ram: 268444224 kbytes

     duplicate: 64946416 pages

     skipped: 64903523 pages

     normal: 7044971 pages

     normal bytes: 28179884 kbytes

    Signed-off-by: Chegu Vinod <chegu_vinod@hp.com>

    Signed-off-by: Juan Quintela <quintela@redhat.com>

diff --git a/arch_init.c b/arch_init.c

index 522caeb..d7a5d7c 100644

--- a/arch_init.c

+++ b/arch_init.c

@@ -104,6 +104,9 @@ int graphic_depth = 15;

 #endif

 const uint32_t arch_type = QEMU_ARCH;

+static bool mig_throttle_on;

+static int dirty_rate_high_cnt;

+static void check_guest_throttling(void);

 /***********************************************************/

 /* ram save/restore */

@@ -378,8 +381,14 @@ static void migration_bitmap_sync(void)

     uint64_t num_dirty_pages_init = migration_dirty_pages;

     MigrationState *s = migrate_get_current();

     static int64_t start_time;

+    static int64_t bytes_xfer_prev;

     static int64_t num_dirty_pages_period;

     int64_t end_time;

+    int64_t bytes_xfer_now;

+

+    if (!bytes_xfer_prev) {

+        bytes_xfer_prev = ram_bytes_transferred();

+    }

     if (!start_time) {

         start_time = qemu_get_clock_ms(rt_clock);

@@ -404,6 +413,25 @@ static void migration_bitmap_sync(void)

     /* more than 1 second = 1000 millisecons */

     if (end_time > start_time + 1000) {

+        if (migrate_auto_converge()) {

+            /* The following detection logic can be refined later. For now:

+               Check to see if the dirtied bytes is 50% more than the approx.

+               amount of bytes that just got transferred since the last time we

+               were in this routine. If that happens >N times (for now N==4)

+               we turn on the throttle down logic */

+            bytes_xfer_now = ram_bytes_transferred();

+            if (s->dirty_pages_rate &&

+               (num_dirty_pages_period * TARGET_PAGE_SIZE >

+                   (bytes_xfer_now - bytes_xfer_prev)/2) &&

+               (dirty_rate_high_cnt++ > 4)) {

+                    trace_migration_throttle();

+                    mig_throttle_on = true;

+                    dirty_rate_high_cnt = 0;

+             }

+             bytes_xfer_prev = bytes_xfer_now;

+        } else {

+             mig_throttle_on = false;

+        }

         s->dirty_pages_rate = num_dirty_pages_period * 1000

             / (end_time - start_time);

         s->dirty_bytes_rate = s->dirty_pages_rate * TARGET_PAGE_SIZE;

@@ -566,6 +594,8 @@ static int ram_save_setup(QEMUFile *f, void *opaque)

     migration_bitmap = bitmap_new(ram_pages);

     bitmap_set(migration_bitmap, 0, ram_pages);

     migration_dirty_pages = ram_pages;

+    mig_throttle_on = false;

+    dirty_rate_high_cnt = 0;

     if (migrate_use_xbzrle()) {

         XBZRLE.cache = cache_init(migrate_xbzrle_cache_size() /

@@ -628,6 +658,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)

         }

         total_sent += bytes_sent;

         acct_info.iterations++;

+        check_guest_throttling();

         /* we want to check in the 1st loop, just in case it was the 1st time

            and we had to sync the dirty bitmap.

            qemu_get_clock_ns() is a bit expensive, so we only check each some

@@ -1097,3 +1128,53 @@ TargetInfo *qmp_query_target(Error **errp)

     return info;

 }

+

+/* Stub function that's gets run on the vcpu when its brought out of the

+   VM to run inside qemu via async_run_on_cpu()*/

+static void mig_sleep_cpu(void *opq)

+{

+    qemu_mutex_unlock_iothread();

+    g_usleep(30*1000);

+    qemu_mutex_lock_iothread();

+}

+

+/* To reduce the dirty rate explicitly disallow the VCPUs from spending

+   much time in the VM. The migration thread will try to catchup.

+   Workload will experience a performance drop.

+*/

+static void mig_throttle_cpu_down(CPUState *cpu, void *data)

+{

+    async_run_on_cpu(cpu, mig_sleep_cpu, NULL);

+}

+

+static void mig_throttle_guest_down(void)

+{

+    qemu_mutex_lock_iothread();

+    qemu_for_each_cpu(mig_throttle_cpu_down, NULL);

+    qemu_mutex_unlock_iothread();

+}

+

+static void check_guest_throttling(void)

+{

+    static int64_t t0;

+    int64_t        t1;

+

+    if (!mig_throttle_on) {

+        return;

+    }

+

+    if (!t0)  {

+        t0 = qemu_get_clock_ns(rt_clock);

+        return;

+    }

+

+    t1 = qemu_get_clock_ns(rt_clock);

+

+    /* If it has been more than 40 ms since the last time the guest

+     * was throttled then do it again.

+     */

+    if (40 < (t1-t0)/1000000) {

+        mig_throttle_guest_down();

+        t0 = t1;

+    }

+}

diff --git a/trace-events b/trace-events

index 9c73931..7cd335d 100644

--- a/trace-events

+++ b/trace-events

@@ -1031,6 +1031,7 @@ savevm_section_end(unsigned int section_id) "section_id %u"

 # arch_init.c

 migration_bitmap_sync_start(void) ""

 migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64""

+migration_throttle(void) ""

 # hw/qxl.c

 disable qxl_interface_set_mm_time(int qid, uint32_t mm_time) "%d %d"