From 298d5857152494dfd31f182ecc16f4aa16c5617f Mon Sep 17 00:00:00 2001

Message-Id: <298d5857152494dfd31f182ecc16f4aa16c5617f@dist-git>

From: "Michael R. Hines" <mrhines@us.ibm.com>

Date: Tue, 23 Sep 2014 15:47:57 +0200

Subject: [PATCH] qemu: RDMA migration support

This patch adds support for RDMA protocol in migration URIs.

USAGE: $ virsh migrate --live --migrateuri rdma://hostname domain qemu+ssh://hostname/system

Since libvirt runs QEMU in a pretty restricted environment, several

files needs to be added to cgroup_device_acl (in qemu.conf) for QEMU to

be able to access the host's infiniband hardware. Full documenation of

the feature can be found on QEMU wiki:

http://wiki.qemu.org/Features/RDMALiveMigration

https://bugzilla.redhat.com/show_bug.cgi?id=1013055

Signed-off-by: Michael R. Hines <mrhines@us.ibm.com>

Signed-off-by: Jiri Denemark <jdenemar@redhat.com>

(cherry picked from commit ed22a4743437b6f646eb33659a212dd252a5b65d)

Signed-off-by: Jiri Denemark <jdenemar@redhat.com>

---

 src/qemu/qemu.conf        |  8 ++++++++

 src/qemu/qemu_command.c   |  8 ++++++++

 src/qemu/qemu_migration.c | 39 +++++++++++++++++++++++++++++++++++++--

 3 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf

index 79bba36..92ca715 100644

--- a/src/qemu/qemu.conf

+++ b/src/qemu/qemu.conf

@@ -274,6 +274,14 @@

 #    "/dev/ptmx", "/dev/kvm", "/dev/kqemu",

 #    "/dev/rtc","/dev/hpet", "/dev/vfio/vfio"

 #]

+#

+# RDMA migration requires the following extra files to be added to the list:

+#   "/dev/infiniband/rdma_cm",

+#   "/dev/infiniband/issm0",

+#   "/dev/infiniband/issm1",

+#   "/dev/infiniband/umad0",

+#   "/dev/infiniband/umad1",

+#   "/dev/infiniband/uverbs0"

 # The default format for Qemu/KVM guest save images is raw; that is, the

diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c

index 05f328a..7d4f3cc 100644

--- a/src/qemu/qemu_command.c

+++ b/src/qemu/qemu_command.c

@@ -9475,6 +9475,14 @@ qemuBuildCommandLine(virConnectPtr conn,

                 goto error;

             }

             virCommandAddArg(cmd, migrateFrom);

+        } else if (STRPREFIX(migrateFrom, "rdma")) {

+            if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) {

+                virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",

+                               _("incoming RDMA migration is not supported "

+                                 "with this QEMU binary"));

+                goto error;

+            }

+            virCommandAddArg(cmd, migrateFrom);

         } else if (STREQ(migrateFrom, "stdio")) {

             if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD)) {

                 virCommandAddArgFormat(cmd, "fd:%d", migrateFd);

diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c

index 6b690b8..3433ec9 100644

--- a/src/qemu/qemu_migration.c

+++ b/src/qemu/qemu_migration.c

@@ -56,6 +56,7 @@

 #include "virhook.h"

 #include "virstring.h"

 #include "virtypedparam.h"

+#include "virprocess.h"

 #define VIR_FROM_THIS VIR_FROM_QEMU

@@ -2653,6 +2654,13 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,

                                        QEMU_MIGRATION_COOKIE_NBD)))

         goto cleanup;

+    if (STREQ(protocol, "rdma") && !vm->def->mem.hard_limit) {

+        virReportError(VIR_ERR_OPERATION_INVALID, "%s",

+                       _("cannot start RDMA migration with no memory hard "

+                         "limit set"));

+        goto cleanup;

+    }

+

     if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN) < 0)

         goto cleanup;

     qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PREPARE);

@@ -2696,6 +2704,11 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,

                                     QEMU_ASYNC_JOB_MIGRATION_IN) < 0)

         goto stop;

+    if (STREQ(protocol, "rdma") &&

+        virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) {

+        goto stop;

+    }

+

     if (mig->lockState) {

         VIR_DEBUG("Received lockstate %s", mig->lockState);

         VIR_FREE(priv->lockState);

@@ -2926,7 +2939,8 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver,

         if (!(uri = qemuMigrationParseURI(uri_in, &well_formed_uri)))

             goto cleanup;

-        if (STRNEQ(uri->scheme, "tcp")) {

+        if (STRNEQ(uri->scheme, "tcp") &&

+            STRNEQ(uri->scheme, "rdma")) {

             virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED,

                            _("unsupported scheme %s in migration URI %s"),

                            uri->scheme, uri_in);

@@ -3547,6 +3561,11 @@ qemuMigrationRun(virQEMUDriverPtr driver,

     switch (spec->destType) {

     case MIGRATION_DEST_HOST:

+        if (STREQ(spec->dest.host.protocol, "rdma") &&

+            virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) {

+            qemuDomainObjExitMonitor(driver, vm);

+            goto cleanup;

+        }

         ret = qemuMonitorMigrateToHost(priv->mon, migrate_flags,

                                        spec->dest.host.protocol,

                                        spec->dest.host.name,

@@ -3719,7 +3738,23 @@ static int doNativeMigrate(virQEMUDriverPtr driver,

     if (!(uribits = qemuMigrationParseURI(uri, NULL)))

         return -1;

-    if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD))

+    if (STREQ(uribits->scheme, "rdma")) {

+        if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) {

+            virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",

+                           _("outgoing RDMA migration is not supported "

+                             "with this QEMU binary"));

+            return -1;

+        }

+        if (!vm->def->mem.hard_limit) {

+            virReportError(VIR_ERR_OPERATION_INVALID, "%s",

+                           _("cannot start RDMA migration with no memory hard "

+                             "limit set"));

+            return -1;

+        }

+    }

+

+    if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD) &&

+        STRNEQ(uribits->scheme, "rdma"))

         spec.destType = MIGRATION_DEST_CONNECT_HOST;

     else

         spec.destType = MIGRATION_DEST_HOST;

-- 

2.1.1