From b858c92c39a76d6a4aedea05b4f9bf801055f824 Mon Sep 17 00:00:00 2001

Message-Id: <b858c92c39a76d6a4aedea05b4f9bf801055f824@dist-git>

From: Peter Krempa <pkrempa@redhat.com>

Date: Wed, 7 Sep 2016 16:12:13 +0200

Subject: [PATCH] qemu: cgroup: Extract temporary relaxing of cgroup setting

 for vcpu hotplug

https://bugzilla.redhat.com/show_bug.cgi?id=1097930

When hot-adding vcpus qemu needs to allocate some structures in the DMA

zone which may be outside of the numa pinning. Extract the code doing

this in a set of helpers so that it can be reused.

(cherry picked from commit eb5dee353479629d8769de0cfe77c59711c70d79)

---

 src/qemu/qemu_cgroup.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++

 src/qemu/qemu_cgroup.h | 11 ++++++

 src/qemu/qemu_driver.c | 36 +++-----------------

 3 files changed, 105 insertions(+), 32 deletions(-)

diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c

index 2dca874..e2b5bab 100644

--- a/src/qemu/qemu_cgroup.c

+++ b/src/qemu/qemu_cgroup.c

@@ -1127,3 +1127,93 @@ qemuRemoveCgroup(virDomainObjPtr vm)

     return virCgroupRemove(priv->cgroup);

 }

+

+

+static void

+qemuCgroupEmulatorAllNodesDataFree(qemuCgroupEmulatorAllNodesDataPtr data)

+{

+    if (!data)

+        return;

+

+    virCgroupFree(&data->emulatorCgroup);

+    VIR_FREE(data->emulatorMemMask);

+    VIR_FREE(data);

+}

+

+

+/**

+ * qemuCgroupEmulatorAllNodesAllow:

+ * @cgroup: domain cgroup pointer

+ * @retData: filled with structure used to roll back the operation

+ *

+ * Allows all NUMA nodes for the qemu emulator thread temporarily. This is

+ * necessary when hotplugging cpus since it requires memory allocated in the

+ * DMA region. Afterwards the operation can be reverted by

+ * qemuCgrouEmulatorAllNodesRestore.

+ *

+ * Returns 0 on success -1 on error

+ */

+int

+qemuCgroupEmulatorAllNodesAllow(virCgroupPtr cgroup,

+                                qemuCgroupEmulatorAllNodesDataPtr *retData)

+{

+    qemuCgroupEmulatorAllNodesDataPtr data = NULL;

+    char *all_nodes_str = NULL;

+    virBitmapPtr all_nodes = NULL;

+    int ret = -1;

+

+    if (!virNumaIsAvailable() ||

+        !virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPUSET))

+        return 0;

+

+    if (!(all_nodes = virNumaGetHostNodeset()))

+        goto cleanup;

+

+    if (!(all_nodes_str = virBitmapFormat(all_nodes)))

+        goto cleanup;

+

+    if (VIR_ALLOC(data) < 0)

+        goto cleanup;

+

+    if (virCgroupNewThread(cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,

+                           false, &data->emulatorCgroup) < 0)

+        goto cleanup;

+

+    if (virCgroupGetCpusetMems(data->emulatorCgroup, &data->emulatorMemMask) < 0 ||

+        virCgroupSetCpusetMems(data->emulatorCgroup, all_nodes_str) < 0)

+        goto cleanup;

+

+    VIR_STEAL_PTR(*retData, data);

+    ret = 0;

+

+ cleanup:

+    VIR_FREE(all_nodes_str);

+    virBitmapFree(all_nodes);

+    qemuCgroupEmulatorAllNodesDataFree(data);

+

+    return ret;

+}

+

+

+/**

+ * qemuCgrouEmulatorAllNodesRestore:

+ * @data: data structure created by qemuCgroupEmulatorAllNodesAllow

+ *

+ * Rolls back the setting done by qemuCgroupEmulatorAllNodesAllow and frees the

+ * associated data.

+ */

+void

+qemuCgrouEmulatorAllNodesRestore(qemuCgroupEmulatorAllNodesDataPtr data)

+{

+    virErrorPtr err;

+

+    if (!data)

+        return;

+

+    err = virSaveLastError();

+    virCgroupSetCpusetMems(data->emulatorCgroup, data->emulatorMemMask);

+    virSetError(err);

+    virFreeError(err);

+

+    qemuCgroupEmulatorAllNodesDataFree(data);

+}

diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h

index dc340a1..e6ebae0 100644

--- a/src/qemu/qemu_cgroup.h

+++ b/src/qemu/qemu_cgroup.h

@@ -57,4 +57,15 @@ int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask);

 int qemuSetupGlobalCpuCgroup(virDomainObjPtr vm);

 int qemuRemoveCgroup(virDomainObjPtr vm);

+typedef struct _qemuCgroupEmulatorAllNodesData qemuCgroupEmulatorAllNodesData;

+typedef qemuCgroupEmulatorAllNodesData *qemuCgroupEmulatorAllNodesDataPtr;

+struct _qemuCgroupEmulatorAllNodesData {

+    virCgroupPtr emulatorCgroup;

+    char *emulatorMemMask;

+};

+

+int qemuCgroupEmulatorAllNodesAllow(virCgroupPtr cgroup,

+                                    qemuCgroupEmulatorAllNodesDataPtr *data);

+void qemuCgrouEmulatorAllNodesRestore(qemuCgroupEmulatorAllNodesDataPtr data);

+

 #endif /* __QEMU_CGROUP_H__ */

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c

index 33bfb67..6775327 100644

--- a/src/qemu/qemu_driver.c

+++ b/src/qemu/qemu_driver.c

@@ -4859,11 +4859,7 @@ qemuDomainSetVcpusLive(virQEMUDriverPtr driver,

                        unsigned int nvcpus)

 {

     qemuDomainObjPrivatePtr priv = vm->privateData;

-    virCgroupPtr cgroup_temp = NULL;

-    char *mem_mask = NULL;

-    char *all_nodes_str = NULL;

-    virBitmapPtr all_nodes = NULL;

-    virErrorPtr err = NULL;

+    qemuCgroupEmulatorAllNodesDataPtr emulatorCgroup = NULL;

     virBitmapPtr vcpumap = NULL;

     ssize_t nextvcpu = -1;

     int rc = 0;

@@ -4872,22 +4868,8 @@ qemuDomainSetVcpusLive(virQEMUDriverPtr driver,

     if (!(vcpumap = qemuDomainSelectHotplugVcpuEntities(vm->def, nvcpus)))

         goto cleanup;

-    if (virNumaIsAvailable() &&

-        virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {

-        if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,

-                               false, &cgroup_temp) < 0)

-            goto cleanup;

-

-        if (!(all_nodes = virNumaGetHostNodeset()))

-            goto cleanup;

-

-        if (!(all_nodes_str = virBitmapFormat(all_nodes)))

-            goto cleanup;

-

-        if (virCgroupGetCpusetMems(cgroup_temp, &mem_mask) < 0 ||

-            virCgroupSetCpusetMems(cgroup_temp, all_nodes_str) < 0)

-            goto cleanup;

-    }

+    if (qemuCgroupEmulatorAllNodesAllow(priv->cgroup, &emulatorCgroup) < 0)

+        goto cleanup;

     if (nvcpus > virDomainDefGetVcpus(vm->def)) {

         while ((nextvcpu = virBitmapNextSetBit(vcpumap, nextvcpu)) != -1) {

@@ -4915,17 +4897,7 @@ qemuDomainSetVcpusLive(virQEMUDriverPtr driver,

     ret = 0;

  cleanup:

-    if (mem_mask) {

-        err = virSaveLastError();

-        virCgroupSetCpusetMems(cgroup_temp, mem_mask);

-        virSetError(err);

-        virFreeError(err);

-        VIR_FREE(mem_mask);

-    }

-

-    VIR_FREE(all_nodes_str);

-    virBitmapFree(all_nodes);

-    virCgroupFree(&cgroup_temp);

+    qemuCgrouEmulatorAllNodesRestore(emulatorCgroup);

     virBitmapFree(vcpumap);

     return ret;

-- 

2.10.0