From b858c92c39a76d6a4aedea05b4f9bf801055f824 Mon Sep 17 00:00:00 2001
Message-Id: <b858c92c39a76d6a4aedea05b4f9bf801055f824@dist-git>
From: Peter Krempa <pkrempa@redhat.com>
Date: Wed, 7 Sep 2016 16:12:13 +0200
Subject: [PATCH] qemu: cgroup: Extract temporary relaxing of cgroup setting
 for vcpu hotplug

https://bugzilla.redhat.com/show_bug.cgi?id=1097930

When hot-adding vcpus qemu needs to allocate some structures in the DMA
zone which may be outside of the numa pinning. Extract the code doing
this in a set of helpers so that it can be reused.

(cherry picked from commit eb5dee353479629d8769de0cfe77c59711c70d79)
---
 src/qemu/qemu_cgroup.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++
 src/qemu/qemu_cgroup.h | 11 ++++++
 src/qemu/qemu_driver.c | 36 +++-----------------
 3 files changed, 105 insertions(+), 32 deletions(-)

diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index 2dca874..e2b5bab 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -1127,3 +1127,93 @@ qemuRemoveCgroup(virDomainObjPtr vm)
 
     return virCgroupRemove(priv->cgroup);
 }
+
+
+static void
+qemuCgroupEmulatorAllNodesDataFree(qemuCgroupEmulatorAllNodesDataPtr data)
+{
+    if (!data)
+        return;
+
+    virCgroupFree(&data->emulatorCgroup);
+    VIR_FREE(data->emulatorMemMask);
+    VIR_FREE(data);
+}
+
+
+/**
+ * qemuCgroupEmulatorAllNodesAllow:
+ * @cgroup: domain cgroup pointer
+ * @retData: filled with structure used to roll back the operation
+ *
+ * Allows all NUMA nodes for the qemu emulator thread temporarily. This is
+ * necessary when hotplugging cpus since it requires memory allocated in the
+ * DMA region. Afterwards the operation can be reverted by
+ * qemuCgrouEmulatorAllNodesRestore.
+ *
+ * Returns 0 on success -1 on error
+ */
+int
+qemuCgroupEmulatorAllNodesAllow(virCgroupPtr cgroup,
+                                qemuCgroupEmulatorAllNodesDataPtr *retData)
+{
+    qemuCgroupEmulatorAllNodesDataPtr data = NULL;
+    char *all_nodes_str = NULL;
+    virBitmapPtr all_nodes = NULL;
+    int ret = -1;
+
+    if (!virNumaIsAvailable() ||
+        !virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
+        return 0;
+
+    if (!(all_nodes = virNumaGetHostNodeset()))
+        goto cleanup;
+
+    if (!(all_nodes_str = virBitmapFormat(all_nodes)))
+        goto cleanup;
+
+    if (VIR_ALLOC(data) < 0)
+        goto cleanup;
+
+    if (virCgroupNewThread(cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,
+                           false, &data->emulatorCgroup) < 0)
+        goto cleanup;
+
+    if (virCgroupGetCpusetMems(data->emulatorCgroup, &data->emulatorMemMask) < 0 ||
+        virCgroupSetCpusetMems(data->emulatorCgroup, all_nodes_str) < 0)
+        goto cleanup;
+
+    VIR_STEAL_PTR(*retData, data);
+    ret = 0;
+
+ cleanup:
+    VIR_FREE(all_nodes_str);
+    virBitmapFree(all_nodes);
+    qemuCgroupEmulatorAllNodesDataFree(data);
+
+    return ret;
+}
+
+
+/**
+ * qemuCgrouEmulatorAllNodesRestore:
+ * @data: data structure created by qemuCgroupEmulatorAllNodesAllow
+ *
+ * Rolls back the setting done by qemuCgroupEmulatorAllNodesAllow and frees the
+ * associated data.
+ */
+void
+qemuCgrouEmulatorAllNodesRestore(qemuCgroupEmulatorAllNodesDataPtr data)
+{
+    virErrorPtr err;
+
+    if (!data)
+        return;
+
+    err = virSaveLastError();
+    virCgroupSetCpusetMems(data->emulatorCgroup, data->emulatorMemMask);
+    virSetError(err);
+    virFreeError(err);
+
+    qemuCgroupEmulatorAllNodesDataFree(data);
+}
diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
index dc340a1..e6ebae0 100644
--- a/src/qemu/qemu_cgroup.h
+++ b/src/qemu/qemu_cgroup.h
@@ -57,4 +57,15 @@ int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask);
 int qemuSetupGlobalCpuCgroup(virDomainObjPtr vm);
 int qemuRemoveCgroup(virDomainObjPtr vm);
 
+typedef struct _qemuCgroupEmulatorAllNodesData qemuCgroupEmulatorAllNodesData;
+typedef qemuCgroupEmulatorAllNodesData *qemuCgroupEmulatorAllNodesDataPtr;
+struct _qemuCgroupEmulatorAllNodesData {
+    virCgroupPtr emulatorCgroup;
+    char *emulatorMemMask;
+};
+
+int qemuCgroupEmulatorAllNodesAllow(virCgroupPtr cgroup,
+                                    qemuCgroupEmulatorAllNodesDataPtr *data);
+void qemuCgrouEmulatorAllNodesRestore(qemuCgroupEmulatorAllNodesDataPtr data);
+
 #endif /* __QEMU_CGROUP_H__ */
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 33bfb67..6775327 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -4859,11 +4859,7 @@ qemuDomainSetVcpusLive(virQEMUDriverPtr driver,
                        unsigned int nvcpus)
 {
     qemuDomainObjPrivatePtr priv = vm->privateData;
-    virCgroupPtr cgroup_temp = NULL;
-    char *mem_mask = NULL;
-    char *all_nodes_str = NULL;
-    virBitmapPtr all_nodes = NULL;
-    virErrorPtr err = NULL;
+    qemuCgroupEmulatorAllNodesDataPtr emulatorCgroup = NULL;
     virBitmapPtr vcpumap = NULL;
     ssize_t nextvcpu = -1;
     int rc = 0;
@@ -4872,22 +4868,8 @@ qemuDomainSetVcpusLive(virQEMUDriverPtr driver,
     if (!(vcpumap = qemuDomainSelectHotplugVcpuEntities(vm->def, nvcpus)))
         goto cleanup;
 
-    if (virNumaIsAvailable() &&
-        virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
-        if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,
-                               false, &cgroup_temp) < 0)
-            goto cleanup;
-
-        if (!(all_nodes = virNumaGetHostNodeset()))
-            goto cleanup;
-
-        if (!(all_nodes_str = virBitmapFormat(all_nodes)))
-            goto cleanup;
-
-        if (virCgroupGetCpusetMems(cgroup_temp, &mem_mask) < 0 ||
-            virCgroupSetCpusetMems(cgroup_temp, all_nodes_str) < 0)
-            goto cleanup;
-    }
+    if (qemuCgroupEmulatorAllNodesAllow(priv->cgroup, &emulatorCgroup) < 0)
+        goto cleanup;
 
     if (nvcpus > virDomainDefGetVcpus(vm->def)) {
         while ((nextvcpu = virBitmapNextSetBit(vcpumap, nextvcpu)) != -1) {
@@ -4915,17 +4897,7 @@ qemuDomainSetVcpusLive(virQEMUDriverPtr driver,
     ret = 0;
 
  cleanup:
-    if (mem_mask) {
-        err = virSaveLastError();
-        virCgroupSetCpusetMems(cgroup_temp, mem_mask);
-        virSetError(err);
-        virFreeError(err);
-        VIR_FREE(mem_mask);
-    }
-
-    VIR_FREE(all_nodes_str);
-    virBitmapFree(all_nodes);
-    virCgroupFree(&cgroup_temp);
+    qemuCgrouEmulatorAllNodesRestore(emulatorCgroup);
     virBitmapFree(vcpumap);
 
     return ret;
-- 
2.10.0