Blob Blame History Raw
From 341e7b3d6b2e1d1bc9c11aa59ebac376f847da87 Mon Sep 17 00:00:00 2001
Message-Id: <341e7b3d6b2e1d1bc9c11aa59ebac376f847da87@dist-git>
From: Martin Kletzander <mkletzan@redhat.com>
Date: Wed, 24 Aug 2016 16:10:54 -0400
Subject: [PATCH] qemu: Add qemuProcessSetupPid() and use it in
 qemuProcessSetupIOThread()

https://bugzilla.redhat.com/show_bug.cgi?id=1097930
https://bugzilla.redhat.com/show_bug.cgi?id=1224341

Setting up cgroups and other things for all kinds of threads (the
emulator thread, vCPU threads, I/O threads) was copy-pasted every time
new thing was added.  Over time each one of those functions changed a
bit differently.  So create one function that does all that setup and
start using it, starting with I/O thread setup.  That will shave some
duplicated code and maybe fix some bugs as well.

Signed-off-by: Martin Kletzander <mkletzan@redhat.com>
(cherry picked from commit 71e419bbeb7d7bf6f9be0c2516c320cfb7492ef3)
---
 src/qemu/qemu_process.c | 200 +++++++++++++++++++++++++++---------------------
 1 file changed, 114 insertions(+), 86 deletions(-)

diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 41e401b..64b1ac9 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -2304,6 +2304,114 @@ qemuProcessSetLinkStates(virQEMUDriverPtr driver,
 }
 
 
+/**
+ * qemuProcessSetupPid:
+ *
+ * This function sets resource properities (affinity, cgroups,
+ * scheduler) for any PID associated with a domain.  It should be used
+ * to set up emulator PIDs as well as vCPU and I/O thread pids to
+ * ensure they are all handled the same way.
+ *
+ * Returns 0 on success, -1 on error.
+ */
+static int
+qemuProcessSetupPid(virDomainObjPtr vm,
+                    pid_t pid,
+                    virCgroupThreadName nameval,
+                    int id,
+                    virBitmapPtr cpumask,
+                    unsigned long long period,
+                    long long quota,
+                    virDomainThreadSchedParamPtr sched)
+{
+    qemuDomainObjPrivatePtr priv = vm->privateData;
+    virDomainNumatuneMemMode mem_mode;
+    virCgroupPtr cgroup = NULL;
+    virBitmapPtr use_cpumask;
+    char *mem_mask = NULL;
+    int ret = -1;
+
+    if ((period || quota) &&
+        !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("cgroup cpu is required for scheduler tuning"));
+        goto cleanup;
+    }
+
+    /* Infer which cpumask shall be used. */
+    if (cpumask)
+        use_cpumask = cpumask;
+    else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
+        use_cpumask = priv->autoCpuset;
+    else
+        use_cpumask = vm->def->cpumask;
+
+    /*
+     * If CPU cgroup controller is not initialized here, then we need
+     * neither period nor quota settings.  And if CPUSET controller is
+     * not initialized either, then there's nothing to do anyway.
+     */
+    if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) ||
+        virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
+
+        if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 &&
+            mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
+            virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
+                                                priv->autoNodeset,
+                                                &mem_mask, -1) < 0)
+            goto cleanup;
+
+        if (virCgroupNewThread(priv->cgroup, nameval, id, true, &cgroup) < 0)
+            goto cleanup;
+
+        if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
+            if (use_cpumask &&
+                qemuSetupCgroupCpusetCpus(cgroup, use_cpumask) < 0)
+                goto cleanup;
+
+            /*
+             * Don't setup cpuset.mems for the emulator, they need to
+             * be set up after initialization in order for kvm
+             * allocations to succeed.
+             */
+            if (nameval != VIR_CGROUP_THREAD_EMULATOR &&
+                mem_mask && virCgroupSetCpusetMems(cgroup, mem_mask) < 0)
+                goto cleanup;
+
+        }
+
+        if ((period || quota) &&
+            qemuSetupCgroupVcpuBW(cgroup, period, quota) < 0)
+            goto cleanup;
+
+        /* Move the thread to the sub dir */
+        if (virCgroupAddTask(cgroup, pid) < 0)
+            goto cleanup;
+
+    }
+
+    /* Setup legacy affinity. */
+    if (use_cpumask && virProcessSetAffinity(pid, use_cpumask) < 0)
+        goto cleanup;
+
+    /* Set scheduler type and priority. */
+    if (sched &&
+        virProcessSetScheduler(pid, sched->policy, sched->priority) < 0)
+        goto cleanup;
+
+    ret = 0;
+ cleanup:
+    VIR_FREE(mem_mask);
+    if (cgroup) {
+        if (ret < 0)
+            virCgroupRemove(cgroup);
+        virCgroupFree(&cgroup);
+    }
+
+    return ret;
+}
+
+
 static int
 qemuProcessSetupEmulator(virDomainObjPtr vm)
 {
@@ -4704,98 +4812,18 @@ qemuProcessSetupVcpus(virDomainObjPtr vm)
 }
 
 
-/**
- * qemuProcessSetupIOThread:
- * @vm: domain object
- * @iothread: iothread data structure to set the data for
- *
- * This function sets resource properities (affinity, cgroups, scheduler) for a
- * IOThread. This function expects that the IOThread is online and the IOThread
- * pids were correctly detected at the point when it's called.
- *
- * Returns 0 on success, -1 on error.
- */
 int
 qemuProcessSetupIOThread(virDomainObjPtr vm,
                          virDomainIOThreadIDDefPtr iothread)
 {
-    qemuDomainObjPrivatePtr priv = vm->privateData;
-    unsigned long long period = vm->def->cputune.period;
-    long long quota = vm->def->cputune.quota;
-    virDomainNumatuneMemMode mem_mode;
-    char *mem_mask = NULL;
-    virCgroupPtr cgroup_iothread = NULL;
-    virBitmapPtr cpumask = NULL;
-    int ret = -1;
 
-    if ((period || quota) &&
-        !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
-        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                       _("cgroup cpu is required for scheduler tuning"));
-        return -1;
-    }
-
-    if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) ||
-        virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
-        if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 &&
-            mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
-            virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
-                                                priv->autoNodeset,
-                                                &mem_mask, -1) < 0)
-            goto cleanup;
-
-        if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_IOTHREAD,
+    return qemuProcessSetupPid(vm, iothread->thread_id,
+                               VIR_CGROUP_THREAD_IOTHREAD,
                                iothread->iothread_id,
-                               true, &cgroup_iothread) < 0)
-            goto cleanup;
-    }
-
-    if (iothread->cpumask)
-        cpumask = iothread->cpumask;
-    else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
-        cpumask = priv->autoCpuset;
-    else
-        cpumask = vm->def->cpumask;
-
-    if (period || quota) {
-        if (qemuSetupCgroupVcpuBW(cgroup_iothread, period, quota) < 0)
-            goto cleanup;
-    }
-
-    if (cgroup_iothread) {
-        if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
-            if (mem_mask &&
-                virCgroupSetCpusetMems(cgroup_iothread, mem_mask) < 0)
-                goto cleanup;
-
-            if (cpumask &&
-                qemuSetupCgroupCpusetCpus(cgroup_iothread, cpumask) < 0)
-                goto cleanup;
-        }
-
-        if (virCgroupAddTask(cgroup_iothread, iothread->thread_id) < 0)
-            goto cleanup;
-    }
-
-    if (cpumask && virProcessSetAffinity(iothread->thread_id, cpumask) < 0)
-        goto cleanup;
-
-    if (iothread->sched.policy != VIR_PROC_POLICY_NONE &&
-        virProcessSetScheduler(iothread->thread_id, iothread->sched.policy,
-                               iothread->sched.priority) < 0)
-        goto cleanup;
-
-    ret = 0;
-
- cleanup:
-    if (cgroup_iothread) {
-        if (ret < 0)
-            virCgroupRemove(cgroup_iothread);
-        virCgroupFree(&cgroup_iothread);
-    }
-
-    VIR_FREE(mem_mask);
-    return ret;
+                               iothread->cpumask,
+                               vm->def->cputune.period,
+                               vm->def->cputune.quota,
+                               &iothread->sched);
 }
 
 
-- 
2.10.0