From c6bfaa39bd6a2d1bfd0e608b38afa39a698c809b Mon Sep 17 00:00:00 2001
Message-Id: <c6bfaa39bd6a2d1bfd0e608b38afa39a698c809b@dist-git>
From: John Ferlan <jferlan@redhat.com>
Date: Thu, 18 Sep 2014 09:29:53 -0400
Subject: [PATCH] qemu_cgroup: Introduce cgroup functions for IOThreads

https://bugzilla.redhat.com/show_bug.cgi?id=1101574

In order to support cpuset setting, introduce qemuSetupCgroupIOThreadsPin
and qemuSetupCgroupForIOThreads to mimic the existing Vcpu API's.

These will support having an 'iotrhreadpin' element in the 'cpuset' in
order to pin named IOThreads to specific CPU's. The IOThread pin names
will follow the IOThread naming scheme starting at 1 (eg "iothread1")
up through an including the def->iothreads value.

(cherry picked from commit 5f6ad32c733a3bd158938aecabb0508a434ece95)

NOTE: Requires commit 938fb12fad6d15c9fdb73f998c4e0ec1e278721f in order
      to build - should be two patches later

Signed-off-by: John Ferlan <jferlan@redhat.com>
Signed-off-by: Jiri Denemark <jdenemar@redhat.com>
---
 src/qemu/qemu_cgroup.c | 104 +++++++++++++++++++++++++++++++++++++++++++++++++
 src/qemu/qemu_cgroup.h |   5 +++
 2 files changed, 109 insertions(+)

diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index 43d14d4..c1d89bb 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -903,6 +903,23 @@ qemuSetupCgroupVcpuPin(virCgroupPtr cgroup,
 }
 
 int
+qemuSetupCgroupIOThreadsPin(virCgroupPtr cgroup,
+                            virDomainVcpuPinDefPtr *iothreadspin,
+                            int niothreadspin,
+                            int iothreadid)
+{
+    size_t i;
+
+    for (i = 0; i < niothreadspin; i++) {
+        if (iothreadid == iothreadspin[i]->vcpuid) {
+            return qemuSetupCgroupEmulatorPin(cgroup, iothreadspin[i]->cpumask);
+        }
+    }
+
+    return -1;
+}
+
+int
 qemuSetupCgroupEmulatorPin(virCgroupPtr cgroup,
                            virBitmapPtr cpumask)
 {
@@ -1083,6 +1100,93 @@ qemuSetupCgroupForEmulator(virQEMUDriverPtr driver,
 }
 
 int
+qemuSetupCgroupForIOThreads(virDomainObjPtr vm)
+{
+    virCgroupPtr cgroup_iothread = NULL;
+    qemuDomainObjPrivatePtr priv = vm->privateData;
+    virDomainDefPtr def = vm->def;
+    size_t i, j;
+    unsigned long long period = vm->def->cputune.period;
+    long long quota = vm->def->cputune.quota;
+
+    if ((period || quota) &&
+        !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("cgroup cpu is required for scheduler tuning"));
+        return -1;
+    }
+
+    /*
+     * If CPU cgroup controller is not initialized here, then we need
+     * neither period nor quota settings.  And if CPUSET controller is
+     * not initialized either, then there's nothing to do anyway.
+     */
+    if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
+        !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
+        return 0;
+
+    /* We are trying to setup cgroups for CPU pinning, which can also be done
+     * with virProcessSetAffinity, thus the lack of cgroups is not fatal here.
+     */
+    if (priv->cgroup == NULL)
+        return 0;
+
+    if (priv->niothreadpids == 0) {
+        VIR_WARN("Unable to get iothreads' pids.");
+        return 0;
+    }
+
+    for (i = 0; i < priv->niothreadpids; i++) {
+        /* IOThreads are numbered 1..n, although the array is 0..n-1,
+         * so we will account for that here
+         */
+        if (virCgroupNewIOThread(priv->cgroup, i+1, true, &cgroup_iothread) < 0)
+            goto cleanup;
+
+        /* move the thread for iothread to sub dir */
+        if (virCgroupAddTask(cgroup_iothread, priv->iothreadpids[i]) < 0)
+            goto cleanup;
+
+        if (period || quota) {
+            if (qemuSetupCgroupVcpuBW(cgroup_iothread, period, quota) < 0)
+                goto cleanup;
+        }
+
+        /* Set iothreadpin in cgroup if iothreadpin xml is provided */
+        if (virCgroupHasController(priv->cgroup,
+                                   VIR_CGROUP_CONTROLLER_CPUSET)) {
+            /* find the right CPU to pin, otherwise
+             * qemuSetupCgroupIOThreadsPin will fail. */
+            for (j = 0; j < def->cputune.niothreadspin; j++) {
+                /* IOThreads are numbered/named 1..n */
+                if (def->cputune.iothreadspin[j]->vcpuid != i+1)
+                    continue;
+
+                if (qemuSetupCgroupIOThreadsPin(cgroup_iothread,
+                                                def->cputune.iothreadspin,
+                                                def->cputune.niothreadspin,
+                                                i+1) < 0)
+                    goto cleanup;
+
+                break;
+            }
+        }
+
+        virCgroupFree(&cgroup_iothread);
+    }
+
+    return 0;
+
+ cleanup:
+    if (cgroup_iothread) {
+        virCgroupRemove(cgroup_iothread);
+        virCgroupFree(&cgroup_iothread);
+    }
+
+    return -1;
+}
+
+int
 qemuRemoveCgroup(virDomainObjPtr vm)
 {
     qemuDomainObjPrivatePtr priv = vm->privateData;
diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
index 7394969..8a2c723 100644
--- a/src/qemu/qemu_cgroup.h
+++ b/src/qemu/qemu_cgroup.h
@@ -57,7 +57,12 @@ int qemuSetupCgroupVcpuPin(virCgroupPtr cgroup,
                            int nvcpupin,
                            int vcpuid);
 int qemuSetupCgroupEmulatorPin(virCgroupPtr cgroup, virBitmapPtr cpumask);
+int qemuSetupCgroupIOThreadsPin(virCgroupPtr cgroup,
+                                virDomainVcpuPinDefPtr *iothreadspin,
+                                int niothreadspin,
+                                int iothreadid);
 int qemuSetupCgroupForVcpu(virDomainObjPtr vm);
+int qemuSetupCgroupForIOThreads(virDomainObjPtr vm);
 int qemuSetupCgroupForEmulator(virQEMUDriverPtr driver,
                                virDomainObjPtr vm,
                                virBitmapPtr nodemask);
-- 
2.1.0