397dc2
From fe08906feb3ab006c4013957895cfb4fa69b7396 Mon Sep 17 00:00:00 2001
397dc2
Message-Id: <fe08906feb3ab006c4013957895cfb4fa69b7396@dist-git>
397dc2
From: Michal Privoznik <mprivozn@redhat.com>
397dc2
Date: Wed, 7 Oct 2020 18:45:43 +0200
397dc2
Subject: [PATCH] qemu: Build HMAT command line
397dc2
MIME-Version: 1.0
397dc2
Content-Type: text/plain; charset=UTF-8
397dc2
Content-Transfer-Encoding: 8bit
397dc2
397dc2
Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1786303
397dc2
397dc2
Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
397dc2
Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>
397dc2
(cherry picked from commit aeecbc87b7317e88a8ef8c82b29bcacd1005c8c2)
397dc2
397dc2
Apart from conflicts below, I had to remove '-cpu qemu64' from
397dc2
tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args to make
397dc2
qemuxml2argvtest happy. This is because
397dc2
3b8feb4793cef66f5dbfb9bdabe4d40834f1e90e isn't backported.
397dc2
397dc2
Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1749518
397dc2
397dc2
Conflicts:
397dc2
- src/qemu/qemu_command.c: Context
397dc2
- src/qemu/qemu_validate.c: The file doesn't exist in downstream.
397dc2
I've made the change to validator that lives in qemu_domain.c.
397dc2
397dc2
Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
397dc2
Message-Id: <0e8dfded8022b564ec7d0563cd745a0d3ffc815f.1602087923.git.mprivozn@redhat.com>
397dc2
Reviewed-by: Ján Tomko <jtomko@redhat.com>
397dc2
---
397dc2
 src/conf/numa_conf.c                          |   7 +
397dc2
 src/qemu/qemu_command.c                       | 171 ++++++++++++++++++
397dc2
 src/qemu/qemu_domain.c                        |   7 +
397dc2
 .../numatune-hmat.x86_64-latest.args          |  52 ++++++
397dc2
 tests/qemuxml2argvtest.c                      |   1 +
397dc2
 tests/qemuxml2xmltest.c                       |   2 +-
397dc2
 6 files changed, 239 insertions(+), 1 deletion(-)
397dc2
 create mode 100644 tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args
397dc2
397dc2
diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c
397dc2
index 277a695d84..5a92eb35cc 100644
397dc2
--- a/src/conf/numa_conf.c
397dc2
+++ b/src/conf/numa_conf.c
397dc2
@@ -1875,6 +1875,13 @@ virDomainNumaGetNodeInitiator(const virDomainNuma *numa,
397dc2
     if (!numa || node >= numa->nmem_nodes)
397dc2
         return -1;
397dc2
 
397dc2
+    /* A NUMA node which has at least one vCPU is initiator to itself by
397dc2
+     * definition. */
397dc2
+    if (numa->mem_nodes[node].cpumask)
397dc2
+        return node;
397dc2
+
397dc2
+    /* For the rest, "NUMA node that has best performance (the lowest
397dc2
+     * latency or largest bandwidth) to this NUMA node." */
397dc2
     for (i = 0; i < numa->ninterconnects; i++) {
397dc2
         const virDomainNumaInterconnect *l = &numa->interconnects[i];
397dc2
 
397dc2
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
397dc2
index ac63d18a42..959207bfea 100644
397dc2
--- a/src/qemu/qemu_command.c
397dc2
+++ b/src/qemu/qemu_command.c
397dc2
@@ -7172,6 +7172,9 @@ qemuBuildMachineCommandLine(virCommandPtr cmd,
397dc2
             virBufferAsprintf(&buf, ",pflash1=%s", priv->pflash1->nodeformat);
397dc2
     }
397dc2
 
397dc2
+    if (virDomainNumaHasHMAT(def->numa))
397dc2
+        virBufferAddLit(&buf, ",hmat=on");
397dc2
+
397dc2
     virCommandAddArgBuffer(cmd, &buf;;
397dc2
 
397dc2
     return 0;
397dc2
@@ -7355,6 +7358,134 @@ qemuBuildIOThreadCommandLine(virCommandPtr cmd,
397dc2
 }
397dc2
 
397dc2
 
397dc2
+static int
397dc2
+qemuBuilNumaCellCache(virCommandPtr cmd,
397dc2
+                      const virDomainDef *def,
397dc2
+                      size_t cell)
397dc2
+{
397dc2
+    size_t ncaches = virDomainNumaGetNodeCacheCount(def->numa, cell);
397dc2
+    size_t i;
397dc2
+
397dc2
+    if (ncaches == 0)
397dc2
+        return 0;
397dc2
+
397dc2
+    for (i = 0; i < ncaches; i++) {
397dc2
+        g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
397dc2
+        unsigned int level;
397dc2
+        unsigned int size;
397dc2
+        unsigned int line;
397dc2
+        virDomainCacheAssociativity associativity;
397dc2
+        virDomainCachePolicy policy;
397dc2
+
397dc2
+        if (virDomainNumaGetNodeCache(def->numa, cell, i,
397dc2
+                                      &level, &size, &line,
397dc2
+                                      &associativity, &policy) < 0) {
397dc2
+            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
397dc2
+                           _("Unable to format NUMA node cache"));
397dc2
+            return -1;
397dc2
+        }
397dc2
+
397dc2
+        virBufferAsprintf(&buf,
397dc2
+                          "hmat-cache,node-id=%zu,size=%uK,level=%u",
397dc2
+                          cell, size, level);
397dc2
+
397dc2
+        switch (associativity) {
397dc2
+        case VIR_DOMAIN_CACHE_ASSOCIATIVITY_NONE:
397dc2
+            virBufferAddLit(&buf, ",associativity=none");
397dc2
+            break;
397dc2
+        case VIR_DOMAIN_CACHE_ASSOCIATIVITY_DIRECT:
397dc2
+            virBufferAddLit(&buf, ",associativity=direct");
397dc2
+            break;
397dc2
+        case VIR_DOMAIN_CACHE_ASSOCIATIVITY_FULL:
397dc2
+            virBufferAddLit(&buf, ",associativity=complex");
397dc2
+            break;
397dc2
+        case VIR_DOMAIN_CACHE_ASSOCIATIVITY_LAST:
397dc2
+            break;
397dc2
+        }
397dc2
+
397dc2
+        switch (policy) {
397dc2
+        case VIR_DOMAIN_CACHE_POLICY_NONE:
397dc2
+            virBufferAddLit(&buf, ",policy=none");
397dc2
+            break;
397dc2
+        case VIR_DOMAIN_CACHE_POLICY_WRITEBACK:
397dc2
+            virBufferAddLit(&buf, ",policy=write-back");
397dc2
+            break;
397dc2
+        case VIR_DOMAIN_CACHE_POLICY_WRITETHROUGH:
397dc2
+            virBufferAddLit(&buf, ",policy=write-through");
397dc2
+            break;
397dc2
+        case VIR_DOMAIN_CACHE_POLICY_LAST:
397dc2
+            break;
397dc2
+        }
397dc2
+
397dc2
+        if (line > 0)
397dc2
+            virBufferAsprintf(&buf, ",line=%u", line);
397dc2
+
397dc2
+        virCommandAddArg(cmd, "-numa");
397dc2
+        virCommandAddArgBuffer(cmd, &buf;;
397dc2
+    }
397dc2
+
397dc2
+    return 0;
397dc2
+}
397dc2
+
397dc2
+
397dc2
+VIR_ENUM_DECL(qemuDomainMemoryHierarchy);
397dc2
+VIR_ENUM_IMPL(qemuDomainMemoryHierarchy,
397dc2
+              4, /* Maximum level of cache */
397dc2
+              "memory", /* Special case, whole memory not specific cache */
397dc2
+              "first-level",
397dc2
+              "second-level",
397dc2
+              "third-level");
397dc2
+
397dc2
+static int
397dc2
+qemuBuildNumaHMATCommandLine(virCommandPtr cmd,
397dc2
+                             const virDomainDef *def)
397dc2
+{
397dc2
+    size_t nlatencies;
397dc2
+    size_t i;
397dc2
+
397dc2
+    if (!def->numa)
397dc2
+        return 0;
397dc2
+
397dc2
+    nlatencies = virDomainNumaGetInterconnectsCount(def->numa);
397dc2
+    for (i = 0; i < nlatencies; i++) {
397dc2
+        g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
397dc2
+        virDomainNumaInterconnectType type;
397dc2
+        unsigned int initiator;
397dc2
+        unsigned int target;
397dc2
+        unsigned int cache;
397dc2
+        virDomainMemoryLatency accessType;
397dc2
+        unsigned long value;
397dc2
+        const char *hierarchyStr;
397dc2
+        const char *accessStr;
397dc2
+
397dc2
+        if (virDomainNumaGetInterconnect(def->numa, i,
397dc2
+                                         &type, &initiator, &target,
397dc2
+                                         &cache, &accessType, &value) < 0)
397dc2
+            return -1;
397dc2
+
397dc2
+        hierarchyStr = qemuDomainMemoryHierarchyTypeToString(cache);
397dc2
+        accessStr = virDomainMemoryLatencyTypeToString(accessType);
397dc2
+        virBufferAsprintf(&buf,
397dc2
+                          "hmat-lb,initiator=%u,target=%u,hierarchy=%s,data-type=%s-",
397dc2
+                          initiator, target, hierarchyStr, accessStr);
397dc2
+
397dc2
+        switch (type) {
397dc2
+        case VIR_DOMAIN_NUMA_INTERCONNECT_TYPE_LATENCY:
397dc2
+            virBufferAsprintf(&buf, "latency,latency=%lu", value);
397dc2
+            break;
397dc2
+        case VIR_DOMAIN_NUMA_INTERCONNECT_TYPE_BANDWIDTH:
397dc2
+            virBufferAsprintf(&buf, "bandwidth,bandwidth=%luK", value);
397dc2
+            break;
397dc2
+        }
397dc2
+
397dc2
+        virCommandAddArg(cmd, "-numa");
397dc2
+        virCommandAddArgBuffer(cmd, &buf;;
397dc2
+    }
397dc2
+
397dc2
+    return 0;
397dc2
+}
397dc2
+
397dc2
+
397dc2
 static int
397dc2
 qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
397dc2
                          virDomainDefPtr def,
397dc2
@@ -7367,13 +7498,20 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
397dc2
     char *next = NULL;
397dc2
     virBufferPtr nodeBackends = NULL;
397dc2
     bool needBackend = false;
397dc2
+    bool hmat = false;
397dc2
     int rc;
397dc2
     int ret = -1;
397dc2
     size_t ncells = virDomainNumaGetNodeCount(def->numa);
397dc2
+    ssize_t masterInitiator = -1;
397dc2
 
397dc2
     if (!virDomainNumatuneNodesetIsAvailable(def->numa, priv->autoNodeset))
397dc2
         goto cleanup;
397dc2
 
397dc2
+    if (virDomainNumaHasHMAT(def->numa)) {
397dc2
+        needBackend = true;
397dc2
+        hmat = true;
397dc2
+    }
397dc2
+
397dc2
     if (VIR_ALLOC_N(nodeBackends, ncells) < 0)
397dc2
         goto cleanup;
397dc2
 
397dc2
@@ -7397,8 +7535,22 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
397dc2
         qemuBuildMemPathStr(cfg, def, cmd, priv) < 0)
397dc2
         goto cleanup;
397dc2
 
397dc2
+    for (i = 0; i < ncells; i++) {
397dc2
+        if (virDomainNumaGetNodeCpumask(def->numa, i)) {
397dc2
+            masterInitiator = i;
397dc2
+            break;
397dc2
+        }
397dc2
+    }
397dc2
+
397dc2
+    if (masterInitiator) {
397dc2
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
397dc2
+                       _("At least one NUMA node has to have CPUs"));
397dc2
+        goto cleanup;
397dc2
+    }
397dc2
+
397dc2
     for (i = 0; i < ncells; i++) {
397dc2
         virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);
397dc2
+        ssize_t initiator = virDomainNumaGetNodeInitiator(def->numa, i);
397dc2
 
397dc2
         if (needBackend) {
397dc2
             virCommandAddArg(cmd, "-object");
397dc2
@@ -7423,6 +7575,13 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
397dc2
             }
397dc2
         }
397dc2
 
397dc2
+        if (hmat) {
397dc2
+            if (initiator < 0)
397dc2
+                initiator = masterInitiator;
397dc2
+
397dc2
+            virBufferAsprintf(&buf, ",initiator=%zd", initiator);
397dc2
+        }
397dc2
+
397dc2
         if (needBackend)
397dc2
             virBufferAsprintf(&buf, ",memdev=ram-node%zu", i);
397dc2
         else
397dc2
@@ -7448,6 +7607,18 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
397dc2
         }
397dc2
     }
397dc2
 
397dc2
+    if (hmat) {
397dc2
+        if (qemuBuildNumaHMATCommandLine(cmd, def) < 0)
397dc2
+            goto cleanup;
397dc2
+
397dc2
+        /* This can't be moved into any of the loops above,
397dc2
+         * because hmat-cache can be specified only after hmat-lb. */
397dc2
+        for (i = 0; i < ncells; i++) {
397dc2
+            if (qemuBuilNumaCellCache(cmd, def, i) < 0)
397dc2
+                goto cleanup;
397dc2
+        }
397dc2
+    }
397dc2
+
397dc2
     ret = 0;
397dc2
 
397dc2
  cleanup:
397dc2
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
397dc2
index be25790f12..e51e176a80 100644
397dc2
--- a/src/qemu/qemu_domain.c
397dc2
+++ b/src/qemu/qemu_domain.c
397dc2
@@ -5904,6 +5904,13 @@ qemuDomainDefValidate(const virDomainDef *def,
397dc2
         }
397dc2
     }
397dc2
 
397dc2
+    if (virDomainNumaHasHMAT(def->numa) &&
397dc2
+        !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA_HMAT)) {
397dc2
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
397dc2
+                       _("HMAT is not supported with this QEMU"));
397dc2
+        return -1;
397dc2
+    }
397dc2
+
397dc2
     if (def->genidRequested &&
397dc2
         !virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_VMGENID)) {
397dc2
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
397dc2
diff --git a/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args b/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args
397dc2
new file mode 100644
397dc2
index 0000000000..413d247a4d
397dc2
--- /dev/null
397dc2
+++ b/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args
397dc2
@@ -0,0 +1,52 @@
397dc2
+LC_ALL=C \
397dc2
+PATH=/bin \
397dc2
+HOME=/tmp/lib/domain--1-QEMUGuest \
397dc2
+USER=test \
397dc2
+LOGNAME=test \
397dc2
+XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \
397dc2
+XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \
397dc2
+XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \
397dc2
+QEMU_AUDIO_DRV=none \
397dc2
+/usr/bin/qemu-system-x86_64 \
397dc2
+-name guest=QEMUGuest,debug-threads=on \
397dc2
+-S \
397dc2
+-object secret,id=masterKey0,format=raw,\
397dc2
+file=/tmp/lib/domain--1-QEMUGuest/master-key.aes \
397dc2
+-machine pc,accel=tcg,usb=off,dump-guest-core=off,hmat=on \
397dc2
+-m 12288 \
397dc2
+-overcommit mem-lock=off \
397dc2
+-smp 12,sockets=12,cores=1,threads=1 \
397dc2
+-object memory-backend-ram,id=ram-node0,size=2147483648 \
397dc2
+-numa node,nodeid=0,cpus=0-3,initiator=0,memdev=ram-node0 \
397dc2
+-object memory-backend-ram,id=ram-node1,size=2147483648 \
397dc2
+-numa node,nodeid=1,cpus=4-7,initiator=1,memdev=ram-node1 \
397dc2
+-object memory-backend-ram,id=ram-node2,size=2147483648 \
397dc2
+-numa node,nodeid=2,cpus=8-11,initiator=2,memdev=ram-node2 \
397dc2
+-object memory-backend-ram,id=ram-node3,size=2147483648 \
397dc2
+-numa node,nodeid=3,initiator=0,memdev=ram-node3 \
397dc2
+-object memory-backend-ram,id=ram-node4,size=2147483648 \
397dc2
+-numa node,nodeid=4,initiator=0,memdev=ram-node4 \
397dc2
+-object memory-backend-ram,id=ram-node5,size=2147483648 \
397dc2
+-numa node,nodeid=5,initiator=0,memdev=ram-node5 \
397dc2
+-numa hmat-lb,initiator=0,target=0,hierarchy=memory,data-type=access-latency,\
397dc2
+latency=5 \
397dc2
+-numa hmat-lb,initiator=0,target=0,hierarchy=first-level,\
397dc2
+data-type=access-latency,latency=10 \
397dc2
+-numa hmat-lb,initiator=0,target=0,hierarchy=memory,data-type=access-bandwidth,\
397dc2
+bandwidth=204800K \
397dc2
+-numa hmat-cache,node-id=0,size=10K,level=1,associativity=direct,\
397dc2
+policy=write-back,line=8 \
397dc2
+-uuid c7a5fdb2-cdaf-9455-926a-d65c16db1809 \
397dc2
+-display none \
397dc2
+-no-user-config \
397dc2
+-nodefaults \
397dc2
+-chardev socket,id=charmonitor,fd=1729,server,nowait \
397dc2
+-mon chardev=charmonitor,id=monitor,mode=control \
397dc2
+-rtc base=utc \
397dc2
+-no-shutdown \
397dc2
+-boot strict=on \
397dc2
+-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \
397dc2
+-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 \
397dc2
+-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,\
397dc2
+resourcecontrol=deny \
397dc2
+-msg timestamp=on
397dc2
diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
397dc2
index 49699e495d..629f5ac100 100644
397dc2
--- a/tests/qemuxml2argvtest.c
397dc2
+++ b/tests/qemuxml2argvtest.c
397dc2
@@ -1813,6 +1813,7 @@ mymain(void)
397dc2
 
397dc2
     DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);
397dc2
     DO_TEST("numatune-no-vcpu", NONE);
397dc2
+    DO_TEST_CAPS_LATEST("numatune-hmat");
397dc2
 
397dc2
     DO_TEST("numatune-auto-nodeset-invalid", NONE);
397dc2
     DO_TEST("numatune-auto-prefer", QEMU_CAPS_OBJECT_MEMORY_RAM,
397dc2
diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c
397dc2
index de1d720e1d..f790bbc6f1 100644
397dc2
--- a/tests/qemuxml2xmltest.c
397dc2
+++ b/tests/qemuxml2xmltest.c
397dc2
@@ -1106,7 +1106,7 @@ mymain(void)
397dc2
     DO_TEST("numatune-memnode-no-memory", QEMU_CAPS_OBJECT_MEMORY_FILE);
397dc2
     DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);
397dc2
     DO_TEST("numatune-no-vcpu", QEMU_CAPS_NUMA);
397dc2
-    DO_TEST("numatune-hmat", NONE);
397dc2
+    DO_TEST("numatune-hmat", QEMU_CAPS_NUMA_HMAT);
397dc2
 
397dc2
     DO_TEST("bios-nvram", NONE);
397dc2
     DO_TEST("bios-nvram-os-interleave", NONE);
397dc2
-- 
397dc2
2.29.2
397dc2