From fe08906feb3ab006c4013957895cfb4fa69b7396 Mon Sep 17 00:00:00 2001

Message-Id: <fe08906feb3ab006c4013957895cfb4fa69b7396@dist-git>

From: Michal Privoznik <mprivozn@redhat.com>

Date: Wed, 7 Oct 2020 18:45:43 +0200

Subject: [PATCH] qemu: Build HMAT command line

MIME-Version: 1.0

Content-Type: text/plain; charset=UTF-8

Content-Transfer-Encoding: 8bit

Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1786303

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>

(cherry picked from commit aeecbc87b7317e88a8ef8c82b29bcacd1005c8c2)

Apart from conflicts below, I had to remove '-cpu qemu64' from

tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args to make

qemuxml2argvtest happy. This is because

3b8feb4793cef66f5dbfb9bdabe4d40834f1e90e isn't backported.

Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1749518

Conflicts:

- src/qemu/qemu_command.c: Context

- src/qemu/qemu_validate.c: The file doesn't exist in downstream.

I've made the change to validator that lives in qemu_domain.c.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

Message-Id: <0e8dfded8022b564ec7d0563cd745a0d3ffc815f.1602087923.git.mprivozn@redhat.com>

Reviewed-by: Ján Tomko <jtomko@redhat.com>

---

 src/conf/numa_conf.c                          |   7 +

 src/qemu/qemu_command.c                       | 171 ++++++++++++++++++

 src/qemu/qemu_domain.c                        |   7 +

 .../numatune-hmat.x86_64-latest.args          |  52 ++++++

 tests/qemuxml2argvtest.c                      |   1 +

 tests/qemuxml2xmltest.c                       |   2 +-

 6 files changed, 239 insertions(+), 1 deletion(-)

 create mode 100644 tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args

diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c

index 277a695d84..5a92eb35cc 100644

--- a/src/conf/numa_conf.c

+++ b/src/conf/numa_conf.c

@@ -1875,6 +1875,13 @@ virDomainNumaGetNodeInitiator(const virDomainNuma *numa,

     if (!numa || node >= numa->nmem_nodes)

         return -1;

+    /* A NUMA node which has at least one vCPU is initiator to itself by

+     * definition. */

+    if (numa->mem_nodes[node].cpumask)

+        return node;

+

+    /* For the rest, "NUMA node that has best performance (the lowest

+     * latency or largest bandwidth) to this NUMA node." */

     for (i = 0; i < numa->ninterconnects; i++) {

         const virDomainNumaInterconnect *l = &numa->interconnects[i];

diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c

index ac63d18a42..959207bfea 100644

--- a/src/qemu/qemu_command.c

+++ b/src/qemu/qemu_command.c

@@ -7172,6 +7172,9 @@ qemuBuildMachineCommandLine(virCommandPtr cmd,

             virBufferAsprintf(&buf, ",pflash1=%s", priv->pflash1->nodeformat);

     }

+    if (virDomainNumaHasHMAT(def->numa))

+        virBufferAddLit(&buf, ",hmat=on");

+

     virCommandAddArgBuffer(cmd, &buf;;

     return 0;

@@ -7355,6 +7358,134 @@ qemuBuildIOThreadCommandLine(virCommandPtr cmd,

 }

+static int

+qemuBuilNumaCellCache(virCommandPtr cmd,

+                      const virDomainDef *def,

+                      size_t cell)

+{

+    size_t ncaches = virDomainNumaGetNodeCacheCount(def->numa, cell);

+    size_t i;

+

+    if (ncaches == 0)

+        return 0;

+

+    for (i = 0; i < ncaches; i++) {

+        g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;

+        unsigned int level;

+        unsigned int size;

+        unsigned int line;

+        virDomainCacheAssociativity associativity;

+        virDomainCachePolicy policy;

+

+        if (virDomainNumaGetNodeCache(def->numa, cell, i,

+                                      &level, &size, &line,

+                                      &associativity, &policy) < 0) {

+            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",

+                           _("Unable to format NUMA node cache"));

+            return -1;

+        }

+

+        virBufferAsprintf(&buf,

+                          "hmat-cache,node-id=%zu,size=%uK,level=%u",

+                          cell, size, level);

+

+        switch (associativity) {

+        case VIR_DOMAIN_CACHE_ASSOCIATIVITY_NONE:

+            virBufferAddLit(&buf, ",associativity=none");

+            break;

+        case VIR_DOMAIN_CACHE_ASSOCIATIVITY_DIRECT:

+            virBufferAddLit(&buf, ",associativity=direct");

+            break;

+        case VIR_DOMAIN_CACHE_ASSOCIATIVITY_FULL:

+            virBufferAddLit(&buf, ",associativity=complex");

+            break;

+        case VIR_DOMAIN_CACHE_ASSOCIATIVITY_LAST:

+            break;

+        }

+

+        switch (policy) {

+        case VIR_DOMAIN_CACHE_POLICY_NONE:

+            virBufferAddLit(&buf, ",policy=none");

+            break;

+        case VIR_DOMAIN_CACHE_POLICY_WRITEBACK:

+            virBufferAddLit(&buf, ",policy=write-back");

+            break;

+        case VIR_DOMAIN_CACHE_POLICY_WRITETHROUGH:

+            virBufferAddLit(&buf, ",policy=write-through");

+            break;

+        case VIR_DOMAIN_CACHE_POLICY_LAST:

+            break;

+        }

+

+        if (line > 0)

+            virBufferAsprintf(&buf, ",line=%u", line);

+

+        virCommandAddArg(cmd, "-numa");

+        virCommandAddArgBuffer(cmd, &buf;;

+    }

+

+    return 0;

+}

+

+

+VIR_ENUM_DECL(qemuDomainMemoryHierarchy);

+VIR_ENUM_IMPL(qemuDomainMemoryHierarchy,

+              4, /* Maximum level of cache */

+              "memory", /* Special case, whole memory not specific cache */

+              "first-level",

+              "second-level",

+              "third-level");

+

+static int

+qemuBuildNumaHMATCommandLine(virCommandPtr cmd,

+                             const virDomainDef *def)

+{

+    size_t nlatencies;

+    size_t i;

+

+    if (!def->numa)

+        return 0;

+

+    nlatencies = virDomainNumaGetInterconnectsCount(def->numa);

+    for (i = 0; i < nlatencies; i++) {

+        g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;

+        virDomainNumaInterconnectType type;

+        unsigned int initiator;

+        unsigned int target;

+        unsigned int cache;

+        virDomainMemoryLatency accessType;

+        unsigned long value;

+        const char *hierarchyStr;

+        const char *accessStr;

+

+        if (virDomainNumaGetInterconnect(def->numa, i,

+                                         &type, &initiator, &target,

+                                         &cache, &accessType, &value) < 0)

+            return -1;

+

+        hierarchyStr = qemuDomainMemoryHierarchyTypeToString(cache);

+        accessStr = virDomainMemoryLatencyTypeToString(accessType);

+        virBufferAsprintf(&buf,

+                          "hmat-lb,initiator=%u,target=%u,hierarchy=%s,data-type=%s-",

+                          initiator, target, hierarchyStr, accessStr);

+

+        switch (type) {

+        case VIR_DOMAIN_NUMA_INTERCONNECT_TYPE_LATENCY:

+            virBufferAsprintf(&buf, "latency,latency=%lu", value);

+            break;

+        case VIR_DOMAIN_NUMA_INTERCONNECT_TYPE_BANDWIDTH:

+            virBufferAsprintf(&buf, "bandwidth,bandwidth=%luK", value);

+            break;

+        }

+

+        virCommandAddArg(cmd, "-numa");

+        virCommandAddArgBuffer(cmd, &buf;;

+    }

+

+    return 0;

+}

+

+

 static int

 qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,

                          virDomainDefPtr def,

@@ -7367,13 +7498,20 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,

     char *next = NULL;

     virBufferPtr nodeBackends = NULL;

     bool needBackend = false;

+    bool hmat = false;

     int rc;

     int ret = -1;

     size_t ncells = virDomainNumaGetNodeCount(def->numa);

+    ssize_t masterInitiator = -1;

     if (!virDomainNumatuneNodesetIsAvailable(def->numa, priv->autoNodeset))

         goto cleanup;

+    if (virDomainNumaHasHMAT(def->numa)) {

+        needBackend = true;

+        hmat = true;

+    }

+

     if (VIR_ALLOC_N(nodeBackends, ncells) < 0)

         goto cleanup;

@@ -7397,8 +7535,22 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,

         qemuBuildMemPathStr(cfg, def, cmd, priv) < 0)

         goto cleanup;

+    for (i = 0; i < ncells; i++) {

+        if (virDomainNumaGetNodeCpumask(def->numa, i)) {

+            masterInitiator = i;

+            break;

+        }

+    }

+

+    if (masterInitiator) {

+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",

+                       _("At least one NUMA node has to have CPUs"));

+        goto cleanup;

+    }

+

     for (i = 0; i < ncells; i++) {

         virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);

+        ssize_t initiator = virDomainNumaGetNodeInitiator(def->numa, i);

         if (needBackend) {

             virCommandAddArg(cmd, "-object");

@@ -7423,6 +7575,13 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,

             }

         }

+        if (hmat) {

+            if (initiator < 0)

+                initiator = masterInitiator;

+

+            virBufferAsprintf(&buf, ",initiator=%zd", initiator);

+        }

+

         if (needBackend)

             virBufferAsprintf(&buf, ",memdev=ram-node%zu", i);

         else

@@ -7448,6 +7607,18 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,

         }

     }

+    if (hmat) {

+        if (qemuBuildNumaHMATCommandLine(cmd, def) < 0)

+            goto cleanup;

+

+        /* This can't be moved into any of the loops above,

+         * because hmat-cache can be specified only after hmat-lb. */

+        for (i = 0; i < ncells; i++) {

+            if (qemuBuilNumaCellCache(cmd, def, i) < 0)

+                goto cleanup;

+        }

+    }

+

     ret = 0;

  cleanup:

diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c

index be25790f12..e51e176a80 100644

--- a/src/qemu/qemu_domain.c

+++ b/src/qemu/qemu_domain.c

@@ -5904,6 +5904,13 @@ qemuDomainDefValidate(const virDomainDef *def,

         }

     }

+    if (virDomainNumaHasHMAT(def->numa) &&

+        !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA_HMAT)) {

+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",

+                       _("HMAT is not supported with this QEMU"));

+        return -1;

+    }

+

     if (def->genidRequested &&

         !virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_VMGENID)) {

         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",

diff --git a/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args b/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args

new file mode 100644

index 0000000000..413d247a4d

--- /dev/null

+++ b/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args

@@ -0,0 +1,52 @@

+LC_ALL=C \

+PATH=/bin \

+HOME=/tmp/lib/domain--1-QEMUGuest \

+USER=test \

+LOGNAME=test \

+XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \

+XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \

+XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \

+QEMU_AUDIO_DRV=none \

+/usr/bin/qemu-system-x86_64 \

+-name guest=QEMUGuest,debug-threads=on \

+-S \

+-object secret,id=masterKey0,format=raw,\

+file=/tmp/lib/domain--1-QEMUGuest/master-key.aes \

+-machine pc,accel=tcg,usb=off,dump-guest-core=off,hmat=on \

+-m 12288 \

+-overcommit mem-lock=off \

+-smp 12,sockets=12,cores=1,threads=1 \

+-object memory-backend-ram,id=ram-node0,size=2147483648 \

+-numa node,nodeid=0,cpus=0-3,initiator=0,memdev=ram-node0 \

+-object memory-backend-ram,id=ram-node1,size=2147483648 \

+-numa node,nodeid=1,cpus=4-7,initiator=1,memdev=ram-node1 \

+-object memory-backend-ram,id=ram-node2,size=2147483648 \

+-numa node,nodeid=2,cpus=8-11,initiator=2,memdev=ram-node2 \

+-object memory-backend-ram,id=ram-node3,size=2147483648 \

+-numa node,nodeid=3,initiator=0,memdev=ram-node3 \

+-object memory-backend-ram,id=ram-node4,size=2147483648 \

+-numa node,nodeid=4,initiator=0,memdev=ram-node4 \

+-object memory-backend-ram,id=ram-node5,size=2147483648 \

+-numa node,nodeid=5,initiator=0,memdev=ram-node5 \

+-numa hmat-lb,initiator=0,target=0,hierarchy=memory,data-type=access-latency,\

+latency=5 \

+-numa hmat-lb,initiator=0,target=0,hierarchy=first-level,\

+data-type=access-latency,latency=10 \

+-numa hmat-lb,initiator=0,target=0,hierarchy=memory,data-type=access-bandwidth,\

+bandwidth=204800K \

+-numa hmat-cache,node-id=0,size=10K,level=1,associativity=direct,\

+policy=write-back,line=8 \

+-uuid c7a5fdb2-cdaf-9455-926a-d65c16db1809 \

+-display none \

+-no-user-config \

+-nodefaults \

+-chardev socket,id=charmonitor,fd=1729,server,nowait \

+-mon chardev=charmonitor,id=monitor,mode=control \

+-rtc base=utc \

+-no-shutdown \

+-boot strict=on \

+-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \

+-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 \

+-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,\

+resourcecontrol=deny \

+-msg timestamp=on

diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c

index 49699e495d..629f5ac100 100644

--- a/tests/qemuxml2argvtest.c

+++ b/tests/qemuxml2argvtest.c

@@ -1813,6 +1813,7 @@ mymain(void)

     DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);

     DO_TEST("numatune-no-vcpu", NONE);

+    DO_TEST_CAPS_LATEST("numatune-hmat");

     DO_TEST("numatune-auto-nodeset-invalid", NONE);

     DO_TEST("numatune-auto-prefer", QEMU_CAPS_OBJECT_MEMORY_RAM,

diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c

index de1d720e1d..f790bbc6f1 100644

--- a/tests/qemuxml2xmltest.c

+++ b/tests/qemuxml2xmltest.c

@@ -1106,7 +1106,7 @@ mymain(void)

     DO_TEST("numatune-memnode-no-memory", QEMU_CAPS_OBJECT_MEMORY_FILE);

     DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);

     DO_TEST("numatune-no-vcpu", QEMU_CAPS_NUMA);

-    DO_TEST("numatune-hmat", NONE);

+    DO_TEST("numatune-hmat", QEMU_CAPS_NUMA_HMAT);

     DO_TEST("bios-nvram", NONE);

     DO_TEST("bios-nvram-os-interleave", NONE);

-- 

2.29.2