From fe08906feb3ab006c4013957895cfb4fa69b7396 Mon Sep 17 00:00:00 2001 Message-Id: From: Michal Privoznik Date: Wed, 7 Oct 2020 18:45:43 +0200 Subject: [PATCH] qemu: Build HMAT command line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1786303 Signed-off-by: Michal Privoznik Reviewed-by: Daniel Henrique Barboza (cherry picked from commit aeecbc87b7317e88a8ef8c82b29bcacd1005c8c2) Apart from conflicts below, I had to remove '-cpu qemu64' from tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args to make qemuxml2argvtest happy. This is because 3b8feb4793cef66f5dbfb9bdabe4d40834f1e90e isn't backported. Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1749518 Conflicts: - src/qemu/qemu_command.c: Context - src/qemu/qemu_validate.c: The file doesn't exist in downstream. I've made the change to validator that lives in qemu_domain.c. Signed-off-by: Michal Privoznik Message-Id: <0e8dfded8022b564ec7d0563cd745a0d3ffc815f.1602087923.git.mprivozn@redhat.com> Reviewed-by: Ján Tomko --- src/conf/numa_conf.c | 7 + src/qemu/qemu_command.c | 171 ++++++++++++++++++ src/qemu/qemu_domain.c | 7 + .../numatune-hmat.x86_64-latest.args | 52 ++++++ tests/qemuxml2argvtest.c | 1 + tests/qemuxml2xmltest.c | 2 +- 6 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c index 277a695d84..5a92eb35cc 100644 --- a/src/conf/numa_conf.c +++ b/src/conf/numa_conf.c @@ -1875,6 +1875,13 @@ virDomainNumaGetNodeInitiator(const virDomainNuma *numa, if (!numa || node >= numa->nmem_nodes) return -1; + /* A NUMA node which has at least one vCPU is initiator to itself by + * definition. */ + if (numa->mem_nodes[node].cpumask) + return node; + + /* For the rest, "NUMA node that has best performance (the lowest + * latency or largest bandwidth) to this NUMA node." */ for (i = 0; i < numa->ninterconnects; i++) { const virDomainNumaInterconnect *l = &numa->interconnects[i]; diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index ac63d18a42..959207bfea 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -7172,6 +7172,9 @@ qemuBuildMachineCommandLine(virCommandPtr cmd, virBufferAsprintf(&buf, ",pflash1=%s", priv->pflash1->nodeformat); } + if (virDomainNumaHasHMAT(def->numa)) + virBufferAddLit(&buf, ",hmat=on"); + virCommandAddArgBuffer(cmd, &buf); return 0; @@ -7355,6 +7358,134 @@ qemuBuildIOThreadCommandLine(virCommandPtr cmd, } +static int +qemuBuilNumaCellCache(virCommandPtr cmd, + const virDomainDef *def, + size_t cell) +{ + size_t ncaches = virDomainNumaGetNodeCacheCount(def->numa, cell); + size_t i; + + if (ncaches == 0) + return 0; + + for (i = 0; i < ncaches; i++) { + g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; + unsigned int level; + unsigned int size; + unsigned int line; + virDomainCacheAssociativity associativity; + virDomainCachePolicy policy; + + if (virDomainNumaGetNodeCache(def->numa, cell, i, + &level, &size, &line, + &associativity, &policy) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to format NUMA node cache")); + return -1; + } + + virBufferAsprintf(&buf, + "hmat-cache,node-id=%zu,size=%uK,level=%u", + cell, size, level); + + switch (associativity) { + case VIR_DOMAIN_CACHE_ASSOCIATIVITY_NONE: + virBufferAddLit(&buf, ",associativity=none"); + break; + case VIR_DOMAIN_CACHE_ASSOCIATIVITY_DIRECT: + virBufferAddLit(&buf, ",associativity=direct"); + break; + case VIR_DOMAIN_CACHE_ASSOCIATIVITY_FULL: + virBufferAddLit(&buf, ",associativity=complex"); + break; + case VIR_DOMAIN_CACHE_ASSOCIATIVITY_LAST: + break; + } + + switch (policy) { + case VIR_DOMAIN_CACHE_POLICY_NONE: + virBufferAddLit(&buf, ",policy=none"); + break; + case VIR_DOMAIN_CACHE_POLICY_WRITEBACK: + virBufferAddLit(&buf, ",policy=write-back"); + break; + case VIR_DOMAIN_CACHE_POLICY_WRITETHROUGH: + virBufferAddLit(&buf, ",policy=write-through"); + break; + case VIR_DOMAIN_CACHE_POLICY_LAST: + break; + } + + if (line > 0) + virBufferAsprintf(&buf, ",line=%u", line); + + virCommandAddArg(cmd, "-numa"); + virCommandAddArgBuffer(cmd, &buf); + } + + return 0; +} + + +VIR_ENUM_DECL(qemuDomainMemoryHierarchy); +VIR_ENUM_IMPL(qemuDomainMemoryHierarchy, + 4, /* Maximum level of cache */ + "memory", /* Special case, whole memory not specific cache */ + "first-level", + "second-level", + "third-level"); + +static int +qemuBuildNumaHMATCommandLine(virCommandPtr cmd, + const virDomainDef *def) +{ + size_t nlatencies; + size_t i; + + if (!def->numa) + return 0; + + nlatencies = virDomainNumaGetInterconnectsCount(def->numa); + for (i = 0; i < nlatencies; i++) { + g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; + virDomainNumaInterconnectType type; + unsigned int initiator; + unsigned int target; + unsigned int cache; + virDomainMemoryLatency accessType; + unsigned long value; + const char *hierarchyStr; + const char *accessStr; + + if (virDomainNumaGetInterconnect(def->numa, i, + &type, &initiator, &target, + &cache, &accessType, &value) < 0) + return -1; + + hierarchyStr = qemuDomainMemoryHierarchyTypeToString(cache); + accessStr = virDomainMemoryLatencyTypeToString(accessType); + virBufferAsprintf(&buf, + "hmat-lb,initiator=%u,target=%u,hierarchy=%s,data-type=%s-", + initiator, target, hierarchyStr, accessStr); + + switch (type) { + case VIR_DOMAIN_NUMA_INTERCONNECT_TYPE_LATENCY: + virBufferAsprintf(&buf, "latency,latency=%lu", value); + break; + case VIR_DOMAIN_NUMA_INTERCONNECT_TYPE_BANDWIDTH: + virBufferAsprintf(&buf, "bandwidth,bandwidth=%luK", value); + break; + } + + virCommandAddArg(cmd, "-numa"); + virCommandAddArgBuffer(cmd, &buf); + } + + return 0; +} + + static int qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, virDomainDefPtr def, @@ -7367,13 +7498,20 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, char *next = NULL; virBufferPtr nodeBackends = NULL; bool needBackend = false; + bool hmat = false; int rc; int ret = -1; size_t ncells = virDomainNumaGetNodeCount(def->numa); + ssize_t masterInitiator = -1; if (!virDomainNumatuneNodesetIsAvailable(def->numa, priv->autoNodeset)) goto cleanup; + if (virDomainNumaHasHMAT(def->numa)) { + needBackend = true; + hmat = true; + } + if (VIR_ALLOC_N(nodeBackends, ncells) < 0) goto cleanup; @@ -7397,8 +7535,22 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, qemuBuildMemPathStr(cfg, def, cmd, priv) < 0) goto cleanup; + for (i = 0; i < ncells; i++) { + if (virDomainNumaGetNodeCpumask(def->numa, i)) { + masterInitiator = i; + break; + } + } + + if (masterInitiator) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("At least one NUMA node has to have CPUs")); + goto cleanup; + } + for (i = 0; i < ncells; i++) { virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i); + ssize_t initiator = virDomainNumaGetNodeInitiator(def->numa, i); if (needBackend) { virCommandAddArg(cmd, "-object"); @@ -7423,6 +7575,13 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, } } + if (hmat) { + if (initiator < 0) + initiator = masterInitiator; + + virBufferAsprintf(&buf, ",initiator=%zd", initiator); + } + if (needBackend) virBufferAsprintf(&buf, ",memdev=ram-node%zu", i); else @@ -7448,6 +7607,18 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, } } + if (hmat) { + if (qemuBuildNumaHMATCommandLine(cmd, def) < 0) + goto cleanup; + + /* This can't be moved into any of the loops above, + * because hmat-cache can be specified only after hmat-lb. */ + for (i = 0; i < ncells; i++) { + if (qemuBuilNumaCellCache(cmd, def, i) < 0) + goto cleanup; + } + } + ret = 0; cleanup: diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index be25790f12..e51e176a80 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -5904,6 +5904,13 @@ qemuDomainDefValidate(const virDomainDef *def, } } + if (virDomainNumaHasHMAT(def->numa) && + !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA_HMAT)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("HMAT is not supported with this QEMU")); + return -1; + } + if (def->genidRequested && !virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_VMGENID)) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", diff --git a/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args b/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args new file mode 100644 index 0000000000..413d247a4d --- /dev/null +++ b/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args @@ -0,0 +1,52 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/tmp/lib/domain--1-QEMUGuest \ +USER=test \ +LOGNAME=test \ +XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \ +XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \ +XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \ +QEMU_AUDIO_DRV=none \ +/usr/bin/qemu-system-x86_64 \ +-name guest=QEMUGuest,debug-threads=on \ +-S \ +-object secret,id=masterKey0,format=raw,\ +file=/tmp/lib/domain--1-QEMUGuest/master-key.aes \ +-machine pc,accel=tcg,usb=off,dump-guest-core=off,hmat=on \ +-m 12288 \ +-overcommit mem-lock=off \ +-smp 12,sockets=12,cores=1,threads=1 \ +-object memory-backend-ram,id=ram-node0,size=2147483648 \ +-numa node,nodeid=0,cpus=0-3,initiator=0,memdev=ram-node0 \ +-object memory-backend-ram,id=ram-node1,size=2147483648 \ +-numa node,nodeid=1,cpus=4-7,initiator=1,memdev=ram-node1 \ +-object memory-backend-ram,id=ram-node2,size=2147483648 \ +-numa node,nodeid=2,cpus=8-11,initiator=2,memdev=ram-node2 \ +-object memory-backend-ram,id=ram-node3,size=2147483648 \ +-numa node,nodeid=3,initiator=0,memdev=ram-node3 \ +-object memory-backend-ram,id=ram-node4,size=2147483648 \ +-numa node,nodeid=4,initiator=0,memdev=ram-node4 \ +-object memory-backend-ram,id=ram-node5,size=2147483648 \ +-numa node,nodeid=5,initiator=0,memdev=ram-node5 \ +-numa hmat-lb,initiator=0,target=0,hierarchy=memory,data-type=access-latency,\ +latency=5 \ +-numa hmat-lb,initiator=0,target=0,hierarchy=first-level,\ +data-type=access-latency,latency=10 \ +-numa hmat-lb,initiator=0,target=0,hierarchy=memory,data-type=access-bandwidth,\ +bandwidth=204800K \ +-numa hmat-cache,node-id=0,size=10K,level=1,associativity=direct,\ +policy=write-back,line=8 \ +-uuid c7a5fdb2-cdaf-9455-926a-d65c16db1809 \ +-display none \ +-no-user-config \ +-nodefaults \ +-chardev socket,id=charmonitor,fd=1729,server,nowait \ +-mon chardev=charmonitor,id=monitor,mode=control \ +-rtc base=utc \ +-no-shutdown \ +-boot strict=on \ +-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \ +-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 \ +-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,\ +resourcecontrol=deny \ +-msg timestamp=on diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 49699e495d..629f5ac100 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -1813,6 +1813,7 @@ mymain(void) DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST); DO_TEST("numatune-no-vcpu", NONE); + DO_TEST_CAPS_LATEST("numatune-hmat"); DO_TEST("numatune-auto-nodeset-invalid", NONE); DO_TEST("numatune-auto-prefer", QEMU_CAPS_OBJECT_MEMORY_RAM, diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c index de1d720e1d..f790bbc6f1 100644 --- a/tests/qemuxml2xmltest.c +++ b/tests/qemuxml2xmltest.c @@ -1106,7 +1106,7 @@ mymain(void) DO_TEST("numatune-memnode-no-memory", QEMU_CAPS_OBJECT_MEMORY_FILE); DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST); DO_TEST("numatune-no-vcpu", QEMU_CAPS_NUMA); - DO_TEST("numatune-hmat", NONE); + DO_TEST("numatune-hmat", QEMU_CAPS_NUMA_HMAT); DO_TEST("bios-nvram", NONE); DO_TEST("bios-nvram-os-interleave", NONE); -- 2.29.2