From 5f6723e71e3765d1d43bfa9ba1c66e0e05e11a48 Mon Sep 17 00:00:00 2001
Message-Id: <5f6723e71e3765d1d43bfa9ba1c66e0e05e11a48@dist-git>
From: Michal Privoznik <mprivozn@redhat.com>
Date: Mon, 9 Nov 2020 17:22:32 +0100
Subject: [PATCH] Allow NUMA nodes without vCPUs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
QEMU allows creating NUMA nodes that have memory only.
These are somehow important for HMAT.
With check done in qemuValidateDomainDef() for QEMU 2.7 or newer
(checked via QEMU_CAPS_NUMA), we can be sure that the vCPUs are
fully assigned to NUMA nodes in domain XML.
Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>
(cherry picked from commit a26f61ee0cffa421b87ef568002b684dd8025432)
Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1749518
Conflicts:
- src/qemu/qemu_validate.c: This file doesn't exist in downstream
yet, so I've moved the change that original patch would do to
qemu_domain.c where the validator lives.
Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
Message-Id: <365508c75e579e9037ad555d6c372068ccd50c95.1604938867.git.mprivozn@redhat.com>
Reviewed-by: Ján Tomko <jtomko@redhat.com>
---
docs/formatdomain.html.in | 2 +
docs/schemas/cputypes.rng | 8 ++-
src/conf/numa_conf.c | 59 ++++++++++---------
src/libxl/xen_xl.c | 10 ++--
src/qemu/qemu_command.c | 26 ++++----
src/qemu/qemu_domain.c | 22 +++----
tests/qemuxml2argvdata/numatune-no-vcpu.args | 33 +++++++++++
tests/qemuxml2argvdata/numatune-no-vcpu.xml | 42 +++++++++++++
tests/qemuxml2argvtest.c | 1 +
tests/qemuxml2xmloutdata/numatune-no-vcpu.xml | 1 +
tests/qemuxml2xmltest.c | 1 +
11 files changed, 149 insertions(+), 56 deletions(-)
create mode 100644 tests/qemuxml2argvdata/numatune-no-vcpu.args
create mode 100644 tests/qemuxml2argvdata/numatune-no-vcpu.xml
create mode 120000 tests/qemuxml2xmloutdata/numatune-no-vcpu.xml
diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in
index 76799f5ffc..4b8d312596 100644
--- a/docs/formatdomain.html.in
+++ b/docs/formatdomain.html.in
@@ -1783,6 +1783,8 @@
<code>cpus</code> specifies the CPU or range of CPUs that are
part of the node. <code>memory</code> specifies the node memory
in kibibytes (i.e. blocks of 1024 bytes).
+ <span class="since">Since 6.6.0</span> the <code>cpus</code> attribute
+ is optional and if omitted a CPU-less NUMA node is created.
<span class="since">Since 1.2.11</span> one can use an additional <a
href="#elementsMemoryAllocation"><code>unit</code></a> attribute to
define units in which <code>memory</code> is specified.
diff --git a/docs/schemas/cputypes.rng b/docs/schemas/cputypes.rng
index e2744acad3..a1682a1003 100644
--- a/docs/schemas/cputypes.rng
+++ b/docs/schemas/cputypes.rng
@@ -115,9 +115,11 @@
<ref name="unsignedInt"/>
</attribute>
</optional>
- <attribute name="cpus">
- <ref name="cpuset"/>
- </attribute>
+ <optional>
+ <attribute name="cpus">
+ <ref name="cpuset"/>
+ </attribute>
+ </optional>
<attribute name="memory">
<ref name="memoryKB"/>
</attribute>
diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c
index c9cc8ac22e..a805336d16 100644
--- a/src/conf/numa_conf.c
+++ b/src/conf/numa_conf.c
@@ -889,32 +889,28 @@ virDomainNumaDefParseXML(virDomainNumaPtr def,
}
VIR_FREE(tmp);
- if (def->mem_nodes[cur_cell].cpumask) {
+ if (def->mem_nodes[cur_cell].mem) {
virReportError(VIR_ERR_XML_ERROR,
_("Duplicate NUMA cell info for cell id '%u'"),
cur_cell);
goto cleanup;
}
- if (!(tmp = virXMLPropString(nodes[i], "cpus"))) {
- virReportError(VIR_ERR_XML_ERROR, "%s",
- _("Missing 'cpus' attribute in NUMA cell"));
- goto cleanup;
- }
+ if ((tmp = virXMLPropString(nodes[i], "cpus"))) {
+ g_autoptr(virBitmap) cpumask = NULL;
- if (virBitmapParse(tmp, &def->mem_nodes[cur_cell].cpumask,
- VIR_DOMAIN_CPUMASK_LEN) < 0)
- goto cleanup;
+ if (virBitmapParse(tmp, &cpumask, VIR_DOMAIN_CPUMASK_LEN) < 0)
+ goto cleanup;
- if (virBitmapIsAllClear(def->mem_nodes[cur_cell].cpumask)) {
- virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
- _("NUMA cell %d has no vCPUs assigned"), cur_cell);
- goto cleanup;
+ if (!virBitmapIsAllClear(cpumask))
+ def->mem_nodes[cur_cell].cpumask = g_steal_pointer(&cpumask);
+ VIR_FREE(tmp);
}
- VIR_FREE(tmp);
for (j = 0; j < n; j++) {
- if (j == cur_cell || !def->mem_nodes[j].cpumask)
+ if (j == cur_cell ||
+ !def->mem_nodes[j].cpumask ||
+ !def->mem_nodes[cur_cell].cpumask)
continue;
if (virBitmapOverlaps(def->mem_nodes[j].cpumask,
@@ -976,7 +972,6 @@ virDomainNumaDefFormatXML(virBufferPtr buf,
{
virDomainMemoryAccess memAccess;
virTristateBool discard;
- char *cpustr;
size_t ncells = virDomainNumaGetNodeCount(def);
size_t i;
@@ -986,17 +981,22 @@ virDomainNumaDefFormatXML(virBufferPtr buf,
virBufferAddLit(buf, "<numa>\n");
virBufferAdjustIndent(buf, 2);
for (i = 0; i < ncells; i++) {
+ virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def, i);
int ndistances;
memAccess = virDomainNumaGetNodeMemoryAccessMode(def, i);
discard = virDomainNumaGetNodeDiscard(def, i);
- if (!(cpustr = virBitmapFormat(virDomainNumaGetNodeCpumask(def, i))))
- return -1;
-
virBufferAddLit(buf, "<cell");
virBufferAsprintf(buf, " id='%zu'", i);
- virBufferAsprintf(buf, " cpus='%s'", cpustr);
+
+ if (cpumask) {
+ g_autofree char *cpustr = virBitmapFormat(cpumask);
+
+ if (!cpustr)
+ return -1;
+ virBufferAsprintf(buf, " cpus='%s'", cpustr);
+ }
virBufferAsprintf(buf, " memory='%llu'",
virDomainNumaGetNodeMemorySize(def, i));
virBufferAddLit(buf, " unit='KiB'");
@@ -1032,8 +1032,6 @@ virDomainNumaDefFormatXML(virBufferPtr buf,
virBufferAdjustIndent(buf, -2);
virBufferAddLit(buf, "</cell>\n");
}
-
- VIR_FREE(cpustr);
}
virBufferAdjustIndent(buf, -2);
virBufferAddLit(buf, "</numa>\n");
@@ -1048,8 +1046,12 @@ virDomainNumaGetCPUCountTotal(virDomainNumaPtr numa)
size_t i;
unsigned int ret = 0;
- for (i = 0; i < numa->nmem_nodes; i++)
- ret += virBitmapCountBits(virDomainNumaGetNodeCpumask(numa, i));
+ for (i = 0; i < numa->nmem_nodes; i++) {
+ virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, i);
+
+ if (cpumask)
+ ret += virBitmapCountBits(cpumask);
+ }
return ret;
}
@@ -1061,11 +1063,14 @@ virDomainNumaGetMaxCPUID(virDomainNumaPtr numa)
unsigned int ret = 0;
for (i = 0; i < numa->nmem_nodes; i++) {
+ virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, i);
int bit;
- bit = virBitmapLastSetBit(virDomainNumaGetNodeCpumask(numa, i));
- if (bit > ret)
- ret = bit;
+ if (cpumask) {
+ bit = virBitmapLastSetBit(cpumask);
+ if (bit > ret)
+ ret = bit;
+ }
}
return ret;
diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
index edea30a86a..752fa925ec 100644
--- a/src/libxl/xen_xl.c
+++ b/src/libxl/xen_xl.c
@@ -1443,19 +1443,21 @@ xenFormatXLVnuma(virConfValuePtr list,
{
int ret = -1;
size_t i;
-
virBuffer buf = VIR_BUFFER_INITIALIZER;
virConfValuePtr numaVnode, tmp;
-
+ virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, node);
size_t nodeSize = virDomainNumaGetNodeMemorySize(numa, node) / 1024;
- char *nodeVcpus = virBitmapFormat(virDomainNumaGetNodeCpumask(numa, node));
+ g_autofree char *nodeVcpus = NULL;
- if (VIR_ALLOC(numaVnode) < 0)
+ if (!cpumask ||
+ VIR_ALLOC(numaVnode) < 0)
goto cleanup;
numaVnode->type = VIR_CONF_LIST;
numaVnode->list = NULL;
+ nodeVcpus = virBitmapFormat(cpumask);
+
/* pnode */
virBufferAsprintf(&buf, "pnode=%zu", node);
xenFormatXLVnode(numaVnode, &buf);
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 1a573c2817..ac63d18a42 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -7364,8 +7364,6 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
size_t i, j;
virQEMUCapsPtr qemuCaps = priv->qemuCaps;
g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
- char *cpumask = NULL;
- char *tmpmask = NULL;
char *next = NULL;
virBufferPtr nodeBackends = NULL;
bool needBackend = false;
@@ -7400,9 +7398,7 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
goto cleanup;
for (i = 0; i < ncells; i++) {
- VIR_FREE(cpumask);
- if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i))))
- goto cleanup;
+ virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);
if (needBackend) {
virCommandAddArg(cmd, "-object");
@@ -7412,11 +7408,19 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
virCommandAddArg(cmd, "-numa");
virBufferAsprintf(&buf, "node,nodeid=%zu", i);
- for (tmpmask = cpumask; tmpmask; tmpmask = next) {
- if ((next = strchr(tmpmask, ',')))
- *(next++) = '\0';
- virBufferAddLit(&buf, ",cpus=");
- virBufferAdd(&buf, tmpmask, -1);
+ if (cpumask) {
+ g_autofree char *cpumaskStr = NULL;
+ char *tmpmask;
+
+ if (!(cpumaskStr = virBitmapFormat(cpumask)))
+ goto cleanup;
+
+ for (tmpmask = cpumaskStr; tmpmask; tmpmask = next) {
+ if ((next = strchr(tmpmask, ',')))
+ *(next++) = '\0';
+ virBufferAddLit(&buf, ",cpus=");
+ virBufferAdd(&buf, tmpmask, -1);
+ }
}
if (needBackend)
@@ -7447,8 +7451,6 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
ret = 0;
cleanup:
- VIR_FREE(cpumask);
-
if (nodeBackends) {
for (i = 0; i < ncells; i++)
virBufferFreeAndReset(&nodeBackends[i]);
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index 35b536868a..be25790f12 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -5373,7 +5373,7 @@ qemuDomainDefValidateNuma(const virDomainDef *def,
}
for (i = 0; i < ncells; i++) {
- g_autofree char * cpumask = NULL;
+ virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);
if (!hasMemoryCap &&
virDomainNumaGetNodeMemoryAccessMode(def->numa, i)) {
@@ -5383,17 +5383,19 @@ qemuDomainDefValidateNuma(const virDomainDef *def,
return -1;
}
- if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i))))
- return -1;
+ if (cpumask) {
+ g_autofree char * cpumaskStr = NULL;
+ if (!(cpumaskStr = virBitmapFormat(cpumask)))
+ return -1;
- if (strchr(cpumask, ',') &&
- !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA)) {
- virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
- _("disjoint NUMA cpu ranges are not supported "
- "with this QEMU"));
- return -1;
+ if (strchr(cpumaskStr, ',') &&
+ !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("disjoint NUMA cpu ranges are not supported "
+ "with this QEMU"));
+ return -1;
+ }
}
-
}
if (virDomainNumaNodesDistancesAreBeingSet(def->numa) &&
diff --git a/tests/qemuxml2argvdata/numatune-no-vcpu.args b/tests/qemuxml2argvdata/numatune-no-vcpu.args
new file mode 100644
index 0000000000..a1f1ee044e
--- /dev/null
+++ b/tests/qemuxml2argvdata/numatune-no-vcpu.args
@@ -0,0 +1,33 @@
+LC_ALL=C \
+PATH=/bin \
+HOME=/tmp/lib/domain--1-QEMUGuest \
+USER=test \
+LOGNAME=test \
+XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \
+XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \
+XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \
+QEMU_AUDIO_DRV=none \
+/usr/bin/qemu-system-x86_64 \
+-name QEMUGuest \
+-S \
+-machine pc,accel=tcg,usb=off,dump-guest-core=off \
+-m 12288 \
+-realtime mlock=off \
+-smp 12,sockets=12,cores=1,threads=1 \
+-numa node,nodeid=0,cpus=0-3,mem=2048 \
+-numa node,nodeid=1,cpus=4-7,mem=2048 \
+-numa node,nodeid=2,cpus=8-11,mem=2048 \
+-numa node,nodeid=3,mem=2048 \
+-numa node,nodeid=4,mem=2048 \
+-numa node,nodeid=5,mem=2048 \
+-uuid c7a5fdb2-cdaf-9455-926a-d65c16db1809 \
+-display none \
+-no-user-config \
+-nodefaults \
+-chardev socket,id=charmonitor,path=/tmp/lib/domain--1-QEMUGuest/monitor.sock,\
+server,nowait \
+-mon chardev=charmonitor,id=monitor,mode=control \
+-rtc base=utc \
+-no-shutdown \
+-usb \
+-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3
diff --git a/tests/qemuxml2argvdata/numatune-no-vcpu.xml b/tests/qemuxml2argvdata/numatune-no-vcpu.xml
new file mode 100644
index 0000000000..f25a07d7ed
--- /dev/null
+++ b/tests/qemuxml2argvdata/numatune-no-vcpu.xml
@@ -0,0 +1,42 @@
+<domain type='qemu'>
+ <name>QEMUGuest</name>
+ <uuid>c7a5fdb2-cdaf-9455-926a-d65c16db1809</uuid>
+ <memory unit='KiB'>12582912</memory>
+ <currentMemory unit='KiB'>12582912</currentMemory>
+ <vcpu placement='static'>12</vcpu>
+ <os>
+ <type arch='x86_64' machine='pc'>hvm</type>
+ <boot dev='hd'/>
+ </os>
+ <features>
+ <acpi/>
+ <apic/>
+ <pae/>
+ </features>
+ <cpu>
+ <numa>
+ <cell id='0' cpus='0-3' memory='2097152' unit='KiB'/>
+ <cell id='1' cpus='4-7' memory='2097152' unit='KiB'/>
+ <cell id='2' cpus='8-11' memory='2097152' unit='KiB'/>
+ <cell id='3' memory='2097152' unit='KiB'/>
+ <cell id='4' memory='2097152' unit='KiB'/>
+ <cell id='5' memory='2097152' unit='KiB'/>
+ </numa>
+ </cpu>
+ <clock offset='utc'/>
+ <on_poweroff>destroy</on_poweroff>
+ <on_reboot>restart</on_reboot>
+ <on_crash>restart</on_crash>
+ <devices>
+ <emulator>/usr/bin/qemu-system-x86_64</emulator>
+ <controller type='usb' index='0'>
+ <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
+ </controller>
+ <controller type='pci' index='0' model='pci-root'/>
+ <input type='mouse' bus='ps2'/>
+ <input type='keyboard' bus='ps2'/>
+ <memballoon model='virtio'>
+ <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+ </memballoon>
+ </devices>
+</domain>
diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
index ff92af606d..49699e495d 100644
--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@@ -1812,6 +1812,7 @@ mymain(void)
DO_TEST_PARSE_ERROR("numatune-memnode-no-memory", NONE);
DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);
+ DO_TEST("numatune-no-vcpu", NONE);
DO_TEST("numatune-auto-nodeset-invalid", NONE);
DO_TEST("numatune-auto-prefer", QEMU_CAPS_OBJECT_MEMORY_RAM,
diff --git a/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml b/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml
new file mode 120000
index 0000000000..f213032685
--- /dev/null
+++ b/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml
@@ -0,0 +1 @@
+../qemuxml2argvdata/numatune-no-vcpu.xml
\ No newline at end of file
diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c
index 6c3f5c4a9e..1ddeba30f0 100644
--- a/tests/qemuxml2xmltest.c
+++ b/tests/qemuxml2xmltest.c
@@ -1105,6 +1105,7 @@ mymain(void)
DO_TEST("numatune-memnode", QEMU_CAPS_NUMA, QEMU_CAPS_OBJECT_MEMORY_FILE);
DO_TEST("numatune-memnode-no-memory", QEMU_CAPS_OBJECT_MEMORY_FILE);
DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);
+ DO_TEST("numatune-no-vcpu", QEMU_CAPS_NUMA);
DO_TEST("bios-nvram", NONE);
DO_TEST("bios-nvram-os-interleave", NONE);
--
2.29.2