From 5fde4ecb99989b91940f1b24a509adb47e3e0894 Mon Sep 17 00:00:00 2001 Message-Id: <5fde4ecb99989b91940f1b24a509adb47e3e0894@dist-git> From: Peter Krempa Date: Wed, 24 Aug 2016 16:11:37 -0400 Subject: [PATCH] qemu: command: Add support for sparse vcpu topologies https://bugzilla.redhat.com/show_bug.cgi?id=1097930 https://bugzilla.redhat.com/show_bug.cgi?id=1224341 Add support for using the new approach to hotplug vcpus using device_add during startup of qemu to allow sparse vcpu topologies. There are a few limitations imposed by qemu on the supported configuration: - vcpu0 needs to be always present and not hotpluggable - non-hotpluggable cpus need to be ordered at the beginning - order of the vcpus needs to be unique for every single hotpluggable entity Qemu also doesn't really allow to query the information necessary to start a VM with the vcpus directly on the commandline. Fortunately they can be hotplugged during startup. The new hotplug code uses the following approach: - non-hotpluggable vcpus are counted and put to the -smp option - qemu is started - qemu is queried for the necessary information - the configuration is checked - the hotpluggable vcpus are hotplugged - vcpus are started This patch adds a lot of checking code and enables the support to specify the individual vcpu element with qemu. (cherry picked from commit 9eb9106ea51b43102ee51132f69780b2c86ccbca) --- docs/formatdomain.html.in | 5 + src/qemu/qemu_command.c | 20 ++- src/qemu/qemu_domain.c | 76 ++++++++- src/qemu/qemu_process.c | 178 +++++++++++++++++++++ .../qemuxml2argv-cpu-hotplug-startup.args | 20 +++ .../qemuxml2argv-cpu-hotplug-startup.xml | 29 ++++ tests/qemuxml2argvtest.c | 2 + 7 files changed, 325 insertions(+), 5 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.xml diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index 252f91c..a838bff 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -580,6 +580,11 @@ Note that providing state for individual cpus may be necessary to enable support of addressable vCPU hotplug and this feature may not be supported by all hypervisors. + + For QEMU the following conditions are required. Vcpu 0 needs to be + enabled and non-hotpluggable. On PPC64 along with it vcpus that are in + the same core need to be enabled as well. All non-hotpluggable cpus + present at boot need to be grouped after vcpu 0. Since 2.2.0 (QEMU only) diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 95055c4..e292f48 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -7109,17 +7109,29 @@ qemuBuildMachineCommandLine(virCommandPtr cmd, static int qemuBuildSmpCommandLine(virCommandPtr cmd, - const virDomainDef *def) + virDomainDefPtr def) { char *smp; virBuffer buf = VIR_BUFFER_INITIALIZER; + unsigned int maxvcpus = virDomainDefGetVcpusMax(def); + unsigned int nvcpus = 0; + virDomainVcpuDefPtr vcpu; + size_t i; + + /* count non-hotpluggable enabled vcpus. Hotpluggable ones will be added + * in a different way */ + for (i = 0; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(def, i); + if (vcpu->online && vcpu->hotpluggable == VIR_TRISTATE_BOOL_NO) + nvcpus++; + } virCommandAddArg(cmd, "-smp"); - virBufferAsprintf(&buf, "%u", virDomainDefGetVcpus(def)); + virBufferAsprintf(&buf, "%u", nvcpus); - if (virDomainDefHasVcpusOffline(def)) - virBufferAsprintf(&buf, ",maxcpus=%u", virDomainDefGetVcpusMax(def)); + if (nvcpus != maxvcpus) + virBufferAsprintf(&buf, ",maxcpus=%u", maxvcpus); /* sockets, cores, and threads are either all zero * or all non-zero, thus checking one of them is enough */ if (def->cpu && def->cpu->sockets) { diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 959b551..8eb6e51 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -2247,6 +2247,76 @@ qemuDomainRecheckInternalPaths(virDomainDefPtr def, static int +qemuDomainDefVcpusPostParse(virDomainDefPtr def) +{ + unsigned int maxvcpus = virDomainDefGetVcpusMax(def); + virDomainVcpuDefPtr vcpu; + virDomainVcpuDefPtr prevvcpu; + size_t i; + bool has_order = false; + + /* vcpu 0 needs to be present, first, and non-hotpluggable */ + vcpu = virDomainDefGetVcpu(def, 0); + if (!vcpu->online) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("vcpu 0 can't be offline")); + return -1; + } + if (vcpu->hotpluggable == VIR_TRISTATE_BOOL_YES) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("vcpu0 can't be hotpluggable")); + return -1; + } + if (vcpu->order != 0 && vcpu->order != 1) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("vcpu0 must be enabled first")); + return -1; + } + + if (vcpu->order != 0) + has_order = true; + + prevvcpu = vcpu; + + /* all online vcpus or non online vcpu need to have order set */ + for (i = 1; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(def, i); + + if (vcpu->online && + (vcpu->order != 0) != has_order) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("all vcpus must have either set or unset order")); + return -1; + } + + /* few conditions for non-hotpluggable (thus online) vcpus */ + if (vcpu->hotpluggable == VIR_TRISTATE_BOOL_NO) { + /* they can be ordered only at the beginning */ + if (prevvcpu->hotpluggable == VIR_TRISTATE_BOOL_YES) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("online non-hotpluggable vcpus need to be " + "ordered prior to hotplugable vcpus")); + return -1; + } + + /* they need to be in order (qemu doesn't support any order yet). + * Also note that multiple vcpus may share order on some platforms */ + if (prevvcpu->order > vcpu->order) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("online non-hotpluggable vcpus must be ordered " + "in ascending order")); + return -1; + } + } + + prevvcpu = vcpu; + } + + return 0; +} + + +static int qemuDomainDefPostParse(virDomainDefPtr def, virCapsPtr caps, unsigned int parseFlags, @@ -2301,6 +2371,9 @@ qemuDomainDefPostParse(virDomainDefPtr def, if (virSecurityManagerVerify(driver->securityManager, def) < 0) goto cleanup; + if (qemuDomainDefVcpusPostParse(def) < 0) + goto cleanup; + ret = 0; cleanup: virObjectUnref(qemuCaps); @@ -2670,7 +2743,8 @@ virDomainDefParserConfig virQEMUDriverDomainDefParserConfig = { .deviceValidateCallback = qemuDomainDeviceDefValidate, .features = VIR_DOMAIN_DEF_FEATURE_MEMORY_HOTPLUG | - VIR_DOMAIN_DEF_FEATURE_OFFLINE_VCPUPIN + VIR_DOMAIN_DEF_FEATURE_OFFLINE_VCPUPIN | + VIR_DOMAIN_DEF_FEATURE_INDIVIDUAL_VCPUS, }; diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index c94eed4..2eac422 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -4719,6 +4719,172 @@ qemuProcessSetupIOThreads(virDomainObjPtr vm) } +static int +qemuProcessValidateHotpluggableVcpus(virDomainDefPtr def) +{ + virDomainVcpuDefPtr vcpu; + virDomainVcpuDefPtr subvcpu; + qemuDomainVcpuPrivatePtr vcpupriv; + unsigned int maxvcpus = virDomainDefGetVcpusMax(def); + size_t i = 0; + size_t j; + virBitmapPtr ordermap = NULL; + int ret = -1; + + if (!(ordermap = virBitmapNew(maxvcpus))) + goto cleanup; + + /* validate: + * - all hotpluggable entities to be hotplugged have the correct data + * - vcpus belonging to a hotpluggable entity share configuration + * - order of the hotpluggable entities is unique + */ + for (i = 0; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(def, i); + vcpupriv = QEMU_DOMAIN_VCPU_PRIVATE(vcpu); + + /* skip over hotpluggable entities */ + if (vcpupriv->vcpus == 0) + continue; + + if (vcpu->order != 0) { + if (virBitmapIsBitSet(ordermap, vcpu->order - 1)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("duplicate vcpu order '%u'"), vcpu->order - 1); + goto cleanup; + } + + ignore_value(virBitmapSetBit(ordermap, vcpu->order - 1)); + } + + + for (j = i + 1; j < (i + vcpupriv->vcpus); j++) { + subvcpu = virDomainDefGetVcpu(def, j); + if (subvcpu->hotpluggable != vcpu->hotpluggable || + subvcpu->online != vcpu->online || + subvcpu->order != vcpu->order) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("vcpus '%zu' and '%zu' are in the same hotplug " + "group but differ in configuration"), i, j); + goto cleanup; + } + } + + if (vcpu->online && vcpu->hotpluggable == VIR_TRISTATE_BOOL_YES) { + if ((vcpupriv->socket_id == -1 && vcpupriv->core_id == -1 && + vcpupriv->thread_id == -1) || + !vcpupriv->type) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("vcpu '%zu' is missing hotplug data"), i); + goto cleanup; + } + } + } + + ret = 0; + cleanup: + virBitmapFree(ordermap); + return ret; +} + + +static int +qemuDomainHasHotpluggableStartupVcpus(virDomainDefPtr def) +{ + size_t maxvcpus = virDomainDefGetVcpusMax(def); + virDomainVcpuDefPtr vcpu; + size_t i; + + for (i = 0; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(def, i); + + if (vcpu->online && vcpu->hotpluggable == VIR_TRISTATE_BOOL_YES) + return true; + } + + return false; +} + + +static int +qemuProcessVcpusSortOrder(const void *a, + const void *b) +{ + virDomainVcpuDefPtr vcpua = *((virDomainVcpuDefPtr *)a); + virDomainVcpuDefPtr vcpub = *((virDomainVcpuDefPtr *)b); + + return vcpua->order - vcpub->order; +} + + +static int +qemuProcessSetupHotpluggableVcpus(virQEMUDriverPtr driver, + virDomainObjPtr vm, + qemuDomainAsyncJob asyncJob) +{ + unsigned int maxvcpus = virDomainDefGetVcpusMax(vm->def); + virDomainVcpuDefPtr vcpu; + qemuDomainVcpuPrivatePtr vcpupriv; + virJSONValuePtr vcpuprops = NULL; + size_t i; + int ret = -1; + int rc; + + virDomainVcpuDefPtr *bootHotplug = NULL; + size_t nbootHotplug = 0; + + for (i = 0; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(vm->def, i); + vcpupriv = QEMU_DOMAIN_VCPU_PRIVATE(vcpu); + + if (vcpu->hotpluggable == VIR_TRISTATE_BOOL_YES && vcpu->online && + vcpupriv->vcpus != 0) { + if (virAsprintf(&vcpupriv->alias, "vcpu%zu", i) < 0) + goto cleanup; + + if (VIR_APPEND_ELEMENT(bootHotplug, nbootHotplug, vcpu) < 0) + goto cleanup; + } + } + + if (nbootHotplug == 0) { + ret = 0; + goto cleanup; + } + + qsort(bootHotplug, nbootHotplug, sizeof(*bootHotplug), + qemuProcessVcpusSortOrder); + + for (i = 0; i < nbootHotplug; i++) { + vcpu = bootHotplug[i]; + + if (!(vcpuprops = qemuBuildHotpluggableCPUProps(vcpu))) + goto cleanup; + + if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0) + goto cleanup; + + rc = qemuMonitorAddDeviceArgs(qemuDomainGetMonitor(vm), vcpuprops); + vcpuprops = NULL; + + if (qemuDomainObjExitMonitor(driver, vm) < 0) + goto cleanup; + + if (rc < 0) + goto cleanup; + + virJSONValueFree(vcpuprops); + } + + ret = 0; + + cleanup: + VIR_FREE(bootHotplug); + virJSONValueFree(vcpuprops); + return ret; +} + + /** * qemuProcessPrepareDomain * @@ -5195,6 +5361,18 @@ qemuProcessLaunch(virConnectPtr conn, if (qemuSetupCpusetMems(vm) < 0) goto cleanup; + VIR_DEBUG("setting up hotpluggable cpus"); + if (qemuDomainHasHotpluggableStartupVcpus(vm->def)) { + if (qemuDomainRefreshVcpuInfo(driver, vm, asyncJob, false) < 0) + goto cleanup; + + if (qemuProcessValidateHotpluggableVcpus(vm->def) < 0) + goto cleanup; + + if (qemuProcessSetupHotpluggableVcpus(driver, vm, asyncJob) < 0) + goto cleanup; + } + VIR_DEBUG("Refreshing VCPU info"); if (qemuDomainRefreshVcpuInfo(driver, vm, asyncJob, false) < 0) goto cleanup; diff --git a/tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.args b/tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.args new file mode 100644 index 0000000..035f250 --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.args @@ -0,0 +1,20 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/home/test \ +USER=test \ +LOGNAME=test \ +QEMU_AUDIO_DRV=none \ +/usr/bin/qemu \ +-name QEMUGuest1 \ +-S \ +-M pc \ +-m 214 \ +-smp 1,maxcpus=6,sockets=3,cores=2,threads=1 \ +-uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \ +-nographic \ +-nodefaults \ +-monitor unix:/tmp/lib/domain--1-QEMUGuest1/monitor.sock,server,nowait \ +-no-acpi \ +-boot n \ +-usb \ +-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 diff --git a/tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.xml b/tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.xml new file mode 100644 index 0000000..58718aa --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.xml @@ -0,0 +1,29 @@ + + QEMUGuest1 + c7a5fdbd-edaf-9455-926a-d65c16db1809 + 219100 + 219100 + 6 + + + + + + + + + + hvm + + + + + + + destroy + restart + destroy + + /usr/bin/qemu + + diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index a9a3e6b..9de2f94 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -2102,6 +2102,8 @@ mymain(void) DO_TEST("intel-iommu", QEMU_CAPS_DEVICE_PCI_BRIDGE, QEMU_CAPS_DEVICE_DMI_TO_PCI_BRIDGE, QEMU_CAPS_DEVICE_INTEL_IOMMU); + DO_TEST("cpu-hotplug-startup", QEMU_CAPS_QUERY_HOTPLUGGABLE_CPUS); + qemuTestDriverFree(&driver); return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE; -- 2.10.0