From bc7971b29f6c152a9c20aa3cda9bd361dcb61f14 Mon Sep 17 00:00:00 2001

Message-Id: <bc7971b29f6c152a9c20aa3cda9bd361dcb61f14@dist-git>

From: Wim ten Have <wim.ten.have@oracle.com>

Date: Mon, 4 Dec 2017 13:38:48 +0100

Subject: [PATCH] xenconfig: add domxml conversions for xen-xl

https://bugzilla.redhat.com/show_bug.cgi?id=1454889

This patch converts NUMA configurations between the Xen libxl

configuration file format and libvirt's XML format.

XML HVM domain on a 4 node (2 cores/socket) configuration:

  <cpu>

    <numa>

      <cell id='0' cpus='0-1' memory='2097152' unit='KiB'>

        <distances>

          <sibling id='0' value='10'/>

          <sibling id='1' value='21'/>

          <sibling id='2' value='31'/>

          <sibling id='3' value='21'/>

        </distances>

      </cell>

      <cell id='1' cpus='2-3' memory='2097152' unit='KiB'>

        <distances>

          <sibling id='0' value='21'/>

          <sibling id='1' value='10'/>

          <sibling id='2' value='21'/>

          <sibling id='3' value='31'/>

        </distances>

      </cell>

      <cell id='2' cpus='3-4' memory='2097152' unit='KiB'>

        <distances>

          <sibling id='0' value='31'/>

          <sibling id='1' value='21'/>

          <sibling id='2' value='10'/>

          <sibling id='3' value='21'/>

        </distances>

      </cell>

      <cell id='3' cpus='5-6' memory='2097152' unit='KiB'>

        <distances>

          <sibling id='0' value='21'/>

          <sibling id='1' value='31'/>

          <sibling id='2' value='21'/>

          <sibling id='3' value='10'/>

        </distances>

      </cell>

    </numa>

  </cpu>

Xen xl.cfg domain configuration:

  vnuma = [["pnode=0","size=2048","vcpus=0-1","vdistances=10,21,31,21"],

           ["pnode=1","size=2048","vcpus=2-3","vdistances=21,10,21,31"],

           ["pnode=2","size=2048","vcpus=4-5","vdistances=31,21,10,21"],

           ["pnode=3","size=2048","vcpus=6-7","vdistances=21,31,21,10"]]

If there is no XML <distances> description amongst the <cell> data the

conversion schema from xml to native will generate 10 for local and 20

for all remote instances.

Signed-off-by: Wim ten Have <wim.ten.have@oracle.com>

Reviewed-by: Jim Fehlig <jfehlig@suse.com>

Signed-off-by: Jim Fehlig <jfehlig@suse.com>

(cherry picked from commit 03d0959af3a72eb2aed47a1d2dcf8ccbb39d4b0e)

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

Signed-off-by: Jiri Denemark <jdenemar@redhat.com>

---

 src/conf/numa_conf.c     | 137 +++++++++++++++++++

 src/conf/numa_conf.h     |  25 ++++

 src/libvirt_private.syms |   5 +

 src/xenconfig/xen_xl.c   | 335 +++++++++++++++++++++++++++++++++++++++++++++++

 4 files changed, 502 insertions(+)

diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c

index 5fbcc72041..7bba4120bd 100644

--- a/src/conf/numa_conf.c

+++ b/src/conf/numa_conf.c

@@ -1114,6 +1114,132 @@ virDomainNumaGetNodeCount(virDomainNumaPtr numa)

 }

+size_t

+virDomainNumaSetNodeCount(virDomainNumaPtr numa, size_t nmem_nodes)

+{

+    if (!nmem_nodes) {

+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",

+                       _("Cannot set an empty mem_nodes set"));

+        return 0;

+    }

+

+    if (numa->mem_nodes) {

+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",

+                       _("Cannot alter an existing mem_nodes set"));

+        return 0;

+    }

+

+    if (VIR_ALLOC_N(numa->mem_nodes, nmem_nodes) < 0)

+        return 0;

+

+    numa->nmem_nodes = nmem_nodes;

+

+    return numa->nmem_nodes;

+}

+

+size_t

+virDomainNumaGetNodeDistance(virDomainNumaPtr numa,

+                             size_t node,

+                             size_t cellid)

+{

+    virDomainNumaDistancePtr distances = NULL;

+

+    if (node < numa->nmem_nodes)

+        distances = numa->mem_nodes[node].distances;

+

+    /*

+     * Present the configured distance value. If

+     * out of range or not available set the platform

+     * defined default for local and remote nodes.

+     */

+    if (!distances ||

+        !distances[cellid].value ||

+        !numa->mem_nodes[node].ndistances)

+        return (node == cellid) ? LOCAL_DISTANCE : REMOTE_DISTANCE;

+

+    return distances[cellid].value;

+}

+

+

+int

+virDomainNumaSetNodeDistance(virDomainNumaPtr numa,

+                             size_t node,

+                             size_t cellid,

+                             unsigned int value)

+{

+    virDomainNumaDistancePtr distances;

+

+    if (node >= numa->nmem_nodes) {

+        virReportError(VIR_ERR_INTERNAL_ERROR,

+                       _("Argument 'node' %zu outranges "

+                         "defined number of NUMA nodes"),

+                       node);

+        return -1;

+    }

+

+    distances = numa->mem_nodes[node].distances;

+    if (!distances ||

+        cellid >= numa->mem_nodes[node].ndistances) {

+        virReportError(VIR_ERR_XML_ERROR, "%s",

+                       _("Arguments under memnode element do not "

+                         "correspond with existing guest's NUMA cell"));

+        return -1;

+    }

+

+    /*

+     * Advanced Configuration and Power Interface

+     * Specification version 6.1. Chapter 5.2.17

+     * System Locality Distance Information Table

+     * ... Distance values of 0-9 are reserved.

+     */

+    if (value < LOCAL_DISTANCE ||

+        value > UNREACHABLE) {

+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,

+                       _("Distance value of %d is not in valid range"),

+                       value);

+        return -1;

+    }

+

+    if (value == LOCAL_DISTANCE && node != cellid) {

+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,

+                       _("Distance value %d under node %zu is "

+                         "LOCAL_DISTANCE and should be set to 10"),

+                       value, node);

+        return -1;

+    }

+

+    distances[cellid].cellid = cellid;

+    distances[cellid].value = value;

+

+    return distances[cellid].value;

+}

+

+

+size_t

+virDomainNumaSetNodeDistanceCount(virDomainNumaPtr numa,

+                                  size_t node,

+                                  size_t ndistances)

+{

+    virDomainNumaDistancePtr distances;

+

+    distances = numa->mem_nodes[node].distances;

+    if (distances) {

+        virReportError(VIR_ERR_INTERNAL_ERROR,

+                       _("Cannot alter an existing nmem_nodes distances set for node: %zu"),

+                       node);

+        return 0;

+    }

+

+    if (VIR_ALLOC_N(distances, ndistances) < 0)

+        return 0;

+

+    numa->mem_nodes[node].distances = distances;

+    numa->mem_nodes[node].ndistances = ndistances;

+

+    return numa->mem_nodes[node].ndistances;

+}

+

+

 virBitmapPtr

 virDomainNumaGetNodeCpumask(virDomainNumaPtr numa,

                             size_t node)

@@ -1122,6 +1248,17 @@ virDomainNumaGetNodeCpumask(virDomainNumaPtr numa,

 }

+virBitmapPtr

+virDomainNumaSetNodeCpumask(virDomainNumaPtr numa,

+                            size_t node,

+                            virBitmapPtr cpumask)

+{

+    numa->mem_nodes[node].cpumask = cpumask;

+

+    return numa->mem_nodes[node].cpumask;

+}

+

+

 virDomainMemoryAccess

 virDomainNumaGetNodeMemoryAccessMode(virDomainNumaPtr numa,

                                      size_t node)

diff --git a/src/conf/numa_conf.h b/src/conf/numa_conf.h

index 378b772e4b..4655de3aa7 100644

--- a/src/conf/numa_conf.h

+++ b/src/conf/numa_conf.h

@@ -87,6 +87,11 @@ int virDomainNumatuneMaybeGetNodeset(virDomainNumaPtr numatune,

 size_t virDomainNumaGetNodeCount(virDomainNumaPtr numa);

+size_t virDomainNumaGetNodeDistance(virDomainNumaPtr numa,

+                                    size_t node,

+                                    size_t sibling)

+    ATTRIBUTE_NONNULL(1);

+

 virBitmapPtr virDomainNumaGetNodeCpumask(virDomainNumaPtr numa,

                                          size_t node)

     ATTRIBUTE_NONNULL(1);

@@ -124,11 +129,31 @@ int virDomainNumatuneSet(virDomainNumaPtr numa,

                          virBitmapPtr nodeset)

     ATTRIBUTE_NONNULL(1);

+size_t virDomainNumaSetNodeCount(virDomainNumaPtr numa,

+                                 size_t nmem_nodes)

+    ATTRIBUTE_NONNULL(1);

+

 void virDomainNumaSetNodeMemorySize(virDomainNumaPtr numa,

                                     size_t node,

                                     unsigned long long size)

     ATTRIBUTE_NONNULL(1);

+int virDomainNumaSetNodeDistance(virDomainNumaPtr numa,

+                                 size_t node,

+                                 size_t sibling,

+                                 unsigned int value)

+    ATTRIBUTE_NONNULL(1);

+

+size_t virDomainNumaSetNodeDistanceCount(virDomainNumaPtr numa,

+                                         size_t node,

+                                         size_t ndistances)

+    ATTRIBUTE_NONNULL(1);

+

+virBitmapPtr virDomainNumaSetNodeCpumask(virDomainNumaPtr numa,

+                                         size_t node,

+                                         virBitmapPtr cpumask)

+    ATTRIBUTE_NONNULL(1);

+

 /*

  * Other accessors

  */

diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms

index ecf7036dff..19543e06ae 100644

--- a/src/libvirt_private.syms

+++ b/src/libvirt_private.syms

@@ -723,9 +723,14 @@ virDomainNumaGetMaxCPUID;

 virDomainNumaGetMemorySize;

 virDomainNumaGetNodeCount;

 virDomainNumaGetNodeCpumask;

+virDomainNumaGetNodeDistance;

 virDomainNumaGetNodeMemoryAccessMode;

 virDomainNumaGetNodeMemorySize;

 virDomainNumaNew;

+virDomainNumaSetNodeCount;

+virDomainNumaSetNodeCpumask;

+virDomainNumaSetNodeDistance;

+virDomainNumaSetNodeDistanceCount;

 virDomainNumaSetNodeMemorySize;

 virDomainNumatuneFormatNodeset;

 virDomainNumatuneFormatXML;

diff --git a/src/xenconfig/xen_xl.c b/src/xenconfig/xen_xl.c

index 8acbfe3f69..81e780586f 100644

--- a/src/xenconfig/xen_xl.c

+++ b/src/xenconfig/xen_xl.c

@@ -309,6 +309,203 @@ xenParseXLSpice(virConfPtr conf, virDomainDefPtr def)

     return -1;

 }

+#ifdef LIBXL_HAVE_VNUMA

+static int

+xenParseXLVnuma(virConfPtr conf,

+                virDomainDefPtr def)

+{

+    int ret = -1;

+    char *tmp = NULL;

+    char **token = NULL;

+    size_t vcpus = 0;

+    size_t nr_nodes = 0;

+    size_t vnodeCnt = 0;

+    virCPUDefPtr cpu = NULL;

+    virConfValuePtr list;

+    virConfValuePtr vnode;

+    virDomainNumaPtr numa;

+

+    numa = def->numa;

+    if (numa == NULL)

+        return -1;

+

+    list = virConfGetValue(conf, "vnuma");

+    if (!list || list->type != VIR_CONF_LIST)

+        return 0;

+

+    vnode = list->list;

+    while (vnode && vnode->type == VIR_CONF_LIST) {

+        vnode = vnode->next;

+        nr_nodes++;

+    }

+

+    if (!virDomainNumaSetNodeCount(numa, nr_nodes))

+        goto cleanup;

+

+    if (VIR_ALLOC(cpu) < 0)

+        goto cleanup;

+

+    list = list->list;

+    while (list) {

+        int pnode = -1;

+        virBitmapPtr cpumask = NULL;

+        unsigned long long kbsize = 0;

+

+        /* Is there a sublist (vnode)? */

+        if (list && list->type == VIR_CONF_LIST) {

+            vnode = list->list;

+

+            while (vnode && vnode->type == VIR_CONF_STRING) {

+                const char *data;

+                const char *str = vnode->str;

+

+                if (!str ||

+                   !(data = strrchr(str, '='))) {

+                    virReportError(VIR_ERR_INTERNAL_ERROR,

+                                   _("vnuma vnode invalid format '%s'"),

+                                   str);

+                    goto cleanup;

+                }

+                data++;

+

+                if (*data) {

+                    size_t len;

+                    char vtoken[64];

+

+                    if (STRPREFIX(str, "pnode")) {

+                        unsigned int cellid;

+

+                        len = strlen(data);

+                        if (!virStrncpy(vtoken, data,

+                                        len, sizeof(vtoken))) {

+                            virReportError(VIR_ERR_INTERNAL_ERROR,

+                                           _("vnuma vnode %zu pnode '%s' too long for destination"),

+                                           vnodeCnt, data);

+                            goto cleanup;

+                        }

+

+                        if ((virStrToLong_ui(vtoken, NULL, 10, &cellid) < 0) ||

+                            (cellid >= nr_nodes)) {

+                            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,

+                                           _("vnuma vnode %zu contains invalid pnode value '%s'"),

+                                           vnodeCnt, data);

+                            goto cleanup;

+                        }

+                        pnode = cellid;

+                    } else if (STRPREFIX(str, "size")) {

+                        len = strlen(data);

+                        if (!virStrncpy(vtoken, data,

+                                        len, sizeof(vtoken))) {

+                            virReportError(VIR_ERR_INTERNAL_ERROR,

+                                           _("vnuma vnode %zu size '%s' too long for destination"),

+                                           vnodeCnt, data);

+                            goto cleanup;

+                        }

+

+                        if (virStrToLong_ull(vtoken, NULL, 10, &kbsize) < 0)

+                            goto cleanup;

+

+                        virDomainNumaSetNodeMemorySize(numa, vnodeCnt, (kbsize * 1024));

+

+                    } else if (STRPREFIX(str, "vcpus")) {

+                        len = strlen(data);

+                        if (!virStrncpy(vtoken, data,

+                                        len, sizeof(vtoken))) {

+                            virReportError(VIR_ERR_INTERNAL_ERROR,

+                                           _("vnuma vnode %zu vcpus '%s' too long for destination"),

+                                           vnodeCnt, data);

+                            goto cleanup;

+                        }

+

+                        if ((virBitmapParse(vtoken, &cpumask, VIR_DOMAIN_CPUMASK_LEN) < 0) ||

+                            (virDomainNumaSetNodeCpumask(numa, vnodeCnt, cpumask) == NULL))

+                            goto cleanup;

+

+                        vcpus += virBitmapCountBits(cpumask);

+

+                    } else if (STRPREFIX(str, "vdistances")) {

+                        size_t i, ndistances;

+                        unsigned int value;

+

+                        len = strlen(data);

+                        if (!virStrncpy(vtoken, data,

+                                        len, sizeof(vtoken))) {

+                            virReportError(VIR_ERR_INTERNAL_ERROR,

+                                           _("vnuma vnode %zu vdistances '%s' too long for destination"),

+                                           vnodeCnt, data);

+                            goto cleanup;

+                        }

+

+                        if (VIR_STRDUP(tmp, vtoken) < 0)

+                            goto cleanup;

+

+                        if (!(token = virStringSplitCount(tmp, ",", 0, &ndistances)))

+                            goto cleanup;

+

+                        if (ndistances != nr_nodes) {

+                            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,

+                                       _("vnuma pnode %d configured '%s' (count %zu) doesn't fit the number of specified vnodes %zu"),

+                                       pnode, str, ndistances, nr_nodes);

+                            goto cleanup;

+                        }

+

+                        if (virDomainNumaSetNodeDistanceCount(numa, vnodeCnt, ndistances) != ndistances)

+                            goto cleanup;

+

+                        for (i = 0; i < ndistances; i++) {

+                            if ((virStrToLong_ui(token[i], NULL, 10, &value) < 0) ||

+                                (virDomainNumaSetNodeDistance(numa, vnodeCnt, i, value) != value))

+                                goto cleanup;

+                        }

+

+                    } else {

+                        virReportError(VIR_ERR_CONF_SYNTAX,

+                                       _("Invalid vnuma configuration for vnode %zu"),

+                                       vnodeCnt);

+                        goto cleanup;

+                    }

+                }

+                vnode = vnode->next;

+            }

+        }

+

+        if ((pnode < 0) ||

+            (cpumask == NULL) ||

+            (kbsize == 0)) {

+            virReportError(VIR_ERR_CONF_SYNTAX,

+                           _("Incomplete vnuma configuration for vnode %zu"),

+                           vnodeCnt);

+            goto cleanup;

+        }

+

+        list = list->next;

+        vnodeCnt++;

+    }

+

+    if (def->maxvcpus == 0)

+        def->maxvcpus = vcpus;

+

+    if (def->maxvcpus < vcpus) {

+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,

+                       _("vnuma configuration contains %zu vcpus, which is greater than %zu maxvcpus"),

+                       vcpus, def->maxvcpus);

+        goto cleanup;

+    }

+

+    cpu->type = VIR_CPU_TYPE_GUEST;

+    def->cpu = cpu;

+

+    ret = 0;

+

+ cleanup:

+    if (ret)

+        VIR_FREE(cpu);

+    virStringListFree(token);

+    VIR_FREE(tmp);

+

+    return ret;

+}

+#endif

 static int

 xenParseXLDiskSrc(virDomainDiskDefPtr disk, char *srcstr)

@@ -863,6 +1060,11 @@ xenParseXL(virConfPtr conf,

     if (xenParseXLOS(conf, def, caps) < 0)

         goto cleanup;

+#ifdef LIBXL_HAVE_VNUMA

+    if (xenParseXLVnuma(conf, def) < 0)

+        goto cleanup;

+#endif

+

     if (xenParseXLDisk(conf, def) < 0)

         goto cleanup;

@@ -1005,6 +1207,134 @@ xenFormatXLOS(virConfPtr conf, virDomainDefPtr def)

     return 0;

 }

+#ifdef LIBXL_HAVE_VNUMA

+static int

+xenFormatXLVnode(virConfValuePtr list,

+                 virBufferPtr buf)

+{

+    int ret = -1;

+    virConfValuePtr numaPnode, tmp;

+

+    if (virBufferCheckError(buf) < 0)

+        goto cleanup;

+

+    if (VIR_ALLOC(numaPnode) < 0)

+        goto cleanup;

+

+    /* Place VNODE directive */

+    numaPnode->type = VIR_CONF_STRING;

+    numaPnode->str = virBufferContentAndReset(buf);

+

+    tmp = list->list;

+    while (tmp && tmp->next)

+        tmp = tmp->next;

+    if (tmp)

+        tmp->next = numaPnode;

+    else

+        list->list = numaPnode;

+    ret = 0;

+

+ cleanup:

+    virBufferFreeAndReset(buf);

+    return ret;

+}

+

+static int

+xenFormatXLVnuma(virConfValuePtr list,

+                 virDomainNumaPtr numa,

+                 size_t node,

+                 size_t nr_nodes)

+{

+    int ret = -1;

+    size_t i;

+

+    virBuffer buf = VIR_BUFFER_INITIALIZER;

+    virConfValuePtr numaVnode, tmp;

+

+    size_t nodeSize = virDomainNumaGetNodeMemorySize(numa, node) / 1024;

+    char *nodeVcpus = virBitmapFormat(virDomainNumaGetNodeCpumask(numa, node));

+

+    if (VIR_ALLOC(numaVnode) < 0)

+        goto cleanup;

+

+    numaVnode->type = VIR_CONF_LIST;

+    numaVnode->list = NULL;

+

+    /* pnode */

+    virBufferAsprintf(&buf, "pnode=%ld", node);

+    xenFormatXLVnode(numaVnode, &buf;;

+

+    /* size */

+    virBufferAsprintf(&buf, "size=%ld", nodeSize);

+    xenFormatXLVnode(numaVnode, &buf;;

+

+    /* vcpus */

+    virBufferAsprintf(&buf, "vcpus=%s", nodeVcpus);

+    xenFormatXLVnode(numaVnode, &buf;;

+

+    /* distances */

+    virBufferAddLit(&buf, "vdistances=");

+    for (i = 0; i < nr_nodes; i++) {

+        virBufferAsprintf(&buf, "%zu",

+            virDomainNumaGetNodeDistance(numa, node, i));

+        if ((nr_nodes - i) > 1)

+            virBufferAddLit(&buf, ",");

+    }

+    xenFormatXLVnode(numaVnode, &buf;;

+

+    tmp = list->list;

+    while (tmp && tmp->next)

+        tmp = tmp->next;

+    if (tmp)

+        tmp->next = numaVnode;

+    else

+        list->list = numaVnode;

+    ret = 0;

+

+ cleanup:

+    VIR_FREE(nodeVcpus);

+    return ret;

+}

+

+static int

+xenFormatXLDomainVnuma(virConfPtr conf,

+                       virDomainDefPtr def)

+{

+    virDomainNumaPtr numa = def->numa;

+    virConfValuePtr vnumaVal;

+    size_t i;

+    size_t nr_nodes;

+

+    if (numa == NULL)

+        return -1;

+

+    if (VIR_ALLOC(vnumaVal) < 0)

+        return -1;

+

+    vnumaVal->type = VIR_CONF_LIST;

+    vnumaVal->list = NULL;

+

+    nr_nodes = virDomainNumaGetNodeCount(numa);

+    for (i = 0; i < nr_nodes; i++) {

+        if (xenFormatXLVnuma(vnumaVal, numa, i, nr_nodes) < 0)

+            goto cleanup;

+    }

+

+    if (vnumaVal->list != NULL) {

+        int ret = virConfSetValue(conf, "vnuma", vnumaVal);

+            vnumaVal = NULL;

+            if (ret < 0)

+                return -1;

+    }

+    VIR_FREE(vnumaVal);

+

+    return 0;

+

+ cleanup:

+    virConfFreeValue(vnumaVal);

+    return -1;

+}

+#endif

 static char *

 xenFormatXLDiskSrcNet(virStorageSourcePtr src)

@@ -1642,6 +1972,11 @@ xenFormatXL(virDomainDefPtr def, virConnectPtr conn)

     if (xenFormatXLOS(conf, def) < 0)

         goto cleanup;

+#ifdef LIBXL_HAVE_VNUMA

+    if (xenFormatXLDomainVnuma(conf, def) < 0)

+        goto cleanup;

+#endif

+

     if (xenFormatXLDomainDisks(conf, def) < 0)

         goto cleanup;

-- 

2.15.1