From 5fb3ec71e0e67f1f1c0157c9c9038c5136820799 Mon Sep 17 00:00:00 2001

Message-Id: <5fb3ec71e0e67f1f1c0157c9c9038c5136820799@dist-git>

From: Martin Kletzander <mkletzan@redhat.com>

Date: Wed, 31 Jan 2018 16:32:29 +0100

Subject: [PATCH] resctrl: Add functions to work with resctrl allocations

With this commit we finally have a way to read and manipulate basic resctrl

settings.  Locking is done only on exposed functions that read/write from/to

resctrlfs.  Not in functions that are exposed in virresctrlpriv.h as those are

only supposed to be used from tests.

More information about how resctrl works:

  https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/x86/intel_rdt_ui.txt

Signed-off-by: Martin Kletzander <mkletzan@redhat.com>

(cherry picked from commit a64c761c275c61517360cf1f5af5461fc5b93138)

Signed-off-by: Martin Kletzander <mkletzan@redhat.com>

https://bugzilla.redhat.com/show_bug.cgi?id=1289368

Conflicts:

 - whitespace due to 3e7db8d3e8538bcd5deb1b111fb1233fc18831aa

Signed-off-by: Martin Kletzander <mkletzan@redhat.com>

---

 src/Makefile.am           |    2 +-

 src/libvirt_private.syms  |   10 +

 src/util/virresctrl.c     | 1174 ++++++++++++++++++++++++++++++++++++++++++++-

 src/util/virresctrl.h     |   49 ++

 src/util/virresctrlpriv.h |   27 ++

 5 files changed, 1260 insertions(+), 2 deletions(-)

 create mode 100644 src/util/virresctrlpriv.h

diff --git a/src/Makefile.am b/src/Makefile.am

index b74856ba78..6b24a47afe 100644

--- a/src/Makefile.am

+++ b/src/Makefile.am

@@ -167,7 +167,7 @@ UTIL_SOURCES =							\

 		util/virprocess.c util/virprocess.h		\

 		util/virqemu.c util/virqemu.h			\

 		util/virrandom.h util/virrandom.c		\

-		util/virresctrl.h util/virresctrl.c		\

+		util/virresctrl.h util/virresctrl.c util/virresctrlpriv.h \

 		util/virrotatingfile.h util/virrotatingfile.c   \

 		util/virscsi.c util/virscsi.h			\

 		util/virscsihost.c util/virscsihost.h		\

diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms

index 02bad6c0ce..ee75e09767 100644

--- a/src/libvirt_private.syms

+++ b/src/libvirt_private.syms

@@ -2547,6 +2547,16 @@ virRandomInt;

 # util/virresctrl.h

 virCacheTypeFromString;

 virCacheTypeToString;

+virResctrlAllocAddPID;

+virResctrlAllocCreate;

+virResctrlAllocForeachSize;

+virResctrlAllocFormat;

+virResctrlAllocGetID;

+virResctrlAllocGetUnused;

+virResctrlAllocNew;

+virResctrlAllocRemove;

+virResctrlAllocSetID;

+virResctrlAllocSetSize;

 virResctrlGetInfo;

 virResctrlInfoGetCache;

 virResctrlInfoNew;

diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c

index d439e7792c..e252aefe31 100644

--- a/src/util/virresctrl.c

+++ b/src/util/virresctrl.c

@@ -18,8 +18,12 @@

 #include <config.h>

-#include "virresctrl.h"

+#include <sys/file.h>

+#include <sys/types.h>

+#include <sys/stat.h>

+#include <fcntl.h>

+#include "virresctrlpriv.h"

 #include "c-ctype.h"

 #include "count-one-bits.h"

 #include "viralloc.h"

@@ -152,6 +156,193 @@ virResctrlInfoNew(void)

 }

+/* Alloc-related definitions and AllocClass-related functions */

+

+/*

+ * virResctrlAlloc represents one allocation (in XML under cputune/cachetune and

+ * consequently a directory under /sys/fs/resctrl).  Since it can have multiple

+ * parts of multiple caches allocated it is represented as bunch of nested

+ * sparse arrays (by sparse I mean array of pointers so that each might be NULL

+ * in case there is no allocation for that particular one (level, cache, ...)).

+ *

+ * Since one allocation can be made for caches on different levels, the first

+ * nested sparse array is of types virResctrlAllocPerLevel.  For example if you

+ * have allocation for level 3 cache, there will be three NULL pointers and then

+ * allocated pointer to virResctrlAllocPerLevel.  That way you can access it by

+ * `alloc[level]` as O(1) is desired instead of crawling through normal arrays

+ * or lists in three nested loops.  The code uses a lot of direct accesses.

+ *

+ * Each virResctrlAllocPerLevel can have allocations for different cache

+ * allocation types.  You can allocate instruction cache (VIR_CACHE_TYPE_CODE),

+ * data cache (VIR_CACHE_TYPE_DATA) or unified cache (VIR_CACHE_TYPE_BOTH).

+ * Those allocations are kept in sparse array of virResctrlAllocPerType pointers.

+ *

+ * For each virResctrlAllocPerType users can request some size of the cache to

+ * be allocated.  That's what the sparse array `sizes` is for.  Non-NULL

+ * pointers represent requested size allocations.  The array is indexed by host

+ * cache id (gotten from `/sys/devices/system/cpu/cpuX/cache/indexY/id`).  Users

+ * can see this information e.g. in the output of `virsh capabilities` (for that

+ * information there's the other struct, namely `virResctrlInfo`).

+ *

+ * When allocation is being created we need to find unused part of the cache for

+ * all of them.  While doing that we store the bitmask in a sparse array of

+ * virBitmaps named `masks` indexed the same way as `sizes`.  The upper bounds

+ * of the sparse arrays are stored in nmasks or nsizes, respectively.

+ */

+typedef struct _virResctrlAllocPerType virResctrlAllocPerType;

+typedef virResctrlAllocPerType *virResctrlAllocPerTypePtr;

+struct _virResctrlAllocPerType {

+    /* There could be bool saying whether this is set or not, but since everything

+     * in virResctrlAlloc (and most of libvirt) goes with pointer arrays we would

+     * have to have one more level of allocation anyway, so this stays faithful to

+     * the concept */

+    unsigned long long **sizes;

+    size_t nsizes;

+

+    /* Mask for each cache */

+    virBitmapPtr *masks;

+    size_t nmasks;

+};

+

+typedef struct _virResctrlAllocPerLevel virResctrlAllocPerLevel;

+typedef virResctrlAllocPerLevel *virResctrlAllocPerLevelPtr;

+struct _virResctrlAllocPerLevel {

+    virResctrlAllocPerTypePtr *types; /* Indexed with enum virCacheType */

+    /* There is no `ntypes` member variable as it is always allocated for

+     * VIR_CACHE_TYPE_LAST number of items */

+};

+

+struct _virResctrlAlloc {

+    virObject parent;

+

+    virResctrlAllocPerLevelPtr *levels;

+    size_t nlevels;

+

+    /* The identifier (any unique string for now) */

+    char *id;

+    /* libvirt-generated path in /sys/fs/resctrl for this particular

+     * allocation */

+    char *path;

+};

+

+static virClassPtr virResctrlAllocClass;

+

+static void

+virResctrlAllocDispose(void *obj)

+{

+    size_t i = 0;

+    size_t j = 0;

+    size_t k = 0;

+

+    virResctrlAllocPtr resctrl = obj;

+

+    for (i = 0; i < resctrl->nlevels; i++) {

+        virResctrlAllocPerLevelPtr level = resctrl->levels[i];

+

+        if (!level)

+            continue;

+

+        for (j = 0; j < VIR_CACHE_TYPE_LAST; j++) {

+            virResctrlAllocPerTypePtr type = level->types[j];

+

+            if (!type)

+                continue;

+

+            for (k = 0; k < type->nsizes; k++)

+                VIR_FREE(type->sizes[k]);

+

+            for (k = 0; k < type->nmasks; k++)

+                virBitmapFree(type->masks[k]);

+

+            VIR_FREE(type->sizes);

+            VIR_FREE(type->masks);

+            VIR_FREE(type);

+        }

+        VIR_FREE(level->types);

+        VIR_FREE(level);

+    }

+

+    VIR_FREE(resctrl->id);

+    VIR_FREE(resctrl->path);

+    VIR_FREE(resctrl->levels);

+}

+

+

+static int

+virResctrlAllocOnceInit(void)

+{

+    if (!(virResctrlAllocClass = virClassNew(virClassForObject(),

+                                             "virResctrlAlloc",

+                                             sizeof(virResctrlAlloc),

+                                             virResctrlAllocDispose)))

+        return -1;

+

+    return 0;

+}

+

+

+VIR_ONCE_GLOBAL_INIT(virResctrlAlloc)

+

+

+virResctrlAllocPtr

+virResctrlAllocNew(void)

+{

+    if (virResctrlAllocInitialize() < 0)

+        return NULL;

+

+    return virObjectNew(virResctrlAllocClass);

+}

+

+

+/* Common functions */

+static int

+virResctrlLockInternal(int op)

+{

+    int fd = open(SYSFS_RESCTRL_PATH, O_DIRECTORY | O_CLOEXEC);

+

+    if (fd < 0) {

+        virReportSystemError(errno, "%s", _("Cannot open resctrlfs"));

+        return -1;

+    }

+

+    if (flock(fd, op) < 0) {

+        virReportSystemError(errno, "%s", _("Cannot lock resctrlfs"));

+        VIR_FORCE_CLOSE(fd);

+        return -1;

+    }

+

+    return fd;

+}

+

+

+static inline int

+virResctrlLockWrite(void)

+{

+    return virResctrlLockInternal(LOCK_EX);

+}

+

+

+static int

+virResctrlUnlock(int fd)

+{

+    if (fd == -1)

+        return 0;

+

+    /* The lock gets unlocked by closing the fd, which we need to do anyway in

+     * order to clean up properly */

+    if (VIR_CLOSE(fd) < 0) {

+        virReportSystemError(errno, "%s", _("Cannot close resctrlfs"));

+

+        /* Trying to save the already broken */

+        if (flock(fd, LOCK_UN) < 0)

+            virReportSystemError(errno, "%s", _("Cannot unlock resctrlfs"));

+        return -1;

+    }

+

+    return 0;

+}

+

+

 /* Info-related functions */

 static bool

 virResctrlInfoIsEmpty(virResctrlInfoPtr resctrl)

@@ -367,3 +558,984 @@ virResctrlInfoGetCache(virResctrlInfoPtr resctrl,

     VIR_FREE(*controls);

     goto cleanup;

 }

+

+

+/* Alloc-related functions */

+bool

+virResctrlAllocIsEmpty(virResctrlAllocPtr resctrl)

+{

+    size_t i = 0;

+    size_t j = 0;

+    size_t k = 0;

+

+    if (!resctrl)

+        return true;

+

+    for (i = 0; i < resctrl->nlevels; i++) {

+        virResctrlAllocPerLevelPtr a_level = resctrl->levels[i];

+

+        if (!a_level)

+            continue;

+

+        for (j = 0; j < VIR_CACHE_TYPE_LAST; j++) {

+            virResctrlAllocPerTypePtr a_type = a_level->types[j];

+

+            if (!a_type)

+                continue;

+

+            for (k = 0; k < a_type->nsizes; k++) {

+                if (a_type->sizes[k])

+                    return false;

+            }

+

+            for (k = 0; k < a_type->nmasks; k++) {

+                if (a_type->masks[k])

+                    return false;

+            }

+        }

+    }

+

+    return true;

+}

+

+

+static virResctrlAllocPerTypePtr

+virResctrlAllocGetType(virResctrlAllocPtr resctrl,

+                       unsigned int level,

+                       virCacheType type)

+{

+    virResctrlAllocPerLevelPtr a_level = NULL;

+

+    if (resctrl->nlevels <= level &&

+        VIR_EXPAND_N(resctrl->levels, resctrl->nlevels, level - resctrl->nlevels + 1) < 0)

+        return NULL;

+

+    if (!resctrl->levels[level]) {

+        virResctrlAllocPerTypePtr *types = NULL;

+

+        if (VIR_ALLOC_N(types, VIR_CACHE_TYPE_LAST) < 0)

+            return NULL;

+

+        if (VIR_ALLOC(resctrl->levels[level]) < 0) {

+            VIR_FREE(types);

+            return NULL;

+        }

+        resctrl->levels[level]->types = types;

+    }

+

+    a_level = resctrl->levels[level];

+

+    if (!a_level->types[type] && VIR_ALLOC(a_level->types[type]) < 0)

+        return NULL;

+

+    return a_level->types[type];

+}

+

+

+static int

+virResctrlAllocUpdateMask(virResctrlAllocPtr resctrl,

+                          unsigned int level,

+                          virCacheType type,

+                          unsigned int cache,

+                          virBitmapPtr mask)

+{

+    virResctrlAllocPerTypePtr a_type = virResctrlAllocGetType(resctrl, level, type);

+

+    if (!a_type)

+        return -1;

+

+    if (a_type->nmasks <= cache &&

+        VIR_EXPAND_N(a_type->masks, a_type->nmasks,

+                     cache - a_type->nmasks + 1) < 0)

+        return -1;

+

+    if (!a_type->masks[cache]) {

+        a_type->masks[cache] = virBitmapNew(virBitmapSize(mask));

+

+        if (!a_type->masks[cache])

+            return -1;

+    }

+

+    return virBitmapCopy(a_type->masks[cache], mask);

+}

+

+

+static int

+virResctrlAllocUpdateSize(virResctrlAllocPtr resctrl,

+                          unsigned int level,

+                          virCacheType type,

+                          unsigned int cache,

+                          unsigned long long size)

+{

+    virResctrlAllocPerTypePtr a_type = virResctrlAllocGetType(resctrl, level, type);

+

+    if (!a_type)

+        return -1;

+

+    if (a_type->nsizes <= cache &&

+        VIR_EXPAND_N(a_type->sizes, a_type->nsizes,

+                     cache - a_type->nsizes + 1) < 0)

+        return -1;

+

+    if (!a_type->sizes[cache] && VIR_ALLOC(a_type->sizes[cache]) < 0)

+        return -1;

+

+    *(a_type->sizes[cache]) = size;

+

+    return 0;

+}

+

+

+/*

+ * Check if there is an allocation for this level/type/cache already.  Called

+ * before updating the structure.  VIR_CACHE_TYPE_BOTH collides with any type,

+ * the other types collide with itself.  This code basically checks if either:

+ * `alloc[level]->types[type]->sizes[cache]`

+ * or

+ * `alloc[level]->types[VIR_CACHE_TYPE_BOTH]->sizes[cache]`

+ * is non-NULL.  All the fuzz around it is checking for NULL pointers along

+ * the way.

+ */

+static bool

+virResctrlAllocCheckCollision(virResctrlAllocPtr alloc,

+                              unsigned int level,

+                              virCacheType type,

+                              unsigned int cache)

+{

+    virResctrlAllocPerLevelPtr a_level = NULL;

+    virResctrlAllocPerTypePtr a_type = NULL;

+

+    if (!alloc)

+        return false;

+

+    if (alloc->nlevels <= level)

+        return false;

+

+    a_level = alloc->levels[level];

+

+    if (!a_level)

+        return false;

+

+    a_type = a_level->types[VIR_CACHE_TYPE_BOTH];

+

+    /* If there is an allocation for type 'both', there can be no other

+     * allocation for the same cache */

+    if (a_type && a_type->nsizes > cache && a_type->sizes[cache])

+        return true;

+

+    if (type == VIR_CACHE_TYPE_BOTH) {

+        a_type = a_level->types[VIR_CACHE_TYPE_CODE];

+

+        if (a_type && a_type->nsizes > cache && a_type->sizes[cache])

+            return true;

+

+        a_type = a_level->types[VIR_CACHE_TYPE_DATA];

+

+        if (a_type && a_type->nsizes > cache && a_type->sizes[cache])

+            return true;

+    } else {

+        a_type = a_level->types[type];

+

+        if (a_type && a_type->nsizes > cache && a_type->sizes[cache])

+            return true;

+    }

+

+    return false;

+}

+

+

+int

+virResctrlAllocSetSize(virResctrlAllocPtr resctrl,

+                       unsigned int level,

+                       virCacheType type,

+                       unsigned int cache,

+                       unsigned long long size)

+{

+    if (virResctrlAllocCheckCollision(resctrl, level, type, cache)) {

+        virReportError(VIR_ERR_XML_ERROR,

+                       _("Colliding cache allocations for cache "

+                         "level '%u' id '%u', type '%s'"),

+                       level, cache, virCacheTypeToString(type));

+        return -1;

+    }

+

+    return virResctrlAllocUpdateSize(resctrl, level, type, cache, size);

+}

+

+

+int

+virResctrlAllocForeachSize(virResctrlAllocPtr resctrl,

+                           virResctrlAllocForeachSizeCallback cb,

+                           void *opaque)

+{

+    int ret = 0;

+    unsigned int level = 0;

+    unsigned int type = 0;

+    unsigned int cache = 0;

+

+    if (!resctrl)

+        return 0;

+

+    for (level = 0; level < resctrl->nlevels; level++) {

+        virResctrlAllocPerLevelPtr a_level = resctrl->levels[level];

+

+        if (!a_level)

+            continue;

+

+        for (type = 0; type < VIR_CACHE_TYPE_LAST; type++) {

+            virResctrlAllocPerTypePtr a_type = a_level->types[type];

+

+            if (!a_type)

+                continue;

+

+            for (cache = 0; cache < a_type->nsizes; cache++) {

+                unsigned long long *size = a_type->sizes[cache];

+

+                if (!size)

+                    continue;

+

+                ret = cb(level, type, cache, *size, opaque);

+                if (ret < 0)

+                    return ret;

+            }

+        }

+    }

+

+    return 0;

+}

+

+

+int

+virResctrlAllocSetID(virResctrlAllocPtr alloc,

+                     const char *id)

+{

+    if (!id) {

+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",

+                       _("Resctrl allocation 'id' cannot be NULL"));

+        return -1;

+    }

+

+    return VIR_STRDUP(alloc->id, id);

+}

+

+

+const char *

+virResctrlAllocGetID(virResctrlAllocPtr alloc)

+{

+    return alloc->id;

+}

+

+

+char *

+virResctrlAllocFormat(virResctrlAllocPtr resctrl)

+{

+    virBuffer buf = VIR_BUFFER_INITIALIZER;

+    unsigned int level = 0;

+    unsigned int type = 0;

+    unsigned int cache = 0;

+

+    if (!resctrl)

+        return NULL;

+

+    for (level = 0; level < resctrl->nlevels; level++) {

+        virResctrlAllocPerLevelPtr a_level = resctrl->levels[level];

+

+        if (!a_level)

+            continue;

+

+        for (type = 0; type < VIR_CACHE_TYPE_LAST; type++) {

+            virResctrlAllocPerTypePtr a_type = a_level->types[type];

+

+            if (!a_type)

+                continue;

+

+            virBufferAsprintf(&buf, "L%u%s:", level, virResctrlTypeToString(type));

+

+            for (cache = 0; cache < a_type->nmasks; cache++) {

+                virBitmapPtr mask = a_type->masks[cache];

+                char *mask_str = NULL;

+

+                if (!mask)

+                    continue;

+

+                mask_str = virBitmapToString(mask, false, true);

+                if (!mask_str) {

+                    virBufferFreeAndReset(&buf;;

+                    return NULL;

+                }

+

+                virBufferAsprintf(&buf, "%u=%s;", cache, mask_str);

+                VIR_FREE(mask_str);

+            }

+

+            virBufferTrim(&buf, ";", 1);

+            virBufferAddChar(&buf, '\n');

+        }

+    }

+

+    virBufferCheckError(&buf;;

+    return virBufferContentAndReset(&buf;;

+}

+

+

+static int

+virResctrlAllocParseProcessCache(virResctrlInfoPtr resctrl,

+                                 virResctrlAllocPtr alloc,

+                                 unsigned int level,

+                                 virCacheType type,

+                                 char *cache)

+{

+    char *tmp = strchr(cache, '=');

+    unsigned int cache_id = 0;

+    virBitmapPtr mask = NULL;

+    int ret = -1;

+

+    if (!tmp)

+        return 0;

+

+    *tmp = '\0';

+    tmp++;

+

+    if (virStrToLong_uip(cache, NULL, 10, &cache_id) < 0) {

+        virReportError(VIR_ERR_INTERNAL_ERROR,

+                       _("Invalid cache id '%s'"), cache);

+        return -1;

+    }

+

+    mask = virBitmapNewString(tmp);

+    if (!mask)

+        return -1;

+

+    virBitmapShrink(mask, resctrl->levels[level]->types[type]->bits);

+

+    if (virResctrlAllocUpdateMask(alloc, level, type, cache_id, mask) < 0)

+        goto cleanup;

+

+    ret = 0;

+ cleanup:

+    virBitmapFree(mask);

+    return ret;

+}

+

+

+static int

+virResctrlAllocParseProcessLine(virResctrlInfoPtr resctrl,

+                                virResctrlAllocPtr alloc,

+                                char *line)

+{

+    char **caches = NULL;

+    char *tmp = NULL;

+    unsigned int level = 0;

+    int type = -1;

+    size_t ncaches = 0;

+    size_t i = 0;

+    int ret = -1;

+

+    /* For no reason there can be spaces */

+    virSkipSpaces((const char **) &line);

+

+    /* Skip lines that don't concern caches, e.g. MB: etc. */

+    if (line[0] != 'L')

+        return 0;

+

+    /* And lines that we can't parse too */

+    tmp = strchr(line, ':');

+    if (!tmp)

+        return 0;

+

+    *tmp = '\0';

+    tmp++;

+

+    if (virStrToLong_uip(line + 1, &line, 10, &level) < 0) {

+        virReportError(VIR_ERR_INTERNAL_ERROR,

+                       _("Cannot parse resctrl schema level '%s'"),

+                       line + 1);

+        return -1;

+    }

+

+    type = virResctrlTypeFromString(line);

+    if (type < 0) {

+        virReportError(VIR_ERR_INTERNAL_ERROR,

+                       _("Cannot parse resctrl schema level '%s'"),

+                       line + 1);

+        return -1;

+    }

+

+    caches = virStringSplitCount(tmp, ";", 0, &ncaches);

+    if (!caches)

+        return 0;

+

+    for (i = 0; i < ncaches; i++) {

+        if (virResctrlAllocParseProcessCache(resctrl, alloc, level, type, caches[i]) < 0)

+            goto cleanup;

+    }

+

+    ret = 0;

+ cleanup:

+    virStringListFree(caches);

+    return ret;

+}

+

+

+static int

+virResctrlAllocParse(virResctrlInfoPtr resctrl,

+                     virResctrlAllocPtr alloc,

+                     const char *schemata)

+{

+    char **lines = NULL;

+    size_t nlines = 0;

+    size_t i = 0;

+    int ret = -1;

+

+    lines = virStringSplitCount(schemata, "\n", 0, &nlines);

+    for (i = 0; i < nlines; i++) {

+        if (virResctrlAllocParseProcessLine(resctrl, alloc, lines[i]) < 0)

+            goto cleanup;