From 41d6769393f449008abf934e815f137360889633 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sun, 23 Jan 2022 16:53:45 -0800 Subject: [PATCH 108/217] cxl/list: Move enabled memdevs underneath their endpoint When a memdev is enabled it means that the kernel was able to validate a CXL connection from the CXL root, through intervening switches, and to the endpoint. Reflect that state by listing memdevs as child objects of endpoints, or aggregated into an array if individual endpoints are not listed. Link: https://lore.kernel.org/r/164298562531.3021641.10620937879296964476.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams Signed-off-by: Vishal Verma --- Documentation/cxl/cxl-list.txt | 11 ++- Documentation/cxl/lib/libcxl.txt | 2 + cxl/filter.c | 130 ++++++++++++++++++++++++------- cxl/json.c | 6 ++ cxl/lib/libcxl.c | 97 +++++++++++++++++++++++ cxl/lib/libcxl.sym | 3 + cxl/libcxl.h | 4 + 7 files changed, 223 insertions(+), 30 deletions(-) diff --git a/Documentation/cxl/cxl-list.txt b/Documentation/cxl/cxl-list.txt index 9c21ab7..1751868 100644 --- a/Documentation/cxl/cxl-list.txt +++ b/Documentation/cxl/cxl-list.txt @@ -19,7 +19,16 @@ Options can be specified to limit the output to specific objects. When a single object type is specified the return json object is an array of just those objects, when multiple objects types are specified the returned the returned object may be an array of arrays with the inner -array named for the given object type. +array named for the given object type. The top-level arrays are ellided +when the objects can nest under a higher object-type in the hierararchy. +The potential top-level array names and their nesting properties are: + +"anon memdevs":: (disabled memory devices) do not nest +"buses":: do not nest +"ports":: nest under buses +"endpoints":: nest under ports or buses (if ports are not emitted) +"memdevs":: nest under endpoints or ports (if endpoints are not + emitted) or buses (if endpoints and ports are not emitted) Filters can by specifed as either a single identidier, a space separated quoted string, or a comma separated list. When multiple filter diff --git a/Documentation/cxl/lib/libcxl.txt b/Documentation/cxl/lib/libcxl.txt index 91fd33e..73b0fb9 100644 --- a/Documentation/cxl/lib/libcxl.txt +++ b/Documentation/cxl/lib/libcxl.txt @@ -41,6 +41,7 @@ struct cxl_memdev *cxl_memdev_get_first(struct cxl_ctx *ctx); struct cxl_memdev *cxl_memdev_get_next(struct cxl_memdev *memdev); struct cxl_ctx *cxl_memdev_get_ctx(struct cxl_memdev *memdev); const char *cxl_memdev_get_host(struct cxl_memdev *memdev) +struct cxl_memdev *cxl_endpoint_get_memdev(struct cxl_endpoint *endpoint); #define cxl_memdev_foreach(ctx, memdev) \ for (memdev = cxl_memdev_get_first(ctx); \ @@ -231,6 +232,7 @@ struct cxl_ctx *cxl_endpoint_get_ctx(struct cxl_endpoint *endpoint); struct cxl_port *cxl_endpoint_get_parent(struct cxl_endpoint *endpoint); struct cxl_port *cxl_endpoint_get_port(struct cxl_endpoint *endpoint); const char *cxl_endpoint_get_host(struct cxl_endpoint *endpoint); +struct cxl_endpoint *cxl_memdev_get_endpoint(struct cxl_memdev *memdev); #define cxl_endpoint_foreach(port, endpoint) \ for (endpoint = cxl_endpoint_get_first(port); endpoint != NULL; \ diff --git a/cxl/filter.c b/cxl/filter.c index 5d80d1b..2130816 100644 --- a/cxl/filter.c +++ b/cxl/filter.c @@ -381,13 +381,16 @@ static struct json_object *pick_array(struct json_object *child, } static void walk_endpoints(struct cxl_port *port, struct cxl_filter_params *p, - struct json_object *jeps, unsigned long flags) + struct json_object *jeps, struct json_object *jdevs, + unsigned long flags) { struct cxl_endpoint *endpoint; cxl_endpoint_foreach(port, endpoint) { struct cxl_port *ep_port = cxl_endpoint_get_port(endpoint); - struct json_object *jendpoint; + const char *devname = cxl_endpoint_get_devname(endpoint); + struct json_object *jendpoint = NULL; + struct cxl_memdev *memdev; if (!util_cxl_endpoint_filter(endpoint, p->endpoint_filter)) continue; @@ -398,24 +401,54 @@ static void walk_endpoints(struct cxl_port *port, struct cxl_filter_params *p, continue; if (!p->idle && !cxl_endpoint_is_enabled(endpoint)) continue; - jendpoint = util_cxl_endpoint_to_json(endpoint, flags); - if (jendpoint) + if (p->endpoints) { + jendpoint = util_cxl_endpoint_to_json(endpoint, flags); + if (!jendpoint) { + err(p, "%s: failed to list\n", devname); + continue; + } json_object_array_add(jeps, jendpoint); + } + if (p->memdevs) { + struct json_object *jobj; + + memdev = cxl_endpoint_get_memdev(endpoint); + if (!memdev) + continue; + if (!util_cxl_memdev_filter(memdev, p->memdev_filter, + p->serial_filter)) + continue; + if (!p->idle && !cxl_memdev_is_enabled(memdev)) + continue; + jobj = util_cxl_memdev_to_json(memdev, flags); + if (!jobj) { + err(p, "failed to json serialize %s\n", + cxl_memdev_get_devname(memdev)); + continue; + } + if (p->endpoints) + json_object_object_add(jendpoint, "memdev", + jobj); + else + json_object_array_add(jdevs, jobj); + } } } static void walk_child_ports(struct cxl_port *parent_port, struct cxl_filter_params *p, struct json_object *jports, - struct json_object *jeps, unsigned long flags) + struct json_object *jeps, + struct json_object *jdevs, unsigned long flags) { struct cxl_port *port; cxl_port_foreach(parent_port, port) { const char *devname = cxl_port_get_devname(port); struct json_object *jport = NULL; + struct json_object *jchilddevs = NULL; struct json_object *jchildports = NULL; - struct json_object *jchildendpoints = NULL; + struct json_object *jchildeps = NULL; if (!util_cxl_port_filter(port, p->port_filter, pf_mode(p))) goto walk_children; @@ -436,28 +469,41 @@ static void walk_child_ports(struct cxl_port *parent_port, devname); continue; } - } - if (p->ports && p->endpoints) { - jchildendpoints = json_object_new_array(); - if (!jchildendpoints) { - err(p, - "%s: failed to enumerate child endpoints\n", - devname); - continue; + if (p->memdevs && !p->endpoints) { + jchilddevs = json_object_new_array(); + if (!jchilddevs) { + err(p, + "%s: failed to enumerate child memdevs\n", + devname); + continue; + } + } + + if (p->endpoints) { + jchildeps = json_object_new_array(); + if (!jchildeps) { + err(p, + "%s: failed to enumerate child endpoints\n", + devname); + continue; + } } } walk_children: - if (p->endpoints) - walk_endpoints(port, p, pick_array(jchildendpoints, jeps), - flags); + if (p->endpoints || p->memdevs) + walk_endpoints(port, p, pick_array(jchildeps, jeps), + pick_array(jchilddevs, jdevs), flags); walk_child_ports(port, p, pick_array(jchildports, jports), - pick_array(jchildendpoints, jeps), flags); + pick_array(jchildeps, jeps), + pick_array(jchilddevs, jdevs), flags); cond_add_put_array_suffix(jport, "ports", devname, jchildports); cond_add_put_array_suffix(jport, "endpoints", devname, - jchildendpoints); + jchildeps); + cond_add_put_array_suffix(jport, "memdevs", devname, + jchilddevs); } } @@ -466,6 +512,7 @@ int cxl_filter_walk(struct cxl_ctx *ctx, struct cxl_filter_params *p) struct json_object *jdevs = NULL, *jbuses = NULL, *jports = NULL; struct json_object *jplatform = json_object_new_array(); unsigned long flags = params_to_flags(p); + struct json_object *janondevs = NULL; struct json_object *jeps = NULL; struct cxl_memdev *memdev; int top_level_objs = 0; @@ -476,8 +523,8 @@ int cxl_filter_walk(struct cxl_ctx *ctx, struct cxl_filter_params *p) return -ENOMEM; } - jdevs = json_object_new_array(); - if (!jdevs) + janondevs = json_object_new_array(); + if (!janondevs) goto err; jbuses = json_object_new_array(); @@ -492,20 +539,28 @@ int cxl_filter_walk(struct cxl_ctx *ctx, struct cxl_filter_params *p) if (!jeps) goto err; + jdevs = json_object_new_array(); + if (!jdevs) + goto err; + dbg(p, "walk memdevs\n"); cxl_memdev_foreach(ctx, memdev) { - struct json_object *jdev; + struct json_object *janondev; if (!util_cxl_memdev_filter(memdev, p->memdev_filter, p->serial_filter)) continue; + if (cxl_memdev_is_enabled(memdev)) + continue; + if (!p->idle) + continue; if (p->memdevs) { - jdev = util_cxl_memdev_to_json(memdev, flags); - if (!jdev) { + janondev = util_cxl_memdev_to_json(memdev, flags); + if (!janondev) { dbg(p, "memdev object allocation failure\n"); continue; } - json_object_array_add(jdevs, jdev); + json_object_array_add(janondevs, janondev); } } @@ -513,6 +568,7 @@ int cxl_filter_walk(struct cxl_ctx *ctx, struct cxl_filter_params *p) cxl_bus_foreach(ctx, bus) { struct json_object *jbus = NULL; struct json_object *jchildports = NULL; + struct json_object *jchilddevs = NULL; struct json_object *jchildeps = NULL; struct cxl_port *port = cxl_bus_get_port(bus); const char *devname = cxl_bus_get_devname(bus); @@ -546,17 +602,29 @@ int cxl_filter_walk(struct cxl_ctx *ctx, struct cxl_filter_params *p) continue; } } + + if (p->memdevs && !p->ports && !p->endpoints) { + jchilddevs = json_object_new_array(); + if (!jchilddevs) { + err(p, + "%s: failed to enumerate child memdevs\n", + devname); + continue; + } + } } walk_children: dbg(p, "walk ports\n"); walk_child_ports(port, p, pick_array(jchildports, jports), - pick_array(jchildeps, jeps), flags); + pick_array(jchildeps, jeps), + pick_array(jchilddevs, jdevs), flags); cond_add_put_array_suffix(jbus, "ports", devname, jchildports); cond_add_put_array_suffix(jbus, "endpoints", devname, jchildeps); + cond_add_put_array_suffix(jbus, "memdevs", devname, jchilddevs); } - if (json_object_array_length(jdevs)) + if (json_object_array_length(janondevs)) top_level_objs++; if (json_object_array_length(jbuses)) top_level_objs++; @@ -564,20 +632,24 @@ walk_children: top_level_objs++; if (json_object_array_length(jeps)) top_level_objs++; + if (json_object_array_length(jdevs)) + top_level_objs++; - splice_array(p, jdevs, jplatform, "anon memdevs", top_level_objs > 1); + splice_array(p, janondevs, jplatform, "anon memdevs", top_level_objs > 1); splice_array(p, jbuses, jplatform, "buses", top_level_objs > 1); splice_array(p, jports, jplatform, "ports", top_level_objs > 1); splice_array(p, jeps, jplatform, "endpoints", top_level_objs > 1); + splice_array(p, jdevs, jplatform, "memdevs", top_level_objs > 1); util_display_json_array(stdout, jplatform, flags); return 0; err: - json_object_put(jdevs); + json_object_put(janondevs); json_object_put(jbuses); json_object_put(jports); json_object_put(jeps); + json_object_put(jdevs); json_object_put(jplatform); return -ENOMEM; } diff --git a/cxl/json.c b/cxl/json.c index 1868686..b809332 100644 --- a/cxl/json.c +++ b/cxl/json.c @@ -224,6 +224,12 @@ struct json_object *util_cxl_memdev_to_json(struct cxl_memdev *memdev, if (jobj) json_object_object_add(jdev, "host", jobj); + if (!cxl_memdev_is_enabled(memdev)) { + jobj = json_object_new_string("disabled"); + if (jobj) + json_object_object_add(jdev, "state", jobj); + } + return jdev; } diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c index c4ddc7d..4523ca6 100644 --- a/cxl/lib/libcxl.c +++ b/cxl/lib/libcxl.c @@ -480,6 +480,60 @@ CXL_EXPORT const char *cxl_memdev_get_firmware_verison(struct cxl_memdev *memdev return memdev->firmware_version; } +static struct cxl_endpoint *cxl_port_find_endpoint(struct cxl_port *parent_port, + struct cxl_memdev *memdev) +{ + struct cxl_endpoint *endpoint; + struct cxl_port *port; + + cxl_port_foreach(parent_port, port) { + cxl_endpoint_foreach(port, endpoint) + if (strcmp(cxl_endpoint_get_host(endpoint), + cxl_memdev_get_devname(memdev)) == 0) + return endpoint; + endpoint = cxl_port_find_endpoint(port, memdev); + if (endpoint) + return endpoint; + } + + return NULL; +} + +CXL_EXPORT struct cxl_endpoint * +cxl_memdev_get_endpoint(struct cxl_memdev *memdev) +{ + struct cxl_ctx *ctx = cxl_memdev_get_ctx(memdev); + struct cxl_endpoint *endpoint = NULL; + struct cxl_bus *bus; + + if (memdev->endpoint) + return memdev->endpoint; + + if (!cxl_memdev_is_enabled(memdev)) + return NULL; + + cxl_bus_foreach (ctx, bus) { + struct cxl_port *port = cxl_bus_get_port(bus); + + endpoint = cxl_port_find_endpoint(port, memdev); + if (endpoint) + break; + } + + if (!endpoint) + return NULL; + + if (endpoint->memdev && endpoint->memdev != memdev) + err(ctx, "%s assigned to %s not %s\n", + cxl_endpoint_get_devname(endpoint), + cxl_memdev_get_devname(endpoint->memdev), + cxl_memdev_get_devname(memdev)); + memdev->endpoint = endpoint; + endpoint->memdev = memdev; + + return endpoint; +} + CXL_EXPORT size_t cxl_memdev_get_label_size(struct cxl_memdev *memdev) { return memdev->lsa_size; @@ -495,6 +549,21 @@ static int is_enabled(const char *drvpath) return 1; } +CXL_EXPORT int cxl_memdev_is_enabled(struct cxl_memdev *memdev) +{ + struct cxl_ctx *ctx = cxl_memdev_get_ctx(memdev); + char *path = memdev->dev_buf; + int len = memdev->buf_len; + + if (snprintf(path, len, "%s/driver", memdev->dev_path) >= len) { + err(ctx, "%s: buffer too small!\n", + cxl_memdev_get_devname(memdev)); + return 0; + } + + return is_enabled(path); +} + CXL_EXPORT int cxl_memdev_nvdimm_bridge_active(struct cxl_memdev *memdev) { struct cxl_ctx *ctx = cxl_memdev_get_ctx(memdev); @@ -660,6 +729,34 @@ CXL_EXPORT int cxl_endpoint_is_enabled(struct cxl_endpoint *endpoint) return cxl_port_is_enabled(&endpoint->port); } +CXL_EXPORT struct cxl_memdev * +cxl_endpoint_get_memdev(struct cxl_endpoint *endpoint) +{ + struct cxl_ctx *ctx = cxl_endpoint_get_ctx(endpoint); + struct cxl_memdev *memdev; + + if (endpoint->memdev) + return endpoint->memdev; + + if (!cxl_endpoint_is_enabled(endpoint)) + return NULL; + + cxl_memdev_foreach(ctx, memdev) + if (strcmp(cxl_memdev_get_devname(memdev), + cxl_endpoint_get_host(endpoint)) == 0) { + if (memdev->endpoint && memdev->endpoint != endpoint) + err(ctx, "%s assigned to %s not %s\n", + cxl_memdev_get_devname(memdev), + cxl_endpoint_get_devname(memdev->endpoint), + cxl_endpoint_get_devname(endpoint)); + endpoint->memdev = memdev; + memdev->endpoint = endpoint; + return memdev; + } + + return NULL; +} + static void *add_cxl_port(void *parent, int id, const char *cxlport_base) { const char *devname = devpath_to_devname(cxlport_base); diff --git a/cxl/lib/libcxl.sym b/cxl/lib/libcxl.sym index 8f0688a..321acac 100644 --- a/cxl/lib/libcxl.sym +++ b/cxl/lib/libcxl.sym @@ -106,4 +106,7 @@ global: cxl_endpoint_get_parent; cxl_endpoint_get_port; cxl_endpoint_get_host; + cxl_endpoint_get_memdev; + cxl_memdev_get_endpoint; + cxl_memdev_is_enabled; } LIBCXL_1; diff --git a/cxl/libcxl.h b/cxl/libcxl.h index 5487b55..790ece8 100644 --- a/cxl/libcxl.h +++ b/cxl/libcxl.h @@ -46,6 +46,8 @@ unsigned long long cxl_memdev_get_pmem_size(struct cxl_memdev *memdev); unsigned long long cxl_memdev_get_ram_size(struct cxl_memdev *memdev); const char *cxl_memdev_get_firmware_verison(struct cxl_memdev *memdev); size_t cxl_memdev_get_label_size(struct cxl_memdev *memdev); +struct cxl_endpoint; +struct cxl_endpoint *cxl_memdev_get_endpoint(struct cxl_memdev *memdev); int cxl_memdev_nvdimm_bridge_active(struct cxl_memdev *memdev); int cxl_memdev_zero_label(struct cxl_memdev *memdev, size_t length, size_t offset); @@ -100,6 +102,8 @@ int cxl_endpoint_is_enabled(struct cxl_endpoint *endpoint); struct cxl_port *cxl_endpoint_get_parent(struct cxl_endpoint *endpoint); struct cxl_port *cxl_endpoint_get_port(struct cxl_endpoint *endpoint); const char *cxl_endpoint_get_host(struct cxl_endpoint *endpoint); +struct cxl_memdev *cxl_endpoint_get_memdev(struct cxl_endpoint *endpoint); +int cxl_memdev_is_enabled(struct cxl_memdev *memdev); #define cxl_endpoint_foreach(port, endpoint) \ for (endpoint = cxl_endpoint_get_first(port); endpoint != NULL; \ -- 2.27.0