From 6cd8155dd5a600f1bb435a49ebddc1fae2f60e73 Mon Sep 17 00:00:00 2001

From: Vishal Verma <vishal.l.verma@intel.com>

Date: Thu, 7 Oct 2021 02:21:24 -0600

Subject: [PATCH 032/217] cxl: add a cxl utility and libcxl library

CXL - or Compute eXpress Link - is a new interconnect that extends PCIe

to support a wide range of devices, including cache coherent memory

expanders. As such, these devices can be new sources of 'persistent

memory', and the 'ndctl' umbrella of tools and libraries needs to be able

to interact with them.

Add a new utility and library for managing these CXL memory devices. This

is an initial bring-up for interacting with CXL devices, and only includes

adding the utility and library infrastructure, parsing device information

from sysfs for CXL devices, and providing a 'cxl-list' command to

display this information in JSON formatted output.

Cc: Ben Widawsky <ben.widawsky@intel.com>

Cc: Dan Williams <dan.j.williams@intel.com>

Reviewed-by: Dan Williams <dan.j.williams@intel.com>

Signed-off-by: Vishal Verma <vishal.l.verma@intel.com>

---

 .clang-format                        |   1 +

 .gitignore                           |   4 +-

 Documentation/cxl/Makefile.am        |  58 +++++

 Documentation/cxl/cxl-list.txt       |  59 +++++

 Documentation/cxl/cxl.txt            |  34 +++

 Documentation/cxl/human-option.txt   |   8 +

 Documentation/cxl/verbose-option.txt |   5 +

 Makefile.am                          |   8 +-

 Makefile.am.in                       |   4 +

 configure.ac                         |   3 +

 cxl/Makefile.am                      |  21 ++

 cxl/builtin.h                        |   8 +

 cxl/cxl.c                            |  96 ++++++++

 cxl/lib/Makefile.am                  |  32 +++

 cxl/lib/libcxl.c                     | 345 +++++++++++++++++++++++++++

 cxl/lib/libcxl.pc.in                 |  11 +

 cxl/lib/libcxl.sym                   |  25 ++

 cxl/lib/private.h                    |  29 +++

 cxl/libcxl.h                         |  55 +++++

 cxl/list.c                           | 121 ++++++++++

 util/filter.c                        |  20 ++

 util/filter.h                        |   2 +

 util/json.c                          |  26 ++

 util/json.h                          |   3 +

 util/main.h                          |   3 +

 25 files changed, 977 insertions(+), 4 deletions(-)

 create mode 100644 Documentation/cxl/Makefile.am

 create mode 100644 Documentation/cxl/cxl-list.txt

 create mode 100644 Documentation/cxl/cxl.txt

 create mode 100644 Documentation/cxl/human-option.txt

 create mode 100644 Documentation/cxl/verbose-option.txt

 create mode 100644 cxl/Makefile.am

 create mode 100644 cxl/builtin.h

 create mode 100644 cxl/cxl.c

 create mode 100644 cxl/lib/Makefile.am

 create mode 100644 cxl/lib/libcxl.c

 create mode 100644 cxl/lib/libcxl.pc.in

 create mode 100644 cxl/lib/libcxl.sym

 create mode 100644 cxl/lib/private.h

 create mode 100644 cxl/libcxl.h

 create mode 100644 cxl/list.c

diff --git a/.clang-format b/.clang-format

index 4e00fff..d2e77d0 100644

--- a/.clang-format

+++ b/.clang-format

@@ -77,6 +77,7 @@ ExperimentalAutoDetectBinPacking: false

 # 		-e 's/\(.*foreach.*\)(.*/\1/' \

 # 	| sort -u)

 ForEachMacros:

+  - 'cxl_memdev_foreach'

   - 'daxctl_dev_foreach'

   - 'daxctl_mapping_foreach'

   - 'daxctl_region_foreach'

diff --git a/.gitignore b/.gitignore

index 53512b2..6a97b92 100644

--- a/.gitignore

+++ b/.gitignore

@@ -16,9 +16,11 @@ Makefile.in

 *.1

 Documentation/daxctl/asciidoc.conf

 Documentation/ndctl/asciidoc.conf

-Documentation/ndctl/attrs.adoc

+Documentation/cxl/asciidoc.conf

 Documentation/daxctl/asciidoctor-extensions.rb

 Documentation/ndctl/asciidoctor-extensions.rb

+Documentation/cxl/asciidoctor-extensions.rb

+Documentation/ndctl/attrs.adoc

 .dirstamp

 daxctl/config.h

 daxctl/daxctl

diff --git a/Documentation/cxl/Makefile.am b/Documentation/cxl/Makefile.am

new file mode 100644

index 0000000..db98dd7

--- /dev/null

+++ b/Documentation/cxl/Makefile.am

@@ -0,0 +1,58 @@

+# SPDX-License-Identifier: GPL-2.0

+# Copyright (C) 2020-2021 Intel Corporation. All rights reserved.

+

+if USE_ASCIIDOCTOR

+

+do_subst = sed -e 's,@Utility@,Cxl,g' -e's,@utility@,cxl,g'

+CONFFILE = asciidoctor-extensions.rb

+asciidoctor-extensions.rb: ../asciidoctor-extensions.rb.in

+	$(AM_V_GEN) $(do_subst) < $< > $@

+

+else

+

+do_subst = sed -e 's,UTILITY,cxl,g'

+CONFFILE = asciidoc.conf

+asciidoc.conf: ../asciidoc.conf.in

+	$(AM_V_GEN) $(do_subst) < $< > $@

+

+endif

+

+man1_MANS = \

+	cxl.1 \

+	cxl-list.1

+

+EXTRA_DIST = $(man1_MANS)

+

+CLEANFILES = $(man1_MANS)

+

+XML_DEPS = \

+	../../version.m4 \

+	../copyright.txt \

+	Makefile \

+	$(CONFFILE)

+

+RM ?= rm -f

+

+if USE_ASCIIDOCTOR

+

+%.1: %.txt $(XML_DEPS)

+	$(AM_V_GEN)$(RM) $@+ $@ && \

+		$(ASCIIDOC) -b manpage -d manpage -acompat-mode \

+		-I. -rasciidoctor-extensions \

+		-amansource=cxl -amanmanual="cxl Manual" \

+		-andctl_version=$(VERSION) -o $@+ $< && \

+		mv $@+ $@

+

+else

+

+%.xml: %.txt $(XML_DEPS)

+	$(AM_V_GEN)$(RM) $@+ $@ && \

+		$(ASCIIDOC) -b docbook -d manpage -f asciidoc.conf \

+		--unsafe -acxl_version=$(VERSION) -o $@+ $< && \

+		mv $@+ $@

+

+%.1: %.xml $(XML_DEPS)

+	$(AM_V_GEN)$(RM) $@ && \

+		$(XMLTO) -o . -m ../manpage-normal.xsl man $<

+

+endif

diff --git a/Documentation/cxl/cxl-list.txt b/Documentation/cxl/cxl-list.txt

new file mode 100644

index 0000000..370d5b8

--- /dev/null

+++ b/Documentation/cxl/cxl-list.txt

@@ -0,0 +1,59 @@

+// SPDX-License-Identifier: GPL-2.0

+

+cxl-list(1)

+===========

+

+NAME

+----

+cxl-list - List CXL capable memory devices, and their attributes in json.

+

+SYNOPSIS

+--------

+[verse]

+'cxl list' [<options>]

+

+Walk the CXL capable device hierarchy in the system and list all device

+instances along with some of their major attributes.

+

+EXAMPLE

+-------

+----

+# cxl list --memdevs

+{

+  "memdev":"mem0",

+  "pmem_size":268435456,

+  "ram_size":0,

+}

+----

+

+OPTIONS

+-------

+-m::

+--memdev=::

+	Specify a cxl memory device name to filter the listing. For example:

+----

+# cxl list --memdev=mem0

+{

+  "memdev":"mem0",

+  "pmem_size":268435456,

+  "ram_size":0,

+}

+----

+

+-M::

+--memdevs::

+	Include CXL memory devices in the listing

+

+-i::

+--idle::

+	Include idle (not enabled / zero-sized) devices in the listing

+

+include::human-option.txt[]

+

+include::verbose-option.txt[]

+

+include::../copyright.txt[]

+

+SEE ALSO

+--------

+linkcxl:ndctl-list[1]

diff --git a/Documentation/cxl/cxl.txt b/Documentation/cxl/cxl.txt

new file mode 100644

index 0000000..41a51c7

--- /dev/null

+++ b/Documentation/cxl/cxl.txt

@@ -0,0 +1,34 @@

+// SPDX-License-Identifier: GPL-2.0

+

+cxl(1)

+======

+

+NAME

+----

+cxl - Provides enumeration and provisioning commands for CXL platforms

+

+SYNOPSIS

+--------

+[verse]

+'cxl' [--version] [--help] COMMAND [ARGS]

+

+OPTIONS

+-------

+-v::

+--version::

+  Display the version of the 'cxl' utility.

+

+-h::

+--help::

+  Run the 'cxl help' command.

+

+DESCRIPTION

+-----------

+The cxl utility provides enumeration and provisioning commands for

+the CXL devices managed by the Linux kernel.

+

+include::../copyright.txt[]

+

+SEE ALSO

+--------

+linkcxl:ndctl[1]

diff --git a/Documentation/cxl/human-option.txt b/Documentation/cxl/human-option.txt

new file mode 100644

index 0000000..2f4de7a

--- /dev/null

+++ b/Documentation/cxl/human-option.txt

@@ -0,0 +1,8 @@

+// SPDX-License-Identifier: GPL-2.0

+

+-u::

+--human::

+	By default the command will output machine-friendly raw-integer

+	data. Instead, with this flag, numbers representing storage size

+	will be formatted as human readable strings with units, other

+	fields are converted to hexadecimal strings.

diff --git a/Documentation/cxl/verbose-option.txt b/Documentation/cxl/verbose-option.txt

new file mode 100644

index 0000000..cb62c8e

--- /dev/null

+++ b/Documentation/cxl/verbose-option.txt

@@ -0,0 +1,5 @@

+// SPDX-License-Identifier: GPL-2.0

+

+-v::

+--verbose::

+	Emit more debug messages

diff --git a/Makefile.am b/Makefile.am

index 60a1998..428fd40 100644

--- a/Makefile.am

+++ b/Makefile.am

@@ -1,9 +1,9 @@

 include Makefile.am.in

 ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}

-SUBDIRS = . daxctl/lib ndctl/lib ndctl daxctl

+SUBDIRS = . cxl/lib daxctl/lib ndctl/lib cxl ndctl daxctl

 if ENABLE_DOCS

-SUBDIRS += Documentation/ndctl Documentation/daxctl

+SUBDIRS += Documentation/ndctl Documentation/daxctl Documentation/cxl

 endif

 SUBDIRS += test

@@ -87,4 +87,6 @@ libutil_a_SOURCES = \

 	util/filter.h \

 	util/bitmap.h

-nobase_include_HEADERS = daxctl/libdaxctl.h

+nobase_include_HEADERS = \

+	daxctl/libdaxctl.h \

+	cxl/libcxl.h

diff --git a/Makefile.am.in b/Makefile.am.in

index bdceda9..aaeee53 100644

--- a/Makefile.am.in

+++ b/Makefile.am.in

@@ -42,3 +42,7 @@ LIBNDCTL_AGE=19

 LIBDAXCTL_CURRENT=6

 LIBDAXCTL_REVISION=0

 LIBDAXCTL_AGE=5

+

+LIBCXL_CURRENT=1

+LIBCXL_REVISION=0

+LIBCXL_AGE=0

diff --git a/configure.ac b/configure.ac

index dc39dbe..dadae0a 100644

--- a/configure.ac

+++ b/configure.ac

@@ -222,12 +222,15 @@ AC_CONFIG_HEADERS(config.h)

 AC_CONFIG_FILES([

         Makefile

         daxctl/lib/Makefile

+        cxl/lib/Makefile

         ndctl/lib/Makefile

         ndctl/Makefile

         daxctl/Makefile

+        cxl/Makefile

         test/Makefile

         Documentation/ndctl/Makefile

         Documentation/daxctl/Makefile

+        Documentation/cxl/Makefile

 ])

 AC_OUTPUT

diff --git a/cxl/Makefile.am b/cxl/Makefile.am

new file mode 100644

index 0000000..98606b9

--- /dev/null

+++ b/cxl/Makefile.am

@@ -0,0 +1,21 @@

+include $(top_srcdir)/Makefile.am.in

+

+bin_PROGRAMS = cxl

+

+DISTCLEANFILES = config.h

+BUILT_SOURCES = config.h

+config.h: $(srcdir)/Makefile.am

+	$(AM_V_GEN) echo "/* Autogenerated by cxl/Makefile.am */" >$@

+

+cxl_SOURCES =\

+		cxl.c \

+		list.c \

+		../util/json.c \

+		builtin.h

+

+cxl_LDADD =\

+	lib/libcxl.la \

+	../libutil.a \

+	$(UUID_LIBS) \

+	$(KMOD_LIBS) \

+	$(JSON_LIBS)

diff --git a/cxl/builtin.h b/cxl/builtin.h

new file mode 100644

index 0000000..3797f98

--- /dev/null

+++ b/cxl/builtin.h

@@ -0,0 +1,8 @@

+/* SPDX-License-Identifier: GPL-2.0 */

+/* Copyright (C) 2020-2021 Intel Corporation. All rights reserved. */

+#ifndef _CXL_BUILTIN_H_

+#define _CXL_BUILTIN_H_

+

+struct cxl_ctx;

+int cmd_list(int argc, const char **argv, struct cxl_ctx *ctx);

+#endif /* _CXL_BUILTIN_H_ */

diff --git a/cxl/cxl.c b/cxl/cxl.c

new file mode 100644

index 0000000..a7725f8

--- /dev/null

+++ b/cxl/cxl.c

@@ -0,0 +1,96 @@

+// SPDX-License-Identifier: GPL-2.0

+/* Copyright (C) 2020-2021 Intel Corporation. All rights reserved. */

+/* Copyright (C) 2005 Andreas Ericsson. All rights reserved. */

+

+/* originally copied from perf and git */

+

+#include <stdio.h>

+#include <errno.h>

+#include <string.h>

+#include <stdlib.h>

+#include <unistd.h>

+#include <sys/stat.h>

+#include <sys/types.h>

+#include <cxl/libcxl.h>

+#include <util/parse-options.h>

+#include <ccan/array_size/array_size.h>

+

+#include <util/strbuf.h>

+#include <util/util.h>

+#include <util/main.h>

+#include <cxl/builtin.h>

+

+const char cxl_usage_string[] = "cxl [--version] [--help] COMMAND [ARGS]";

+const char cxl_more_info_string[] =

+	"See 'cxl help COMMAND' for more information on a specific command.\n"

+	" cxl --list-cmds to see all available commands";

+

+static int cmd_version(int argc, const char **argv, struct cxl_ctx *ctx)

+{

+	printf("%s\n", VERSION);

+	return 0;

+}

+

+static int cmd_help(int argc, const char **argv, struct cxl_ctx *ctx)

+{

+	const char * const builtin_help_subcommands[] = {

+		"list",

+		NULL,

+	};

+	struct option builtin_help_options[] = {

+		OPT_END(),

+	};

+	const char *builtin_help_usage[] = {

+		"cxl help [command]",

+		NULL

+	};

+

+	argc = parse_options_subcommand(argc, argv, builtin_help_options,

+			builtin_help_subcommands, builtin_help_usage, 0);

+

+	if (!argv[0]) {

+		printf("\n usage: %s\n\n", cxl_usage_string);

+		printf("\n %s\n\n", cxl_more_info_string);

+		return 0;

+	}

+

+	return help_show_man_page(argv[0], "cxl", "CXL_MAN_VIEWER");

+}

+

+static struct cmd_struct commands[] = {

+	{ "version", .c_fn = cmd_version },

+	{ "list", .c_fn = cmd_list },

+	{ "help", .c_fn = cmd_help },

+};

+

+int main(int argc, const char **argv)

+{

+	struct cxl_ctx *ctx;

+	int rc;

+

+	/* Look for flags.. */

+	argv++;

+	argc--;

+	main_handle_options(&argv, &argc, cxl_usage_string, commands,

+			ARRAY_SIZE(commands));

+

+	if (argc > 0) {

+		if (!prefixcmp(argv[0], "--"))

+			argv[0] += 2;

+	} else {

+		/* The user didn't specify a command; give them help */

+		printf("\n usage: %s\n\n", cxl_usage_string);

+		printf("\n %s\n\n", cxl_more_info_string);

+		goto out;

+	}

+

+	rc = cxl_new(&ctx;;

+	if (rc)

+		goto out;

+	main_handle_internal_command(argc, argv, ctx, commands,

+			ARRAY_SIZE(commands), PROG_CXL);

+	cxl_unref(ctx);

+	fprintf(stderr, "Unknown command: '%s'\n", argv[0]);

+out:

+	return 1;

+}

diff --git a/cxl/lib/Makefile.am b/cxl/lib/Makefile.am

new file mode 100644

index 0000000..277f0cd

--- /dev/null

+++ b/cxl/lib/Makefile.am

@@ -0,0 +1,32 @@

+include $(top_srcdir)/Makefile.am.in

+

+%.pc: %.pc.in Makefile

+	$(SED_PROCESS)

+

+pkginclude_HEADERS = ../libcxl.h

+lib_LTLIBRARIES = libcxl.la

+

+libcxl_la_SOURCES =\

+	../libcxl.h \

+	private.h \

+	../../util/sysfs.c \

+	../../util/sysfs.h \

+	../../util/log.c \

+	../../util/log.h \

+	libcxl.c

+

+libcxl_la_LIBADD =\

+	$(UUID_LIBS) \

+	$(KMOD_LIBS)

+

+EXTRA_DIST += libcxl.sym

+

+libcxl_la_LDFLAGS = $(AM_LDFLAGS) \

+	-version-info $(LIBCXL_CURRENT):$(LIBCXL_REVISION):$(LIBCXL_AGE) \

+	-Wl,--version-script=$(top_srcdir)/cxl/lib/libcxl.sym

+libcxl_la_DEPENDENCIES = libcxl.sym

+

+pkgconfigdir = $(libdir)/pkgconfig

+pkgconfig_DATA = libcxl.pc

+EXTRA_DIST += libcxl.pc.in

+CLEANFILES += libcxl.pc

diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c

new file mode 100644

index 0000000..c15e987

--- /dev/null

+++ b/cxl/lib/libcxl.c

@@ -0,0 +1,345 @@

+// SPDX-License-Identifier: LGPL-2.1

+// Copyright (C) 2020-2021, Intel Corporation. All rights reserved.

+#include <stdio.h>

+#include <errno.h>

+#include <limits.h>

+#include <libgen.h>

+#include <stdlib.h>

+#include <dirent.h>

+#include <unistd.h>

+#include <sys/stat.h>

+#include <sys/types.h>

+#include <sys/sysmacros.h>

+#include <uuid/uuid.h>

+#include <ccan/list/list.h>

+#include <ccan/array_size/array_size.h>

+

+#include <util/log.h>

+#include <util/sysfs.h>

+#include <util/bitmap.h>

+#include <cxl/libcxl.h>

+#include "private.h"

+

+/**

+ * struct cxl_ctx - library user context to find "nd" instances

+ *

+ * Instantiate with cxl_new(), which takes an initial reference.  Free

+ * the context by dropping the reference count to zero with

+ * cxl_unref(), or take additional references with cxl_ref()

+ * @timeout: default library timeout in milliseconds

+ */

+struct cxl_ctx {

+	/* log_ctx must be first member for cxl_set_log_fn compat */

+	struct log_ctx ctx;

+	int refcount;

+	void *userdata;

+	int memdevs_init;

+	struct list_head memdevs;

+	struct kmod_ctx *kmod_ctx;

+	void *private_data;

+};

+

+static void free_memdev(struct cxl_memdev *memdev, struct list_head *head)

+{

+	if (head)

+		list_del_from(head, &memdev->list);

+	kmod_module_unref(memdev->module);

+	free(memdev->firmware_version);

+	free(memdev->dev_buf);

+	free(memdev->dev_path);

+	free(memdev);

+}

+

+/**

+ * cxl_get_userdata - retrieve stored data pointer from library context

+ * @ctx: cxl library context

+ *

+ * This might be useful to access from callbacks like a custom logging

+ * function.

+ */

+CXL_EXPORT void *cxl_get_userdata(struct cxl_ctx *ctx)

+{

+	if (ctx == NULL)

+		return NULL;

+	return ctx->userdata;

+}

+

+/**

+ * cxl_set_userdata - store custom @userdata in the library context

+ * @ctx: cxl library context

+ * @userdata: data pointer

+ */

+CXL_EXPORT void cxl_set_userdata(struct cxl_ctx *ctx, void *userdata)

+{

+	if (ctx == NULL)

+		return;

+	ctx->userdata = userdata;

+}

+

+CXL_EXPORT void cxl_set_private_data(struct cxl_ctx *ctx, void *data)

+{

+	ctx->private_data = data;

+}

+

+CXL_EXPORT void *cxl_get_private_data(struct cxl_ctx *ctx)

+{

+	return ctx->private_data;

+}

+

+/**

+ * cxl_new - instantiate a new library context

+ * @ctx: context to establish

+ *

+ * Returns zero on success and stores an opaque pointer in ctx.  The

+ * context is freed by cxl_unref(), i.e. cxl_new() implies an

+ * internal cxl_ref().

+ */

+CXL_EXPORT int cxl_new(struct cxl_ctx **ctx)

+{

+	struct kmod_ctx *kmod_ctx;

+	struct cxl_ctx *c;

+	int rc = 0;

+

+	c = calloc(1, sizeof(struct cxl_ctx));

+	if (!c)

+		return -ENOMEM;

+

+	kmod_ctx = kmod_new(NULL, NULL);

+	if (check_kmod(kmod_ctx) != 0) {

+		rc = -ENXIO;

+		goto out;

+	}

+

+	c->refcount = 1;

+	log_init(&c->ctx, "libcxl", "CXL_LOG");

+	info(c, "ctx %p created\n", c);

+	dbg(c, "log_priority=%d\n", c->ctx.log_priority);

+	*ctx = c;

+	list_head_init(&c->memdevs);

+	c->kmod_ctx = kmod_ctx;

+

+	return 0;

+out:

+	free(c);

+	return rc;

+}

+

+/**

+ * cxl_ref - take an additional reference on the context

+ * @ctx: context established by cxl_new()

+ */

+CXL_EXPORT struct cxl_ctx *cxl_ref(struct cxl_ctx *ctx)

+{

+	if (ctx == NULL)

+		return NULL;

+	ctx->refcount++;

+	return ctx;

+}

+

+/**

+ * cxl_unref - drop a context reference count

+ * @ctx: context established by cxl_new()

+ *

+ * Drop a reference and if the resulting reference count is 0 destroy

+ * the context.

+ */

+CXL_EXPORT void cxl_unref(struct cxl_ctx *ctx)

+{

+	struct cxl_memdev *memdev, *_d;

+

+	if (ctx == NULL)

+		return;

+	ctx->refcount--;

+	if (ctx->refcount > 0)

+		return;

+

+	list_for_each_safe(&ctx->memdevs, memdev, _d, list)

+		free_memdev(memdev, &ctx->memdevs);

+

+	kmod_unref(ctx->kmod_ctx);

+	info(ctx, "context %p released\n", ctx);

+	free(ctx);

+}

+

+/**

+ * cxl_set_log_fn - override default log routine

+ * @ctx: cxl library context

+ * @log_fn: function to be called for logging messages

+ *

+ * The built-in logging writes to stderr. It can be overridden by a

+ * custom function, to plug log messages into the user's logging

+ * functionality.

+ */

+CXL_EXPORT void cxl_set_log_fn(struct cxl_ctx *ctx,

+		void (*cxl_log_fn)(struct cxl_ctx *ctx, int priority,

+			const char *file, int line, const char *fn,

+			const char *format, va_list args))

+{

+	ctx->ctx.log_fn = (log_fn) cxl_log_fn;

+	info(ctx, "custom logging function %p registered\n", cxl_log_fn);

+}

+

+/**

+ * cxl_get_log_priority - retrieve current library loglevel (syslog)

+ * @ctx: cxl library context

+ */

+CXL_EXPORT int cxl_get_log_priority(struct cxl_ctx *ctx)

+{

+	return ctx->ctx.log_priority;

+}