From c47191feb83a737f0923e8002cffa28577f34d66 Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Feb 08 2021 15:32:48 +0000 Subject: Build nvml engines for ppc64le; fix segfault with external engines --- diff --git a/0001-fio-fix-dlopen-refcounting-of-dynamic-engines.patch b/0001-fio-fix-dlopen-refcounting-of-dynamic-engines.patch new file mode 100644 index 0000000..fb8bf13 --- /dev/null +++ b/0001-fio-fix-dlopen-refcounting-of-dynamic-engines.patch @@ -0,0 +1,84 @@ +From 48ff7df9daea86c82a572b0a840bb8371b6b1a29 Mon Sep 17 00:00:00 2001 +From: Eric Sandeen +Date: Mon, 25 Jan 2021 13:23:48 -0600 +Subject: [PATCH] fio: fix dlopen refcounting of dynamic engines + +ioengine_load() will dlclose the dynamic library if it matches one +that we've already got open, but this defeats the built-in refcounting +done by dlopen/dlclose. As each thread exits, it calls free_ioengine(), +and this may do a final dlclose on a dynamic ioengine that is still +in use if we don't have the proper reference count. + +Fix this by dropping the explicit dlclose of a "matching" dlopened +dynamic engine library, and let each dlclose decrement the refcount +on the engine library as is normal. + +This also adds/modifies a couple of debug messages to help track this. + +Signed-off-by: Eric Sandeen +Signed-off-by: Jens Axboe +--- + init.c | 5 +---- + ioengines.c | 8 ++++++-- + 2 files changed, 7 insertions(+), 6 deletions(-) + +diff --git a/init.c b/init.c +index ab38b334..d6dbaf7c 100644 +--- a/init.c ++++ b/init.c +@@ -1109,11 +1109,8 @@ int ioengine_load(struct thread_data *td) + if (!ops) + goto fail; + +- if (ops == td->io_ops && dlhandle == td->io_ops->dlhandle) { +- if (dlhandle) +- dlclose(dlhandle); ++ if (ops == td->io_ops && dlhandle == td->io_ops->dlhandle) + return 0; +- } + + if (dlhandle && dlhandle != td->io_ops->dlhandle) + dlclose(dlhandle); +diff --git a/ioengines.c b/ioengines.c +index dcc9496d..f88b0537 100644 +--- a/ioengines.c ++++ b/ioengines.c +@@ -95,6 +95,7 @@ static void *dlopen_external(struct thread_data *td, const char *engine) + + sprintf(engine_path, "%s/fio-%s.so", FIO_EXT_ENG_DIR, engine); + ++ dprint(FD_IO, "dlopen external %s\n", engine_path); + dlhandle = dlopen(engine_path, RTLD_LAZY); + if (!dlhandle) + log_info("Engine %s not found; Either name is invalid, was not built, or fio-engine-%s package is missing.\n", +@@ -116,7 +117,7 @@ static struct ioengine_ops *dlopen_ioengine(struct thread_data *td, + !strncmp(engine_lib, "aio", 3)) + engine_lib = "libaio"; + +- dprint(FD_IO, "dload engine %s\n", engine_lib); ++ dprint(FD_IO, "dlopen engine %s\n", engine_lib); + + dlerror(); + dlhandle = dlopen(engine_lib, RTLD_LAZY); +@@ -194,7 +195,9 @@ struct ioengine_ops *load_ioengine(struct thread_data *td) + * so as not to break job files not using the prefix. + */ + ops = __load_ioengine(td->o.ioengine); +- if (!ops) ++ ++ /* We do re-dlopen existing handles, for reference counting */ ++ if (!ops || ops->dlhandle) + ops = dlopen_ioengine(td, name); + + /* +@@ -229,6 +232,7 @@ void free_ioengine(struct thread_data *td) + } + + if (td->io_ops->dlhandle) { ++ dprint(FD_IO, "dlclose ioengine %s\n", td->io_ops->name); + dlclose(td->io_ops->dlhandle); + td->io_ops->dlhandle = NULL; + } +-- +2.26.2 + diff --git a/0001-fio-move-dynamic-library-handle-to-io_ops-structure.patch b/0001-fio-move-dynamic-library-handle-to-io_ops-structure.patch new file mode 100644 index 0000000..57ae690 --- /dev/null +++ b/0001-fio-move-dynamic-library-handle-to-io_ops-structure.patch @@ -0,0 +1,110 @@ +From f6931a1dd35896433c8cc2e10de51372a2c496c4 Mon Sep 17 00:00:00 2001 +From: Eric Sandeen +Date: Mon, 25 Jan 2021 13:18:31 -0600 +Subject: [PATCH] fio: move dynamic library handle to io_ops structure + +Keeping a dynamic engine's dlopen'd dlhandle on a thread structure doesn't +make sense; that thread may exit while others are still using the engine. + +Move the dlhandle onto the ops structure itself. + +We still only call dlopen for the first thead, which leaves a refcounting +issue which will be fixed in the next patch. + +Signed-off-by: Eric Sandeen +Signed-off-by: Jens Axboe +--- + fio.h | 1 - + init.c | 6 +++--- + ioengines.c | 8 ++++---- + ioengines.h | 3 ++- + 4 files changed, 9 insertions(+), 9 deletions(-) + +diff --git a/fio.h b/fio.h +index ee582a72..062abfa7 100644 +--- a/fio.h ++++ b/fio.h +@@ -281,7 +281,6 @@ struct thread_data { + * IO engine private data and dlhandle. + */ + void *io_ops_data; +- void *io_ops_dlhandle; + + /* + * Queue depth of io_u's that fio MIGHT do +diff --git a/init.c b/init.c +index 1d14df16..ab38b334 100644 +--- a/init.c ++++ b/init.c +@@ -1104,18 +1104,18 @@ int ioengine_load(struct thread_data *td) + * for this name and see if they match. If they do, then + * the engine is unchanged. + */ +- dlhandle = td->io_ops_dlhandle; ++ dlhandle = td->io_ops->dlhandle; + ops = load_ioengine(td); + if (!ops) + goto fail; + +- if (ops == td->io_ops && dlhandle == td->io_ops_dlhandle) { ++ if (ops == td->io_ops && dlhandle == td->io_ops->dlhandle) { + if (dlhandle) + dlclose(dlhandle); + return 0; + } + +- if (dlhandle && dlhandle != td->io_ops_dlhandle) ++ if (dlhandle && dlhandle != td->io_ops->dlhandle) + dlclose(dlhandle); + + /* Unload the old engine. */ +diff --git a/ioengines.c b/ioengines.c +index 5ac512ae..dcc9496d 100644 +--- a/ioengines.c ++++ b/ioengines.c +@@ -155,7 +155,7 @@ static struct ioengine_ops *dlopen_ioengine(struct thread_data *td, + return NULL; + } + +- td->io_ops_dlhandle = dlhandle; ++ ops->dlhandle = dlhandle; + return ops; + } + +@@ -228,9 +228,9 @@ void free_ioengine(struct thread_data *td) + td->eo = NULL; + } + +- if (td->io_ops_dlhandle) { +- dlclose(td->io_ops_dlhandle); +- td->io_ops_dlhandle = NULL; ++ if (td->io_ops->dlhandle) { ++ dlclose(td->io_ops->dlhandle); ++ td->io_ops->dlhandle = NULL; + } + + td->io_ops = NULL; +diff --git a/ioengines.h b/ioengines.h +index a928b211..839b318d 100644 +--- a/ioengines.h ++++ b/ioengines.h +@@ -8,7 +8,7 @@ + #include "io_u.h" + #include "zbd_types.h" + +-#define FIO_IOOPS_VERSION 27 ++#define FIO_IOOPS_VERSION 28 + + #ifndef CONFIG_DYNAMIC_ENGINES + #define FIO_STATIC static +@@ -30,6 +30,7 @@ struct ioengine_ops { + const char *name; + int version; + int flags; ++ void *dlhandle; + int (*setup)(struct thread_data *); + int (*init)(struct thread_data *); + int (*post_init)(struct thread_data *); +-- +2.26.2 + diff --git a/fio.spec b/fio.spec index f0112d8..419eae6 100644 --- a/fio.spec +++ b/fio.spec @@ -1,12 +1,14 @@ Name: fio Version: 3.25 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Multithreaded IO generation tool License: GPLv2 URL: http://git.kernel.dk/?p=fio.git;a=summary Source: http://brick.kernel.dk/snaps/%{name}-%{version}.tar.bz2 +Patch0: 0001-fio-move-dynamic-library-handle-to-io_ops-structure.patch +Patch1: 0001-fio-fix-dlopen-refcounting-of-dynamic-engines.patch BuildRequires: gcc BuildRequires: libaio-devel @@ -15,7 +17,7 @@ BuildRequires: python3-devel BuildRequires: libnbd-devel BuildRequires: libcurl-devel BuildRequires: openssl-devel -%ifarch x86_64 +%ifarch x86_64 ppc64le BuildRequires: libpmem-devel BuildRequires: libpmemblk-devel %endif @@ -40,7 +42,7 @@ BuildRequires: make Recommends: %{name}-engine-libaio Recommends: %{name}-engine-http Recommends: %{name}-engine-nbd -%ifarch x86-64 +%ifarch x86-64 ppc64le Recommends: %{name}-engine-dev-dax Recommends: %{name}-engine-pmemblk Recommends: %{name}-engine-libpmem @@ -82,7 +84,7 @@ Requires: %{name}%{?_isa} = %{version}-%{release} %description engine-nbd Network Block Device (NBD) engine for %{name}. -%ifarch x86_64 +%ifarch x86_64 ppc64le %package engine-dev-dax Summary: PMDK dev-dax engine for %{name}. Requires: %{name}%{?_isa} = %{version}-%{release} @@ -93,7 +95,7 @@ Read and write using device DAX to a persistent memory device (e.g., /dev/dax0.0) through the PMDK libpmem library. %endif -%ifarch x86_64 +%ifarch x86_64 ppc64le %package engine-pmemblk Summary: PMDK pmemblk engine for %{name}. Requires: %{name}%{?_isa} = %{version}-%{release} @@ -104,7 +106,7 @@ Read and write using filesystem DAX to a file on a filesystem mounted with DAX on a persistent memory device through the PMDK libpmemblk library. %endif -%ifarch x86_64 +%ifarch x86_64 ppc64le %package engine-libpmem Summary: PMDK pmemblk engine for %{name}. Requires: %{name}%{?_isa} = %{version}-%{release} @@ -170,7 +172,7 @@ make install prefix=%{_prefix} mandir=%{_mandir} libdir=%{_libdir}/fio DESTDIR=$ %{_mandir}/man1/* %{_datadir}/%{name}/* -%ifarch x86_64 +%ifarch x86_64 ppc64le %files engine-dev-dax %{_libdir}/fio/fio-dev-dax.so %endif @@ -181,7 +183,7 @@ make install prefix=%{_prefix} mandir=%{_mandir} libdir=%{_libdir}/fio DESTDIR=$ %files engine-libaio %{_libdir}/fio/fio-libaio.so -%ifarch x86_64 +%ifarch x86_64 ppc64le %files engine-libpmem %{_libdir}/fio/fio-libpmem.so %endif @@ -189,7 +191,7 @@ make install prefix=%{_prefix} mandir=%{_mandir} libdir=%{_libdir}/fio DESTDIR=$ %files engine-nbd %{_libdir}/fio/fio-nbd.so -%ifarch x86_64 +%ifarch x86_64 ppc64le %files engine-pmemblk %{_libdir}/fio/fio-pmemblk.so %endif @@ -208,6 +210,10 @@ make install prefix=%{_prefix} mandir=%{_mandir} libdir=%{_libdir}/fio DESTDIR=$ %endif %changelog +* Mon Feb 08 2021 Eric Sandeen 3.25-3 +- Fix segfault with external IO engines and multiple threads +- Enable dev-dax, pmemblk, libpmem engines for ppc64le + * Tue Jan 26 2021 Fedora Release Engineering - 3.25-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_34_Mass_Rebuild