From 4f299eb033ad1ffbfdc7796dffdc0c41713ed122 Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Jul 28 2020 10:32:57 +0000 Subject: import nbdkit-1.16.2-4.module+el8.3.0+6922+fd575af8 --- diff --git a/.gitignore b/.gitignore index 46b8f8c..9bf8ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ SOURCES/libguestfs.keyring -SOURCES/nbdkit-1.4.2.tar.gz +SOURCES/nbdkit-1.16.2.tar.gz diff --git a/.nbdkit.metadata b/.nbdkit.metadata index a3d1ed4..fdcb45a 100644 --- a/.nbdkit.metadata +++ b/.nbdkit.metadata @@ -1,2 +1,2 @@ 1bbc40f501a7fef9eef2a39b701a71aee2fea7c4 SOURCES/libguestfs.keyring -183de3c31d768148c49456732c3a86350126c101 SOURCES/nbdkit-1.4.2.tar.gz +42a5761cd3403c02c43cdf7d541ff3faaf8b4769 SOURCES/nbdkit-1.16.2.tar.gz diff --git a/SOURCES/0001-server-Allow-D-nbdkit.-debug-flags-for-the-core-serv.patch b/SOURCES/0001-server-Allow-D-nbdkit.-debug-flags-for-the-core-serv.patch new file mode 100644 index 0000000..758f2f0 --- /dev/null +++ b/SOURCES/0001-server-Allow-D-nbdkit.-debug-flags-for-the-core-serv.patch @@ -0,0 +1,75 @@ +From d7836fb0a7131c725e3c02be7e48e99c671637c3 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 12 Dec 2019 08:57:15 +0000 +Subject: [PATCH 01/19] server: Allow -D nbdkit.* debug flags for the core + server. + +These work like plugin/filter debug flags, but apply to the internals +of the server. + +(cherry picked from commit 3b45db234a691f8ff926a6fef583e11c3601f112) +--- + docs/nbdkit.pod | 7 +++++++ + docs/synopsis.txt | 2 +- + server/main.c | 3 +++ + server/nbdkit.syms | 2 ++ + 4 files changed, 13 insertions(+), 1 deletion(-) + +diff --git a/docs/nbdkit.pod b/docs/nbdkit.pod +index a2e72b13..346d8332 100644 +--- a/docs/nbdkit.pod ++++ b/docs/nbdkit.pod +@@ -177,6 +177,13 @@ Display brief command line usage information and exit. + Set the plugin or filter Debug Flag called C to the integer + value C. See L. + ++=item B<-D> nbdkit.FLAG=N ++ ++=item B<--debug> nbdkit.FLAG=N ++ ++Set the nbdkit server Debug Flag called C to the integer value ++C. ++ + =item B<--dump-config> + + Dump out the compile-time configuration values and exit. +diff --git a/docs/synopsis.txt b/docs/synopsis.txt +index 3c239373..c3675422 100644 +--- a/docs/synopsis.txt ++++ b/docs/synopsis.txt +@@ -1,4 +1,4 @@ +-nbdkit [-D|--debug PLUGIN|FILTER.FLAG=N] ++nbdkit [-D|--debug PLUGIN|FILTER|nbdkit.FLAG=N] + [-e|--exportname EXPORTNAME] [--exit-with-parent] + [--filter FILTER ...] [-f|--foreground] + [-g|--group GROUP] [-i|--ipaddr IPADDR] +diff --git a/server/main.c b/server/main.c +index d39941b1..11ba1e6d 100644 +--- a/server/main.c ++++ b/server/main.c +@@ -563,6 +563,9 @@ main (int argc, char *argv[]) + free (t); + } + ++ /* Apply nbdkit.* flags for the server. */ ++ apply_debug_flags (NULL, "nbdkit"); ++ + /* Check all debug flags were used, and free them. */ + free_debug_flags (); + +diff --git a/server/nbdkit.syms b/server/nbdkit.syms +index 390972e2..96c22c07 100644 +--- a/server/nbdkit.syms ++++ b/server/nbdkit.syms +@@ -67,6 +67,8 @@ + nbdkit_vdebug; + nbdkit_verror; + ++ nbdkit_debug_*; ++ + # Everything else is hidden. + local: *; + }; +-- +2.18.2 + diff --git a/SOURCES/0001-vddk-Remove-vimapiver-parameter.patch b/SOURCES/0001-vddk-Remove-vimapiver-parameter.patch deleted file mode 100644 index 9150380..0000000 --- a/SOURCES/0001-vddk-Remove-vimapiver-parameter.patch +++ /dev/null @@ -1,117 +0,0 @@ -From f8b15da031cbbc9ec3d21d056cd1cdf673416bbc Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 24 Jul 2018 12:08:41 +0100 -Subject: [PATCH] vddk: Remove vimapiver parameter. - -VDDK never used this, setting it is useless. - -(cherry picked from commit ecface865aa121a601c571831d78f4ea1f0574b8) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 12 ++---------- - plugins/vddk/vddk.c | 20 +++----------------- - 2 files changed, 5 insertions(+), 27 deletions(-) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index 4d9c6e9..c5486a3 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -10,7 +10,7 @@ nbdkit-vddk-plugin - VMware VDDK plugin for nbdkit - [vm=moref=ID] [server=HOSTNAME] [user=USERNAME] - [password=PASSWORD | password=- | password=+FILENAME] - [cookie=COOKIE] [thumbprint=THUMBPRINT] -- [vimapiver=APIVER] [port=PORT] [nfchostport=PORT] -+ [port=PORT] [nfchostport=PORT] - [snapshot=MOREF] [transports=MODE:MODE:...] - nbdkit vddk --dump-plugin - -@@ -177,10 +177,7 @@ L - --Optional. Specify the VIM API version. If not given it defaults to --the current version. -- --(Only supported in VDDK ≥ 6.5.0) -+This parameter is ignored for backwards compatibility. - - =back - -@@ -271,11 +268,6 @@ at runtime. - If this is printed then the C parameter is supported - by this build. - --=item C -- --If this is printed then the C parameter is supported --by this build. -- - =back - - =head1 DEBUGGING VDDK -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 8bc1517..7e0590e 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -1,5 +1,5 @@ - /* nbdkit -- * Copyright (C) 2013-2017 Red Hat Inc. -+ * Copyright (C) 2013-2018 Red Hat Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without -@@ -59,7 +59,6 @@ static const char *snapshot_moref = NULL; /* snapshot */ - static const char *thumb_print = NULL; /* thumbprint */ - static const char *transport_modes = NULL; /* transports */ - static const char *username = NULL; /* user */ --static const char *vim_api_ver = NULL; /* vimapiver */ - static const char *vmx_spec = NULL; /* vm */ - static int is_remote = 0; - -@@ -208,12 +207,7 @@ vddk_config (const char *key, const char *value) - username = value; - } - else if (strcmp (key, "vimapiver") == 0) { --#if HAVE_VIXDISKLIBCONNECTPARAMS_VIMAPIVER -- vim_api_ver = value; --#else -- nbdkit_error ("this version of VDDK is too old to support vimapiver"); -- return -1; --#endif -+ /* Ignored for backwards compatibility. */ - } - else if (strcmp (key, "vm") == 0) { - vmx_spec = value; -@@ -248,8 +242,7 @@ vddk_config_complete (void) - cookie || - thumb_print || - port || -- nfc_host_port || -- vim_api_ver; -+ nfc_host_port; - - if (is_remote) { - #define missing(test, param) \ -@@ -281,10 +274,6 @@ vddk_dump_plugin (void) - printf ("vddk_has_nfchostport=1\n"); - #endif - --#if HAVE_VIXDISKLIBCONNECTPARAMS_VIMAPIVER -- printf ("vddk_has_vimapiver=1\n"); --#endif -- - /* XXX We really need to print the version of the dynamically - * linked library here, but VDDK does not provide it. - */ -@@ -337,9 +326,6 @@ vddk_open (int readonly) - #if HAVE_VIXDISKLIBCONNECTPARAMS_NFCHOSTPORT - params.nfcHostPort = nfc_host_port; - #endif --#if HAVE_VIXDISKLIBCONNECTPARAMS_VIMAPIVER -- params.vimApiVer = (char *) vim_api_ver; --#endif - } - - /* XXX Some documentation suggests we should call --- -1.8.3.1 - diff --git a/SOURCES/0002-server-Allow-D-debug-flags-to-contain-dots-for-names.patch b/SOURCES/0002-server-Allow-D-debug-flags-to-contain-dots-for-names.patch new file mode 100644 index 0000000..c7f1786 --- /dev/null +++ b/SOURCES/0002-server-Allow-D-debug-flags-to-contain-dots-for-names.patch @@ -0,0 +1,67 @@ +From e5d2d44fff9214725506cbc84e7b3c035ec0eae9 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 12 Dec 2019 11:06:36 +0000 +Subject: [PATCH 02/19] server: Allow -D debug flags to contain dots for + namespacing. + +This is just a convenience. Either of: + + -D myplugin.foo_bar=1 + -D myplugin.foo.bar=1 + +correspond to the same plugin variable "myplugin_debug_foo_bar". + +(cherry picked from commit a895fa84aaa50f52af68319523020046394c789f) +--- + docs/nbdkit-plugin.pod | 8 ++++++++ + server/debug-flags.c | 10 +++++++++- + 2 files changed, 17 insertions(+), 1 deletion(-) + +diff --git a/docs/nbdkit-plugin.pod b/docs/nbdkit-plugin.pod +index b69cb825..879ddf09 100644 +--- a/docs/nbdkit-plugin.pod ++++ b/docs/nbdkit-plugin.pod +@@ -1298,6 +1298,14 @@ You should only use this feature for debug settings. For general + settings use ordinary plugin parameters. Debug Flags can only be C + ints. They are not supported by non-C language plugins. + ++For convenience C<'.'> characters are replaced with C<'_'> characters ++in the variable name, so both of these parameters: ++ ++ -D myplugin.foo_bar=1 ++ -D myplugin.foo.bar=1 ++ ++correspond to the plugin variable C. ++ + =head1 INSTALLING THE PLUGIN + + The plugin is a C<*.so> file and possibly a manual page. You can of +diff --git a/server/debug-flags.c b/server/debug-flags.c +index 9344d85c..5e06f5ed 100644 +--- a/server/debug-flags.c ++++ b/server/debug-flags.c +@@ -56,12 +56,20 @@ static char * + symbol_of_debug_flag (const char *name, const char *flag) + { + char *var; ++ size_t i; ++ int len; + +- if (asprintf (&var, "%s_debug_%s", name, flag) == -1) { ++ if ((len = asprintf (&var, "%s_debug_%s", name, flag)) == -1) { + perror ("asprintf"); + exit (EXIT_FAILURE); + } + ++ /* If there are any '.'s remaining in the name, convert them to '_'. */ ++ for (i = 0; i < (size_t) len; ++i) { ++ if (var[i] == '.') ++ var[i] = '_'; ++ } ++ + return var; /* caller frees */ + } + +-- +2.18.2 + diff --git a/SOURCES/0002-vddk-Remove-compile-time-dependency-on-VDDK-library.patch b/SOURCES/0002-vddk-Remove-compile-time-dependency-on-VDDK-library.patch deleted file mode 100644 index b676355..0000000 --- a/SOURCES/0002-vddk-Remove-compile-time-dependency-on-VDDK-library.patch +++ /dev/null @@ -1,600 +0,0 @@ -From 15c68a414dee20d5ac60936bddb640fb1262c641 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 24 Jul 2018 12:11:17 +0100 -Subject: [PATCH] vddk: Remove compile-time dependency on VDDK library. -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Allow the plugin to be compiled without needing the library. Instead -of linking to the library at compile time we open the library at -runtime using dlopen. - -The plugin is now compiled unconditionally, unless you use -‘./configure --disable-vddk’. (Of course you still need the VDDK -library if you want to use the plugin. We cannot even test the plugin -loads without the library.). - -This change also moves the initialization of VDDK (calling InitEx) -into the config_complete method instead of the load method. This -later initialization allows the "config=FILENAME" and -"libdir=PATHNAME" parameters to have an effect whereas previously they -were silently ignored. - -(cherry picked from commit 8d7f7c26eb435334d7fa35e84ceee7d266dfae4c) ---- - README | 4 -- - configure.ac | 75 ++-------------------- - plugins/vddk/Makefile.am | 13 ++-- - plugins/vddk/README.VDDK | 25 ++------ - plugins/vddk/nbdkit-vddk-plugin.pod | 48 ++++++++++---- - plugins/vddk/vddk-structs.h | 125 ++++++++++++++++++++++++++++++++++++ - plugins/vddk/vddk.c | 89 ++++++++++++++++++------- - 7 files changed, 244 insertions(+), 135 deletions(-) - create mode 100644 plugins/vddk/vddk-structs.h - -diff --git a/README b/README -index 9ef251d..ae79dec 100644 ---- a/README -+++ b/README -@@ -79,10 +79,6 @@ For the ext2 plugin: - - - com_err - --For the VDDK plugin: -- -- - VDDK (see plugins/vddk/README.VDDK) -- - For the Perl, example4 and tar plugins: - - - perl interpreter -diff --git a/configure.ac b/configure.ac -index a970451..6bb9405 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -502,77 +502,14 @@ AS_IF([test "$with_ext2" != "no"], [ - AM_CONDITIONAL([HAVE_EXT2], - [test "x$EXT2FS_LIBS" != "x" && test "x$COM_ERR_LIBS" != "x"]) - -+dnl Check if the user wants to disable VDDK support. - dnl See plugins/vddk/README.VDDK. --AC_CHECK_SIZEOF([size_t]) --AS_IF([test "x$ac_cv_sizeof_size_t" = "x4"],[bits=32],[bits=64]) --AC_ARG_WITH([vddk],[ -- AS_HELP_STRING([--with-vddk], -- [enable VMware VDDK plugin @<:@default=no@:>@])], -+AC_ARG_ENABLE([vddk],[ -+ AS_HELP_STRING([--disable-vddk], -+ [disable VMware VDDK plugin])], - [], -- [with_vddk=no]) --AS_IF([test "$with_vddk" = "yes"],[ -- VDDK_CFLAGS= -- VDDK_LIBS="-lvixDiskLib" -- # XXX Warning: stupid VMware API. -- VDDK_LIBDIR="$libdir/vmware-vix-disklib" -- AC_MSG_NOTICE([VDDK plugin enabled from $VDDK_LIBDIR]) -- ],[ -- AS_IF([test "$with_vddk" != "no"], [ -- VDDK_CFLAGS="-I$with_vddk/include" -- VDDK_LIBS="-L$with_vddk/lib$bits -lvixDiskLib" -- VDDK_LIBDIR="$with_vddk" -- AC_MSG_NOTICE([VDDK plugin enabled from $with_vddk]) -- ], -- [AC_MSG_NOTICE([VDDK plugin disabled]) -- ]) --]) -- --dnl If the VDDK plugin was enabled, compile and link a test program to make --dnl sure the library really works. --AS_IF([test "x$VDDK_LIBS" != "x"],[ -- # Save CFLAGS etc while we do this test. -- acx_nbdkit_save_CFLAGS="${CFLAGS}" -- acx_nbdkit_save_LIBS="${LIBS}" -- CFLAGS="$CFLAGS $VDDK_CFLAGS" -- LIBS="$VDDK_LIBS $LIBS" -- -- AC_MSG_CHECKING([if we can link to VDDK]) -- AC_LINK_IFELSE([ -- AC_LANG_SOURCE([[ --#include --#include --#include -- --int --main () --{ -- VixDiskLib_Exit (); --} --]]) -- ],[ -- AC_MSG_RESULT([yes]) -- ],[ -- AC_MSG_RESULT([no]) -- AC_MSG_ERROR([could not link to VDDK, see ‘config.log’ for more information]) -- ]) -- -- dnl Check for optional fields in VixDiskLibConnectParams struct. -- AC_CHECK_MEMBERS([VixDiskLibConnectParams.nfcHostPort, VixDiskLibConnectParams.vimApiVer], -- [], [], [[ --#include --#include --#include --]]) -- -- dnl Restore CFLAGS, etc. -- CFLAGS="${acx_nbdkit_save_CFLAGS}" -- LIBS="${acx_nbdkit_save_LIBS}" --]) -- --AC_SUBST([VDDK_CFLAGS]) --AC_SUBST([VDDK_LIBS]) --AC_DEFINE_UNQUOTED([VDDK_LIBDIR],["$VDDK_LIBDIR"],[VDDK 'libDir'.]) --AM_CONDITIONAL([HAVE_VDDK],[test "x$VDDK_LIBS" != "x"]) -+ [enable_vddk=yes]) -+AM_CONDITIONAL([HAVE_VDDK], [test "x$enable_vddk" = "xyes"]) - - dnl Produce output files. - AC_CONFIG_HEADERS([config.h]) -diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am -index 3b3e8aa..0f06768 100644 ---- a/plugins/vddk/Makefile.am -+++ b/plugins/vddk/Makefile.am -@@ -1,5 +1,5 @@ - # nbdkit --# Copyright (C) 2013 Red Hat Inc. -+# Copyright (C) 2013-2018 Red Hat Inc. - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without -@@ -42,15 +42,16 @@ plugin_LTLIBRARIES = nbdkit-vddk-plugin.la - - nbdkit_vddk_plugin_la_SOURCES = \ - vddk.c \ -+ vddk-structs.h \ - $(top_srcdir)/include/nbdkit-plugin.h - - nbdkit_vddk_plugin_la_CPPFLAGS = \ -- -I$(top_srcdir)/include -+ -I$(top_srcdir)/include \ -+ -DVDDK_LIBDIR=\"$(libdir)/vmware-vix-disklib\" - nbdkit_vddk_plugin_la_CFLAGS = \ -- $(WARNINGS_CFLAGS) \ -- $(VDDK_CFLAGS) -+ $(WARNINGS_CFLAGS) - nbdkit_vddk_plugin_la_LIBADD = \ -- $(VDDK_LIBS) -+ -ldl - nbdkit_vddk_plugin_la_LDFLAGS = \ - -module -avoid-version -shared - -@@ -66,4 +67,4 @@ nbdkit-vddk-plugin.1: nbdkit-vddk-plugin.pod - - endif - --endif -+endif HAVE_VDDK -diff --git a/plugins/vddk/README.VDDK b/plugins/vddk/README.VDDK -index d8c62b1..1e56631 100644 ---- a/plugins/vddk/README.VDDK -+++ b/plugins/vddk/README.VDDK -@@ -11,27 +11,14 @@ account and download it from: - This directory contains an nbdkit plugin which uses this library to - export VMDK files and VMware disks over NBD. - --VDDK >= 6.5 is required. -- --Note: VDDK can do NBD on its own, so nbdkit might not be needed unless --you want the extra features and flexibility of nbdkit. -- --It is never compiled by default. To enable it you have to do: -- -- ./configure --with-vddk -- --If the VDDK library is located in a non-standard location, use this --instead: -- -- ./configure --with-vddk=/path/to/vmware-vix-disklib-distrib -- --(This looks for include/ and lib{32,64}/ subdirectories of the given --path for header files and libraries respectively.) -+You do NOT require VDDK to compile the plugin, and the plugin does not -+contain any VMware code. You only need VDDK at runtime. The plugin -+uses dlopen to load the library from LD_LIBRARY_PATH (or else the -+standard shared library paths). - - After building nbdkit-vddk-plugin.so, read the man page to find out --how to use it (nbdkit-vddk-plugin(1)). -- --You'll probably also want to read the VDDK developer documentation. -+how to use it (nbdkit-vddk-plugin(1)). You'll probably also want to -+read the VDDK developer documentation. - - Bugs - ---- -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index c5486a3..ba7806d 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -16,21 +16,41 @@ nbdkit-vddk-plugin - VMware VDDK plugin for nbdkit - - =head1 DESCRIPTION - --C is a L plugin that serves files from -+C is an L plugin that serves files from - local VMware VMDK files, VMware ESXi servers, VMware VCenter servers, --and other sources by using VMware's proprietary VDDK library. -+and other sources. It requires VMware's proprietary VDDK library that -+you must download yourself separately. - - The plugin can serve read-only (if the I<-r> option is used) or - read/write. - --=head1 LIBRARY LOCATION -+=head1 LIBRARY AND CONFIG FILE LOCATIONS - --If the VDDK library (C) is located on a non-standard --path, you may need to set C or modify --C before this plugin will work. -+If the VDDK library (F) is located on a -+non-standard path, you may need to set C or modify -+F before this plugin will work. In addition you may -+want to set the C parameter so that the VDDK library can load -+plugins like Advanced Transport. - --The VDDK library may depend on C or other libraries --which you may have to install yourself. -+For 64 bit platforms pass the F subdirectory: -+ -+ export LD_LIBRARY_PATH=/path/to/vmware-vix-disklib-distrib/lib64 -+ -+For 32 bit platforms pass the F subdirectory: -+ -+ export LD_LIBRARY_PATH=/path/to/vmware-vix-disklib-distrib/lib32 -+ -+Then pass the VDDK distribution directory as C along with -+other parameters as required: -+ -+ nbdkit vddk \ -+ libdir=/path/to/vmware-vix-disklib-distrib \ -+ file=file.vmdk -+ -+VDDK itself looks in a few default locations for the optional -+configuration file, usually including F and -+F<$HOME/.vmware/config>, but you can override this using the C -+parameter. - - =head1 PARAMETERS - -@@ -67,11 +87,14 @@ L below). - - =item B - --Optional. This sets the path of the VMware VDDK library. It must be --an absolute path. -+Optional. This sets the path of the VMware VDDK distribution. It -+must be an absolute path. -+ -+VDDK uses this to load its own plugins, if this path is unspecified or -+wrong then VDDK will work with reduced functionality. - - If the parameter is not given, then a hard-coded path determined at --compile time is used. -+compile time is used, see L below. - - =item B - -@@ -307,7 +330,6 @@ support this. - - =head1 SEE ALSO - --L, - L, - L, - L, -@@ -319,7 +341,7 @@ Richard W.M. Jones - - =head1 COPYRIGHT - --Copyright (C) 2013-2017 Red Hat Inc. -+Copyright (C) 2013-2018 Red Hat Inc. - - =head1 LICENSE - -diff --git a/plugins/vddk/vddk-structs.h b/plugins/vddk/vddk-structs.h -new file mode 100644 -index 0000000..3e7a3c6 ---- /dev/null -+++ b/plugins/vddk/vddk-structs.h -@@ -0,0 +1,125 @@ -+/* nbdkit -+ * Copyright (C) 2013-2018 Red Hat Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+/* Types and structs that we pass to or return from the VDDK API. -+ * -+ * Updated to VDDK 6.7 -+ */ -+ -+#ifndef NBDKIT_VDDK_STRUCTS_H -+#define NBDKIT_VDDK_STRUCTS_H -+ -+typedef uint64_t VixError; -+#define VIX_OK 0 -+ -+#define VIXDISKLIB_FLAG_OPEN_READ_ONLY 4 -+#define VIXDISKLIB_SECTOR_SIZE 512 -+ -+typedef void *VixDiskLibConnection; -+typedef void *VixDiskLibHandle; -+ -+typedef void VixDiskLibGenericLogFunc (const char *fmt, va_list args); -+ -+enum VixDiskLibCredType { -+ VIXDISKLIB_CRED_UID = 1, -+ VIXDISKLIB_CRED_SESSIONID = 2, -+ VIXDISKLIB_CRED_TICKETID = 3, -+ VIXDISKLIB_CRED_SSPI = 4, -+ VIXDISKLIB_CRED_UNKNOWN = 256 -+}; -+ -+enum VixDiskLibSpecType { -+ VIXDISKLIB_SPEC_VMX = 0, -+ VIXDISKLIB_SPEC_VSTORAGE_OBJECT = 1, -+ VIXDISKLIB_SPEC_UNKNOWN = 2 -+}; -+ -+struct VixDiskLibVStorageObjectSpec { -+ char *id; -+ char *datastoreMoRef; -+ char *ssId; -+}; -+ -+typedef struct VixDiskLibConnectParams { -+ char *vmxSpec; -+ char *serverName; -+ char *thumbPrint; -+ long reserved1; -+ enum VixDiskLibCredType credType; -+ union { -+ struct { -+ char *userName; -+ char *password; -+ } uid; -+ struct { -+ char *cookie; -+ char *userName; -+ char *key; -+ } sessionId; -+ void *reserved2; -+ } creds; -+ uint32_t port; -+ uint32_t nfcHostPort; -+ char *reserved3; -+ char reserved4[8]; -+ void *reserved5; -+ union { -+ struct VixDiskLibVStorageObjectSpec vStorageObjSpec; -+ } spec; -+ enum VixDiskLibSpecType specType; -+} VixDiskLibConnectParams; -+ -+struct VixDiskLibGeometry { -+ uint32_t cylinders; -+ uint32_t heads; -+ uint32_t sectors; -+}; -+ -+enum VixDiskLibAdapterType { -+ VIXDISKLIB_ADAPTER_IDE = 1, -+ VIXDISKLIB_ADAPTER_SCSI_BUSLOGIC = 2, -+ VIXDISKLIB_ADAPTER_SCSI_LSILOGIC = 3, -+ VIXDISKLIB_ADAPTER_UNKNOWN = 256 -+}; -+ -+typedef struct VixDiskLibInfo { -+ struct VixDiskLibGeometry biosGeo; -+ struct VixDiskLibGeometry physGeo; -+ uint64_t capacity; -+ enum VixDiskLibAdapterType adapterType; -+ int numLinks; -+ char *parentFileNameHint; -+ char *uuid; -+} VixDiskLibInfo; -+ -+#endif /* NBDKIT_VDDK_STRUCTS_H */ -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 7e0590e..748f1b3 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -39,14 +39,36 @@ - #include - #include - #include -+#include - - #include - --#include -+#include "vddk-structs.h" - -+/* The VDDK APIs that we call. These globals are initialized when the -+ * plugin is loaded (by vddk_load). -+ */ -+char *(*VixDiskLib_GetErrorText) (VixError err, const char *unused); -+void (*VixDiskLib_FreeErrorText) (char *text); -+VixError (*VixDiskLib_InitEx) (uint32_t major, uint32_t minor, VixDiskLibGenericLogFunc *log_function, VixDiskLibGenericLogFunc *warn_function, VixDiskLibGenericLogFunc *panic_function, const char *lib_dir, const char *config_file); -+void (*VixDiskLib_Exit) (void); -+VixError (*VixDiskLib_ConnectEx) (const VixDiskLibConnectParams *params, char read_only, const char *snapshot_ref, const char *transport_modes, VixDiskLibConnection *connection); -+VixError (*VixDiskLib_Open) (const VixDiskLibConnection connection, const char *path, uint32_t flags, VixDiskLibHandle *handle); -+const char *(*VixDiskLib_GetTransportMode) (VixDiskLibHandle handle); -+VixError (*VixDiskLib_Close) (VixDiskLibHandle handle); -+VixError (*VixDiskLib_Disconnect) (VixDiskLibConnection connection); -+VixError (*VixDiskLib_GetInfo) (VixDiskLibHandle handle, VixDiskLibInfo **info); -+void (*VixDiskLib_FreeInfo) (VixDiskLibInfo *info); -+VixError (*VixDiskLib_Read) (VixDiskLibHandle handle, uint64_t start_sector, uint64_t nr_sectors, unsigned char *buf); -+VixError (*VixDiskLib_Write) (VixDiskLibHandle handle, uint64_t start_sector, uint64_t nr_sectors, const unsigned char *buf); -+ -+/* Parameters passed to InitEx. */ - #define VDDK_MAJOR 5 - #define VDDK_MINOR 1 - -+static void *dl = NULL; /* dlopen handle */ -+static int init_called = 0; /* was InitEx called */ -+ - static char *config = NULL; /* config */ - static const char *cookie = NULL; /* cookie */ - static const char *filename = NULL; /* file */ -@@ -120,27 +142,39 @@ error_function (const char *fs, va_list args) - static void - vddk_load (void) - { -- VixError err; -+ const char *soname = "libvixDiskLib.so.6"; - -- DEBUG_CALL ("VixDiskLib_InitEx", -- "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s", -- VDDK_MAJOR, VDDK_MINOR, libdir, config ? : "NULL"); -- err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR, -- &debug_function, /* log function */ -- &error_function, /* warn function */ -- &error_function, /* panic function */ -- libdir, config); -- if (err != VIX_OK) { -- VDDK_ERROR (err, "VixDiskLib_InitEx"); -+ /* Load the plugin and set the entry points. */ -+ dl = dlopen (soname, RTLD_NOW); -+ if (dl == NULL) { -+ nbdkit_error ("%s: %s", soname, dlerror ()); - exit (EXIT_FAILURE); - } -+ -+ VixDiskLib_GetErrorText = dlsym (dl, "VixDiskLib_GetErrorText"); -+ VixDiskLib_FreeErrorText = dlsym (dl, "VixDiskLib_FreeErrorText"); -+ VixDiskLib_InitEx = dlsym (dl, "VixDiskLib_InitEx"); -+ VixDiskLib_Exit = dlsym (dl, "VixDiskLib_Exit"); -+ VixDiskLib_ConnectEx = dlsym (dl, "VixDiskLib_ConnectEx"); -+ VixDiskLib_Open = dlsym (dl, "VixDiskLib_Open"); -+ VixDiskLib_GetTransportMode = dlsym (dl, "VixDiskLib_GetTransportMode"); -+ VixDiskLib_Close = dlsym (dl, "VixDiskLib_Close"); -+ VixDiskLib_Disconnect = dlsym (dl, "VixDiskLib_Disconnect"); -+ VixDiskLib_GetInfo = dlsym (dl, "VixDiskLib_GetInfo"); -+ VixDiskLib_FreeInfo = dlsym (dl, "VixDiskLib_FreeInfo"); -+ VixDiskLib_Read = dlsym (dl, "VixDiskLib_Read"); -+ VixDiskLib_Write = dlsym (dl, "VixDiskLib_Write"); - } - - static void - vddk_unload (void) - { -- DEBUG_CALL ("VixDiskLib_Exit", ""); -- VixDiskLib_Exit (); -+ if (init_called) { -+ DEBUG_CALL ("VixDiskLib_Exit", ""); -+ VixDiskLib_Exit (); -+ } -+ if (dl) -+ dlclose (dl); - free (config); - free (password); - } -@@ -170,15 +204,10 @@ vddk_config (const char *key, const char *value) - libdir = value; - } - else if (strcmp (key, "nfchostport") == 0) { --#if HAVE_VIXDISKLIBCONNECTPARAMS_NFCHOSTPORT - if (sscanf (value, "%d", &nfc_host_port) != 1) { - nbdkit_error ("cannot parse nfchostport: %s", value); - return -1; - } --#else -- nbdkit_error ("this version of VDDK is too old to support nfchostpost"); -- return -1; --#endif - } - else if (strcmp (key, "password") == 0) { - free (password); -@@ -223,6 +252,8 @@ vddk_config (const char *key, const char *value) - static int - vddk_config_complete (void) - { -+ VixError err; -+ - if (filename == NULL) { - nbdkit_error ("you must supply the file= parameter after the plugin name on the command line"); - return -1; -@@ -258,6 +289,21 @@ vddk_config_complete (void) - #undef missing - } - -+ /* Initialize VDDK library. */ -+ DEBUG_CALL ("VixDiskLib_InitEx", -+ "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s", -+ VDDK_MAJOR, VDDK_MINOR, libdir, config ? : "NULL"); -+ err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR, -+ &debug_function, /* log function */ -+ &error_function, /* warn function */ -+ &error_function, /* panic function */ -+ libdir, config); -+ if (err != VIX_OK) { -+ VDDK_ERROR (err, "VixDiskLib_InitEx"); -+ exit (EXIT_FAILURE); -+ } -+ init_called = 1; -+ - return 0; - } - -@@ -269,10 +315,7 @@ static void - vddk_dump_plugin (void) - { - printf ("vddk_default_libdir=%s\n", VDDK_LIBDIR); -- --#if HAVE_VIXDISKLIBCONNECTPARAMS_NFCHOSTPORT - printf ("vddk_has_nfchostport=1\n"); --#endif - - /* XXX We really need to print the version of the dynamically - * linked library here, but VDDK does not provide it. -@@ -323,9 +366,7 @@ vddk_open (int readonly) - } - params.thumbPrint = (char *) thumb_print; - params.port = port; --#if HAVE_VIXDISKLIBCONNECTPARAMS_NFCHOSTPORT - params.nfcHostPort = nfc_host_port; --#endif - } - - /* XXX Some documentation suggests we should call --- -1.8.3.1 - diff --git a/SOURCES/0003-server-Add-D-nbdkit.backend.controlpath-and-D-nbdkit.patch b/SOURCES/0003-server-Add-D-nbdkit.backend.controlpath-and-D-nbdkit.patch new file mode 100644 index 0000000..7aac76a --- /dev/null +++ b/SOURCES/0003-server-Add-D-nbdkit.backend.controlpath-and-D-nbdkit.patch @@ -0,0 +1,451 @@ +From 83c72d9bf9d6a9ccf6939b4ebd0028b62673a78a Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 12 Dec 2019 10:57:52 +0000 +Subject: [PATCH 03/19] server: Add -D nbdkit.backend.controlpath and -D + nbdkit.backend.datapath. + +These can be used to suppress verbose debugging messages from the +backend. + +BugLink: https://bugzilla.redhat.com/1782868 + +Cherry picked from commit 231717e8cd5f27d76631be6651062d5a5ccf7fdc. +Remove use of nofilter from the test. +--- + docs/nbdkit.pod | 35 ++++++++++++- + server/backend.c | 83 ++++++++++++++++++------------ + tests/Makefile.am | 4 ++ + tests/test-nbdkit-backend-debug.sh | 70 +++++++++++++++++++++++++ + 4 files changed, 158 insertions(+), 34 deletions(-) + create mode 100755 tests/test-nbdkit-backend-debug.sh + +diff --git a/docs/nbdkit.pod b/docs/nbdkit.pod +index 346d8332..38e6bfca 100644 +--- a/docs/nbdkit.pod ++++ b/docs/nbdkit.pod +@@ -182,7 +182,7 @@ value C. See L. + =item B<--debug> nbdkit.FLAG=N + + Set the nbdkit server Debug Flag called C to the integer value +-C. ++C. See L below. + + =item B<--dump-config> + +@@ -527,6 +527,39 @@ languages. The file should be executable. For example: + + (see L for a full example). + ++=head1 SERVER DEBUG FLAGS ++ ++As well as enabling or disabling debugging in the server using ++I<--verbose> you can control extra debugging in the server using the ++C<-D nbdkit.*> flags listed in this section. Note these flags are an ++internal implementation detail of the server and may be changed or ++removed at any time in the future. ++ ++=over 4 ++ ++=item B<-D nbdkit.backend.controlpath=0> ++ ++=item B<-D nbdkit.backend.controlpath=1> ++ ++=item B<-D nbdkit.backend.datapath=0> ++ ++=item B<-D nbdkit.backend.datapath=1> ++ ++These flags control the verbosity of nbdkit backend debugging messages ++(the ones which show every request processed by the server). The ++default for both settings is C<1> (normal debugging) but you can set ++them to C<0> to suppress these messages. ++ ++C<-D nbdkit.backend.datapath=0> is the more useful setting which lets you ++suppress messages about pread, pwrite, zero, trim, etc. commands. ++When transferring large amounts of data these messages are numerous ++and not usually very interesting. ++ ++C<-D nbdkit.backend.controlpath=0> suppresses the non-datapath ++commands (config, open, close, can_write, etc.) ++ ++=back ++ + =head1 SIGNALS + + nbdkit responds to the following signals: +diff --git a/server/backend.c b/server/backend.c +index b9fe2a21..208c07b1 100644 +--- a/server/backend.c ++++ b/server/backend.c +@@ -46,6 +46,22 @@ + + /* Helpers for registering a new backend. */ + ++/* Use: ++ * -D nbdkit.backend.controlpath=0 to suppress control path debugging. ++ * -D nbdkit.backend.datapath=0 to suppress data path debugging. ++ */ ++int nbdkit_debug_backend_controlpath = 1; ++int nbdkit_debug_backend_datapath = 1; ++ ++#define controlpath_debug(fs, ...) \ ++ do { \ ++ if (nbdkit_debug_backend_controlpath) debug ((fs), ##__VA_ARGS__); \ ++ } while (0) ++#define datapath_debug(fs, ...) \ ++ do { \ ++ if (nbdkit_debug_backend_datapath) debug ((fs), ##__VA_ARGS__); \ ++ } while (0) ++ + void + backend_init (struct backend *b, struct backend *next, size_t index, + const char *filename, void *dl, const char *type) +@@ -108,7 +124,7 @@ backend_load (struct backend *b, const char *name, void (*load) (void)) + apply_debug_flags (b->dl, name); + + /* Call the on-load callback if it exists. */ +- debug ("%s: load", name); ++ controlpath_debug ("%s: load", name); + if (load) + load (); + } +@@ -121,7 +137,7 @@ backend_unload (struct backend *b, void (*unload) (void)) + */ + lock_unload (); + +- debug ("%s: unload %s", b->name, b->type); ++ controlpath_debug ("%s: unload %s", b->name, b->type); + if (unload) + unload (); + +@@ -139,7 +155,7 @@ backend_open (struct backend *b, struct connection *conn, int readonly) + { + struct b_conn_handle *h = &conn->handles[b->i]; + +- debug ("%s: open readonly=%d", b->name, readonly); ++ controlpath_debug ("%s: open readonly=%d", b->name, readonly); + + assert (h->handle == NULL); + assert ((h->state & HANDLE_OPEN) == 0); +@@ -151,7 +167,7 @@ backend_open (struct backend *b, struct connection *conn, int readonly) + * inner-to-outer ordering. + */ + h->handle = b->open (b, conn, readonly); +- debug ("%s: open returned handle %p", b->name, h->handle); ++ controlpath_debug ("%s: open returned handle %p", b->name, h->handle); + + if (h->handle == NULL) { + if (b->i) /* Do not strand backend if this layer failed */ +@@ -179,7 +195,7 @@ backend_prepare (struct backend *b, struct connection *conn) + if (b->i && backend_prepare (b->next, conn) == -1) + return -1; + +- debug ("%s: prepare readonly=%d", b->name, h->can_write == 0); ++ controlpath_debug ("%s: prepare readonly=%d", b->name, h->can_write == 0); + + if (b->prepare (b, conn, h->handle, h->can_write == 0) == -1) + return -1; +@@ -196,7 +212,7 @@ backend_finalize (struct backend *b, struct connection *conn) + * filter furthest away from the plugin, and matching .close order. + */ + +- debug ("%s: finalize", b->name); ++ controlpath_debug ("%s: finalize", b->name); + + /* Once finalize fails, we can do nothing further on this connection */ + if (h->state & HANDLE_FAILED) +@@ -223,7 +239,7 @@ backend_close (struct backend *b, struct connection *conn) + struct b_conn_handle *h = &conn->handles[b->i]; + + /* outer-to-inner order, opposite .open */ +- debug ("%s: close", b->name); ++ controlpath_debug ("%s: close", b->name); + + if (h->handle) { + assert (h->state & HANDLE_OPEN); +@@ -252,7 +268,7 @@ backend_valid_range (struct backend *b, struct connection *conn, + int + backend_reopen (struct backend *b, struct connection *conn, int readonly) + { +- debug ("%s: reopen readonly=%d", b->name, readonly); ++ controlpath_debug ("%s: reopen readonly=%d", b->name, readonly); + + if (backend_finalize (b, conn) == -1) + return -1; +@@ -274,7 +290,7 @@ backend_get_size (struct backend *b, struct connection *conn) + { + struct b_conn_handle *h = &conn->handles[b->i]; + +- debug ("%s: get_size", b->name); ++ controlpath_debug ("%s: get_size", b->name); + + assert (h->handle && (h->state & HANDLE_CONNECTED)); + if (h->exportsize == -1) +@@ -287,7 +303,7 @@ backend_can_write (struct backend *b, struct connection *conn) + { + struct b_conn_handle *h = &conn->handles[b->i]; + +- debug ("%s: can_write", b->name); ++ controlpath_debug ("%s: can_write", b->name); + + assert (h->handle && (h->state & HANDLE_CONNECTED)); + if (h->can_write == -1) +@@ -300,7 +316,7 @@ backend_can_flush (struct backend *b, struct connection *conn) + { + struct b_conn_handle *h = &conn->handles[b->i]; + +- debug ("%s: can_flush", b->name); ++ controlpath_debug ("%s: can_flush", b->name); + + assert (h->handle && (h->state & HANDLE_CONNECTED)); + if (h->can_flush == -1) +@@ -313,7 +329,7 @@ backend_is_rotational (struct backend *b, struct connection *conn) + { + struct b_conn_handle *h = &conn->handles[b->i]; + +- debug ("%s: is_rotational", b->name); ++ controlpath_debug ("%s: is_rotational", b->name); + + assert (h->handle && (h->state & HANDLE_CONNECTED)); + if (h->is_rotational == -1) +@@ -327,7 +343,7 @@ backend_can_trim (struct backend *b, struct connection *conn) + struct b_conn_handle *h = &conn->handles[b->i]; + int r; + +- debug ("%s: can_trim", b->name); ++ controlpath_debug ("%s: can_trim", b->name); + + assert (h->handle && (h->state & HANDLE_CONNECTED)); + if (h->can_trim == -1) { +@@ -347,7 +363,7 @@ backend_can_zero (struct backend *b, struct connection *conn) + struct b_conn_handle *h = &conn->handles[b->i]; + int r; + +- debug ("%s: can_zero", b->name); ++ controlpath_debug ("%s: can_zero", b->name); + + assert (h->handle && (h->state & HANDLE_CONNECTED)); + if (h->can_zero == -1) { +@@ -367,7 +383,7 @@ backend_can_fast_zero (struct backend *b, struct connection *conn) + struct b_conn_handle *h = &conn->handles[b->i]; + int r; + +- debug ("%s: can_fast_zero", b->name); ++ controlpath_debug ("%s: can_fast_zero", b->name); + + assert (h->handle && (h->state & HANDLE_CONNECTED)); + if (h->can_fast_zero == -1) { +@@ -386,7 +402,7 @@ backend_can_extents (struct backend *b, struct connection *conn) + { + struct b_conn_handle *h = &conn->handles[b->i]; + +- debug ("%s: can_extents", b->name); ++ controlpath_debug ("%s: can_extents", b->name); + + assert (h->handle && (h->state & HANDLE_CONNECTED)); + if (h->can_extents == -1) +@@ -400,7 +416,7 @@ backend_can_fua (struct backend *b, struct connection *conn) + struct b_conn_handle *h = &conn->handles[b->i]; + int r; + +- debug ("%s: can_fua", b->name); ++ controlpath_debug ("%s: can_fua", b->name); + + assert (h->handle && (h->state & HANDLE_CONNECTED)); + if (h->can_fua == -1) { +@@ -420,7 +436,7 @@ backend_can_multi_conn (struct backend *b, struct connection *conn) + struct b_conn_handle *h = &conn->handles[b->i]; + + assert (h->handle && (h->state & HANDLE_CONNECTED)); +- debug ("%s: can_multi_conn", b->name); ++ controlpath_debug ("%s: can_multi_conn", b->name); + + if (h->can_multi_conn == -1) + h->can_multi_conn = b->can_multi_conn (b, conn, h->handle); +@@ -432,7 +448,7 @@ backend_can_cache (struct backend *b, struct connection *conn) + { + struct b_conn_handle *h = &conn->handles[b->i]; + +- debug ("%s: can_cache", b->name); ++ controlpath_debug ("%s: can_cache", b->name); + + assert (h->handle && (h->state & HANDLE_CONNECTED)); + if (h->can_cache == -1) +@@ -451,8 +467,8 @@ backend_pread (struct backend *b, struct connection *conn, + assert (h->handle && (h->state & HANDLE_CONNECTED)); + assert (backend_valid_range (b, conn, offset, count)); + assert (flags == 0); +- debug ("%s: pread count=%" PRIu32 " offset=%" PRIu64, +- b->name, count, offset); ++ datapath_debug ("%s: pread count=%" PRIu32 " offset=%" PRIu64, ++ b->name, count, offset); + + r = b->pread (b, conn, h->handle, buf, count, offset, flags, err); + if (r == -1) +@@ -475,8 +491,8 @@ backend_pwrite (struct backend *b, struct connection *conn, + assert (!(flags & ~NBDKIT_FLAG_FUA)); + if (fua) + assert (h->can_fua > NBDKIT_FUA_NONE); +- debug ("%s: pwrite count=%" PRIu32 " offset=%" PRIu64 " fua=%d", +- b->name, count, offset, fua); ++ datapath_debug ("%s: pwrite count=%" PRIu32 " offset=%" PRIu64 " fua=%d", ++ b->name, count, offset, fua); + + r = b->pwrite (b, conn, h->handle, buf, count, offset, flags, err); + if (r == -1) +@@ -494,7 +510,7 @@ backend_flush (struct backend *b, struct connection *conn, + assert (h->handle && (h->state & HANDLE_CONNECTED)); + assert (h->can_flush == 1); + assert (flags == 0); +- debug ("%s: flush", b->name); ++ datapath_debug ("%s: flush", b->name); + + r = b->flush (b, conn, h->handle, flags, err); + if (r == -1) +@@ -518,8 +534,8 @@ backend_trim (struct backend *b, struct connection *conn, + assert (!(flags & ~NBDKIT_FLAG_FUA)); + if (fua) + assert (h->can_fua > NBDKIT_FUA_NONE); +- debug ("%s: trim count=%" PRIu32 " offset=%" PRIu64 " fua=%d", +- b->name, count, offset, fua); ++ datapath_debug ("%s: trim count=%" PRIu32 " offset=%" PRIu64 " fua=%d", ++ b->name, count, offset, fua); + + r = b->trim (b, conn, h->handle, count, offset, flags, err); + if (r == -1) +@@ -547,9 +563,10 @@ backend_zero (struct backend *b, struct connection *conn, + assert (h->can_fua > NBDKIT_FUA_NONE); + if (fast) + assert (h->can_fast_zero == 1); +- debug ("%s: zero count=%" PRIu32 " offset=%" PRIu64 +- " may_trim=%d fua=%d fast=%d", +- b->name, count, offset, !!(flags & NBDKIT_FLAG_MAY_TRIM), fua, fast); ++ datapath_debug ("%s: zero count=%" PRIu32 " offset=%" PRIu64 ++ " may_trim=%d fua=%d fast=%d", ++ b->name, count, offset, ++ !!(flags & NBDKIT_FLAG_MAY_TRIM), fua, fast); + + r = b->zero (b, conn, h->handle, count, offset, flags, err); + if (r == -1) { +@@ -572,8 +589,8 @@ backend_extents (struct backend *b, struct connection *conn, + assert (h->can_extents >= 0); + assert (backend_valid_range (b, conn, offset, count)); + assert (!(flags & ~NBDKIT_FLAG_REQ_ONE)); +- debug ("%s: extents count=%" PRIu32 " offset=%" PRIu64 " req_one=%d", +- b->name, count, offset, !!(flags & NBDKIT_FLAG_REQ_ONE)); ++ datapath_debug ("%s: extents count=%" PRIu32 " offset=%" PRIu64 " req_one=%d", ++ b->name, count, offset, !!(flags & NBDKIT_FLAG_REQ_ONE)); + + if (h->can_extents == 0) { + /* By default it is safe assume that everything in the range is +@@ -602,8 +619,8 @@ backend_cache (struct backend *b, struct connection *conn, + assert (h->can_cache > NBDKIT_CACHE_NONE); + assert (backend_valid_range (b, conn, offset, count)); + assert (flags == 0); +- debug ("%s: cache count=%" PRIu32 " offset=%" PRIu64, +- b->name, count, offset); ++ datapath_debug ("%s: cache count=%" PRIu32 " offset=%" PRIu64, ++ b->name, count, offset); + + if (h->can_cache == NBDKIT_CACHE_EMULATE) { + static char buf[MAX_REQUEST_SIZE]; /* data sink, never read */ +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 01341973..d225cc63 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -135,6 +135,7 @@ EXTRA_DIST = \ + test-nbd-extents.sh \ + test-nbd-tls.sh \ + test-nbd-tls-psk.sh \ ++ test-nbdkit-backend-debug.sh \ + test-nozero.sh \ + test-null-extents.sh \ + test_ocaml_plugin.ml \ +@@ -746,6 +747,9 @@ endif HAVE_VDDK + # zero plugin test. + TESTS += test-zero.sh + ++# -D nbdkit.backend.* settings. ++TESTS += test-nbdkit-backend-debug.sh ++ + #---------------------------------------------------------------------- + # Tests of language plugins. + +diff --git a/tests/test-nbdkit-backend-debug.sh b/tests/test-nbdkit-backend-debug.sh +new file mode 100755 +index 00000000..69a69a7c +--- /dev/null ++++ b/tests/test-nbdkit-backend-debug.sh +@@ -0,0 +1,70 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2019 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -x ++set -e ++ ++requires qemu-img --version ++ ++out="test-nbdkit-backend-debug.out" ++debug="test-nbdkit-backend-debug.debug" ++files="$out $debug" ++rm -f $files ++cleanup_fn rm -f $files ++ ++nbdkit -U - \ ++ -v \ ++ memory 10M \ ++ --run "qemu-img convert \$nbd $out" |& tee $debug ++ ++# Should contain all debugging messages. ++grep '^nbdkit:.*debug: memory: open' $debug ++grep '^nbdkit:.*debug: memory: pread' $debug ++ ++nbdkit -U - \ ++ -v -D nbdkit.backend.controlpath=0 \ ++ memory 10M \ ++ --run "qemu-img convert \$nbd $out" |& tee $debug ++ ++# Should contain only datapath messages. ++grep -v '^nbdkit:.*debug: memory: open' $debug ++grep '^nbdkit:.*debug: memory: pread' $debug ++ ++nbdkit -U - \ ++ -v -D nbdkit.backend.datapath=0 \ ++ memory 10M \ ++ --run "qemu-img convert \$nbd $out" |& tee $debug ++ ++# Should contain only controlpath messages. ++grep '^nbdkit:.*debug: memory: open' $debug ++grep -v '^nbdkit:.*debug: memory: pread' $debug +-- +2.18.2 + diff --git a/SOURCES/0003-vddk-Add-comment-about-my-experiment-with-PrepareFor.patch b/SOURCES/0003-vddk-Add-comment-about-my-experiment-with-PrepareFor.patch deleted file mode 100644 index 5722b60..0000000 --- a/SOURCES/0003-vddk-Add-comment-about-my-experiment-with-PrepareFor.patch +++ /dev/null @@ -1,29 +0,0 @@ -From 1041402b8d919bd794ee0ca72017fa6e04d4675b Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 24 Jul 2018 15:35:20 +0100 -Subject: [PATCH] vddk: Add comment about my experiment with PrepareForAccess. - -(cherry picked from commit ba593d2dfa3b3ccd4073f7bad7bcd2d67ce23b64) ---- - plugins/vddk/vddk.c | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 748f1b3..54b95fb 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -370,8 +370,9 @@ vddk_open (int readonly) - } - - /* XXX Some documentation suggests we should call -- * VixDiskLib_PrepareForAccess here. However we need the true VM -- * name to do that. -+ * VixDiskLib_PrepareForAccess here. It may be required for -+ * Advanced Transport modes, but I could not make it work with -+ * either ESXi or vCenter servers. - */ - - DEBUG_CALL ("VixDiskLib_ConnectEx", --- -1.8.3.1 - diff --git a/SOURCES/0004-python-Add-various-constants-to-the-API.patch b/SOURCES/0004-python-Add-various-constants-to-the-API.patch new file mode 100644 index 0000000..8268475 --- /dev/null +++ b/SOURCES/0004-python-Add-various-constants-to-the-API.patch @@ -0,0 +1,65 @@ +From b646050b8da51c39cf21f95fa847c12784a1169c Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Nov 2019 15:02:44 +0000 +Subject: [PATCH 04/19] python: Add various constants to the API. + +These are accessible from the plugin by: + + import nbdkit + + if flags & nbdkit.FLAG_MAY_TRIM: + &c. + +Many (all?) of these are not yet useful for plugins, some will never +be useful, but they only consume a tiny bit of memory and it's nice to +have the complete set available for future use. + +(cherry picked from commit 14b7fe2e0de881e3dfc8803484ade29a61e323c9) +--- + plugins/python/python.c | 30 ++++++++++++++++++++++++++++++ + 1 file changed, 30 insertions(+) + +diff --git a/plugins/python/python.c b/plugins/python/python.c +index 7052aac0..47da0838 100644 +--- a/plugins/python/python.c ++++ b/plugins/python/python.c +@@ -231,6 +231,36 @@ create_nbdkit_module (void) + nbdkit_error ("could not create the nbdkit API module"); + exit (EXIT_FAILURE); + } ++ ++ /* Constants corresponding to various flags. */ ++#define ADD_INT_CONSTANT(name) \ ++ if (PyModule_AddIntConstant (m, #name, NBDKIT_##name) == -1) { \ ++ nbdkit_error ("could not add constant %s to nbdkit API module", \ ++ #name); \ ++ exit (EXIT_FAILURE); \ ++ } ++ ADD_INT_CONSTANT (THREAD_MODEL_SERIALIZE_CONNECTIONS); ++ ADD_INT_CONSTANT (THREAD_MODEL_SERIALIZE_ALL_REQUESTS); ++ ADD_INT_CONSTANT (THREAD_MODEL_SERIALIZE_REQUESTS); ++ ADD_INT_CONSTANT (THREAD_MODEL_PARALLEL); ++ ++ ADD_INT_CONSTANT (FLAG_MAY_TRIM); ++ ADD_INT_CONSTANT (FLAG_FUA); ++ ADD_INT_CONSTANT (FLAG_REQ_ONE); ++ ADD_INT_CONSTANT (FLAG_FAST_ZERO); ++ ++ ADD_INT_CONSTANT (FUA_NONE); ++ ADD_INT_CONSTANT (FUA_EMULATE); ++ ADD_INT_CONSTANT (FUA_NATIVE); ++ ++ ADD_INT_CONSTANT (CACHE_NONE); ++ ADD_INT_CONSTANT (CACHE_EMULATE); ++ ADD_INT_CONSTANT (CACHE_NATIVE); ++ ++ ADD_INT_CONSTANT (EXTENT_HOLE); ++ ADD_INT_CONSTANT (EXTENT_ZERO); ++#undef ADD_INT_CONSTANT ++ + return m; + } + +-- +2.18.2 + diff --git a/SOURCES/0004-vddk-Make-dlsym-variables-static.patch b/SOURCES/0004-vddk-Make-dlsym-variables-static.patch deleted file mode 100644 index 3655fff..0000000 --- a/SOURCES/0004-vddk-Make-dlsym-variables-static.patch +++ /dev/null @@ -1,50 +0,0 @@ -From 901f42bd3fc61af0a4b864d4961fc4b02bec0541 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 25 Jul 2018 09:10:03 +0100 -Subject: [PATCH] vddk: Make dlsym variables static. - -Fixes commit 8d7f7c26eb435334d7fa35e84ceee7d266dfae4c. - -(cherry picked from commit 168364eff47004e64d0880516de5744fecaa8047) ---- - plugins/vddk/vddk.c | 22 +++++++++++----------- - 1 file changed, 11 insertions(+), 11 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 54b95fb..e8b1be5 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -48,19 +48,19 @@ - /* The VDDK APIs that we call. These globals are initialized when the - * plugin is loaded (by vddk_load). - */ --char *(*VixDiskLib_GetErrorText) (VixError err, const char *unused); --void (*VixDiskLib_FreeErrorText) (char *text); --VixError (*VixDiskLib_InitEx) (uint32_t major, uint32_t minor, VixDiskLibGenericLogFunc *log_function, VixDiskLibGenericLogFunc *warn_function, VixDiskLibGenericLogFunc *panic_function, const char *lib_dir, const char *config_file); -+static char *(*VixDiskLib_GetErrorText) (VixError err, const char *unused); -+static void (*VixDiskLib_FreeErrorText) (char *text); -+static VixError (*VixDiskLib_InitEx) (uint32_t major, uint32_t minor, VixDiskLibGenericLogFunc *log_function, VixDiskLibGenericLogFunc *warn_function, VixDiskLibGenericLogFunc *panic_function, const char *lib_dir, const char *config_file); - void (*VixDiskLib_Exit) (void); --VixError (*VixDiskLib_ConnectEx) (const VixDiskLibConnectParams *params, char read_only, const char *snapshot_ref, const char *transport_modes, VixDiskLibConnection *connection); --VixError (*VixDiskLib_Open) (const VixDiskLibConnection connection, const char *path, uint32_t flags, VixDiskLibHandle *handle); -+static VixError (*VixDiskLib_ConnectEx) (const VixDiskLibConnectParams *params, char read_only, const char *snapshot_ref, const char *transport_modes, VixDiskLibConnection *connection); -+static VixError (*VixDiskLib_Open) (const VixDiskLibConnection connection, const char *path, uint32_t flags, VixDiskLibHandle *handle); - const char *(*VixDiskLib_GetTransportMode) (VixDiskLibHandle handle); --VixError (*VixDiskLib_Close) (VixDiskLibHandle handle); --VixError (*VixDiskLib_Disconnect) (VixDiskLibConnection connection); --VixError (*VixDiskLib_GetInfo) (VixDiskLibHandle handle, VixDiskLibInfo **info); --void (*VixDiskLib_FreeInfo) (VixDiskLibInfo *info); --VixError (*VixDiskLib_Read) (VixDiskLibHandle handle, uint64_t start_sector, uint64_t nr_sectors, unsigned char *buf); --VixError (*VixDiskLib_Write) (VixDiskLibHandle handle, uint64_t start_sector, uint64_t nr_sectors, const unsigned char *buf); -+static VixError (*VixDiskLib_Close) (VixDiskLibHandle handle); -+static VixError (*VixDiskLib_Disconnect) (VixDiskLibConnection connection); -+static VixError (*VixDiskLib_GetInfo) (VixDiskLibHandle handle, VixDiskLibInfo **info); -+static void (*VixDiskLib_FreeInfo) (VixDiskLibInfo *info); -+static VixError (*VixDiskLib_Read) (VixDiskLibHandle handle, uint64_t start_sector, uint64_t nr_sectors, unsigned char *buf); -+static VixError (*VixDiskLib_Write) (VixDiskLibHandle handle, uint64_t start_sector, uint64_t nr_sectors, const unsigned char *buf); - - /* Parameters passed to InitEx. */ - #define VDDK_MAJOR 5 --- -1.8.3.1 - diff --git a/SOURCES/0005-python-Implement-nbdkit-API-version-2.patch b/SOURCES/0005-python-Implement-nbdkit-API-version-2.patch new file mode 100644 index 0000000..1fa6ee2 --- /dev/null +++ b/SOURCES/0005-python-Implement-nbdkit-API-version-2.patch @@ -0,0 +1,558 @@ +From 49ef7e7d7c3602cc8e53d2052fce9d3a12840ea2 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Nov 2019 15:44:39 +0000 +Subject: [PATCH 05/19] python: Implement nbdkit API version 2. + +To avoid breaking existing plugins, Python plugins wishing to use +version 2 of the API must opt in by declaring: + + API_VERSION = 2 + +(Plugins which do not do this are assumed to want API version 1). + +For v2 API, we also avoid a copy by passing a buffer into pread. + +It's more efficient if we pass the C buffer directly to Python code. +In some cases the Python code will be able to write directly into the +C buffer using functions like file.readinto and socket.recv_into. +This avoids an extra copy. + +Thanks: Nir Soffer +https://www.redhat.com/archives/libguestfs/2019-November/thread.html#00220 +(cherry picked from commit a9b2637cf4f00fb8a25ffaf31ee83be5fe019ae2) +--- + plugins/python/example.py | 20 +++- + plugins/python/nbdkit-python-plugin.pod | 69 +++++++----- + plugins/python/python.c | 139 +++++++++++++++++++----- + tests/python-exception.py | 4 +- + tests/shebang.py | 5 +- + tests/test.py | 28 +++-- + 6 files changed, 190 insertions(+), 75 deletions(-) + +diff --git a/plugins/python/example.py b/plugins/python/example.py +index 60f9d7f0..c04b7e29 100644 +--- a/plugins/python/example.py ++++ b/plugins/python/example.py +@@ -34,6 +34,12 @@ import errno + disk = bytearray(1024 * 1024) + + ++# There are several variants of the API. nbdkit will call this ++# function first to determine which one you want to use. This is the ++# latest version at the time this example was written. ++API_VERSION = 2 ++ ++ + # This just prints the extra command line parameters, but real plugins + # should parse them and reject any unknown parameters. + def config(key, value): +@@ -54,20 +60,22 @@ def get_size(h): + return len(disk) + + +-def pread(h, count, offset): ++def pread(h, buf, offset, flags): + global disk +- return disk[offset:offset+count] ++ end = offset + len(buf) ++ buf[:] = disk[offset:end] ++ # or if reading from a file you can use: ++ #f.readinto(buf) + +- +-def pwrite(h, buf, offset): ++def pwrite(h, buf, offset, flags): + global disk + end = offset + len(buf) + disk[offset:end] = buf + + +-def zero(h, count, offset, may_trim): ++def zero(h, count, offset, flags): + global disk +- if may_trim: ++ if flags & nbdkit.FLAG_MAY_TRIM: + disk[offset:offset+count] = bytearray(count) + else: + nbdkit.set_error(errno.EOPNOTSUPP) +diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod +index 3680fd65..4923d9da 100644 +--- a/plugins/python/nbdkit-python-plugin.pod ++++ b/plugins/python/nbdkit-python-plugin.pod +@@ -33,11 +33,12 @@ To write a Python nbdkit plugin, you create a Python file which + contains at least the following required functions (in the top level + C<__main__> module): + ++ API_VERSION = 2 + def open(readonly): + # see below + def get_size(h): + # see below +- def pread(h, count, offset): ++ def pread(h, buf, offset, flags): + # see below + + Note that the subroutines must have those literal names (like C), +@@ -82,6 +83,18 @@ I<--dump-plugin> option, eg: + python_version=3.7.0 + python_pep_384_abi_version=3 + ++=head2 API versions ++ ++The nbdkit API has evolved and new versions are released periodically. ++To ensure backwards compatibility plugins have to opt in to the new ++version. From Python you do this by declaring a constant in your ++module: ++ ++ API_VERSION = 2 ++ ++(where 2 is the latest version at the time this documentation was ++written). All newly written Python modules must have this constant. ++ + =head2 Executable script + + If you want you can make the script executable and include a "shebang" +@@ -199,16 +212,12 @@ contents will be garbage collected. + + (Required) + +- def pread(h, count, offset): +- # construct a buffer of length count bytes and return it ++ def pread(h, buf, offset, flags): ++ # read into the buffer + +-The body of your C function should construct a buffer of length +-(at least) C bytes. You should read C bytes from the +-disk starting at C. +- +-The returned buffer can be any type compatible with the Python 3 +-buffer protocol, such as bytearray, bytes or memoryview +-(L) ++The body of your C function should read exactly C ++bytes of data starting at disk C and write it into the buffer ++C. C is always 0. + + NBD only supports whole reads, so your function should try to read + the whole region (perhaps requiring a loop). If the read fails or +@@ -219,13 +228,13 @@ C first. + + (Optional) + +- def pwrite(h, buf, offset): ++ def pwrite(h, buf, offset, flags): + length = len (buf) + # no return value + + The body of your C function should write the buffer C to + the disk. You should write C bytes to the disk starting at +-C. ++C. C may contain C. + + NBD only supports whole writes, so your function should try to + write the whole region (perhaps requiring a loop). If the write +@@ -236,11 +245,12 @@ fails or is partial, your function should throw an exception, + + (Optional) + +- def flush(h): ++ def flush(h, flags): + # no return value + + The body of your C function should do a L or + L or equivalent on the backing store. ++C is always 0. + + If the flush fails, your function should throw an exception, optionally + using C first. +@@ -249,32 +259,35 @@ using C first. + + (Optional) + +- def trim(h, count, offset): ++ def trim(h, count, offset, flags): + # no return value + +-The body of your C function should "punch a hole" in the +-backing store. If the trim fails, your function should throw an +-exception, optionally using C first. ++The body of your C function should "punch a hole" in the backing ++store. C may contain C. If the trim fails, ++your function should throw an exception, optionally using ++C first. + + =item C + + (Optional) + +- def zero(h, count, offset, may_trim): ++ def zero(h, count, offset, flags): + # no return value + +-The body of your C function should ensure that C bytes +-of the disk, starting at C, will read back as zero. If +-C is true, the operation may be optimized as a trim as long +-as subsequent reads see zeroes. ++The body of your C function should ensure that C bytes of ++the disk, starting at C, will read back as zero. C is ++a bitmask which may include C, ++C, C. + + NBD only supports whole writes, so your function should try to +-write the whole region (perhaps requiring a loop). If the write +-fails or is partial, your function should throw an exception, +-optionally using C first. In particular, if +-you would like to automatically fall back to C (perhaps +-because there is nothing to optimize if C is false), +-use C. ++write the whole region (perhaps requiring a loop). ++ ++If the write fails or is partial, your function should throw an ++exception, optionally using C first. In particular, ++if you would like to automatically fall back to C (perhaps ++because there is nothing to optimize if ++S> is false), use ++S>. + + =back + +diff --git a/plugins/python/python.c b/plugins/python/python.c +index 47da0838..0f28595f 100644 +--- a/plugins/python/python.c ++++ b/plugins/python/python.c +@@ -46,6 +46,8 @@ + #define PY_SSIZE_T_CLEAN 1 + #include + ++#define NBDKIT_API_VERSION 2 ++ + #include + + #include "cleanup.h" +@@ -60,6 +62,7 @@ + */ + static const char *script; + static PyObject *module; ++static int py_api_version = 1; + + static int last_error; + +@@ -285,9 +288,14 @@ py_dump_plugin (void) + PyObject *fn; + PyObject *r; + ++ /* Python version and ABI. */ + printf ("python_version=%s\n", PY_VERSION); + printf ("python_pep_384_abi_version=%d\n", PYTHON_ABI_VERSION); + ++ /* Maximum nbdkit API version supported. */ ++ printf ("nbdkit_python_maximum_api_version=%d\n", NBDKIT_API_VERSION); ++ ++ /* If the script has a dump_plugin function, call it. */ + if (script && callback_defined ("dump_plugin", &fn)) { + PyErr_Clear (); + +@@ -297,6 +305,30 @@ py_dump_plugin (void) + } + } + ++static int ++get_py_api_version (void) ++{ ++ PyObject *obj; ++ long value; ++ ++ obj = PyObject_GetAttrString (module, "API_VERSION"); ++ if (obj == NULL) ++ return 1; /* Default to API version 1. */ ++ ++ value = PyLong_AsLong (obj); ++ Py_DECREF (obj); ++ ++ if (value < 1 || value > NBDKIT_API_VERSION) { ++ nbdkit_error ("%s: API_VERSION requested unknown version: %ld. " ++ "This plugin supports API versions between 1 and %d.", ++ script, value, NBDKIT_API_VERSION); ++ return -1; ++ } ++ ++ nbdkit_debug ("module requested API_VERSION %ld", value); ++ return (int) value; ++} ++ + static int + py_config (const char *key, const char *value) + { +@@ -359,6 +391,11 @@ py_config (const char *key, const char *value) + "nbdkit requires these callbacks.", script); + return -1; + } ++ ++ /* Get the API version. */ ++ py_api_version = get_py_api_version (); ++ if (py_api_version == -1) ++ return -1; + } + else if (callback_defined ("config", &fn)) { + /* Other parameters are passed to the Python .config callback. */ +@@ -469,8 +506,8 @@ py_get_size (void *handle) + } + + static int +-py_pread (void *handle, void *buf, +- uint32_t count, uint64_t offset) ++py_pread (void *handle, void *buf, uint32_t count, uint64_t offset, ++ uint32_t flags) + { + PyObject *obj = handle; + PyObject *fn; +@@ -485,24 +522,40 @@ py_pread (void *handle, void *buf, + + PyErr_Clear (); + +- r = PyObject_CallFunction (fn, "OiL", obj, count, offset); ++ switch (py_api_version) { ++ case 1: ++ r = PyObject_CallFunction (fn, "OiL", obj, count, offset); ++ break; ++ case 2: ++ r = PyObject_CallFunction (fn, "ONLI", obj, ++ PyMemoryView_FromMemory ((char *)buf, count, PyBUF_WRITE), ++ offset, flags); ++ break; ++ default: abort (); ++ } + Py_DECREF (fn); + if (check_python_failure ("pread") == -1) + return ret; + +- if (PyObject_GetBuffer (r, &view, PyBUF_SIMPLE) == -1) { +- nbdkit_error ("%s: value returned from pread does not support the " +- "buffer protocol", +- script); +- goto out; +- } ++ if (py_api_version == 1) { ++ /* In API v1 the Python pread function had to return a buffer ++ * protocol compatible function. In API v2+ it writes directly to ++ * the C buffer so this code is not used. ++ */ ++ if (PyObject_GetBuffer (r, &view, PyBUF_SIMPLE) == -1) { ++ nbdkit_error ("%s: value returned from pread does not support the " ++ "buffer protocol", ++ script); ++ goto out; ++ } + +- if (view.len < count) { +- nbdkit_error ("%s: buffer returned from pread is too small", script); +- goto out; +- } ++ if (view.len < count) { ++ nbdkit_error ("%s: buffer returned from pread is too small", script); ++ goto out; ++ } + +- memcpy (buf, view.buf, count); ++ memcpy (buf, view.buf, count); ++ } + ret = 0; + + out: +@@ -515,8 +568,8 @@ out: + } + + static int +-py_pwrite (void *handle, const void *buf, +- uint32_t count, uint64_t offset) ++py_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, ++ uint32_t flags) + { + PyObject *obj = handle; + PyObject *fn; +@@ -525,9 +578,19 @@ py_pwrite (void *handle, const void *buf, + if (callback_defined ("pwrite", &fn)) { + PyErr_Clear (); + +- r = PyObject_CallFunction (fn, "ONL", obj, ++ switch (py_api_version) { ++ case 1: ++ r = PyObject_CallFunction (fn, "ONL", obj, + PyMemoryView_FromMemory ((char *)buf, count, PyBUF_READ), + offset); ++ break; ++ case 2: ++ r = PyObject_CallFunction (fn, "ONLI", obj, ++ PyMemoryView_FromMemory ((char *)buf, count, PyBUF_READ), ++ offset, flags); ++ break; ++ default: abort (); ++ } + Py_DECREF (fn); + if (check_python_failure ("pwrite") == -1) + return -1; +@@ -542,7 +605,7 @@ py_pwrite (void *handle, const void *buf, + } + + static int +-py_flush (void *handle) ++py_flush (void *handle, uint32_t flags) + { + PyObject *obj = handle; + PyObject *fn; +@@ -551,7 +614,15 @@ py_flush (void *handle) + if (callback_defined ("flush", &fn)) { + PyErr_Clear (); + +- r = PyObject_CallFunctionObjArgs (fn, obj, NULL); ++ switch (py_api_version) { ++ case 1: ++ r = PyObject_CallFunctionObjArgs (fn, obj, NULL); ++ break; ++ case 2: ++ r = PyObject_CallFunction (fn, "OI", obj, flags); ++ break; ++ default: abort (); ++ } + Py_DECREF (fn); + if (check_python_failure ("flush") == -1) + return -1; +@@ -566,7 +637,7 @@ py_flush (void *handle) + } + + static int +-py_trim (void *handle, uint32_t count, uint64_t offset) ++py_trim (void *handle, uint32_t count, uint64_t offset, uint32_t flags) + { + PyObject *obj = handle; + PyObject *fn; +@@ -575,7 +646,15 @@ py_trim (void *handle, uint32_t count, uint64_t offset) + if (callback_defined ("trim", &fn)) { + PyErr_Clear (); + +- r = PyObject_CallFunction (fn, "OiL", obj, count, offset); ++ switch (py_api_version) { ++ case 1: ++ r = PyObject_CallFunction (fn, "OiL", obj, count, offset); ++ break; ++ case 2: ++ r = PyObject_CallFunction (fn, "OiLI", obj, count, offset, flags); ++ break; ++ default: abort (); ++ } + Py_DECREF (fn); + if (check_python_failure ("trim") == -1) + return -1; +@@ -590,7 +669,7 @@ py_trim (void *handle, uint32_t count, uint64_t offset) + } + + static int +-py_zero (void *handle, uint32_t count, uint64_t offset, int may_trim) ++py_zero (void *handle, uint32_t count, uint64_t offset, uint32_t flags) + { + PyObject *obj = handle; + PyObject *fn; +@@ -600,9 +679,19 @@ py_zero (void *handle, uint32_t count, uint64_t offset, int may_trim) + PyErr_Clear (); + + last_error = 0; +- r = PyObject_CallFunction (fn, "OiLO", +- obj, count, offset, +- may_trim ? Py_True : Py_False); ++ switch (py_api_version) { ++ case 1: { ++ int may_trim = flags & NBDKIT_FLAG_MAY_TRIM; ++ r = PyObject_CallFunction (fn, "OiLO", ++ obj, count, offset, ++ may_trim ? Py_True : Py_False); ++ break; ++ } ++ case 2: ++ r = PyObject_CallFunction (fn, "OiLI", obj, count, offset, flags); ++ break; ++ default: abort (); ++ } + Py_DECREF (fn); + if (last_error == EOPNOTSUPP || last_error == ENOTSUP) { + /* When user requests this particular error, we want to +diff --git a/tests/python-exception.py b/tests/python-exception.py +index d0c79bb0..ee4a3f3a 100644 +--- a/tests/python-exception.py ++++ b/tests/python-exception.py +@@ -62,5 +62,5 @@ def get_size(h): + return 0 + + +-def pread(h, count, offset): +- return "" ++def pread(h, buf, offset): ++ buf[:] = bytearray(len(buf)) +diff --git a/tests/shebang.py b/tests/shebang.py +index 6f336230..0634589a 100755 +--- a/tests/shebang.py ++++ b/tests/shebang.py +@@ -13,6 +13,7 @@ def get_size(h): + return len(disk) + + +-def pread(h, count, offset): ++def pread(h, buf, offset): + global disk +- return disk[offset:offset+count] ++ end = offset + len(buf) ++ buf[:] = disk[offset:end] +diff --git a/tests/test.py b/tests/test.py +index 9a2e947d..4db56623 100644 +--- a/tests/test.py ++++ b/tests/test.py +@@ -3,6 +3,9 @@ import nbdkit + disk = bytearray(1024*1024) + + ++API_VERSION = 2 ++ ++ + def config_complete(): + print ("set_error = %r" % nbdkit.set_error) + +@@ -32,25 +35,26 @@ def can_trim(h): + return True + + +-def pread(h, count, offset): ++def pread(h, buf, offset, flags): + global disk +- return disk[offset:offset+count] ++ end = offset + len(buf) ++ buf[:] = disk[offset:end] + + +-def pwrite(h, buf, offset): ++def pwrite(h, buf, offset, flags): + global disk + end = offset + len(buf) + disk[offset:end] = buf + + +-def zero(h, count, offset, may_trim=False): ++def flush(h, flags): ++ pass ++ ++ ++def trim(h, count, offset, flags): ++ pass ++ ++ ++def zero(h, count, offset, flags): + global disk + disk[offset:offset+count] = bytearray(count) +- +- +-def flush(h): +- pass +- +- +-def trim(h, count, offset): +- pass +-- +2.18.2 + diff --git a/SOURCES/0005-vddk-Improve-error-message-if-the-proprietary-librar.patch b/SOURCES/0005-vddk-Improve-error-message-if-the-proprietary-librar.patch deleted file mode 100644 index e7e3090..0000000 --- a/SOURCES/0005-vddk-Improve-error-message-if-the-proprietary-librar.patch +++ /dev/null @@ -1,37 +0,0 @@ -From a4e54de2e1ddbf7bd52e2b2e9857402ad93d5286 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 25 Jul 2018 09:28:04 +0100 -Subject: [PATCH] vddk: Improve error message if the proprietary library cannot - be found. - -(cherry picked from commit 94e791f87c6029983befa6199771345fd9cdfcc9) ---- - plugins/vddk/vddk.c | 8 ++++++-- - 1 file changed, 6 insertions(+), 2 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index e8b1be5..67aaa61 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -142,12 +142,16 @@ error_function (const char *fs, va_list args) - static void - vddk_load (void) - { -- const char *soname = "libvixDiskLib.so.6"; -+ static const char soname[] = "libvixDiskLib.so.6"; - - /* Load the plugin and set the entry points. */ - dl = dlopen (soname, RTLD_NOW); - if (dl == NULL) { -- nbdkit_error ("%s: %s", soname, dlerror ()); -+ nbdkit_error ("%s\n\n" -+ "If '%s' is located on a non-standard path you may need to\n" -+ "set $LD_LIBRARY_PATH or edit /etc/ld.so.conf.\n\n" -+ "See the nbdkit-vddk-plugin(1) man page for details.", -+ dlerror (), soname); - exit (EXIT_FAILURE); - } - --- -1.8.3.1 - diff --git a/SOURCES/0006-python-Implement-cache.patch b/SOURCES/0006-python-Implement-cache.patch new file mode 100644 index 0000000..032327d --- /dev/null +++ b/SOURCES/0006-python-Implement-cache.patch @@ -0,0 +1,98 @@ +From c5b1fac4c67078f0164bd23eab6d4d2b8c9830b0 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Nov 2019 16:42:02 +0000 +Subject: [PATCH 06/19] python: Implement cache. + +However this does not implement can_cache, since that is not a simple +boolean. + +(cherry picked from commit e61ffb73c7a0af0c383184fdb8f08d30784a195e) +--- + plugins/python/nbdkit-python-plugin.pod | 14 ++++++++++- + plugins/python/python.c | 31 +++++++++++++++++++++++++ + 2 files changed, 44 insertions(+), 1 deletion(-) + +diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod +index 4923d9da..0ea8deef 100644 +--- a/plugins/python/nbdkit-python-plugin.pod ++++ b/plugins/python/nbdkit-python-plugin.pod +@@ -289,6 +289,19 @@ because there is nothing to optimize if + S> is false), use + S>. + ++=item C ++ ++(Optional) ++ ++ def cache(h, count, offset, flags): ++ # no return value ++ ++The body of your C function should prefetch data in the ++indicated range. ++ ++If the cache operation fails, your function should throw an exception, ++optionally using C first. ++ + =back + + =head2 Missing callbacks +@@ -317,7 +330,6 @@ C, + C, + C, + C, +-C, + C. + + These are not yet supported. +diff --git a/plugins/python/python.c b/plugins/python/python.c +index 0f28595f..c5cf38e5 100644 +--- a/plugins/python/python.c ++++ b/plugins/python/python.c +@@ -714,6 +714,36 @@ py_zero (void *handle, uint32_t count, uint64_t offset, uint32_t flags) + return -1; + } + ++static int ++py_cache (void *handle, uint32_t count, uint64_t offset, uint32_t flags) ++{ ++ PyObject *obj = handle; ++ PyObject *fn; ++ PyObject *r; ++ ++ if (callback_defined ("cache", &fn)) { ++ PyErr_Clear (); ++ ++ switch (py_api_version) { ++ case 1: ++ case 2: ++ r = PyObject_CallFunction (fn, "OiLI", obj, count, offset, flags, NULL); ++ break; ++ default: abort (); ++ } ++ Py_DECREF (fn); ++ if (check_python_failure ("cache") == -1) ++ return -1; ++ Py_DECREF (r); ++ } ++ else { ++ nbdkit_error ("%s not implemented", "cache"); ++ return -1; ++ } ++ ++ return 0; ++} ++ + static int + boolean_callback (void *handle, const char *can_fn, const char *plain_fn) + { +@@ -799,6 +829,7 @@ static struct nbdkit_plugin plugin = { + .flush = py_flush, + .trim = py_trim, + .zero = py_zero, ++ .cache = py_cache, + }; + + NBDKIT_REGISTER_PLUGIN (plugin) +-- +2.18.2 + diff --git a/SOURCES/0006-vddk-If-relative-libdir-parameter-is-passed-make-it-.patch b/SOURCES/0006-vddk-If-relative-libdir-parameter-is-passed-make-it-.patch deleted file mode 100644 index 41839c4..0000000 --- a/SOURCES/0006-vddk-If-relative-libdir-parameter-is-passed-make-it-.patch +++ /dev/null @@ -1,79 +0,0 @@ -From 7aa9fbe2dc6ef46b4701f13584c88d657255bdbf Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 25 Jul 2018 09:28:32 +0100 -Subject: [PATCH] vddk: If relative libdir parameter is passed, make it - absolute. - -(cherry picked from commit 8838497c44d51f2c3ea12adad89fd836116af201) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 3 +-- - plugins/vddk/vddk.c | 14 ++++++++++---- - 2 files changed, 11 insertions(+), 6 deletions(-) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index ba7806d..57a039f 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -87,8 +87,7 @@ L below). - - =item B - --Optional. This sets the path of the VMware VDDK distribution. It --must be an absolute path. -+Optional. This sets the path of the VMware VDDK distribution. - - VDDK uses this to load its own plugins, if this path is unspecified or - wrong then VDDK will work with reduced functionality. -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 67aaa61..a8216fc 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -72,7 +72,7 @@ static int init_called = 0; /* was InitEx called */ - static char *config = NULL; /* config */ - static const char *cookie = NULL; /* cookie */ - static const char *filename = NULL; /* file */ --static const char *libdir = VDDK_LIBDIR; /* libdir */ -+static char *libdir = NULL; /* libdir */ - static int nfc_host_port = 0; /* nfchostport */ - static char *password = NULL; /* password */ - static int port = 0; /* port */ -@@ -180,6 +180,7 @@ vddk_unload (void) - if (dl) - dlclose (dl); - free (config); -+ free (libdir); - free (password); - } - -@@ -205,7 +206,11 @@ vddk_config (const char *key, const char *value) - filename = value; - } - else if (strcmp (key, "libdir") == 0) { -- libdir = value; -+ /* See FILENAMES AND PATHS in nbdkit-plugin(3). */ -+ free (libdir); -+ libdir = nbdkit_realpath (value); -+ if (!libdir) -+ return -1; - } - else if (strcmp (key, "nfchostport") == 0) { - if (sscanf (value, "%d", &nfc_host_port) != 1) { -@@ -296,12 +301,13 @@ vddk_config_complete (void) - /* Initialize VDDK library. */ - DEBUG_CALL ("VixDiskLib_InitEx", - "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s", -- VDDK_MAJOR, VDDK_MINOR, libdir, config ? : "NULL"); -+ VDDK_MAJOR, VDDK_MINOR, -+ libdir ? : VDDK_LIBDIR, config ? : "NULL"); - err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR, - &debug_function, /* log function */ - &error_function, /* warn function */ - &error_function, /* panic function */ -- libdir, config); -+ libdir ? : VDDK_LIBDIR, config); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_InitEx"); - exit (EXIT_FAILURE); --- -1.8.3.1 - diff --git a/SOURCES/0007-python-Implement-can_zero-can_fast_zero.patch b/SOURCES/0007-python-Implement-can_zero-can_fast_zero.patch new file mode 100644 index 0000000..e6a82a4 --- /dev/null +++ b/SOURCES/0007-python-Implement-can_zero-can_fast_zero.patch @@ -0,0 +1,80 @@ +From 17721b316dd66b0a1ed792eeccd2489fb97828df Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Nov 2019 16:42:59 +0000 +Subject: [PATCH 07/19] python: Implement can_zero, can_fast_zero. + +(cherry picked from commit 039f600d2ad7a9ff04523a165eb2fe41b9c87c01) +--- + plugins/python/nbdkit-python-plugin.pod | 16 ++++++++++++++-- + plugins/python/python.c | 14 ++++++++++++++ + 2 files changed, 28 insertions(+), 2 deletions(-) + +diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod +index 0ea8deef..1f1c30f6 100644 +--- a/plugins/python/nbdkit-python-plugin.pod ++++ b/plugins/python/nbdkit-python-plugin.pod +@@ -208,6 +208,20 @@ contents will be garbage collected. + def can_trim(h): + # return a boolean + ++=item C ++ ++(Optional) ++ ++ def can_zero(h): ++ # return a boolean ++ ++=item C ++ ++(Optional) ++ ++ def can_fast_zero(h): ++ # return a boolean ++ + =item C + + (Required) +@@ -326,8 +340,6 @@ C, + C, + C, + C, +-C, +-C, + C, + C, + C. +diff --git a/plugins/python/python.c b/plugins/python/python.c +index c5cf38e5..38fc1193 100644 +--- a/plugins/python/python.c ++++ b/plugins/python/python.c +@@ -797,6 +797,18 @@ py_can_trim (void *handle) + return boolean_callback (handle, "can_trim", "trim"); + } + ++static int ++py_can_zero (void *handle) ++{ ++ return boolean_callback (handle, "can_zero", "zero"); ++} ++ ++static int ++py_can_fast_zero (void *handle) ++{ ++ return boolean_callback (handle, "can_fast_zero", NULL); ++} ++ + #define py_config_help \ + "script= (required) The Python plugin to run.\n" \ + "[other arguments may be used by the plugin that you load]" +@@ -823,6 +835,8 @@ static struct nbdkit_plugin plugin = { + .can_write = py_can_write, + .can_flush = py_can_flush, + .can_trim = py_can_trim, ++ .can_zero = py_can_zero, ++ .can_fast_zero = py_can_fast_zero, + + .pread = py_pread, + .pwrite = py_pwrite, +-- +2.18.2 + diff --git a/SOURCES/0007-vddk-Two-more-static-dlsym-variables.patch b/SOURCES/0007-vddk-Two-more-static-dlsym-variables.patch deleted file mode 100644 index b4c32fe..0000000 --- a/SOURCES/0007-vddk-Two-more-static-dlsym-variables.patch +++ /dev/null @@ -1,33 +0,0 @@ -From f44a56ae2818daf71851aab1c6e930b365ee9012 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 25 Jul 2018 14:02:32 +0100 -Subject: [PATCH] vddk: Two more static dlsym variables. - -Fixes commit 168364eff47004e64d0880516de5744fecaa8047 -and commit 8d7f7c26eb435334d7fa35e84ceee7d266dfae4c. - -(cherry picked from commit b776d1f5e59faef659f0d6e7fbffec614d58a368) ---- - plugins/vddk/vddk.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index a8216fc..df7a7e0 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -51,10 +51,10 @@ - static char *(*VixDiskLib_GetErrorText) (VixError err, const char *unused); - static void (*VixDiskLib_FreeErrorText) (char *text); - static VixError (*VixDiskLib_InitEx) (uint32_t major, uint32_t minor, VixDiskLibGenericLogFunc *log_function, VixDiskLibGenericLogFunc *warn_function, VixDiskLibGenericLogFunc *panic_function, const char *lib_dir, const char *config_file); --void (*VixDiskLib_Exit) (void); -+static void (*VixDiskLib_Exit) (void); - static VixError (*VixDiskLib_ConnectEx) (const VixDiskLibConnectParams *params, char read_only, const char *snapshot_ref, const char *transport_modes, VixDiskLibConnection *connection); - static VixError (*VixDiskLib_Open) (const VixDiskLibConnection connection, const char *path, uint32_t flags, VixDiskLibHandle *handle); --const char *(*VixDiskLib_GetTransportMode) (VixDiskLibHandle handle); -+static const char *(*VixDiskLib_GetTransportMode) (VixDiskLibHandle handle); - static VixError (*VixDiskLib_Close) (VixDiskLibHandle handle); - static VixError (*VixDiskLib_Disconnect) (VixDiskLibConnection connection); - static VixError (*VixDiskLib_GetInfo) (VixDiskLibHandle handle, VixDiskLibInfo **info); --- -1.8.3.1 - diff --git a/SOURCES/0008-python-Implement-can_multi_conn.patch b/SOURCES/0008-python-Implement-can_multi_conn.patch new file mode 100644 index 0000000..d854076 --- /dev/null +++ b/SOURCES/0008-python-Implement-can_multi_conn.patch @@ -0,0 +1,65 @@ +From 2a85ce81ad95eb2f9b2f29666480b814ea0f80d9 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Nov 2019 16:46:11 +0000 +Subject: [PATCH 08/19] python: Implement can_multi_conn. + +(cherry picked from commit 21dd7bf49d3238c7e75918d4bf324b617f458d83) +--- + plugins/python/nbdkit-python-plugin.pod | 8 +++++++- + plugins/python/python.c | 7 +++++++ + 2 files changed, 14 insertions(+), 1 deletion(-) + +diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod +index 1f1c30f6..b92bb56a 100644 +--- a/plugins/python/nbdkit-python-plugin.pod ++++ b/plugins/python/nbdkit-python-plugin.pod +@@ -187,6 +187,13 @@ contents will be garbage collected. + def is_rotational(h): + # return a boolean + ++=item C ++ ++(Optional) ++ ++ def can_multi_conn(h): ++ # return a boolean ++ + =item C + + (Optional) +@@ -341,7 +348,6 @@ C, + C, + C, + C, +-C, + C. + + These are not yet supported. +diff --git a/plugins/python/python.c b/plugins/python/python.c +index 38fc1193..b186b991 100644 +--- a/plugins/python/python.c ++++ b/plugins/python/python.c +@@ -779,6 +779,12 @@ py_is_rotational (void *handle) + return boolean_callback (handle, "is_rotational", NULL); + } + ++static int ++py_can_multi_conn (void *handle) ++{ ++ return boolean_callback (handle, "can_multi_conn", NULL); ++} ++ + static int + py_can_write (void *handle) + { +@@ -832,6 +838,7 @@ static struct nbdkit_plugin plugin = { + + .get_size = py_get_size, + .is_rotational = py_is_rotational, ++ .can_multi_conn = py_can_multi_conn, + .can_write = py_can_write, + .can_flush = py_can_flush, + .can_trim = py_can_trim, +-- +2.18.2 + diff --git a/SOURCES/0008-vddk-Add-a-very-simple-test.patch b/SOURCES/0008-vddk-Add-a-very-simple-test.patch deleted file mode 100644 index 82f3b34..0000000 --- a/SOURCES/0008-vddk-Add-a-very-simple-test.patch +++ /dev/null @@ -1,179 +0,0 @@ -From 80b83f39a8b365455880d8dabbcb86249c1ecd6b Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 25 Jul 2018 14:09:58 +0100 -Subject: [PATCH] vddk: Add a very simple test. - -We cannot do anything like a real test without the proprietary -library. However by making a dummy library which contains some stub -functions we can test --dump-plugin output. - -(cherry picked from commit 70f7227ecc9b7c8d628987cb12ca7541bf485d66) ---- - tests/Makefile.am | 21 ++++++++++++++++++ - tests/dummy-vddk.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ - tests/test-vddk.sh | 45 +++++++++++++++++++++++++++++++++++++++ - 3 files changed, 128 insertions(+) - create mode 100644 tests/dummy-vddk.c - create mode 100755 tests/test-vddk.sh - -diff --git a/tests/Makefile.am b/tests/Makefile.am -index c0c2155..2973268 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -75,6 +75,7 @@ EXTRA_DIST = \ - test-random-sock.sh \ - test-tls.sh \ - test-tls-psk.sh \ -+ test-vddk.sh \ - test-version.sh \ - test-version-filter.sh \ - test-version-plugin.sh \ -@@ -365,6 +366,26 @@ test_streaming_SOURCES = test-streaming.c test.h - test_streaming_CFLAGS = $(WARNINGS_CFLAGS) $(LIBGUESTFS_CFLAGS) - test_streaming_LDADD = libtest.la $(LIBGUESTFS_LIBS) - -+# VDDK plugin test. -+# This only tests that the plugin can be loaded against a -+# dummy VDDK library, it is not a detailed test. -+ -+# check_LTLIBRARIES won't build a shared library (see automake manual). -+# So we have to do this and add a dependency. -+noinst_LTLIBRARIES += libvixDiskLib.la -+TESTS += test-vddk.sh -+ -+libvixDiskLib_la_SOURCES = \ -+ dummy-vddk.c -+libvixDiskLib_la_CPPFLAGS = \ -+ -I$(top_srcdir)/plugins/vddk -+libvixDiskLib_la_CXXFLAGS = \ -+ $(WARNINGS_CFLAGS) -+# For use of the -rpath option, see: -+# https://lists.gnu.org/archive/html/libtool/2007-07/msg00067.html -+libvixDiskLib_la_LDFLAGS = \ -+ -shared -version-number 6:0:0 -rpath /nowhere -+ - # xz plugin test. - if HAVE_LIBLZMA - if HAVE_GUESTFISH -diff --git a/tests/dummy-vddk.c b/tests/dummy-vddk.c -new file mode 100644 -index 0000000..e9069c9 ---- /dev/null -+++ b/tests/dummy-vddk.c -@@ -0,0 +1,62 @@ -+/* nbdkit -+ * Copyright (C) 2018 Red Hat Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+/* This file pretends to be libvixDiskLib.so.6. -+ * -+ * In fact because we don't check the result from dlsym and because we -+ * only call a few APIs in the --dump-plugin path there are only a few -+ * stub functions needed. -+ */ -+ -+#include -+#include -+#include -+ -+#include "vddk-structs.h" -+ -+VixError -+VixDiskLib_InitEx (uint32_t major, uint32_t minor, -+ VixDiskLibGenericLogFunc *log_function, -+ VixDiskLibGenericLogFunc *warn_function, -+ VixDiskLibGenericLogFunc *panic_function, -+ const char *lib_dir, const char *config_file) -+{ -+ /* Do nothing, only exit with no error. */ -+ return VIX_OK; -+} -+ -+void -+VixDiskLib_Exit (void) -+{ -+ /* Do nothing. */ -+} -diff --git a/tests/test-vddk.sh b/tests/test-vddk.sh -new file mode 100755 -index 0000000..5ccfff1 ---- /dev/null -+++ b/tests/test-vddk.sh -@@ -0,0 +1,45 @@ -+#!/bin/bash - -+# nbdkit -+# Copyright (C) 2018 Red Hat Inc. -+# All rights reserved. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+set -x -+set -e -+ -+rm -f test-vddk.out -+ -+LD_LIBRARY_PATH=.libs:$LD_LIBRARY_PATH \ -+nbdkit vddk --dump-plugin > test-vddk.out -+cat test-vddk.out -+ -+grep ^vddk_default_libdir= test-vddk.out -+ -+rm test-vddk.out --- -1.8.3.1 - diff --git a/SOURCES/0009-python-Implement-can_fua-and-can_cache.patch b/SOURCES/0009-python-Implement-can_fua-and-can_cache.patch new file mode 100644 index 0000000..f979c4e --- /dev/null +++ b/SOURCES/0009-python-Implement-can_fua-and-can_cache.patch @@ -0,0 +1,126 @@ +From 38124a137974e1433d68732640ca7f88664557da Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 22 Nov 2019 19:25:53 +0000 +Subject: [PATCH 09/19] python: Implement can_fua and can_cache. + +(cherry picked from commit 97c46f885edec5a61a96ac86eccb9d8c874c602e) +--- + plugins/python/nbdkit-python-plugin.pod | 18 +++++++- + plugins/python/python.c | 58 +++++++++++++++++++++++++ + 2 files changed, 74 insertions(+), 2 deletions(-) + +diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod +index b92bb56a..4065ec75 100644 +--- a/plugins/python/nbdkit-python-plugin.pod ++++ b/plugins/python/nbdkit-python-plugin.pod +@@ -229,6 +229,22 @@ contents will be garbage collected. + def can_fast_zero(h): + # return a boolean + ++=item C ++ ++(Optional) ++ ++ def can_fua(h): ++ # return nbdkit.FUA_NONE or nbdkit.FUA_EMULATE ++ # or nbdkit.FUA_NATIVE ++ ++=item C ++ ++(Optional) ++ ++ def can_cache(h): ++ # return nbdkit.CACHE_NONE or nbdkit.CACHE_EMULATE ++ # or nbdkit.CACHE_NATIVE ++ + =item C + + (Required) +@@ -345,8 +361,6 @@ C, + C, + C, + C, +-C, +-C, + C, + C. + +diff --git a/plugins/python/python.c b/plugins/python/python.c +index b186b991..5e2e5269 100644 +--- a/plugins/python/python.c ++++ b/plugins/python/python.c +@@ -815,6 +815,62 @@ py_can_fast_zero (void *handle) + return boolean_callback (handle, "can_fast_zero", NULL); + } + ++static int ++py_can_fua (void *handle) ++{ ++ PyObject *obj = handle; ++ PyObject *fn; ++ PyObject *r; ++ int ret; ++ ++ if (callback_defined ("can_fua", &fn)) { ++ PyErr_Clear (); ++ ++ r = PyObject_CallFunctionObjArgs (fn, obj, NULL); ++ Py_DECREF (fn); ++ if (check_python_failure ("can_fua") == -1) ++ return -1; ++ ret = PyLong_AsLong (r); ++ Py_DECREF (r); ++ return ret; ++ } ++ /* No Python ‘can_fua’, but check if there's a Python ‘flush’ ++ * callback defined. (In C modules, nbdkit would do this). ++ */ ++ else if (callback_defined ("flush", NULL)) ++ return NBDKIT_FUA_EMULATE; ++ else ++ return NBDKIT_FUA_NONE; ++} ++ ++static int ++py_can_cache (void *handle) ++{ ++ PyObject *obj = handle; ++ PyObject *fn; ++ PyObject *r; ++ int ret; ++ ++ if (callback_defined ("can_cache", &fn)) { ++ PyErr_Clear (); ++ ++ r = PyObject_CallFunctionObjArgs (fn, obj, NULL); ++ Py_DECREF (fn); ++ if (check_python_failure ("can_cache") == -1) ++ return -1; ++ ret = PyLong_AsLong (r); ++ Py_DECREF (r); ++ return ret; ++ } ++ /* No Python ‘can_cache’, but check if there's a Python ‘cache’ ++ * callback defined. (In C modules, nbdkit would do this). ++ */ ++ else if (callback_defined ("cache", NULL)) ++ return NBDKIT_CACHE_NATIVE; ++ else ++ return NBDKIT_CACHE_NONE; ++} ++ + #define py_config_help \ + "script= (required) The Python plugin to run.\n" \ + "[other arguments may be used by the plugin that you load]" +@@ -844,6 +900,8 @@ static struct nbdkit_plugin plugin = { + .can_trim = py_can_trim, + .can_zero = py_can_zero, + .can_fast_zero = py_can_fast_zero, ++ .can_fua = py_can_fua, ++ .can_cache = py_can_cache, + + .pread = py_pread, + .pwrite = py_pwrite, +-- +2.18.2 + diff --git a/SOURCES/0009-python-Try-harder-to-print-the-full-traceback-on-err.patch b/SOURCES/0009-python-Try-harder-to-print-the-full-traceback-on-err.patch deleted file mode 100644 index 2bb34cb..0000000 --- a/SOURCES/0009-python-Try-harder-to-print-the-full-traceback-on-err.patch +++ /dev/null @@ -1,229 +0,0 @@ -From c713e3337d1227db68a4088096cd19ffed746e9f Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 8 Aug 2018 13:50:23 +0100 -Subject: [PATCH] python: Try harder to print the full traceback on error. -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -The tracebacks are compressed into a single line because we're using -PyObject_Str, but they are just about usable if not very readable. -For example you would see an error like this: - -nbdkit: error: ./python-exception.py: config_complete: error: ['Traceback (most recent call last):\n', ' File "./python-exception.py", line 54, in config_complete\n raise_error1()\n', ' File "./python-exception.py", line 48, in raise_error1\n raise_error2()\n', ' File "./python-exception.py", line 45, in raise_error2\n raise RuntimeError("this is the test string")\n', 'RuntimeError: this is the test string\n'] - -which can be read by manually unfolding the exception in an editor as: - -nbdkit: error: ./python-exception.py: config_complete: error: -Traceback (most recent call last): - File "./python-exception.py", line 54, in config_complete - raise_error1() - File "./python-exception.py", line 48, in raise_error1 - raise_error2() - File "./python-exception.py", line 45, in raise_error2 - raise RuntimeError("this is the test string") -RuntimeError: this is the test string - -This also fixes the Python exception test: - -(1) It originally was not testing anything. Adding ‘set -e’ fixes -that. - -(2) The valgrind test is always broken because of Python itself. -Skip the test under valgrind. - -(3) This now tests both simple exceptions and full tracebacks. - -Tested with Python 2.7.15 & 3.6.6. - -(cherry picked from commit 72c0d64a47db642cafa89884f2ee554bd0b8e822) ---- - plugins/python/python.c | 93 +++++++++++++++++++++++++++++++++++------- - tests/python-exception.py | 20 ++++++++- - tests/test-python-exception.sh | 20 ++++++++- - 3 files changed, 117 insertions(+), 16 deletions(-) - -diff --git a/plugins/python/python.c b/plugins/python/python.c -index 7eb91d7..ef1a2cf 100644 ---- a/plugins/python/python.c -+++ b/plugins/python/python.c -@@ -129,27 +129,92 @@ python_to_string (PyObject *str) - return NULL; - } - -+/* This is the fallback in case we cannot get the full traceback. */ -+static void -+print_python_error (const char *callback, PyObject *error) -+{ -+ PyObject *error_str; -+ char *error_cstr = NULL; -+ -+ error_str = PyObject_Str (error); -+ error_cstr = python_to_string (error_str); -+ nbdkit_error ("%s: %s: error: %s", -+ script, callback, -+ error_cstr ? error_cstr : ""); -+ Py_DECREF (error_str); -+ free (error_cstr); -+} -+ -+/* Convert the Python traceback to a string and call nbdkit_error. -+ * https://stackoverflow.com/a/15907460/7126113 -+ */ -+static int -+print_python_traceback (const char *callback, -+ PyObject *type, PyObject *error, PyObject *traceback) -+{ -+ PyObject *module_name, *traceback_module, *format_exception_fn, *rv, -+ *traceback_str; -+ char *traceback_cstr; -+ -+#ifdef HAVE_PYSTRING_FROMSTRING -+ module_name = PyString_FromString ("traceback"); -+#else -+ module_name = PyUnicode_FromString ("traceback"); -+#endif -+ traceback_module = PyImport_Import (module_name); -+ Py_DECREF (module_name); -+ -+ /* couldn't 'import traceback' */ -+ if (traceback_module == NULL) -+ return -1; -+ -+ format_exception_fn = PyObject_GetAttrString (traceback_module, -+ "format_exception"); -+ if (format_exception_fn == NULL) -+ return -1; -+ if (!PyCallable_Check (format_exception_fn)) -+ return -1; -+ -+ rv = PyObject_CallFunctionObjArgs (format_exception_fn, -+ type, error, traceback, NULL); -+ traceback_str = PyObject_Str (rv); -+ Py_DECREF (rv); -+ traceback_cstr = python_to_string (traceback_str); -+ if (traceback_cstr == NULL) { -+ Py_DECREF (traceback_str); -+ return -1; -+ } -+ -+ nbdkit_error ("%s: %s: error: %s", -+ script, callback, -+ traceback_cstr); -+ Py_DECREF (traceback_str); -+ free (traceback_cstr); -+ -+ /* This means we succeeded in calling nbdkit_error. */ -+ return 0; -+} -+ - static int - check_python_failure (const char *callback) - { - if (PyErr_Occurred ()) { -- PyObject *type, *error, *traceback, *error_str; -- char *error_cstr; -+ PyObject *type, *error, *traceback; - -- /* Convert the Python exception to a string. -- * https://stackoverflow.com/a/1418703 -- * But forget about the traceback, it's very hard to print. -- * https://stackoverflow.com/q/1796510 -- */ - PyErr_Fetch (&type, &error, &traceback); - PyErr_NormalizeException (&type, &error, &traceback); -- error_str = PyObject_Str (error); -- error_cstr = python_to_string (error_str); -- nbdkit_error ("%s: %s: error: %s", -- script, callback, -- error_cstr ? error_cstr : ""); -- Py_DECREF (error_str); -- free (error_cstr); -+ -+ /* Try to print the full traceback. */ -+ if (print_python_traceback (callback, type, error, traceback) == -1) { -+ /* Couldn't do that, so fall back to converting the Python error -+ * to a string. -+ */ -+ print_python_error (callback, error); -+ } -+ -+ /* In all cases this returns -1 to indicate that a Python error -+ * occurred. -+ */ - return -1; - } - return 0; -diff --git a/tests/python-exception.py b/tests/python-exception.py -index 1debf51..739057f 100644 ---- a/tests/python-exception.py -+++ b/tests/python-exception.py -@@ -32,10 +32,28 @@ - - # A dummy python plugin which just raises an exception in config_complete. - -+test = "simple" - --def config_complete(): -+def config(k, v): -+ global test -+ if k == "test": -+ test = v -+ else: -+ raise RuntimeError("unknown config parameter") -+ -+def raise_error2(): - raise RuntimeError("this is the test string") - -+def raise_error1(): -+ raise_error2() -+ -+def config_complete(): -+ if test == "simple": -+ raise RuntimeError("this is the test string") -+ elif test == "traceback": -+ raise_error1() -+ else: -+ raise RuntimeError("unknown test") - - def open(readonly): - return 1 -diff --git a/tests/test-python-exception.sh b/tests/test-python-exception.sh -index 83999af..fd94827 100755 ---- a/tests/test-python-exception.sh -+++ b/tests/test-python-exception.sh -@@ -31,12 +31,30 @@ - # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - # SUCH DAMAGE. - -+set -e -+set -x -+ -+# Python language leaks like a sieve as well as a lot of worrying -+# "Conditional jump or move depends on uninitialised value(s)". -+if test -n "$NBDKIT_VALGRIND"; then -+ echo "$0: skipping Python test under valgrind." -+ exit 77 -+fi -+ - output=test-python-exception.out - - rm -f $output - --nbdkit -f -v python ./python-exception.py > $output 2>&1 ||: -+nbdkit -f -v python ./python-exception.py test=simple > $output 2>&1 ||: -+cat $output - - grep 'this is the test string' $output - -+nbdkit -f -v python ./python-exception.py test=traceback > $output 2>&1 ||: -+cat $output -+ -+grep 'raise_error1' $output -+grep 'raise_error2' $output -+grep 'this is the test string' $output -+ - rm $output --- -1.8.3.1 - diff --git a/SOURCES/0010-tests-Test-the-Python-plugin-thoroughly.patch b/SOURCES/0010-tests-Test-the-Python-plugin-thoroughly.patch new file mode 100644 index 0000000..27270af --- /dev/null +++ b/SOURCES/0010-tests-Test-the-Python-plugin-thoroughly.patch @@ -0,0 +1,597 @@ +From 7cb79aef2a12f29f1286caf3858001e47214f871 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Nov 2019 20:54:41 +0000 +Subject: [PATCH 10/19] tests: Test the Python plugin thoroughly. + +This tests the Python plugin thoroughly by issuing client commands +through libnbd and checking we get the expected results. + +(cherry picked from commit 8ead4a82ec3227dbecb6cbfc419f1a18f2817d62) +--- + .gitignore | 1 + + README | 2 + + tests/Makefile.am | 15 +-- + tests/test-lang-plugins.c | 3 +- + tests/test-python-plugin.py | 133 +++++++++++++++++++++ + tests/test-python.sh | 49 ++++++++ + tests/test.py | 60 ---------- + tests/test_python.py | 222 ++++++++++++++++++++++++++++++++++++ + 8 files changed, 413 insertions(+), 72 deletions(-) + create mode 100644 tests/test-python-plugin.py + create mode 100755 tests/test-python.sh + delete mode 100644 tests/test.py + create mode 100755 tests/test_python.py + +diff --git a/.gitignore b/.gitignore +index b25ac7fe..e25bd99b 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -71,6 +71,7 @@ Makefile.in + /server/synopsis.c + /server/test-public + /stamp-h1 ++/tests/__pycache__/ + /tests/disk + /tests/disk.gz + /tests/disk.xz +diff --git a/README b/README +index 40f4cd37..05f1e060 100644 +--- a/README ++++ b/README +@@ -130,6 +130,8 @@ For the Python plugin: + + - python development libraries + ++ - python unittest to run the test suite ++ + For the OCaml plugin: + + - OCaml >= 4.02.2 +diff --git a/tests/Makefile.am b/tests/Makefile.am +index d225cc63..09103fbb 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -67,6 +67,7 @@ EXTRA_PROGRAMS = + TESTS_ENVIRONMENT = \ + PATH=$(abs_top_builddir):$(PATH) \ + SRCDIR=$(srcdir) \ ++ PYTHON=$(PYTHON) \ + LIBGUESTFS_ATTACH_METHOD=appliance \ + LIBGUESTFS_DEBUG=1 \ + LIBGUESTFS_TRACE=1 \ +@@ -160,7 +161,9 @@ EXTRA_DIST = \ + test-probe-plugin.sh \ + test-python-exception.sh \ + test.pl \ +- test.py \ ++ test_python.py \ ++ test-python-plugin.py \ ++ test-python.sh \ + test-rate.sh \ + test-rate-dynamic.sh \ + test.rb \ +@@ -801,18 +804,10 @@ endif HAVE_PERL + if HAVE_PYTHON + + TESTS += \ ++ test-python.sh \ + test-python-exception.sh \ + test-shebang-python.sh \ + $(NULL) +-LIBGUESTFS_TESTS += test-python +- +-test_python_SOURCES = test-lang-plugins.c test.h +-test_python_CFLAGS = \ +- -DLANG='"python"' -DSCRIPT='"$(srcdir)/test.py"' \ +- $(WARNINGS_CFLAGS) \ +- $(LIBGUESTFS_CFLAGS) \ +- $(NULL) +-test_python_LDADD = libtest.la $(LIBGUESTFS_LIBS) + + endif HAVE_PYTHON + +diff --git a/tests/test-lang-plugins.c b/tests/test-lang-plugins.c +index ffb19180..93f99381 100644 +--- a/tests/test-lang-plugins.c ++++ b/tests/test-lang-plugins.c +@@ -56,8 +56,7 @@ main (int argc, char *argv[]) + */ + s = getenv ("NBDKIT_VALGRIND"); + if (s && strcmp (s, "1") == 0 && +- (strcmp (LANG, "python") == 0 || +- strcmp (LANG, "ruby") == 0 || ++ (strcmp (LANG, "ruby") == 0 || + strcmp (LANG, "tcl") == 0)) { + fprintf (stderr, "%s test skipped under valgrind.\n", LANG); + exit (77); /* Tells automake to skip the test. */ +diff --git a/tests/test-python-plugin.py b/tests/test-python-plugin.py +new file mode 100644 +index 00000000..8e90bc23 +--- /dev/null ++++ b/tests/test-python-plugin.py +@@ -0,0 +1,133 @@ ++# nbdkit test plugin ++# Copyright (C) 2019 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++"""See test-python.py.""" ++ ++import nbdkit ++import sys ++import pickle ++import base64 ++ ++API_VERSION = 2 ++ ++cfg = {} ++ ++def config (k, v): ++ global cfg ++ if k == "cfg": ++ cfg = pickle.loads (base64.b64decode (v.encode())) ++ ++def config_complete (): ++ print ("set_error = %r" % nbdkit.set_error) ++ ++def open (readonly): ++ return { ++ 'disk': bytearray (cfg.get ('size', 0)) ++ } ++ ++def get_size (h): ++ return len (h['disk']) ++ ++def is_rotational (h): ++ return cfg.get ('is_rotational', False) ++ ++def can_multi_conn (h): ++ return cfg.get ('can_multi_conn', False) ++ ++def can_write (h): ++ return cfg.get ('can_write', True) ++ ++def can_flush (h): ++ return cfg.get ('can_flush', False) ++ ++def can_trim (h): ++ return cfg.get ('can_trim', False) ++ ++def can_zero (h): ++ return cfg.get ('can_zero', False) ++ ++def can_fast_zero (h): ++ return cfg.get ('can_fast_zero', False) ++ ++def can_fua (h): ++ fua = cfg.get ('can_fua', "none") ++ if fua == "none": ++ return nbdkit.FUA_NONE ++ elif fua == "emulate": ++ return nbdkit.FUA_EMULATE ++ elif fua == "native": ++ return nbdkit.FUA_NATIVE ++ ++def can_cache (h): ++ cache = cfg.get ('can_cache', "none") ++ if cache == "none": ++ return nbdkit.CACHE_NONE ++ elif cache == "emulate": ++ return nbdkit.CACHE_EMULATE ++ elif cache == "native": ++ return nbdkit.CACHE_NATIVE ++ ++def pread (h, buf, offset, flags): ++ assert flags == 0 ++ end = offset + len(buf) ++ buf[:] = h['disk'][offset:end] ++ ++def pwrite (h, buf, offset, flags): ++ expect_fua = cfg.get ('pwrite_expect_fua', False) ++ actual_fua = bool (flags & nbdkit.FLAG_FUA) ++ assert expect_fua == actual_fua ++ end = offset + len(buf) ++ h['disk'][offset:end] = buf ++ ++def flush (h, flags): ++ assert flags == 0 ++ ++def trim (h, count, offset, flags): ++ expect_fua = cfg.get ('trim_expect_fua', False) ++ actual_fua = bool (flags & nbdkit.FLAG_FUA) ++ assert expect_fua == actual_fua ++ h['disk'][offset:offset+count] = bytearray(count) ++ ++def zero (h, count, offset, flags): ++ expect_fua = cfg.get ('zero_expect_fua', False) ++ actual_fua = bool (flags & nbdkit.FLAG_FUA) ++ assert expect_fua == actual_fua ++ expect_may_trim = cfg.get ('zero_expect_may_trim', False) ++ actual_may_trim = bool (flags & nbdkit.FLAG_MAY_TRIM) ++ assert expect_may_trim == actual_may_trim ++ expect_fast_zero = cfg.get ('zero_expect_fast_zero', False) ++ actual_fast_zero = bool (flags & nbdkit.FLAG_FAST_ZERO) ++ assert expect_fast_zero == actual_fast_zero ++ h['disk'][offset:offset+count] = bytearray(count) ++ ++def cache (h, count, offset, flags): ++ assert flags == 0 ++ # do nothing +diff --git a/tests/test-python.sh b/tests/test-python.sh +new file mode 100755 +index 00000000..50324d0f +--- /dev/null ++++ b/tests/test-python.sh +@@ -0,0 +1,49 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2019 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires $PYTHON --version ++requires $PYTHON -c 'import unittest' ++requires $PYTHON -c 'import nbd' ++requires test -f test_python.py ++requires test -f test-python-plugin.py ++ ++# Python has proven very difficult to valgrind, therefore it is disabled. ++if [ "$NBDKIT_VALGRIND" = "1" ]; then ++ echo "$0: skipping Python test under valgrind." ++ exit 77 ++fi ++ ++$PYTHON -m unittest test_python +diff --git a/tests/test.py b/tests/test.py +deleted file mode 100644 +index 4db56623..00000000 +--- a/tests/test.py ++++ /dev/null +@@ -1,60 +0,0 @@ +-import nbdkit +- +-disk = bytearray(1024*1024) +- +- +-API_VERSION = 2 +- +- +-def config_complete(): +- print ("set_error = %r" % nbdkit.set_error) +- +- +-def open(readonly): +- return 1 +- +- +-def get_size(h): +- global disk +- return len(disk) +- +- +-def can_write(h): +- return True +- +- +-def can_flush(h): +- return True +- +- +-def is_rotational(h): +- return False +- +- +-def can_trim(h): +- return True +- +- +-def pread(h, buf, offset, flags): +- global disk +- end = offset + len(buf) +- buf[:] = disk[offset:end] +- +- +-def pwrite(h, buf, offset, flags): +- global disk +- end = offset + len(buf) +- disk[offset:end] = buf +- +- +-def flush(h, flags): +- pass +- +- +-def trim(h, count, offset, flags): +- pass +- +- +-def zero(h, count, offset, flags): +- global disk +- disk[offset:offset+count] = bytearray(count) +diff --git a/tests/test_python.py b/tests/test_python.py +new file mode 100755 +index 00000000..6b9f2979 +--- /dev/null ++++ b/tests/test_python.py +@@ -0,0 +1,222 @@ ++#!/usr/bin/env python3 ++# nbdkit ++# Copyright (C) 2019 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++""" ++This tests the Python plugin thoroughly by issuing client commands ++through libnbd and checking we get the expected results. It uses an ++associated plugin (test-python-plugin.sh). ++""" ++ ++import os ++import sys ++import nbd ++import unittest ++import pickle ++import base64 ++ ++class Test (unittest.TestCase): ++ def setUp (self): ++ self.h = nbd.NBD () ++ ++ def tearDown (self): ++ del self.h ++ ++ def connect (self, cfg): ++ cfg = base64.b64encode (pickle.dumps (cfg)).decode() ++ cmd = ["nbdkit", "-v", "-s", "--exit-with-parent", ++ "python", "test-python-plugin.py", "cfg=" + cfg] ++ self.h.connect_command (cmd) ++ ++ def test_none (self): ++ """ ++ Test we can send an empty pickled test configuration and do ++ nothing else. This is just to ensure the machinery of the ++ test works. ++ """ ++ self.connect ({}) ++ ++ def test_size_512 (self): ++ """Test the size.""" ++ self.connect ({"size": 512}) ++ assert self.h.get_size() == 512 ++ ++ def test_size_1m (self): ++ """Test the size.""" ++ self.connect ({"size": 1024*1024}) ++ assert self.h.get_size() == 1024*1024 ++ ++ # Test each flag call. ++ def test_is_rotational_true (self): ++ self.connect ({"size": 512, "is_rotational": True}) ++ assert self.h.is_rotational() ++ ++ def test_is_rotational_false (self): ++ self.connect ({"size": 512, "is_rotational": False}) ++ assert not self.h.is_rotational() ++ ++ def test_can_multi_conn_true (self): ++ self.connect ({"size": 512, "can_multi_conn": True}) ++ assert self.h.can_multi_conn() ++ ++ def test_can_multi_conn_false (self): ++ self.connect ({"size": 512, "can_multi_conn": False}) ++ assert not self.h.can_multi_conn() ++ ++ def test_read_write (self): ++ self.connect ({"size": 512, "can_write": True}) ++ assert not self.h.is_read_only() ++ ++ def test_read_only (self): ++ self.connect ({"size": 512, "can_write": False}) ++ assert self.h.is_read_only() ++ ++ def test_can_flush_true (self): ++ self.connect ({"size": 512, "can_flush": True}) ++ assert self.h.can_flush() ++ ++ def test_can_flush_false (self): ++ self.connect ({"size": 512, "can_flush": False}) ++ assert not self.h.can_flush() ++ ++ def test_can_trim_true (self): ++ self.connect ({"size": 512, "can_trim": True}) ++ assert self.h.can_trim() ++ ++ def test_can_trim_false (self): ++ self.connect ({"size": 512, "can_trim": False}) ++ assert not self.h.can_trim() ++ ++ # nbdkit can always zero because it emulates it. ++ #self.connect ({"size": 512, "can_zero": True}) ++ #assert self.h.can_zero() ++ #self.connect ({"size": 512, "can_zero": False}) ++ #assert not self.h.can_zero() ++ ++ def test_can_fast_zero_true (self): ++ self.connect ({"size": 512, "can_fast_zero": True}) ++ assert self.h.can_fast_zero() ++ ++ def test_can_fast_zero_false (self): ++ self.connect ({"size": 512, "can_fast_zero": False}) ++ assert not self.h.can_fast_zero() ++ ++ def test_can_fua_none (self): ++ self.connect ({"size": 512, "can_fua": "none"}) ++ assert not self.h.can_fua() ++ ++ def test_can_fua_emulate (self): ++ self.connect ({"size": 512, "can_fua": "emulate"}) ++ assert self.h.can_fua() ++ ++ def test_can_fua_native (self): ++ self.connect ({"size": 512, "can_fua": "native"}) ++ assert self.h.can_fua() ++ ++ def test_can_cache_none (self): ++ self.connect ({"size": 512, "can_cache": "none"}) ++ assert not self.h.can_cache() ++ ++ def test_can_cache_emulate (self): ++ self.connect ({"size": 512, "can_cache": "emulate"}) ++ assert self.h.can_cache() ++ ++ def test_can_cache_native (self): ++ self.connect ({"size": 512, "can_cache": "native"}) ++ assert self.h.can_cache() ++ ++ # Not yet implemented: can_extents. ++ ++ def test_pread (self): ++ """Test pread.""" ++ self.connect ({"size": 512}) ++ buf = self.h.pread (512, 0) ++ assert buf == bytearray (512) ++ ++ # Test pwrite + flags. ++ def test_pwrite (self): ++ self.connect ({"size": 512}) ++ buf = bytearray (512) ++ self.h.pwrite (buf, 0) ++ ++ def test_pwrite_fua (self): ++ self.connect ({"size": 512, ++ "can_fua": "native", ++ "pwrite_expect_fua": True}) ++ buf = bytearray (512) ++ self.h.pwrite (buf, 0, nbd.CMD_FLAG_FUA) ++ ++ def test_flush (self): ++ """Test flush.""" ++ self.connect ({"size": 512, "can_flush": True}) ++ self.h.flush () ++ ++ # Test trim + flags. ++ def test_trim (self): ++ self.connect ({"size": 512, "can_trim": True}) ++ self.h.trim (512, 0) ++ ++ def test_trim_fua (self): ++ self.connect ({"size": 512, ++ "can_trim": True, ++ "can_fua": "native", ++ "trim_expect_fua": True}) ++ self.h.trim (512, 0, nbd.CMD_FLAG_FUA) ++ ++ # Test zero + flags. ++ def test_zero (self): ++ self.connect ({"size": 512, "can_zero": True}) ++ self.h.zero (512, 0, nbd.CMD_FLAG_NO_HOLE) ++ ++ def test_zero_fua (self): ++ self.connect ({"size": 512, ++ "can_zero": True, ++ "can_fua": "native", ++ "zero_expect_fua": True}) ++ self.h.zero (512, 0, nbd.CMD_FLAG_NO_HOLE | nbd.CMD_FLAG_FUA) ++ ++ def test_zero_may_trim (self): ++ self.connect ({"size": 512, ++ "can_zero": True, ++ "zero_expect_may_trim": True}) ++ self.h.zero (512, 0, 0) # absence of nbd.CMD_FLAG_NO_HOLE ++ ++ def test_zero_fast_zero (self): ++ self.connect ({"size": 512, ++ "can_zero": True, ++ "can_fast_zero": True, ++ "zero_expect_fast_zero": True}) ++ self.h.zero (512, 0, nbd.CMD_FLAG_NO_HOLE | nbd.CMD_FLAG_FAST_ZERO) ++ ++ def test_cache (self): ++ """Test cache.""" ++ self.connect ({"size": 512, "can_cache": "native"}) ++ self.h.cache (512, 0) +-- +2.18.2 + diff --git a/SOURCES/0011-New-filter-extentlist.patch b/SOURCES/0011-New-filter-extentlist.patch new file mode 100644 index 0000000..d33eb00 --- /dev/null +++ b/SOURCES/0011-New-filter-extentlist.patch @@ -0,0 +1,790 @@ +From e744dcb38cc52cbe64977efcdd4bc60e802d1b17 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 23 Jan 2020 19:52:00 +0000 +Subject: [PATCH 11/19] New filter: extentlist. + +Allows a list of extents to be placed on top of an existing plugin. + +(cherry picked from commit 3e770b6d6620a62546849a2863638041c0b00640) +--- + TODO | 4 + + configure.ac | 2 + + .../nbdkit-cacheextents-filter.pod | 1 + + filters/extentlist/Makefile.am | 67 ++++ + filters/extentlist/extentlist.c | 326 ++++++++++++++++++ + .../extentlist/nbdkit-extentlist-filter.pod | 90 +++++ + filters/noextents/nbdkit-noextents-filter.pod | 1 + + tests/Makefile.am | 4 + + tests/test-extentlist.sh | 175 ++++++++++ + 9 files changed, 670 insertions(+) + create mode 100644 filters/extentlist/Makefile.am + create mode 100644 filters/extentlist/extentlist.c + create mode 100644 filters/extentlist/nbdkit-extentlist-filter.pod + create mode 100755 tests/test-extentlist.sh + +diff --git a/TODO b/TODO +index d2aca440..2a3e89dc 100644 +--- a/TODO ++++ b/TODO +@@ -187,6 +187,10 @@ Suggestions for filters + MBs of extra data) + https://github.com/facebook/zstd/issues/395#issuecomment-535875379 + ++* nbdkit-extentlist-filter could read the extents generated by ++ qemu-img map, allowing extents to be ported from a qemu block ++ device. ++ + nbdkit-rate-filter: + + * allow other kinds of traffic shaping such as VBR +diff --git a/configure.ac b/configure.ac +index fde498b8..41e68de3 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -896,6 +896,7 @@ filters="\ + cow \ + delay \ + error \ ++ extentlist \ + fua \ + log \ + nocache \ +@@ -979,6 +980,7 @@ AC_CONFIG_FILES([Makefile + filters/cow/Makefile + filters/delay/Makefile + filters/error/Makefile ++ filters/extentlist/Makefile + filters/fua/Makefile + filters/log/Makefile + filters/nocache/Makefile +diff --git a/filters/cacheextents/nbdkit-cacheextents-filter.pod b/filters/cacheextents/nbdkit-cacheextents-filter.pod +index fdd2285a..bb2514a4 100644 +--- a/filters/cacheextents/nbdkit-cacheextents-filter.pod ++++ b/filters/cacheextents/nbdkit-cacheextents-filter.pod +@@ -52,6 +52,7 @@ C first appeared in nbdkit 1.14. + + L, + L, ++L, + L, + L, + L, +diff --git a/filters/extentlist/Makefile.am b/filters/extentlist/Makefile.am +new file mode 100644 +index 00000000..88a9afe1 +--- /dev/null ++++ b/filters/extentlist/Makefile.am +@@ -0,0 +1,67 @@ ++# nbdkit ++# Copyright (C) 2019-2020 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++include $(top_srcdir)/common-rules.mk ++ ++EXTRA_DIST = nbdkit-extentlist-filter.pod ++ ++filter_LTLIBRARIES = nbdkit-extentlist-filter.la ++ ++nbdkit_extentlist_filter_la_SOURCES = \ ++ extentlist.c \ ++ $(top_srcdir)/include/nbdkit-filter.h \ ++ $(NULL) ++ ++nbdkit_extentlist_filter_la_CPPFLAGS = \ ++ -I$(top_srcdir)/include \ ++ -I$(top_srcdir)/common/include \ ++ -I$(top_srcdir)/common/utils \ ++ $(NULL) ++nbdkit_extentlist_filter_la_CFLAGS = $(WARNINGS_CFLAGS) ++nbdkit_extentlist_filter_la_LDFLAGS = \ ++ -module -avoid-version -shared \ ++ -Wl,--version-script=$(top_srcdir)/filters/filters.syms \ ++ $(NULL) ++nbdkit_extentlist_filter_la_LIBADD = \ ++ $(top_builddir)/common/utils/libutils.la \ ++ $(NULL) ++ ++if HAVE_POD ++ ++man_MANS = nbdkit-extentlist-filter.1 ++CLEANFILES += $(man_MANS) ++ ++nbdkit-extentlist-filter.1: nbdkit-extentlist-filter.pod ++ $(PODWRAPPER) --section=1 --man $@ \ ++ --html $(top_builddir)/html/$@.html \ ++ $< ++ ++endif HAVE_POD +diff --git a/filters/extentlist/extentlist.c b/filters/extentlist/extentlist.c +new file mode 100644 +index 00000000..5f4990b3 +--- /dev/null ++++ b/filters/extentlist/extentlist.c +@@ -0,0 +1,326 @@ ++/* nbdkit ++ * Copyright (C) 2019-2020 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "cleanup.h" ++#include "minmax.h" ++ ++#define HOLE (NBDKIT_EXTENT_HOLE|NBDKIT_EXTENT_ZERO) ++ ++static const char *extentlist; ++ ++/* List of extents. Once we've finally parsed them this will be ++ * ordered, non-overlapping and have no gaps. ++ */ ++struct extent { ++ uint64_t offset, length; ++ uint32_t type; ++}; ++static struct extent *extents; ++static size_t nr_extents, allocated; ++ ++/* Insert an extent before i. If i = nr_extents, inserts at the end. */ ++static void ++insert_extent (size_t i, struct extent new_extent) ++{ ++ if (nr_extents >= allocated) { ++ allocated = allocated == 0 ? 1 : allocated * 2; ++ extents = realloc (extents, (sizeof (struct extent) * allocated)); ++ if (extents == NULL) { ++ nbdkit_error ("realloc: %m"); ++ exit (EXIT_FAILURE); ++ } ++ } ++ memmove (&extents[i+1], &extents[i], ++ sizeof (struct extent) * (nr_extents-i)); ++ extents[i] = new_extent; ++ nr_extents++; ++} ++ ++static void ++extentlist_unload (void) ++{ ++ free (extents); ++} ++ ++/* Called for each key=value passed on the command line. */ ++static int ++extentlist_config (nbdkit_next_config *next, void *nxdata, ++ const char *key, const char *value) ++{ ++ if (strcmp (key, "extentlist") == 0) { ++ if (extentlist != NULL) { ++ nbdkit_error ("extentlist cannot appear twice"); ++ exit (EXIT_FAILURE); ++ } ++ extentlist = value; ++ return 0; ++ } ++ else ++ return next (nxdata, key, value); ++} ++ ++static int ++compare_offsets (const void *ev1, const void *ev2) ++{ ++ const struct extent *e1 = ev1; ++ const struct extent *e2 = ev2; ++ ++ if (e1->offset < e2->offset) ++ return -1; ++ else if (e1->offset > e2->offset) ++ return 1; ++ else ++ return 0; ++} ++ ++static int ++compare_ranges (const void *ev1, const void *ev2) ++{ ++ const struct extent *e1 = ev1; ++ const struct extent *e2 = ev2; ++ ++ if (e1->offset < e2->offset) ++ return -1; ++ else if (e1->offset >= e2->offset + e2->length) ++ return 1; ++ else ++ return 0; ++} ++ ++/* Similar to parse_extents in plugins/sh/methods.c */ ++static void ++parse_extentlist (void) ++{ ++ FILE *fp; ++ CLEANUP_FREE char *line = NULL; ++ size_t linelen = 0; ++ ssize_t len; ++ size_t i; ++ uint64_t end; ++ ++ assert (extentlist != NULL); ++ assert (extents == NULL); ++ assert (nr_extents == 0); ++ ++ fp = fopen (extentlist, "r"); ++ if (!fp) { ++ nbdkit_error ("open: %s: %m", extentlist); ++ exit (EXIT_FAILURE); ++ } ++ ++ while ((len = getline (&line, &linelen, fp)) != -1) { ++ const char *delim = " \t"; ++ char *sp, *p; ++ int64_t offset, length; ++ uint32_t type; ++ ++ if (len > 0 && line[len-1] == '\n') { ++ line[len-1] = '\0'; ++ len--; ++ } ++ ++ if ((p = strtok_r (line, delim, &sp)) == NULL) { ++ parse_error: ++ nbdkit_error ("%s: cannot parse %s", extentlist, line); ++ exit (EXIT_FAILURE); ++ } ++ offset = nbdkit_parse_size (p); ++ if (offset == -1) ++ exit (EXIT_FAILURE); ++ ++ if ((p = strtok_r (NULL, delim, &sp)) == NULL) ++ goto parse_error; ++ length = nbdkit_parse_size (p); ++ if (length == -1) ++ exit (EXIT_FAILURE); ++ ++ /* Skip zero length extents. Makes the rest of the code easier. */ ++ if (length == 0) ++ continue; ++ ++ if ((p = strtok_r (NULL, delim, &sp)) == NULL) ++ /* empty type field means allocated data (0) */ ++ type = 0; ++ else if (sscanf (p, "%" SCNu32, &type) == 1) ++ ; ++ else { ++ type = 0; ++ if (strstr (p, "hole") != NULL) ++ type |= NBDKIT_EXTENT_HOLE; ++ if (strstr (p, "zero") != NULL) ++ type |= NBDKIT_EXTENT_ZERO; ++ } ++ ++ insert_extent (nr_extents, ++ (struct extent){.offset = offset, .length=length, ++ .type=type}); ++ } ++ ++ fclose (fp); ++ ++ /* Sort the extents by offset. */ ++ qsort (extents, nr_extents, sizeof (struct extent), compare_offsets); ++ ++ /* There must not be overlaps at this point. */ ++ end = 0; ++ for (i = 0; i < nr_extents; ++i) { ++ if (extents[i].offset < end || ++ extents[i].offset + extents[i].length < extents[i].offset) { ++ nbdkit_error ("extents in the extent list are overlapping"); ++ exit (EXIT_FAILURE); ++ } ++ end = extents[i].offset + extents[i].length; ++ } ++ ++ /* If there's a gap at the beginning, insert a hole|zero extent. */ ++ if (nr_extents == 0 || extents[0].offset > 0) { ++ end = nr_extents == 0 ? UINT64_MAX : extents[0].offset; ++ insert_extent (0, (struct extent){.offset = 0, .length = end, ++ .type = HOLE}); ++ } ++ ++ /* Now insert hole|zero extents after every extent where there ++ * is a gap between that extent and the next one. ++ */ ++ for (i = 0; i < nr_extents-1; ++i) { ++ end = extents[i].offset + extents[i].length; ++ if (end < extents[i+1].offset) ++ insert_extent (i+1, (struct extent){.offset = end, ++ .length = extents[i+1].offset - end, ++ .type = HOLE}); ++ } ++ ++ /* If there's a gap at the end, insert a hole|zero extent. */ ++ end = extents[nr_extents-1].offset + extents[nr_extents-1].length; ++ if (end < UINT64_MAX) ++ insert_extent (nr_extents, (struct extent){.offset = end, ++ .length = UINT64_MAX-end, ++ .type = HOLE}); ++ ++ /* Debug the final list. */ ++ for (i = 0; i < nr_extents; ++i) { ++ nbdkit_debug ("extentlist: " ++ "extent[%zu] = %" PRIu64 "-%" PRIu64 " (length %" PRIu64 ")" ++ " type %" PRIu32, ++ i, extents[i].offset, ++ extents[i].offset + extents[i].length - 1, ++ extents[i].length, ++ extents[i].type); ++ } ++} ++ ++static int ++extentlist_config_complete (nbdkit_next_config_complete *next, void *nxdata) ++{ ++ if (extentlist == NULL) { ++ nbdkit_error ("you must supply the extentlist parameter " ++ "on the command line"); ++ return -1; ++ } ++ ++ parse_extentlist (); ++ ++ return next (nxdata); ++} ++ ++static int ++extentlist_can_extents (struct nbdkit_next_ops *next_ops, void *nxdata, ++ void *handle) ++{ ++ return 1; ++} ++ ++/* Use ‘-D extentlist.lookup=1’ to debug the function below. */ ++int extentlist_debug_lookup = 0; ++ ++/* Read extents. */ ++static int ++extentlist_extents (struct nbdkit_next_ops *next_ops, void *nxdata, ++ void *handle, uint32_t count, uint64_t offset, ++ uint32_t flags, ++ struct nbdkit_extents *ret_extents, ++ int *err) ++{ ++ const struct extent eoffset = { .offset = offset }; ++ struct extent *p; ++ ssize_t i; ++ uint64_t end; ++ ++ /* Find the starting point in the extents list. */ ++ p = bsearch (&eoffset, extents, ++ nr_extents, sizeof (struct extent), compare_ranges); ++ assert (p != NULL); ++ i = p - extents; ++ ++ /* Add extents to the output. */ ++ while (count > 0) { ++ if (extentlist_debug_lookup) ++ nbdkit_debug ("extentlist lookup: " ++ "loop i=%zd count=%" PRIu32 " offset=%" PRIu64, ++ i, count, offset); ++ ++ end = extents[i].offset + extents[i].length; ++ if (nbdkit_add_extent (ret_extents, offset, end - offset, ++ extents[i].type) == -1) ++ return -1; ++ ++ count -= MIN (count, end-offset); ++ offset = end; ++ i++; ++ } ++ ++ return 0; ++} ++ ++static struct nbdkit_filter filter = { ++ .name = "extentlist", ++ .longname = "nbdkit extentlist filter", ++ .unload = extentlist_unload, ++ .config = extentlist_config, ++ .config_complete = extentlist_config_complete, ++ .can_extents = extentlist_can_extents, ++ .extents = extentlist_extents, ++}; ++ ++NBDKIT_REGISTER_FILTER(filter) +diff --git a/filters/extentlist/nbdkit-extentlist-filter.pod b/filters/extentlist/nbdkit-extentlist-filter.pod +new file mode 100644 +index 00000000..adfb4ad8 +--- /dev/null ++++ b/filters/extentlist/nbdkit-extentlist-filter.pod +@@ -0,0 +1,90 @@ ++=head1 NAME ++ ++nbdkit-extentlist-filter - place extent list over a plugin ++ ++=head1 SYNOPSIS ++ ++ nbdkit --filter=extentlist plugin extentlist=FILENAME ++ ++=head1 DESCRIPTION ++ ++C is an nbdkit filter lets you place a ++static list of extents on top of an existing plugin. Extents record ++whether or not specific parts of the disk are allocated or sparse. ++ ++You can use this with plugins which cannot get extent information ++themselves, but you can get this information from another source. One ++place where it is useful is with L because the ++sftp protocol does not support reading sparseness information, but you ++may be able to get this information directly from the source disk on ++the remote server. ++ ++=head1 FILE FORMAT ++ ++The list of extents is specified in a text file. There is one extent ++specified per line. Each line has the format: ++ ++ offset length type ++ ++The C and C fields may use any format understood by ++C. The optional C field may be an integer, ++missing (same as 0), or a comma-separated list of the words C ++and C. (The fields correspond to the inputs of the ++C function, see L). ++ ++An example of a valid set of extents covering a C<10M> disk where the ++first megabyte only is allocated data: ++ ++ 0 1M ++ 1M 9M hole,zero ++ ++Or you could omit the C extent since any gaps are assumed ++to be holes with that type: ++ ++ 0 1M ++ ++The extent list need not cover the whole disk, and does not need to be ++in ascending order, but it must I contain overlapping extents. ++ ++=head1 PARAMETERS ++ ++=over 4 ++ ++=item BFILENAME ++ ++Specify the file containing the extent list, in the format described ++in L above. ++ ++=back ++ ++=head1 FILES ++ ++=over 4 ++ ++=item F<$filterdir/nbdkit-extentlist-filter.so> ++ ++The filter. ++ ++Use C to find the location of C<$filterdir>. ++ ++=back ++ ++=head1 VERSION ++ ++C first appeared in nbdkit 1.18. ++ ++=head1 SEE ALSO ++ ++L, ++L, ++L, ++L, ++L. ++ ++=head1 AUTHORS ++ ++Richard W.M. Jones ++ ++=head1 COPYRIGHT ++ ++Copyright (C) 2020 Red Hat Inc. +diff --git a/filters/noextents/nbdkit-noextents-filter.pod b/filters/noextents/nbdkit-noextents-filter.pod +index 991ecfe8..0260a5cf 100644 +--- a/filters/noextents/nbdkit-noextents-filter.pod ++++ b/filters/noextents/nbdkit-noextents-filter.pod +@@ -47,6 +47,7 @@ C first appeared in nbdkit 1.14. + + L, + L, ++L, + L, + L, + L, +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 09103fbb..b99952f4 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -110,6 +110,7 @@ EXTRA_DIST = \ + test-error100.sh \ + test-error-triggered.sh \ + test-export-name.sh \ ++ test-extentlist.sh \ + test-file-extents.sh \ + test-floppy.sh \ + test-foreground.sh \ +@@ -1009,6 +1010,9 @@ TESTS += \ + test-error-triggered.sh \ + $(NULL) + ++# extentlist filter test. ++TESTS += test-extentlist.sh ++ + # fua filter test. + TESTS += test-fua.sh + +diff --git a/tests/test-extentlist.sh b/tests/test-extentlist.sh +new file mode 100755 +index 00000000..7d05de4f +--- /dev/null ++++ b/tests/test-extentlist.sh +@@ -0,0 +1,175 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2016-2020 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++# Test the extentlist filter. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires jq --version ++requires qemu-img --version ++requires qemu-img map --help ++ ++out=test-extentlist.out ++input=test-extentlist.in ++expected=test-extentlist.expected ++files="$out $input $expected" ++rm -f $files ++cleanup_fn rm $files ++ ++test () ++{ ++ nbdkit -v -D extentlist.lookup=1 \ ++ -U - \ ++ --filter=extentlist \ ++ null size=$1 extentlist=$input \ ++ --run 'qemu-img map -f raw --output=json $nbd' | ++ jq -c '.[] | {start:.start, length:.length, data:.data, zero:.zero}' \ ++ > $out ++ diff -u $out $expected ++} ++ ++# Empty extent list. ++cat > $input <<'EOF' ++EOF ++ ++cat > $expected <<'EOF' ++{"start":0,"length":0,"data":false,"zero":false} ++EOF ++test 0 ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":false,"zero":true} ++EOF ++test 1M ++ ++# Extent list covering 0-1M with data. ++cat > $input <<'EOF' ++0 1M ++EOF ++ ++cat > $expected <<'EOF' ++{"start":0,"length":0,"data":false,"zero":false} ++EOF ++test 0 ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":true,"zero":false} ++EOF ++test 1M ++ ++# Extent list covering 1-2M with data. ++cat > $input <<'EOF' ++1M 1M ++EOF ++ ++cat > $expected <<'EOF' ++{"start":0,"length":0,"data":false,"zero":false} ++EOF ++test 0 ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":false,"zero":true} ++EOF ++test 1M ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":false,"zero":true} ++{"start":1048576,"length":1048576,"data":true,"zero":false} ++EOF ++test 2M ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":false,"zero":true} ++{"start":1048576,"length":1048576,"data":true,"zero":false} ++{"start":2097152,"length":1048576,"data":false,"zero":true} ++EOF ++test 3M ++ ++# Extent list covering 1-2M with data, but in a more fragmented ++# way than the above. ++cat > $input <<'EOF' ++1024K 512K ++1536K 512K ++EOF ++ ++cat > $expected <<'EOF' ++{"start":0,"length":0,"data":false,"zero":false} ++EOF ++test 0 ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":false,"zero":true} ++EOF ++test 1M ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":false,"zero":true} ++{"start":1048576,"length":1048576,"data":true,"zero":false} ++EOF ++test 2M ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":false,"zero":true} ++{"start":1048576,"length":1048576,"data":true,"zero":false} ++{"start":2097152,"length":1048576,"data":false,"zero":true} ++EOF ++test 3M ++ ++# Adjacent data and holes. ++cat > $input <<'EOF' ++0 1M ++2M 1M ++4M 1M ++EOF ++ ++cat > $expected <<'EOF' ++{"start":0,"length":0,"data":false,"zero":false} ++EOF ++test 0 ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":true,"zero":false} ++EOF ++test 1M ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":true,"zero":false} ++{"start":1048576,"length":1048576,"data":false,"zero":true} ++EOF ++test 2M ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":true,"zero":false} ++{"start":1048576,"length":1048576,"data":false,"zero":true} ++{"start":2097152,"length":1048576,"data":true,"zero":false} ++EOF ++test 3M ++cat > $expected <<'EOF' ++{"start":0,"length":1048576,"data":true,"zero":false} ++{"start":1048576,"length":1048576,"data":false,"zero":true} ++{"start":2097152,"length":1048576,"data":true,"zero":false} ++{"start":3145728,"length":1048576,"data":false,"zero":true} ++{"start":4194304,"length":1048576,"data":true,"zero":false} ++{"start":5242880,"length":1048576,"data":false,"zero":true} ++EOF ++test 6M +-- +2.18.2 + diff --git a/SOURCES/0012-extentlist-Documentation-and-test-fixes.patch b/SOURCES/0012-extentlist-Documentation-and-test-fixes.patch new file mode 100644 index 0000000..62bdab0 --- /dev/null +++ b/SOURCES/0012-extentlist-Documentation-and-test-fixes.patch @@ -0,0 +1,125 @@ +From 2a3e909e9e1ccb608bde75b76524acd753b33889 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 25 Jan 2020 11:38:14 +0000 +Subject: [PATCH 12/19] extentlist: Documentation and test fixes. + +Updates commit 3e770b6d6620a62546849a2863638041c0b00640. + +(cherry picked from commit c16709ef663a5ed9fd9ddef4e379f316d84c9a07) +--- + TODO | 12 +++++++---- + .../extentlist/nbdkit-extentlist-filter.pod | 21 +++++++++++++------ + plugins/curl/nbdkit-curl-plugin.pod | 1 + + plugins/ssh/nbdkit-ssh-plugin.pod | 1 + + tests/test-extentlist.sh | 2 +- + 5 files changed, 26 insertions(+), 11 deletions(-) + +diff --git a/TODO b/TODO +index 2a3e89dc..e1ac71cd 100644 +--- a/TODO ++++ b/TODO +@@ -187,10 +187,6 @@ Suggestions for filters + MBs of extra data) + https://github.com/facebook/zstd/issues/395#issuecomment-535875379 + +-* nbdkit-extentlist-filter could read the extents generated by +- qemu-img map, allowing extents to be ported from a qemu block +- device. +- + nbdkit-rate-filter: + + * allow other kinds of traffic shaping such as VBR +@@ -216,6 +212,14 @@ nbdkit-retry-filter: + + * subsecond times + ++nbdkit-extentlist-filter: ++ ++* read the extents generated by qemu-img map, allowing extents to be ++ ported from a qemu block device ++ ++* make non-read-only access safe by updating the extent list when the ++ filter sees writes and trims ++ + Filters for security + -------------------- + +diff --git a/filters/extentlist/nbdkit-extentlist-filter.pod b/filters/extentlist/nbdkit-extentlist-filter.pod +index adfb4ad8..5d1a38ae 100644 +--- a/filters/extentlist/nbdkit-extentlist-filter.pod ++++ b/filters/extentlist/nbdkit-extentlist-filter.pod +@@ -4,7 +4,7 @@ nbdkit-extentlist-filter - place extent list over a plugin + + =head1 SYNOPSIS + +- nbdkit --filter=extentlist plugin extentlist=FILENAME ++ nbdkit -r --filter=extentlist plugin extentlist=FILENAME + + =head1 DESCRIPTION + +@@ -13,11 +13,20 @@ static list of extents on top of an existing plugin. Extents record + whether or not specific parts of the disk are allocated or sparse. + + You can use this with plugins which cannot get extent information +-themselves, but you can get this information from another source. One +-place where it is useful is with L because the +-sftp protocol does not support reading sparseness information, but you +-may be able to get this information directly from the source disk on +-the remote server. ++themselves, but where you can get this information from another ++source. One place where it is useful is with L ++because the sftp protocol does not support reading sparseness ++information, but you may be able to get this information directly from ++the source disk on the remote server using commands such as ++L. A similar situation applies to ++L. ++ ++Note that the extent list is read-only. This filter does not monitor ++writes and trims in order to update the extent list. What can happen ++is that you would write to a “hole” in the disk, but would not be able ++to read it back because the NBD client would still think that part of ++the disk is a hole. So it is generally only safe to use this filter ++in read-only mode (I<-r> option). + + =head1 FILE FORMAT + +diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod +index 827e0bd1..d3c85248 100644 +--- a/plugins/curl/nbdkit-curl-plugin.pod ++++ b/plugins/curl/nbdkit-curl-plugin.pod +@@ -182,6 +182,7 @@ L, + L + L, + L, ++L, + L, + L, + L, +diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod +index 0a0421d5..3fc3146a 100644 +--- a/plugins/ssh/nbdkit-ssh-plugin.pod ++++ b/plugins/ssh/nbdkit-ssh-plugin.pod +@@ -316,6 +316,7 @@ C first appeared in nbdkit 1.12. + + L, + L, ++L, + L, + L, + L, +diff --git a/tests/test-extentlist.sh b/tests/test-extentlist.sh +index 7d05de4f..73ce3ca6 100755 +--- a/tests/test-extentlist.sh ++++ b/tests/test-extentlist.sh +@@ -50,7 +50,7 @@ cleanup_fn rm $files + test () + { + nbdkit -v -D extentlist.lookup=1 \ +- -U - \ ++ -r -U - \ + --filter=extentlist \ + null size=$1 extentlist=$input \ + --run 'qemu-img map -f raw --output=json $nbd' | +-- +2.18.2 + diff --git a/SOURCES/0013-vddk-Update-for-VDDK-7.0-RHBZ-1831969.patch b/SOURCES/0013-vddk-Update-for-VDDK-7.0-RHBZ-1831969.patch new file mode 100644 index 0000000..dcfd092 --- /dev/null +++ b/SOURCES/0013-vddk-Update-for-VDDK-7.0-RHBZ-1831969.patch @@ -0,0 +1,51 @@ +From bf1eabb211004f3dc74dd243e2adf52a13290377 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 6 May 2020 09:33:32 +0100 +Subject: [PATCH 13/19] vddk: Update for VDDK 7.0 (RHBZ#1831969). + +This version of VDDK changes the soname to libvixDiskLib.so.7. + +Thanks: Ming Xie +(cherry picked from commit 7f53999179af98aa47569c6771517f7dfa08c5d0) +--- + plugins/vddk/vddk-structs.h | 4 +++- + plugins/vddk/vddk.c | 1 + + 2 files changed, 4 insertions(+), 1 deletion(-) + +diff --git a/plugins/vddk/vddk-structs.h b/plugins/vddk/vddk-structs.h +index 86087871..fff7201b 100644 +--- a/plugins/vddk/vddk-structs.h ++++ b/plugins/vddk/vddk-structs.h +@@ -32,7 +32,7 @@ + + /* Types and structs that we pass to or return from the VDDK API. + * +- * Updated to VDDK 6.7 ++ * Updated to VDDK 7.0 + */ + + #ifndef NBDKIT_VDDK_STRUCTS_H +@@ -127,6 +127,8 @@ typedef struct VixDiskLibInfo { + int numLinks; + char *parentFileNameHint; + char *uuid; ++ uint32_t logicalSectorSize; /* Added in 7.0. */ ++ uint32_t physicalSectorSize; /* Added in 7.0. */ + } VixDiskLibInfo; + + typedef struct { +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 5d3764d6..97ef5297 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -149,6 +149,7 @@ vddk_load (void) + { + static const char *sonames[] = { + /* Prefer the newest library in case multiple exist. */ ++ "libvixDiskLib.so.7", + "libvixDiskLib.so.6", + "libvixDiskLib.so.5", + }; +-- +2.18.2 + diff --git a/SOURCES/0014-common-include-Add-ASCII-only-ctype-header-and-ascii.patch b/SOURCES/0014-common-include-Add-ASCII-only-ctype-header-and-ascii.patch new file mode 100644 index 0000000..6383c63 --- /dev/null +++ b/SOURCES/0014-common-include-Add-ASCII-only-ctype-header-and-ascii.patch @@ -0,0 +1,354 @@ +From cb3d83d0606d5267752895151bb3c229c48d6fb6 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 19 May 2020 12:03:23 +0100 +Subject: [PATCH 14/19] common/include: Add ASCII-only ctype header and + ascii_is* functions. + +Our existing uses of were not necessarily safe if the locale +was changed. + +Also I removed the unnecessary use of isascii, deprecated by POSIX.1-2008. + +(cherry picked from commit 9f34db74786fdc92b290a7d47e4b003bd84fec69) +--- + .gitignore | 1 + + common/include/Makefile.am | 6 +++ + common/include/ascii-ctype.h | 60 ++++++++++++++++++++++++++ + common/include/test-ascii-ctype.c | 63 ++++++++++++++++++++++++++++ + plugins/partitioning/partition-gpt.c | 12 +++--- + plugins/sh/Makefile.am | 1 + + plugins/sh/call.c | 6 +-- + server/backend.c | 7 ++-- + server/public.c | 4 +- + 9 files changed, 146 insertions(+), 14 deletions(-) + create mode 100644 common/include/ascii-ctype.h + create mode 100644 common/include/test-ascii-ctype.c + +diff --git a/.gitignore b/.gitignore +index e25bd99b..523894b7 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -26,6 +26,7 @@ Makefile.in + /aclocal.m4 + /autom4te.cache + /common/bitmap/test-bitmap ++/common/include/test-ascii-ctype + /common/include/test-byte-swapping + /common/include/test-current-dir-name + /common/include/test-isaligned +diff --git a/common/include/Makefile.am b/common/include/Makefile.am +index 4482de37..d7b0d7a8 100644 +--- a/common/include/Makefile.am ++++ b/common/include/Makefile.am +@@ -34,6 +34,7 @@ include $(top_srcdir)/common-rules.mk + # These headers contain only common code shared by the core server, + # plugins and/or filters. They are not installed. + EXTRA_DIST = \ ++ ascii-ctype.h \ + byte-swapping.h \ + exit-with-parent.h \ + get-current-dir-name.h \ +@@ -50,6 +51,7 @@ EXTRA_DIST = \ + # Unit tests. + + TESTS = \ ++ test-ascii-ctype \ + test-byte-swapping \ + test-current-dir-name \ + test-isaligned \ +@@ -62,6 +64,10 @@ TESTS = \ + $(NULL) + check_PROGRAMS = $(TESTS) + ++test_ascii_ctype_SOURCES = test-ascii-ctype.c ascii-ctype.h ++test_ascii_ctype_CPPFLAGS = -I$(srcdir) ++test_ascii_ctype_CFLAGS = $(WARNINGS_CFLAGS) ++ + test_byte_swapping_SOURCES = test-byte-swapping.c byte-swapping.h + test_byte_swapping_CPPFLAGS = -I$(srcdir) + test_byte_swapping_CFLAGS = $(WARNINGS_CFLAGS) +diff --git a/common/include/ascii-ctype.h b/common/include/ascii-ctype.h +new file mode 100644 +index 00000000..5e8bf237 +--- /dev/null ++++ b/common/include/ascii-ctype.h +@@ -0,0 +1,60 @@ ++/* nbdkit ++ * Copyright (C) 2013-2020 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++/* Normal ctype functions are affected by the current locale. For ++ * example isupper() might recognize Ä in some but not all locales. ++ * These functions match only 7 bit ASCII characters. ++ */ ++ ++#ifndef NBDKIT_ASCII_CTYPE_H ++#define NBDKIT_ASCII_CTYPE_H ++ ++#define ascii_isalnum(c) (ascii_isalpha (c) || ascii_isdigit (c)) ++ ++#define ascii_isalpha(c) \ ++ (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z')) ++ ++#define ascii_isdigit(c) \ ++ ((c) >= '0' && (c) <= '9') ++ ++#define ascii_isspace(c) \ ++ ((c) == '\t' || (c) == '\n' || (c) == '\f' || (c) == '\r' || (c) == ' ') ++ ++#define ascii_isxdigit(c) \ ++ ((c) == '0' || (c) == '1' || (c) == '2' || (c) == '3' || (c) == '4' || \ ++ (c) == '5' || (c) == '6' || (c) == '7' || (c) == '8' || (c) == '9' || \ ++ (c) == 'a' || (c) == 'b' || (c) == 'c' || \ ++ (c) == 'd' || (c) == 'e' || (c) == 'f' || \ ++ (c) == 'A' || (c) == 'B' || (c) == 'C' || \ ++ (c) == 'D' || (c) == 'E' || (c) == 'F') ++ ++#endif /* NBDKIT_ASCII_CTYPE_H */ +diff --git a/common/include/test-ascii-ctype.c b/common/include/test-ascii-ctype.c +new file mode 100644 +index 00000000..edf27aa6 +--- /dev/null ++++ b/common/include/test-ascii-ctype.c +@@ -0,0 +1,63 @@ ++/* nbdkit ++ * Copyright (C) 2020 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++ ++#include "ascii-ctype.h" ++ ++int ++main (void) ++{ ++ assert (ascii_isspace (' ')); ++ assert (ascii_isspace ('\t')); ++ assert (ascii_isspace ('\n')); ++ assert (! ascii_isspace ('a')); ++ ++ assert (ascii_isalpha ('a')); ++ assert (ascii_isalpha ('Z')); ++ assert (ascii_isalpha ('z')); ++ assert (! ascii_isalpha (' ')); ++ assert (! ascii_isalpha ('0')); ++ { const char *s = "Ä"; assert (! ascii_isalpha (s[0])); } ++ { const char *s = "®"; assert (! ascii_isalpha (s[0])); } ++ ++ assert (ascii_isdigit ('0')); ++ assert (ascii_isdigit ('9')); ++ { const char *s = "Ø"; assert (! ascii_isdigit (s[0])); } /* U+00D8 */ ++ { const char *s = "９"; assert (! ascii_isdigit (s[0])); } /* U+FF19 */ ++ ++ exit (EXIT_SUCCESS); ++} +diff --git a/plugins/partitioning/partition-gpt.c b/plugins/partitioning/partition-gpt.c +index 75b4643a..819e9abe 100644 +--- a/plugins/partitioning/partition-gpt.c ++++ b/plugins/partitioning/partition-gpt.c +@@ -36,12 +36,12 @@ + #include + #include + #include +-#include + #include + #include + + #include + ++#include "ascii-ctype.h" + #include "byte-swapping.h" + + #include "efi-crc32.h" +@@ -244,19 +244,19 @@ parse_guid (const char *str, char *out) + return -1; + + for (i = 0; i < 8; ++i) +- if (!isxdigit (str[i])) ++ if (!ascii_isxdigit (str[i])) + return -1; + for (i = 9; i < 13; ++i) +- if (!isxdigit (str[i])) ++ if (!ascii_isxdigit (str[i])) + return -1; + for (i = 14; i < 18; ++i) +- if (!isxdigit (str[i])) ++ if (!ascii_isxdigit (str[i])) + return -1; + for (i = 19; i < 23; ++i) +- if (!isxdigit (str[i])) ++ if (!ascii_isxdigit (str[i])) + return -1; + for (i = 24; i < 36; ++i) +- if (!isxdigit (str[i])) ++ if (!ascii_isxdigit (str[i])) + return -1; + + /* The first, second and third blocks are parsed as little endian, +diff --git a/plugins/sh/Makefile.am b/plugins/sh/Makefile.am +index 445cdcd5..1f42b64c 100644 +--- a/plugins/sh/Makefile.am ++++ b/plugins/sh/Makefile.am +@@ -48,6 +48,7 @@ nbdkit_sh_plugin_la_SOURCES = \ + + nbdkit_sh_plugin_la_CPPFLAGS = \ + -I$(top_srcdir)/include \ ++ -I$(top_srcdir)/common/include \ + -I$(top_srcdir)/common/utils \ + $(NULL) + nbdkit_sh_plugin_la_CFLAGS = $(WARNINGS_CFLAGS) +diff --git a/plugins/sh/call.c b/plugins/sh/call.c +index 2d99a120..ae0cc0ac 100644 +--- a/plugins/sh/call.c ++++ b/plugins/sh/call.c +@@ -44,10 +44,10 @@ + #include + #include + #include +-#include + + #include + ++#include "ascii-ctype.h" + #include "cleanup.h" + #include "utils.h" + +@@ -392,7 +392,7 @@ handle_script_error (const char *argv0, char *ebuf, size_t len) + } + + if (skip && ebuf[skip]) { +- if (!isspace ((unsigned char) ebuf[skip])) { ++ if (!ascii_isspace ((unsigned char) ebuf[skip])) { + /* Treat 'EINVALID' as EIO, not EINVAL */ + err = EIO; + skip = 0; +@@ -400,7 +400,7 @@ handle_script_error (const char *argv0, char *ebuf, size_t len) + else + do + skip++; +- while (isspace ((unsigned char) ebuf[skip])); ++ while (ascii_isspace ((unsigned char) ebuf[skip])); + } + + while (len > 0 && ebuf[len-1] == '\n') +diff --git a/server/backend.c b/server/backend.c +index 208c07b1..9add341f 100644 +--- a/server/backend.c ++++ b/server/backend.c +@@ -37,13 +37,14 @@ + #include + #include + #include +-#include + + #include + +-#include "internal.h" ++#include "ascii-ctype.h" + #include "minmax.h" + ++#include "internal.h" ++ + /* Helpers for registering a new backend. */ + + /* Use: +@@ -100,7 +101,7 @@ backend_load (struct backend *b, const char *name, void (*load) (void)) + for (i = 0; i < len; ++i) { + unsigned char c = name[i]; + +- if (!(isascii (c) && isalnum (c))) { ++ if (! ascii_isalnum (c)) { + fprintf (stderr, + "%s: %s: %s.name ('%s') field " + "must contain only ASCII alphanumeric characters\n", +diff --git a/server/public.c b/server/public.c +index 418945fe..98b78482 100644 +--- a/server/public.c ++++ b/server/public.c +@@ -45,13 +45,13 @@ + #include + #include + #include +-#include + #include + #include + #include + #include + #include + ++#include "ascii-ctype.h" + #include "get-current-dir-name.h" + + #include "internal.h" +@@ -210,7 +210,7 @@ nbdkit_parse_int64_t (const char *what, const char *str, int64_t *rp) + */ + #define PARSE_ERROR_IF_NEGATIVE \ + do { \ +- while (isspace (*str)) \ ++ while (ascii_isspace (*str)) \ + str++; \ + if (*str == '-') { \ + nbdkit_error ("%s: negative numbers are not allowed", what); \ +-- +2.18.2 + diff --git a/SOURCES/0015-curl-Remove-some-useless-debug-messages.patch b/SOURCES/0015-curl-Remove-some-useless-debug-messages.patch new file mode 100644 index 0000000..fe49773 --- /dev/null +++ b/SOURCES/0015-curl-Remove-some-useless-debug-messages.patch @@ -0,0 +1,46 @@ +From 0aa8e873e626c8af5f47e2e9896f33dcff4d7bb6 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 28 Mar 2020 16:59:55 +0000 +Subject: [PATCH 15/19] curl: Remove some useless debug messages. + +You can get the same information by setting -D curl.verbose=1 + +(cherry picked from commit e3539d55241adcbf1bc8102c019a0dd0ae8a3407) +--- + plugins/curl/curl.c | 6 ------ + 1 file changed, 6 deletions(-) + +diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c +index 007449bc..8b341ae0 100644 +--- a/plugins/curl/curl.c ++++ b/plugins/curl/curl.c +@@ -311,8 +311,6 @@ curl_open (int readonly) + goto err; + } + +- nbdkit_debug ("opened libcurl easy handle"); +- + /* Note this writes the output to stderr directly. We should + * consider using CURLOPT_DEBUGFUNCTION so we can handle it with + * nbdkit_debug. +@@ -340,8 +338,6 @@ curl_open (int readonly) + goto err; + } + +- nbdkit_debug ("set libcurl URL: %s", url); +- + curl_easy_setopt (h->c, CURLOPT_AUTOREFERER, 1); + curl_easy_setopt (h->c, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt (h->c, CURLOPT_FAILONERROR, 1); +@@ -436,8 +432,6 @@ curl_open (int readonly) + curl_easy_setopt (h->c, CURLOPT_READDATA, h); + } + +- nbdkit_debug ("returning new handle %p", h); +- + return h; + + err: +-- +2.18.2 + diff --git a/SOURCES/0016-curl-Fix-D-curl.verbose-1-option.patch b/SOURCES/0016-curl-Fix-D-curl.verbose-1-option.patch new file mode 100644 index 0000000..51e0233 --- /dev/null +++ b/SOURCES/0016-curl-Fix-D-curl.verbose-1-option.patch @@ -0,0 +1,182 @@ +From f421e599d3507f22d3d06b2dab070811e7e4f41c Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 28 Mar 2020 18:22:42 +0000 +Subject: [PATCH 16/19] curl: Fix -D curl.verbose=1 option. + +It didn't work previously for various reasons: + + - Passed int instead of long to curl_easy_setopt. + + - No CURLOPT_DEBUGFUNCTION callback was supplied, so messages were + sent to stderr, which meant they were never logged for the majority + of use cases. + +This also removes extra debugging in the regular header callback. +This is no longer needed as the now-working -D curl.verbose=1 option +will log the headers. + +Fixes commit 2ba11ee8f154ad1c84e10b43479b265fca2e996b. + +(cherry picked from commit 6791c69bddf76577b65fa3ddfde652c0594ce340) +--- + plugins/curl/Makefile.am | 2 ++ + plugins/curl/curl.c | 73 ++++++++++++++++++++++++++++++---------- + tests/test-curl.c | 3 +- + 3 files changed, 60 insertions(+), 18 deletions(-) + +diff --git a/plugins/curl/Makefile.am b/plugins/curl/Makefile.am +index 6595eb95..024ddb6d 100644 +--- a/plugins/curl/Makefile.am ++++ b/plugins/curl/Makefile.am +@@ -44,6 +44,7 @@ nbdkit_curl_plugin_la_SOURCES = \ + + nbdkit_curl_plugin_la_CPPFLAGS = \ + -I$(top_srcdir)/include \ ++ -I$(top_srcdir)/common/utils \ + $(NULL) + nbdkit_curl_plugin_la_CFLAGS = \ + $(WARNINGS_CFLAGS) \ +@@ -51,6 +52,7 @@ nbdkit_curl_plugin_la_CFLAGS = \ + $(NULL) + nbdkit_curl_plugin_la_LIBADD = \ + $(CURL_LIBS) \ ++ $(top_builddir)/common/utils/libutils.la \ + $(NULL) + nbdkit_curl_plugin_la_LDFLAGS = \ + -module -avoid-version -shared \ +diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c +index 8b341ae0..b1693dc0 100644 +--- a/plugins/curl/curl.c ++++ b/plugins/curl/curl.c +@@ -56,6 +56,8 @@ + + #include + ++#include "cleanup.h" ++ + static const char *url = NULL; + static const char *user = NULL; + static char *password = NULL; +@@ -283,6 +285,8 @@ struct curl_handle { + curl_easy_strerror ((r)), (h)->errbuf); \ + } while (0) + ++static int debug_cb (CURL *handle, curl_infotype type, ++ const char *data, size_t size, void *); + static size_t header_cb (void *ptr, size_t size, size_t nmemb, void *opaque); + static size_t write_cb (char *ptr, size_t size, size_t nmemb, void *opaque); + static size_t read_cb (void *ptr, size_t size, size_t nmemb, void *opaque); +@@ -311,11 +315,13 @@ curl_open (int readonly) + goto err; + } + +- /* Note this writes the output to stderr directly. We should +- * consider using CURLOPT_DEBUGFUNCTION so we can handle it with +- * nbdkit_debug. +- */ +- curl_easy_setopt (h->c, CURLOPT_VERBOSE, curl_debug_verbose); ++ if (curl_debug_verbose) { ++ /* NB: Constants must be explicitly long because the parameter is ++ * varargs. ++ */ ++ curl_easy_setopt (h->c, CURLOPT_VERBOSE, 1L); ++ curl_easy_setopt (h->c, CURLOPT_DEBUGFUNCTION, debug_cb); ++ } + + curl_easy_setopt (h->c, CURLOPT_ERRORBUFFER, h->errbuf); + +@@ -441,12 +447,56 @@ curl_open (int readonly) + return NULL; + } + ++/* When using CURLOPT_VERBOSE, this callback is used to redirect ++ * messages to nbdkit_debug (instead of stderr). ++ */ ++static int ++debug_cb (CURL *handle, curl_infotype type, ++ const char *data, size_t size, void *opaque) ++{ ++ size_t origsize = size; ++ CLEANUP_FREE char *str; ++ ++ /* The data parameter passed is NOT \0-terminated, but also it may ++ * have \n or \r\n line endings. The only sane way to deal with ++ * this is to copy the string. (The data strings may also be ++ * multi-line, but we don't deal with that here). ++ */ ++ str = malloc (size + 1); ++ if (str == NULL) ++ goto out; ++ memcpy (str, data, size); ++ str[size] = '\0'; ++ ++ while (size > 0 && (str[size-1] == '\n' || str[size-1] == '\r')) { ++ str[size-1] = '\0'; ++ size--; ++ } ++ ++ switch (type) { ++ case CURLINFO_TEXT: ++ nbdkit_debug ("%s", str); ++ break; ++ case CURLINFO_HEADER_IN: ++ nbdkit_debug ("S: %s", str); ++ break; ++ case CURLINFO_HEADER_OUT: ++ nbdkit_debug ("C: %s", str); ++ break; ++ default: ++ /* Assume everything else is binary data that we cannot print. */ ++ nbdkit_debug ("", origsize); ++ } ++ ++ out: ++ return 0; ++} ++ + static size_t + header_cb (void *ptr, size_t size, size_t nmemb, void *opaque) + { + struct curl_handle *h = opaque; + size_t realsize = size * nmemb; +- size_t len; + const char *accept_line = "Accept-Ranges: bytes"; + const char *line = ptr; + +@@ -454,17 +504,6 @@ header_cb (void *ptr, size_t size, size_t nmemb, void *opaque) + strncmp (line, accept_line, strlen (accept_line)) == 0) + h->accept_range = true; + +- /* Useful to print the server headers when debugging. However we +- * must strip off trailing \r?\n from each line. +- */ +- len = realsize; +- if (len > 0 && line[len-1] == '\n') +- len--; +- if (len > 0 && line[len-1] == '\r') +- len--; +- if (len > 0) +- nbdkit_debug ("S: %.*s", (int) len, line); +- + return realsize; + } + +diff --git a/tests/test-curl.c b/tests/test-curl.c +index 2b7e3beb..165edb35 100644 +--- a/tests/test-curl.c ++++ b/tests/test-curl.c +@@ -74,9 +74,10 @@ main (int argc, char *argv[]) + exit (EXIT_FAILURE); + } + if (test_start_nbdkit ("curl", ++ "-D", "curl.verbose=1", ++ "http://localhost/disk", + "cookie=foo=bar; baz=1;", + usp_param, /* unix-socket-path=... */ +- "http://localhost/disk", + NULL) == -1) + exit (EXIT_FAILURE); + +-- +2.18.2 + diff --git a/SOURCES/0017-curl-Case-insensitive-check-for-accept-ranges-RHBZ-1.patch b/SOURCES/0017-curl-Case-insensitive-check-for-accept-ranges-RHBZ-1.patch new file mode 100644 index 0000000..0e83b5d --- /dev/null +++ b/SOURCES/0017-curl-Case-insensitive-check-for-accept-ranges-RHBZ-1.patch @@ -0,0 +1,116 @@ +From aa62596afcc9143aa663bf834d305441cdd4cc70 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 19 May 2020 11:15:07 +0100 +Subject: [PATCH 17/19] curl: Case insensitive check for accept-ranges + (RHBZ#1837337). +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +When accessing an HTTP/2 server we read lowercase headers so the +existing test for byte range support did not work. You would see an +error like this: + + nbdkit: curl[1]: error: server does not support 'range' (byte range) requests + +This commit copies the bug fix which was recently added to qemu’s +block/curl.c. + +qemu commits: + + commit 69032253c33ae1774233c63cedf36d32242a85fc + Author: David Edmondson + Date: Mon Feb 24 10:13:10 2020 +0000 + + block/curl: HTTP header field names are case insensitive + + commit 7788a319399f17476ff1dd43164c869e320820a2 + Author: David Edmondson + Date: Mon Feb 24 10:13:09 2020 +0000 + + block/curl: HTTP header fields allow whitespace around values + +Thanks: David Edmondson, Pino Toscano, Zi Liu +(cherry picked from commit c1260ec1f6538831e10f164567b53054a2ec0c2a) +--- + plugins/curl/Makefile.am | 1 + + plugins/curl/curl.c | 29 ++++++++++++++++++++++++----- + tests/web-server.c | 2 +- + 3 files changed, 26 insertions(+), 6 deletions(-) + +diff --git a/plugins/curl/Makefile.am b/plugins/curl/Makefile.am +index 024ddb6d..3dbe3ca8 100644 +--- a/plugins/curl/Makefile.am ++++ b/plugins/curl/Makefile.am +@@ -44,6 +44,7 @@ nbdkit_curl_plugin_la_SOURCES = \ + + nbdkit_curl_plugin_la_CPPFLAGS = \ + -I$(top_srcdir)/include \ ++ -I$(top_srcdir)/common/include \ + -I$(top_srcdir)/common/utils \ + $(NULL) + nbdkit_curl_plugin_la_CFLAGS = \ +diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c +index b1693dc0..ac30cbdd 100644 +--- a/plugins/curl/curl.c ++++ b/plugins/curl/curl.c +@@ -57,6 +57,7 @@ + #include + + #include "cleanup.h" ++#include "ascii-ctype.h" + + static const char *url = NULL; + static const char *user = NULL; +@@ -497,12 +498,30 @@ header_cb (void *ptr, size_t size, size_t nmemb, void *opaque) + { + struct curl_handle *h = opaque; + size_t realsize = size * nmemb; +- const char *accept_line = "Accept-Ranges: bytes"; +- const char *line = ptr; ++ const char *header = ptr; ++ const char *end = header + realsize; ++ const char *accept_ranges = "accept-ranges:"; ++ const char *bytes = "bytes"; + +- if (realsize >= strlen (accept_line) && +- strncmp (line, accept_line, strlen (accept_line)) == 0) +- h->accept_range = true; ++ if (realsize >= strlen (accept_ranges) && ++ strncasecmp (header, accept_ranges, strlen (accept_ranges)) == 0) { ++ const char *p = strchr (header, ':') + 1; ++ ++ /* Skip whitespace between the header name and value. */ ++ while (p < end && *p && ascii_isspace (*p)) ++ p++; ++ ++ if (end - p >= strlen (bytes) ++ && strncmp (p, bytes, strlen (bytes)) == 0) { ++ /* Check that there is nothing but whitespace after the value. */ ++ p += strlen (bytes); ++ while (p < end && *p && ascii_isspace (*p)) ++ p++; ++ ++ if (p == end || !*p) ++ h->accept_range = true; ++ } ++ } + + return realsize; + } +diff --git a/tests/web-server.c b/tests/web-server.c +index f27ee70d..f9f10917 100644 +--- a/tests/web-server.c ++++ b/tests/web-server.c +@@ -235,7 +235,7 @@ handle_request (int s, bool headers_only) + const char response1_ok[] = "HTTP/1.1 200 OK\r\n"; + const char response1_partial[] = "HTTP/1.1 206 Partial Content\r\n"; + const char response2[] = +- "Accept-Ranges: bytes\r\n" ++ "Accept-rANGES: bytes\r\n" /* See RHBZ#1837337 */ + "Connection: keep-alive\r\n" + "Content-Type: application/octet-stream\r\n"; + char response3[64]; +-- +2.18.2 + diff --git a/SOURCES/0018-sh-Don-t-need-to-cast-parameter-of-ascii_is-to-unsig.patch b/SOURCES/0018-sh-Don-t-need-to-cast-parameter-of-ascii_is-to-unsig.patch new file mode 100644 index 0000000..5bbed35 --- /dev/null +++ b/SOURCES/0018-sh-Don-t-need-to-cast-parameter-of-ascii_is-to-unsig.patch @@ -0,0 +1,41 @@ +From d5947881c2918196d61d7795adba0abb881a86b9 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 19 May 2020 15:29:55 +0100 +Subject: [PATCH 18/19] sh: Don't need to cast parameter of ascii_is* to + (unsigned char). + +Our replacements for these functions are not undefined for negative +values. + +Thanks: Eric Blake +Fixes: commit 9f34db74786fdc92b290a7d47e4b003bd84fec69. +(cherry picked from commit 06a79b8bb8cfd97a272223c967601d8858acb817) +--- + plugins/sh/call.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/plugins/sh/call.c b/plugins/sh/call.c +index ae0cc0ac..ba9f055f 100644 +--- a/plugins/sh/call.c ++++ b/plugins/sh/call.c +@@ -392,7 +392,7 @@ handle_script_error (const char *argv0, char *ebuf, size_t len) + } + + if (skip && ebuf[skip]) { +- if (!ascii_isspace ((unsigned char) ebuf[skip])) { ++ if (!ascii_isspace (ebuf[skip])) { + /* Treat 'EINVALID' as EIO, not EINVAL */ + err = EIO; + skip = 0; +@@ -400,7 +400,7 @@ handle_script_error (const char *argv0, char *ebuf, size_t len) + else + do + skip++; +- while (ascii_isspace ((unsigned char) ebuf[skip])); ++ while (ascii_isspace (ebuf[skip])); + } + + while (len > 0 && ebuf[len-1] == '\n') +-- +2.18.2 + diff --git a/SOURCES/0019-common-include-Add-locale-safe-ascii_strcasecmp-and-.patch b/SOURCES/0019-common-include-Add-locale-safe-ascii_strcasecmp-and-.patch new file mode 100644 index 0000000..5437c9e --- /dev/null +++ b/SOURCES/0019-common-include-Add-locale-safe-ascii_strcasecmp-and-.patch @@ -0,0 +1,559 @@ +From 9a99549f5df6ef69dd1d2b509c13aaff4e431742 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 19 May 2020 16:11:28 +0100 +Subject: [PATCH 19/19] common/include: Add locale-safe ascii_strcasecmp and + ascii_strncasecmp. + +These are derived from the FreeBSD functions here: +https://github.com/freebsd/freebsd/blob/master/sys/libkern/strcasecmp.c + +Thanks: Eric Blake. +(cherry picked from commit 46a29b8e91d69e812d78df53e91b526a560000fe) +--- + .gitignore | 1 + + common/include/Makefile.am | 6 +++ + common/include/ascii-ctype.h | 6 +++ + common/include/ascii-string.h | 77 ++++++++++++++++++++++++++++ + common/include/test-ascii-string.c | 79 +++++++++++++++++++++++++++++ + plugins/curl/curl.c | 7 +-- + plugins/info/info.c | 17 ++++--- + plugins/nbd/nbd.c | 8 +-- + plugins/partitioning/partitioning.c | 10 ++-- + plugins/sh/call.c | 25 ++++----- + server/main.c | 8 +-- + server/public.c | 21 ++++---- + 12 files changed, 222 insertions(+), 43 deletions(-) + create mode 100644 common/include/ascii-string.h + create mode 100644 common/include/test-ascii-string.c + +diff --git a/.gitignore b/.gitignore +index 523894b7..aa148f04 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -27,6 +27,7 @@ Makefile.in + /autom4te.cache + /common/bitmap/test-bitmap + /common/include/test-ascii-ctype ++/common/include/test-ascii-string + /common/include/test-byte-swapping + /common/include/test-current-dir-name + /common/include/test-isaligned +diff --git a/common/include/Makefile.am b/common/include/Makefile.am +index d7b0d7a8..eff71863 100644 +--- a/common/include/Makefile.am ++++ b/common/include/Makefile.am +@@ -35,6 +35,7 @@ include $(top_srcdir)/common-rules.mk + # plugins and/or filters. They are not installed. + EXTRA_DIST = \ + ascii-ctype.h \ ++ ascii-string.h \ + byte-swapping.h \ + exit-with-parent.h \ + get-current-dir-name.h \ +@@ -52,6 +53,7 @@ EXTRA_DIST = \ + + TESTS = \ + test-ascii-ctype \ ++ test-ascii-string \ + test-byte-swapping \ + test-current-dir-name \ + test-isaligned \ +@@ -68,6 +70,10 @@ test_ascii_ctype_SOURCES = test-ascii-ctype.c ascii-ctype.h + test_ascii_ctype_CPPFLAGS = -I$(srcdir) + test_ascii_ctype_CFLAGS = $(WARNINGS_CFLAGS) + ++test_ascii_string_SOURCES = test-ascii-string.c ascii-string.h ++test_ascii_string_CPPFLAGS = -I$(srcdir) ++test_ascii_string_CFLAGS = $(WARNINGS_CFLAGS) ++ + test_byte_swapping_SOURCES = test-byte-swapping.c byte-swapping.h + test_byte_swapping_CPPFLAGS = -I$(srcdir) + test_byte_swapping_CFLAGS = $(WARNINGS_CFLAGS) +diff --git a/common/include/ascii-ctype.h b/common/include/ascii-ctype.h +index 5e8bf237..a700563e 100644 +--- a/common/include/ascii-ctype.h ++++ b/common/include/ascii-ctype.h +@@ -49,6 +49,9 @@ + #define ascii_isspace(c) \ + ((c) == '\t' || (c) == '\n' || (c) == '\f' || (c) == '\r' || (c) == ' ') + ++#define ascii_isupper(c) \ ++ ((c) >= 'A' && (c) <= 'Z') ++ + #define ascii_isxdigit(c) \ + ((c) == '0' || (c) == '1' || (c) == '2' || (c) == '3' || (c) == '4' || \ + (c) == '5' || (c) == '6' || (c) == '7' || (c) == '8' || (c) == '9' || \ +@@ -57,4 +60,7 @@ + (c) == 'A' || (c) == 'B' || (c) == 'C' || \ + (c) == 'D' || (c) == 'E' || (c) == 'F') + ++#define ascii_tolower(c) \ ++ (ascii_isupper ((c)) ? (c) - 'A' + 'a' : (c)) ++ + #endif /* NBDKIT_ASCII_CTYPE_H */ +diff --git a/common/include/ascii-string.h b/common/include/ascii-string.h +new file mode 100644 +index 00000000..0a60d5f4 +--- /dev/null ++++ b/common/include/ascii-string.h +@@ -0,0 +1,77 @@ ++/* nbdkit ++ * Copyright (C) 2013-2020 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++/* Case insensitive string comparison functions (like strcasecmp, ++ * strncasecmp) which work correctly in any locale. They can only be ++ * used for comparison when one or both strings is 7 bit ASCII. ++ */ ++ ++#ifndef NBDKIT_ASCII_STRING_H ++#define NBDKIT_ASCII_STRING_H ++ ++#include "ascii-ctype.h" ++ ++static inline int ++ascii_strcasecmp (const char *s1, const char *s2) ++{ ++ const unsigned char *us1 = (const unsigned char *)s1; ++ const unsigned char *us2 = (const unsigned char *)s2; ++ ++ while (ascii_tolower (*us1) == ascii_tolower (*us2)) { ++ if (*us1++ == '\0') ++ return 0; ++ us2++; ++ } ++ ++ return ascii_tolower (*us1) - ascii_tolower (*us2); ++} ++ ++static inline int ++ascii_strncasecmp (const char *s1, const char *s2, size_t n) ++{ ++ if (n != 0) { ++ const unsigned char *us1 = (const unsigned char *)s1; ++ const unsigned char *us2 = (const unsigned char *)s2; ++ ++ do { ++ if (ascii_tolower (*us1) != ascii_tolower (*us2)) ++ return ascii_tolower (*us1) - ascii_tolower (*us2); ++ if (*us1++ == '\0') ++ break; ++ us2++; ++ } while (--n != 0); ++ } ++ ++ return 0; ++} ++ ++#endif /* NBDKIT_ASCII_STRING_H */ +diff --git a/common/include/test-ascii-string.c b/common/include/test-ascii-string.c +new file mode 100644 +index 00000000..0fa4a483 +--- /dev/null ++++ b/common/include/test-ascii-string.c +@@ -0,0 +1,79 @@ ++/* nbdkit ++ * Copyright (C) 2020 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++ ++#include "ascii-string.h" ++ ++int ++main (void) ++{ ++ assert (ascii_strcasecmp ("", "") == 0); ++ assert (ascii_strcasecmp ("a", "a") == 0); ++ assert (ascii_strcasecmp ("abc", "abc") == 0); ++ assert (ascii_strcasecmp ("a", "b") < 0); ++ assert (ascii_strcasecmp ("b", "a") > 0); ++ assert (ascii_strcasecmp ("aa", "a") > 0); ++ ++ /* Second string contains Turkish dotless lowercase letter ı. */ ++ assert (ascii_strcasecmp ("hi", "hı") != 0); ++ ++ /* Check that we got our rounding behaviour correct. */ ++ assert (ascii_strcasecmp ("\x1", "\x7f") < 0); ++ assert (ascii_strcasecmp ("\x1", "\x80") < 0); ++ assert (ascii_strcasecmp ("\x1", "\x81") < 0); ++ assert (ascii_strcasecmp ("\x1", "\xff") < 0); ++ ++ assert (ascii_strncasecmp ("", "", 0) == 0); ++ assert (ascii_strncasecmp ("a", "a", 1) == 0); ++ assert (ascii_strncasecmp ("abc", "abc", 3) == 0); ++ assert (ascii_strncasecmp ("abc", "def", 0) == 0); ++ assert (ascii_strncasecmp ("abc", "abd", 2) == 0); ++ assert (ascii_strncasecmp ("a", "b", 1) < 0); ++ assert (ascii_strncasecmp ("b", "a", 1) > 0); ++ assert (ascii_strncasecmp ("aa", "a", 2) > 0); ++ assert (ascii_strncasecmp ("aa", "a", 100) > 0); ++ ++ assert (ascii_strncasecmp ("hi", "hı", 1) == 0); ++ assert (ascii_strncasecmp ("hi", "hı", 2) != 0); ++ ++ assert (ascii_strncasecmp ("\x1", "\x7f", 1) < 0); ++ assert (ascii_strncasecmp ("\x1", "\x80", 1) < 0); ++ assert (ascii_strncasecmp ("\x1", "\x81", 1) < 0); ++ assert (ascii_strncasecmp ("\x1", "\xff", 1) < 0); ++ ++ exit (EXIT_SUCCESS); ++} +diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c +index ac30cbdd..00f0628a 100644 +--- a/plugins/curl/curl.c ++++ b/plugins/curl/curl.c +@@ -58,6 +58,7 @@ + + #include "cleanup.h" + #include "ascii-ctype.h" ++#include "ascii-string.h" + + static const char *url = NULL; + static const char *user = NULL; +@@ -419,8 +420,8 @@ curl_open (int readonly) + #endif + nbdkit_debug ("content length: %" PRIi64, h->exportsize); + +- if (strncasecmp (url, "http://", strlen ("http://")) == 0 || +- strncasecmp (url, "https://", strlen ("https://")) == 0) { ++ if (ascii_strncasecmp (url, "http://", strlen ("http://")) == 0 || ++ ascii_strncasecmp (url, "https://", strlen ("https://")) == 0) { + if (!h->accept_range) { + nbdkit_error ("server does not support 'range' (byte range) requests"); + goto err; +@@ -504,7 +505,7 @@ header_cb (void *ptr, size_t size, size_t nmemb, void *opaque) + const char *bytes = "bytes"; + + if (realsize >= strlen (accept_ranges) && +- strncasecmp (header, accept_ranges, strlen (accept_ranges)) == 0) { ++ ascii_strncasecmp (header, accept_ranges, strlen (accept_ranges)) == 0) { + const char *p = strchr (header, ':') + 1; + + /* Skip whitespace between the header name and value. */ +diff --git a/plugins/info/info.c b/plugins/info/info.c +index 329a3684..e04d672b 100644 +--- a/plugins/info/info.c ++++ b/plugins/info/info.c +@@ -49,6 +49,7 @@ + + #include + ++#include "ascii-string.h" + #include "byte-swapping.h" + #include "tvdiff.h" + +@@ -76,12 +77,12 @@ static int + info_config (const char *key, const char *value) + { + if (strcmp (key, "mode") == 0) { +- if (strcasecmp (value, "exportname") == 0 || +- strcasecmp (value, "export-name") == 0) { ++ if (ascii_strcasecmp (value, "exportname") == 0 || ++ ascii_strcasecmp (value, "export-name") == 0) { + mode = MODE_EXPORTNAME; + } +- else if (strcasecmp (value, "base64exportname") == 0 || +- strcasecmp (value, "base64-export-name") == 0) { ++ else if (ascii_strcasecmp (value, "base64exportname") == 0 || ++ ascii_strcasecmp (value, "base64-export-name") == 0) { + #ifdef HAVE_BASE64 + mode = MODE_BASE64EXPORTNAME; + #else +@@ -89,13 +90,13 @@ info_config (const char *key, const char *value) + return -1; + #endif + } +- else if (strcasecmp (value, "address") == 0) ++ else if (ascii_strcasecmp (value, "address") == 0) + mode = MODE_ADDRESS; +- else if (strcasecmp (value, "time") == 0) ++ else if (ascii_strcasecmp (value, "time") == 0) + mode = MODE_TIME; +- else if (strcasecmp (value, "uptime") == 0) ++ else if (ascii_strcasecmp (value, "uptime") == 0) + mode = MODE_UPTIME; +- else if (strcasecmp (value, "conntime") == 0) ++ else if (ascii_strcasecmp (value, "conntime") == 0) + mode = MODE_CONNTIME; + else { + nbdkit_error ("unknown mode: '%s'", value); +diff --git a/plugins/nbd/nbd.c b/plugins/nbd/nbd.c +index d020beec..980ce8ec 100644 +--- a/plugins/nbd/nbd.c ++++ b/plugins/nbd/nbd.c +@@ -52,6 +52,8 @@ + #define NBDKIT_API_VERSION 2 + + #include ++ ++#include "ascii-string.h" + #include "byte-swapping.h" + #include "cleanup.h" + #include "utils.h" +@@ -152,9 +154,9 @@ nbdplug_config (const char *key, const char *value) + shared = r; + } + else if (strcmp (key, "tls") == 0) { +- if (strcasecmp (value, "require") == 0 || +- strcasecmp (value, "required") == 0 || +- strcasecmp (value, "force") == 0) ++ if (ascii_strcasecmp (value, "require") == 0 || ++ ascii_strcasecmp (value, "required") == 0 || ++ ascii_strcasecmp (value, "force") == 0) + tls = LIBNBD_TLS_REQUIRE; + else { + r = nbdkit_parse_bool (value); +diff --git a/plugins/partitioning/partitioning.c b/plugins/partitioning/partitioning.c +index 6e426b93..e35764dc 100644 +--- a/plugins/partitioning/partitioning.c ++++ b/plugins/partitioning/partitioning.c +@@ -48,6 +48,7 @@ + + #include + ++#include "ascii-string.h" + #include "byte-swapping.h" + #include "isaligned.h" + #include "iszero.h" +@@ -176,9 +177,10 @@ partitioning_config (const char *key, const char *value) + nr_files++; + } + else if (strcmp (key, "partition-type") == 0) { +- if (strcasecmp (value, "mbr") == 0 || strcasecmp (value, "dos") == 0) ++ if (ascii_strcasecmp (value, "mbr") == 0 || ++ ascii_strcasecmp (value, "dos") == 0) + parttype = PARTTYPE_MBR; +- else if (strcasecmp (value, "gpt") == 0) ++ else if (ascii_strcasecmp (value, "gpt") == 0) + parttype = PARTTYPE_GPT; + else { + nbdkit_error ("unknown partition-type: %s", value); +@@ -209,13 +211,13 @@ partitioning_config (const char *key, const char *value) + alignment = r; + } + else if (strcmp (key, "mbr-id") == 0) { +- if (strcasecmp (value, "default") == 0) ++ if (ascii_strcasecmp (value, "default") == 0) + mbr_id = DEFAULT_MBR_ID; + else if (nbdkit_parse_uint8_t ("mbr-id", value, &mbr_id) == -1) + return -1; + } + else if (strcmp (key, "type-guid") == 0) { +- if (strcasecmp (value, "default") == 0) ++ if (ascii_strcasecmp (value, "default") == 0) + parse_guid (DEFAULT_TYPE_GUID, type_guid); + else if (parse_guid (value, type_guid) == -1) { + nbdkit_error ("could not validate GUID: %s", value); +diff --git a/plugins/sh/call.c b/plugins/sh/call.c +index ba9f055f..554e2f78 100644 +--- a/plugins/sh/call.c ++++ b/plugins/sh/call.c +@@ -48,6 +48,7 @@ + #include + + #include "ascii-ctype.h" ++#include "ascii-string.h" + #include "cleanup.h" + #include "utils.h" + +@@ -332,48 +333,48 @@ handle_script_error (const char *argv0, char *ebuf, size_t len) + } + + /* Recognize the errno values that match NBD protocol errors */ +- if (strncasecmp (ebuf, "EPERM", 5) == 0) { ++ if (ascii_strncasecmp (ebuf, "EPERM", 5) == 0) { + err = EPERM; + skip = 5; + } +- else if (strncasecmp (ebuf, "EIO", 3) == 0) { ++ else if (ascii_strncasecmp (ebuf, "EIO", 3) == 0) { + err = EIO; + skip = 3; + } +- else if (strncasecmp (ebuf, "ENOMEM", 6) == 0) { ++ else if (ascii_strncasecmp (ebuf, "ENOMEM", 6) == 0) { + err = ENOMEM; + skip = 6; + } +- else if (strncasecmp (ebuf, "EINVAL", 6) == 0) { ++ else if (ascii_strncasecmp (ebuf, "EINVAL", 6) == 0) { + err = EINVAL; + skip = 6; + } +- else if (strncasecmp (ebuf, "ENOSPC", 6) == 0) { ++ else if (ascii_strncasecmp (ebuf, "ENOSPC", 6) == 0) { + err = ENOSPC; + skip = 6; + } +- else if (strncasecmp (ebuf, "EOVERFLOW", 9) == 0) { ++ else if (ascii_strncasecmp (ebuf, "EOVERFLOW", 9) == 0) { + err = EOVERFLOW; + skip = 9; + } +- else if (strncasecmp (ebuf, "ESHUTDOWN", 9) == 0) { ++ else if (ascii_strncasecmp (ebuf, "ESHUTDOWN", 9) == 0) { + err = ESHUTDOWN; + skip = 9; + } +- else if (strncasecmp (ebuf, "ENOTSUP", 7) == 0) { ++ else if (ascii_strncasecmp (ebuf, "ENOTSUP", 7) == 0) { + err = ENOTSUP; + skip = 7; + } +- else if (strncasecmp (ebuf, "EOPNOTSUPP", 10) == 0) { ++ else if (ascii_strncasecmp (ebuf, "EOPNOTSUPP", 10) == 0) { + err = EOPNOTSUPP; + skip = 10; + } + /* Other errno values that server/protocol.c treats specially */ +- else if (strncasecmp (ebuf, "EROFS", 5) == 0) { ++ else if (ascii_strncasecmp (ebuf, "EROFS", 5) == 0) { + err = EROFS; + skip = 5; + } +- else if (strncasecmp (ebuf, "EDQUOT", 6) == 0) { ++ else if (ascii_strncasecmp (ebuf, "EDQUOT", 6) == 0) { + #ifdef EDQUOT + err = EDQUOT; + #else +@@ -381,7 +382,7 @@ handle_script_error (const char *argv0, char *ebuf, size_t len) + #endif + skip = 6; + } +- else if (strncasecmp (ebuf, "EFBIG", 5) == 0) { ++ else if (ascii_strncasecmp (ebuf, "EFBIG", 5) == 0) { + err = EFBIG; + skip = 5; + } +diff --git a/server/main.c b/server/main.c +index 11ba1e6d..08116d74 100644 +--- a/server/main.c ++++ b/server/main.c +@@ -55,6 +55,8 @@ + + #include + ++#include "ascii-string.h" ++ + #include "internal.h" + #include "nbd-protocol.h" + #include "options.h" +@@ -282,9 +284,9 @@ main (int argc, char *argv[]) + + case TLS_OPTION: + tls_set_on_cli = true; +- if (strcasecmp (optarg, "require") == 0 || +- strcasecmp (optarg, "required") == 0 || +- strcasecmp (optarg, "force") == 0) ++ if (ascii_strcasecmp (optarg, "require") == 0 || ++ ascii_strcasecmp (optarg, "required") == 0 || ++ ascii_strcasecmp (optarg, "force") == 0) + tls = 2; + else { + tls = nbdkit_parse_bool (optarg); +diff --git a/server/public.c b/server/public.c +index 98b78482..919f082d 100644 +--- a/server/public.c ++++ b/server/public.c +@@ -52,6 +52,7 @@ + #include + + #include "ascii-ctype.h" ++#include "ascii-string.h" + #include "get-current-dir-name.h" + + #include "internal.h" +@@ -385,19 +386,19 @@ int + nbdkit_parse_bool (const char *str) + { + if (!strcmp (str, "1") || +- !strcasecmp (str, "true") || +- !strcasecmp (str, "t") || +- !strcasecmp (str, "yes") || +- !strcasecmp (str, "y") || +- !strcasecmp (str, "on")) ++ !ascii_strcasecmp (str, "true") || ++ !ascii_strcasecmp (str, "t") || ++ !ascii_strcasecmp (str, "yes") || ++ !ascii_strcasecmp (str, "y") || ++ !ascii_strcasecmp (str, "on")) + return 1; + + if (!strcmp (str, "0") || +- !strcasecmp (str, "false") || +- !strcasecmp (str, "f") || +- !strcasecmp (str, "no") || +- !strcasecmp (str, "n") || +- !strcasecmp (str, "off")) ++ !ascii_strcasecmp (str, "false") || ++ !ascii_strcasecmp (str, "f") || ++ !ascii_strcasecmp (str, "no") || ++ !ascii_strcasecmp (str, "n") || ++ !ascii_strcasecmp (str, "off")) + return 0; + + nbdkit_error ("could not decipher boolean (%s)", str); +-- +2.18.2 + diff --git a/SOURCES/nbdkit-1.16.2.tar.gz.sig b/SOURCES/nbdkit-1.16.2.tar.gz.sig new file mode 100644 index 0000000..1360653 --- /dev/null +++ b/SOURCES/nbdkit-1.16.2.tar.gz.sig @@ -0,0 +1,17 @@ +-----BEGIN PGP SIGNATURE----- + +iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAl4wFvoRHHJpY2hAYW5u +ZXhpYS5vcmcACgkQkXOPc+G3aKASexAAmpZw61rCI7SY8zm4O0gb+pIx7oLYx0Lq +2puIftzxUUw9Q6pFJJyXSvlsvHy3qUF7HiMVdpW61ItIChV1xBDVKEPAacNzsZh4 +30CI7kfJMfj6u+hpOCVlLk4uJFjZkmIpEKkDpEBemxLMME4JsLJdawKzKhjT2PI7 +dWMjYkOeD4NkAzQLQGskEswoIgZQ0twuyPUErjEL9fcXw4OjxFvQJG85FsIF2lR6 +FUDQg5y9YLzeMJMsjW4rO+LAz2c1mJwYR1EgYP43avm/pJfd1mVQLGRoLb7NwMSw +6mkwhJ4Kvq6BN0PSqpKqQtXZrDoElWN8cVJVf+dAjONcvzYi0gsHWDL+FZ731Q2M +s4nq0aRscBTL2DOaE9DzBY2AO1jKUB/+02qRpidWTYBmsmL2QQI8n33Q7JuDuEXX +bVm1RDA4ike4PUXXY5KJ6MZhKID5453SVFausFse+u4MCQHQPFYspkXmaNWRhjgs +yu2zPc9jHdBkpzNov/CCZoFketFRz/BKexBeH2vcfTYfREVf9lEZi7qEa0kQHDn9 +EMTFsCqmGat9TEVbt9t8c/tODTeRE00MFx4gPspzy+m4YP+Gl3ySHsAbPQ90uBGX +c8xggwqWXr1GAP5HbAhs/Bs7USrWMMgqii1ppnzoAkHh+j4rsdL4dS2dmhxX756u +IKP/JC2oA8U= +=mV8z +-----END PGP SIGNATURE----- diff --git a/SOURCES/nbdkit-1.4.2.tar.gz.sig b/SOURCES/nbdkit-1.4.2.tar.gz.sig deleted file mode 100644 index 77556c9..0000000 --- a/SOURCES/nbdkit-1.4.2.tar.gz.sig +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN PGP SIGNATURE----- - -iQIcBAABAgAGBQJbYeqVAAoJEJFzj3Pht2ig7A8P/3KbLzc8T0Nkjky1oJytaKU9 -uOJyyma0AGW9WtWJ7Xe3xOAh89B8o6rq4eVv4TTHL1stxCuurv/eObOUBvRyFZd0 -QV5ptkvK2B3Kwbtki67p+nzoYs7fxdsQ2sC0J9vfAiAk2b0eDVXlJO/Xqy+SKxnQ -Dr+iKI63cFaVyrAAc4H7kYd9Fxx1h47WW4HfSxb3EHZB8fUijQ6gq/LybFslm5Zc -vXWaq26ngb+4U7uaU/rdV2NfLoql1pKHcdnmb8e+CuE/HXIfGHyiT2mrYhQpPPwP -RXaMl1bZtOvM5CdyLxOUATPhsGMpYR+7DvOQrjV3ovsjB6/hIQOxy6iCwqe8tSOb -ZYc/AXbv/1FwGkJDVh+0Pi8RiF5GQHktMJsFV91zkw2fzSFHkdEXITcsrp+SaweJ -FEH5LfJkRe6ir0cKiWl+VYN0SbLQjrE/BsZdr9+vBbGDalBJQl8xJ53g5yMEoKTx -ksQQ+czJPbYlj9F9lyMMcncztxiPMIgcsZ81flWlmX2PfoWCEgD2A1e6cR3HWixF -fFW4Ya6gQMZ/3KzaHt7X3nDGpTg5bEJNvegIGC5XZiUCTZ8Uxn5d9DE4EEFQa2un -NRHSZgYP6+cvncs8OQiCrZtf+15e4q8wV8X+J8xXhpXmnTnJM/F9g+zqs4+eFTMq -+i/bx8RPkXTXtbg6x48W -=stIl ------END PGP SIGNATURE----- diff --git a/SPECS/nbdkit.spec b/SPECS/nbdkit.spec index 36b7ac4..416c628 100644 --- a/SPECS/nbdkit.spec +++ b/SPECS/nbdkit.spec @@ -1,6 +1,6 @@ %global _hardened_build 1 -%ifarch aarch64 %{arm} %{ix86} x86_64 ppc %{power64} +%ifarch aarch64 %{arm} x86_64 ppc %{power64} %global have_libguestfs 1 %endif @@ -11,6 +11,11 @@ # often broken on non-x86_64 arches. %global complete_test_arches x86_64 +# Disable libvirt on riscv64 for now. +%ifnarch riscv64 +%global have_libvirt 1 +%endif + # If we should verify tarball signature with GPGv2. %global verify_tarball_signature 1 @@ -18,11 +23,11 @@ %global patches_touch_autotools 1 # The source directory. -%global source_directory 1.4-stable +%global source_directory 1.16-stable Name: nbdkit -Version: 1.4.2 -Release: 5%{?dist} +Version: 1.16.2 +Release: 4%{?dist} Summary: NBD server License: BSD @@ -32,22 +37,36 @@ Source0: http://libguestfs.org/download/nbdkit/%{source_directory}/%{name %if 0%{verify_tarball_signature} Source1: http://libguestfs.org/download/nbdkit/%{source_directory}/%{name}-%{version}.tar.gz.sig # Keyring used to verify tarball signature. -Source2: libguestfs.keyring +Source2: libguestfs.keyring %endif -# Patches come from: -# https://github.com/libguestfs/nbdkit/tree/rhel-8.0 +# Patches come from this upstream branch: +# https://github.com/libguestfs/nbdkit/tree/rhel-8.2 # Patches. -Patch0001: 0001-vddk-Remove-vimapiver-parameter.patch -Patch0002: 0002-vddk-Remove-compile-time-dependency-on-VDDK-library.patch -Patch0003: 0003-vddk-Add-comment-about-my-experiment-with-PrepareFor.patch -Patch0004: 0004-vddk-Make-dlsym-variables-static.patch -Patch0005: 0005-vddk-Improve-error-message-if-the-proprietary-librar.patch -Patch0006: 0006-vddk-If-relative-libdir-parameter-is-passed-make-it-.patch -Patch0007: 0007-vddk-Two-more-static-dlsym-variables.patch -Patch0008: 0008-vddk-Add-a-very-simple-test.patch -Patch0009: 0009-python-Try-harder-to-print-the-full-traceback-on-err.patch +Patch0001: 0001-server-Allow-D-nbdkit.-debug-flags-for-the-core-serv.patch +Patch0002: 0002-server-Allow-D-debug-flags-to-contain-dots-for-names.patch +Patch0003: 0003-server-Add-D-nbdkit.backend.controlpath-and-D-nbdkit.patch +Patch0004: 0004-python-Add-various-constants-to-the-API.patch +Patch0005: 0005-python-Implement-nbdkit-API-version-2.patch +Patch0006: 0006-python-Implement-cache.patch +Patch0007: 0007-python-Implement-can_zero-can_fast_zero.patch +Patch0008: 0008-python-Implement-can_multi_conn.patch +Patch0009: 0009-python-Implement-can_fua-and-can_cache.patch +Patch0010: 0010-tests-Test-the-Python-plugin-thoroughly.patch +Patch0011: 0011-New-filter-extentlist.patch +Patch0012: 0012-extentlist-Documentation-and-test-fixes.patch +Patch0013: 0013-vddk-Update-for-VDDK-7.0-RHBZ-1831969.patch +Patch0014: 0014-common-include-Add-ASCII-only-ctype-header-and-ascii.patch +Patch0015: 0015-curl-Remove-some-useless-debug-messages.patch +Patch0016: 0016-curl-Fix-D-curl.verbose-1-option.patch +Patch0017: 0017-curl-Case-insensitive-check-for-accept-ranges-RHBZ-1.patch +Patch0018: 0018-sh-Don-t-need-to-cast-parameter-of-ascii_is-to-unsig.patch +Patch0019: 0019-common-include-Add-locale-safe-ascii_strcasecmp-and-.patch + +%if 0%{patches_touch_autotools} +BuildRequires: autoconf, automake, libtool +%endif %if 0%{patches_touch_autotools} BuildRequires: autoconf, automake, libtool @@ -67,9 +86,14 @@ BuildRequires: libselinux-devel %if 0%{?have_libguestfs} BuildRequires: libguestfs-devel %endif +%if 0%{?have_libvirt} BuildRequires: libvirt-devel +%endif BuildRequires: xz-devel BuildRequires: zlib-devel +BuildRequires: libcurl-devel +BuildRequires: libssh-devel +BuildRequires: e2fsprogs, e2fsprogs-devel BuildRequires: bash-completion BuildRequires: perl-devel BuildRequires: perl(ExtUtils::Embed) @@ -80,96 +104,121 @@ BuildRequires: gnupg2 # Only for running the test suite: BuildRequires: /usr/bin/certtool +BuildRequires: jq +BuildRequires: /usr/bin/nbdsh BuildRequires: /usr/bin/qemu-img BuildRequires: /usr/bin/socat BuildRequires: /usr/sbin/ss +BuildRequires: /usr/bin/ssh-keygen + +# nbdkit is a metapackage pulling the server and a useful subset +# of the plugins and filters. +Requires: nbdkit-server%{?_isa} = %{version}-%{release} +Requires: nbdkit-basic-plugins%{?_isa} = %{version}-%{release} +Requires: nbdkit-basic-filters%{?_isa} = %{version}-%{release} + %description NBD is a protocol for accessing block devices (hard disks and disk-like things) over the network. -'nbdkit' is a toolkit for creating NBD servers. +nbdkit is a toolkit for creating NBD servers. The key features are: * Multithreaded NBD server written in C with good performance. -* Well-documented, simple plugin API with a stable ABI guarantee. - Allows you to export "unconventional" block devices easily. +* Minimal dependencies for the basic server. * Liberal license (BSD) allows nbdkit to be linked to proprietary libraries or included in proprietary code. -You probably want to install one of more plugins (%{name}-plugin-*). +* Well-documented, simple plugin API with a stable ABI guarantee. + Lets you to export "unconventional" block devices easily. + +* You can write plugins in C or many other languages. -To develop plugins, install the %{name}-devel package and start by +* Filters can be stacked in front of plugins to transform the output. + +In Red Hat Enterprise Linux, '%{name}' is a meta-package which pulls +in the core server and a useful subset of plugins and filters. + +If you want just the server, install '%{name}-server'. + +To develop plugins, install the '%{name}-devel' package and start by reading the nbdkit(1) and nbdkit-plugin(3) manual pages. +%package server +Summary: The %{name} server +License: BSD + +Conflicts: nbdkit < 1.12 + + +%description server +This package contains the %{name} server with no plugins or filters. + + %package basic-plugins Summary: Basic plugins for %{name} License: BSD -Requires: %{name}%{?_isa} = %{version}-%{release} - -# For upgrade path, remove these in Fedora 30. -Obsoletes: %{name}-plugin-file < 1.1.19-1 -Obsoletes: %{name}-plugin-nbd < 1.1.19-1 -Obsoletes: %{name}-plugin-streaming < 1.1.19-1 +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Provides: %{name}-data-plugin = %{version}-%{release} +Provides: %{name}-file-plugin = %{version}-%{release} +Provides: %{name}-floppy-plugin = %{version}-%{release} +Provides: %{name}-full-plugin = %{version}-%{release} +Provides: %{name}-info-plugin = %{version}-%{release} +Provides: %{name}-memory-plugin = %{version}-%{release} +Provides: %{name}-null-plugin = %{version}-%{release} +Provides: %{name}-pattern-plugin = %{version}-%{release} +Provides: %{name}-partitioning-plugin = %{version}-%{release} +Provides: %{name}-random-plugin = %{version}-%{release} +Provides: %{name}-sh-plugin = %{version}-%{release} +Provides: %{name}-split-plugin = %{version}-%{release} +Provides: %{name}-streaming-plugin = %{version}-%{release} +Provides: %{name}-zero-plugin = %{version}-%{release} %description basic-plugins This package contains some basic plugins for %{name} which have only trivial dependencies. -* nbdkit-file-plugin - - A file serving plugin. - -* nbdkit-memory-plugin +nbdkit-data-plugin Serve small amounts of data from the command line. - A virtual memory plugin. +nbdkit-file-plugin The normal file plugin for serving files. -* nbdkit-nbd-plugin +nbdkit-floppy-plugin Create a virtual floppy disk from a directory. - An NBD forwarding plugin. +nbdkit-full-plugin A virtual disk that returns ENOSPC errors. - It provides an NBD server that forwards all traffic as a client to - another existing NBD server. A primary usage of this setup is to - alter the set of features available to the ultimate end client, - without having to change the original server (for example, to - convert between oldstyle and newtyle, or to add TLS support where - the original server lacks it). +nbdkit-info-plugin Serve client and server information. -* nbdkit-null-plugin +nbdkit-memory-plugin A virtual memory plugin. - A null (bitbucket) plugin. +nbdkit-null-plugin A null (bitbucket) plugin. -* nbdkit-random-plugin +nbdkit-pattern-plugin Fixed test pattern. - Random content plugin for testing. +nbdkit-partitioning-plugin Create virtual disks from partitions. -* nbdkit-split-plugin +nbdkit-random-plugin Random content plugin for testing. - Concatenate one or more files into a single virtual disk. +nbdkit-sh-plugin Write plugins as shell scripts or executables. -* nbdkit-streaming-plugin +nbdkit-split-plugin Concatenate one or more files. - A streaming file serving plugin. +nbdkit-streaming-plugin A streaming file serving plugin. -* nbdkit-zero-plugin - - Zero-length plugin for testing. +nbdkit-zero-plugin Zero-length plugin for testing. %package example-plugins Summary: Example plugins for %{name} License: BSD -Requires: %{name}%{?_isa} = %{version}-%{release} - -# For upgrade path, remove this in Fedora 30. -Obsoletes: %{name}-plugin-examples < 1.1.19-1 +Requires: %{name}-server%{?_isa} = %{version}-%{release} %description example-plugins @@ -179,73 +228,182 @@ This package contains example plugins for %{name}. # The plugins below have non-trivial dependencies are so are # packaged separately. -%package plugin-gzip +%package curl-plugin +Summary: HTTP/FTP (cURL) plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description curl-plugin +This package contains cURL (HTTP/FTP) support for %{name}. + + +%package gzip-plugin Summary: GZip file serving plugin for %{name} License: BSD -Requires: %{name}%{?_isa} = %{version}-%{release} +# Upgrade path from RHEL 8.0 +Provides: %{name}-plugin-gzip = %{version}-%{release} +Obsoletes: %{name}-plugin-gzip <= %{version}-%{release} + +Requires: %{name}-server%{?_isa} = %{version}-%{release} -%description plugin-gzip +%description gzip-plugin This package is a gzip file serving plugin for %{name}. -%package plugin-python-common -Summary: Python 2 and 3 plugin common files for %{name} +%package linuxdisk-plugin +Summary: Virtual Linux disk plugin for %{name} License: BSD -Requires: %{name}%{?_isa} = %{version}-%{release} +Requires: %{name}-server%{?_isa} = %{version}-%{release} +# for mke2fs +Requires: e2fsprogs -%description plugin-python-common -This package contains common files for Python %{name} plugins. +%description linuxdisk-plugin +This package is a virtual Linux disk plugin for %{name}. -You should not install this package directly. Instead install -%{name}-plugin-python3. - -%package plugin-python3 +%package python-plugin Summary: Python 3 plugin for %{name} License: BSD -Requires: %{name}%{?_isa} = %{version}-%{release} -Requires: %{name}-plugin-python-common = %{version}-%{release} +# Upgrade path from RHEL 8.0 +Provides: %{name}-plugin-python-common = %{version}-%{release} +Obsoletes: %{name}-plugin-python-common <= %{version}-%{release} +Provides: %{name}-plugin-python3 = %{version}-%{release} +Obsoletes: %{name}-plugin-python3 <= %{version}-%{release} + +Requires: %{name}-server%{?_isa} = %{version}-%{release} -%description plugin-python3 +%description python-plugin This package lets you write Python 3 plugins for %{name}. +%package ssh-plugin +Summary: SSH plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description ssh-plugin +This package contains SSH support for %{name}. + + %ifarch %{ix86} x86_64 -%package plugin-vddk +%package vddk-plugin Summary: VMware VDDK plugin for %{name} License: BSD -Requires: %{name}%{?_isa} = %{version}-%{release} +# Upgrade path from RHEL 8.0 +Provides: %{name}-plugin-vddk = %{version}-%{release} +Obsoletes: %{name}-plugin-vddk <= %{version}-%{release} + +Requires: %{name}-server%{?_isa} = %{version}-%{release} -%description plugin-vddk +%description vddk-plugin This package is a plugin for %{name} which connects to VMware VDDK for accessing VMware disks and servers. %endif -%package plugin-xz -Summary: XZ file serving plugin for %{name} +%package basic-filters +Summary: Basic filters for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Provides: %{name}-blocksize-filter = %{version}-%{release} +Provides: %{name}-cache-filter = %{version}-%{release} +Provides: %{name}-cacheextents-filter = %{version}-%{release} +Provides: %{name}-cow-filter = %{version}-%{release} +Provides: %{name}-delay-filter = %{version}-%{release} +Provides: %{name}-error-filter = %{version}-%{release} +Provides: %{name}-extentlist-filter = %{version}-%{release} +Provides: %{name}-fua-filter = %{version}-%{release} +Provides: %{name}-log-filter = %{version}-%{release} +Provides: %{name}-nocache-filter = %{version}-%{release} +Provides: %{name}-noextents-filter = %{version}-%{release} +Provides: %{name}-noparallel-filter = %{version}-%{release} +Provides: %{name}-nozero-filter = %{version}-%{release} +Provides: %{name}-offset-filter = %{version}-%{release} +Provides: %{name}-partition-filter = %{version}-%{release} +Provides: %{name}-rate-filter = %{version}-%{release} +Provides: %{name}-readahead-filter = %{version}-%{release} +Provides: %{name}-retry-filter = %{version}-%{release} +Provides: %{name}-stats-filter = %{version}-%{release} +Provides: %{name}-truncate-filter = %{version}-%{release} + +%description basic-filters +This package contains some basic filters for %{name} which have only +trivial dependencies. + +nbdkit-blocksize-filter Adjust block size of requests sent to plugins. + +nbdkit-cache-filter Server-side cache. + +nbdkit-cacheextents-filter Cache extents. + +nbdkit-cow-filter Copy-on-write overlay for read-only plugins. + +nbdkit-delay-filter Inject read and write delays. + +nbdkit-error-filter Inject errors. + +nbdkit-extentlist-filter Place extent list over a plugin. + +nbdkit-fua-filter Modify flush behaviour in plugins. + +nbdkit-log-filter Log all transactions to a file. + +nbdkit-nocache-filter Disable cache requests in the underlying plugin. + +nbdkit-noextents-filter Disable extents in the underlying plugin. + +nbdkit-noparallel-filter Serialize requests to the underlying plugin. + +nbdkit-nozero-filter Adjust handling of zero requests by plugins. + +nbdkit-offset-filter Serve an offset and range. + +nbdkit-partition-filter Serve a single partition. + +nbdkit-rate-filter Limit bandwidth by connection or server. + +nbdkit-readahead-filter Prefetch data when reading sequentially. + +nbdkit-retry-filter Reopen connection on error. + +nbdkit-stats-filter Display statistics about operations. + +nbdkit-truncate-filter Truncate, expand, round up or round down size. + + +%package xz-filter +Summary: XZ filter for %{name} License: BSD -Requires: %{name}%{?_isa} = %{version}-%{release} +# Upgrade path from RHEL 8.0 +Provides: %{name}-plugin-xz = %{version}-%{release} +Obsoletes: %{name}-plugin-xz <= %{version}-%{release} + +Requires: %{name}-server%{?_isa} = %{version}-%{release} -%description plugin-xz -This package is a xz file serving plugin for %{name}. +%description xz-filter +This package is the xz filter for %{name}. %package devel Summary: Development files and documentation for %{name} License: BSD -Requires: %{name}%{?_isa} = %{version}-%{release} +Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: pkgconfig @@ -259,7 +417,7 @@ plugins for %{name}. Summary: Bash tab-completion for %{name} BuildArch: noarch Requires: bash-completion >= 2.0 -Requires: %{name} = %{version}-%{release} +Requires: %{name}-server = %{version}-%{release} %description bash-completion @@ -286,60 +444,74 @@ autoreconf -i %build -export PYTHON=%{__python3} -%configure --disable-static \ - --with-tls-priority=@NBDKIT,SYSTEM \ +%configure \ + PYTHON=%{_bindir}/python3 \ + --disable-static \ + --disable-lua \ --disable-perl \ + --disable-nbd-plugin \ --disable-ocaml \ --disable-ruby \ + --disable-rust \ + --disable-ruby \ --disable-tcl \ - --without-curl \ + --without-ext2 \ + --without-iso \ + --without-libguestfs \ --without-libvirt \ - --without-libguestfs + --with-tls-priority=@NBDKIT,SYSTEM + # Verify that it picked the correct version of Python # to avoid RHBZ#1404631 happening again silently. grep '^PYTHON_VERSION = 3' Makefile + make %{?_smp_mflags} -unset PYTHON %install %make_install -pushd $RPM_BUILD_ROOT%{_libdir}/nbdkit/plugins/ -mv nbdkit-python-plugin.so nbdkit-python3-plugin.so -popd - -# Disable built-in filters but leave the empty directory. -rm -r $RPM_BUILD_ROOT%{_libdir}/%{name}/filters/nbdkit-*-filter.so -rm -r $RPM_BUILD_ROOT%{_mandir}/man1/nbdkit-*-filter.1* - # Delete libtool crap. find $RPM_BUILD_ROOT -name '*.la' -delete -# Delete the VDDK plugin on !x86 architectures since it is not -# applicable there. -%ifnarch %{ix86} x86_64 -rm $RPM_BUILD_ROOT%{_libdir}/%{name}/plugins/nbdkit-vddk-plugin.so -rm $RPM_BUILD_ROOT%{_mandir}/man1/nbdkit-vddk-plugin.1* -%endif +# If cargo happens to be installed on the machine then the +# rust plugin is built. Delete it if this happens. +rm -f $RPM_BUILD_ROOT%{_mandir}/man3/nbdkit-rust-plugin.3* %check # Workaround for broken libvirt (RHBZ#1138604). mkdir -p $HOME/.cache/libvirt +# tests/test-captive.sh is racy especially on s390x. We need to +# rethink this test upstream. +truncate -s 0 tests/test-captive.sh + +# Temporarily kill tests/test-shutdown.sh because this test is racy on +# slow hardware. +truncate -s 0 tests/test-shutdown.sh + +%ifarch s390x +# Temporarily kill tests/test-cache-max-size.sh since it fails +# sometimes on s390x for unclear reasons. +truncate -s 0 tests/test-cache-max-size.sh +%endif + # Make sure we can see the debug messages (RHBZ#1230160). export LIBGUESTFS_DEBUG=1 export LIBGUESTFS_TRACE=1 -make check -j1 || { +make %{?_smp_mflags} check || { cat tests/test-suite.log exit 1 } %files +# metapackage so empty + + +%files server %doc README %license LICENSE %{_sbindir}/nbdkit @@ -347,24 +519,43 @@ make check -j1 || { %dir %{_libdir}/%{name}/plugins %dir %{_libdir}/%{name}/filters %{_mandir}/man1/nbdkit.1* +%{_mandir}/man1/nbdkit-captive.1* +%{_mandir}/man1/nbdkit-loop.1* +%{_mandir}/man1/nbdkit-probing.1* +%{_mandir}/man1/nbdkit-protocol.1* +%{_mandir}/man1/nbdkit-service.1* +%{_mandir}/man1/nbdkit-security.1* +%{_mandir}/man1/nbdkit-tls.1* %files basic-plugins %doc README %license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-data-plugin.so %{_libdir}/%{name}/plugins/nbdkit-file-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-floppy-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-full-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-info-plugin.so %{_libdir}/%{name}/plugins/nbdkit-memory-plugin.so -%{_libdir}/%{name}/plugins/nbdkit-nbd-plugin.so %{_libdir}/%{name}/plugins/nbdkit-null-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-partitioning-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-pattern-plugin.so %{_libdir}/%{name}/plugins/nbdkit-random-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-sh-plugin.so %{_libdir}/%{name}/plugins/nbdkit-split-plugin.so %{_libdir}/%{name}/plugins/nbdkit-streaming-plugin.so %{_libdir}/%{name}/plugins/nbdkit-zero-plugin.so +%{_mandir}/man1/nbdkit-data-plugin.1* %{_mandir}/man1/nbdkit-file-plugin.1* +%{_mandir}/man1/nbdkit-floppy-plugin.1* +%{_mandir}/man1/nbdkit-full-plugin.1* +%{_mandir}/man1/nbdkit-info-plugin.1* %{_mandir}/man1/nbdkit-memory-plugin.1* -%{_mandir}/man1/nbdkit-nbd-plugin.1* %{_mandir}/man1/nbdkit-null-plugin.1* +%{_mandir}/man1/nbdkit-partitioning-plugin.1* +%{_mandir}/man1/nbdkit-pattern-plugin.1* %{_mandir}/man1/nbdkit-random-plugin.1* +%{_mandir}/man3/nbdkit-sh-plugin.3* %{_mandir}/man1/nbdkit-split-plugin.1* %{_mandir}/man1/nbdkit-streaming-plugin.1* %{_mandir}/man1/nbdkit-zero-plugin.1* @@ -377,25 +568,43 @@ make check -j1 || { %{_mandir}/man1/nbdkit-example*-plugin.1* -%files plugin-gzip +%files curl-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-curl-plugin.so +%{_mandir}/man1/nbdkit-curl-plugin.1* + + +%files gzip-plugin %doc README %license LICENSE %{_libdir}/%{name}/plugins/nbdkit-gzip-plugin.so %{_mandir}/man1/nbdkit-gzip-plugin.1* -%files plugin-python-common +%files linuxdisk-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-linuxdisk-plugin.so +%{_mandir}/man1/nbdkit-linuxdisk-plugin.1* + + +%files python-plugin %doc README %license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-python-plugin.so %{_mandir}/man3/nbdkit-python-plugin.3* -%files plugin-python3 -%{_libdir}/%{name}/plugins/nbdkit-python3-plugin.so +%files ssh-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-ssh-plugin.so +%{_mandir}/man1/nbdkit-ssh-plugin.1* %ifarch %{ix86} x86_64 -%files plugin-vddk +%files vddk-plugin %doc README %license LICENSE %{_libdir}/%{name}/plugins/nbdkit-vddk-plugin.so @@ -403,23 +612,73 @@ make check -j1 || { %endif -%files plugin-xz +%files basic-filters +%doc README +%license LICENSE +%{_libdir}/%{name}/filters/nbdkit-blocksize-filter.so +%{_libdir}/%{name}/filters/nbdkit-cache-filter.so +%{_libdir}/%{name}/filters/nbdkit-cacheextents-filter.so +%{_libdir}/%{name}/filters/nbdkit-cow-filter.so +%{_libdir}/%{name}/filters/nbdkit-delay-filter.so +%{_libdir}/%{name}/filters/nbdkit-error-filter.so +%{_libdir}/%{name}/filters/nbdkit-extentlist-filter.so +%{_libdir}/%{name}/filters/nbdkit-fua-filter.so +%{_libdir}/%{name}/filters/nbdkit-log-filter.so +%{_libdir}/%{name}/filters/nbdkit-nocache-filter.so +%{_libdir}/%{name}/filters/nbdkit-noextents-filter.so +%{_libdir}/%{name}/filters/nbdkit-noparallel-filter.so +%{_libdir}/%{name}/filters/nbdkit-nozero-filter.so +%{_libdir}/%{name}/filters/nbdkit-offset-filter.so +%{_libdir}/%{name}/filters/nbdkit-partition-filter.so +%{_libdir}/%{name}/filters/nbdkit-rate-filter.so +%{_libdir}/%{name}/filters/nbdkit-readahead-filter.so +%{_libdir}/%{name}/filters/nbdkit-retry-filter.so +%{_libdir}/%{name}/filters/nbdkit-stats-filter.so +%{_libdir}/%{name}/filters/nbdkit-truncate-filter.so +%{_mandir}/man1/nbdkit-blocksize-filter.1* +%{_mandir}/man1/nbdkit-cache-filter.1* +%{_mandir}/man1/nbdkit-cacheextents-filter.1* +%{_mandir}/man1/nbdkit-cow-filter.1* +%{_mandir}/man1/nbdkit-delay-filter.1* +%{_mandir}/man1/nbdkit-error-filter.1* +%{_mandir}/man1/nbdkit-extentlist-filter.1* +%{_mandir}/man1/nbdkit-fua-filter.1* +%{_mandir}/man1/nbdkit-log-filter.1* +%{_mandir}/man1/nbdkit-nocache-filter.1* +%{_mandir}/man1/nbdkit-noextents-filter.1* +%{_mandir}/man1/nbdkit-noparallel-filter.1* +%{_mandir}/man1/nbdkit-nozero-filter.1* +%{_mandir}/man1/nbdkit-offset-filter.1* +%{_mandir}/man1/nbdkit-partition-filter.1* +%{_mandir}/man1/nbdkit-rate-filter.1* +%{_mandir}/man1/nbdkit-readahead-filter.1* +%{_mandir}/man1/nbdkit-retry-filter.1* +%{_mandir}/man1/nbdkit-stats-filter.1* +%{_mandir}/man1/nbdkit-truncate-filter.1* + + +%files xz-filter %doc README %license LICENSE -%{_libdir}/%{name}/plugins/nbdkit-xz-plugin.so -%{_mandir}/man1/nbdkit-xz-plugin.1* +%{_libdir}/%{name}/filters/nbdkit-xz-filter.so +%{_mandir}/man1/nbdkit-xz-filter.1* %files devel -%doc OTHER_PLUGINS README TODO +%doc BENCHMARKING OTHER_PLUGINS README SECURITY TODO %license LICENSE # Include the source of the example plugins in the documentation. %doc plugins/example*/*.c +%doc plugins/python/example.py +%doc plugins/sh/example.sh %{_includedir}/nbdkit-common.h %{_includedir}/nbdkit-filter.h %{_includedir}/nbdkit-plugin.h +%{_includedir}/nbdkit-version.h +%{_includedir}/nbd-protocol.h %{_mandir}/man3/nbdkit-filter.3* %{_mandir}/man3/nbdkit-plugin.3* +%{_mandir}/man1/nbdkit-release-notes-1.*.1* %{_libdir}/pkgconfig/nbdkit.pc @@ -430,6 +689,14 @@ make check -j1 || { %changelog +* Thu Jun 04 2020 Danilo C. L. de Paula - 1.16.2 +- Resolves: bz#1810193 + (Upgrade components in virt:rhel module:stream for RHEL-8.3 release) + +* Mon Apr 27 2020 Danilo C. L. de Paula - 1.16.2 +- Resolves: bz#1810193 + (Upgrade components in virt:rhel module:stream for RHEL-8.3 release) + * Fri Jun 28 2019 Danilo de Paula - 1.4.2-5 - Rebuild all virt packages to fix RHEL's upgrade path - Resolves: rhbz#1695587