diff --git a/.gitignore b/.gitignore index fe99bfd..b9d7401 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ SOURCES/libguestfs.keyring -SOURCES/nbdkit-1.28.5.tar.gz +SOURCES/nbdkit-1.30.8.tar.gz diff --git a/.nbdkit.metadata b/.nbdkit.metadata index 8374831..5561c3f 100644 --- a/.nbdkit.metadata +++ b/.nbdkit.metadata @@ -1,2 +1,2 @@ cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring -60f2c2021658a94d778eb9cde0123d1c092ff15d SOURCES/nbdkit-1.28.5.tar.gz +6c4607ff13e13460cfdf67f47b9fea9ac0a8ebc3 SOURCES/nbdkit-1.30.8.tar.gz diff --git a/SOURCES/0001-ssh-Allow-the-remote-file-to-be-created.patch b/SOURCES/0001-ssh-Allow-the-remote-file-to-be-created.patch new file mode 100644 index 0000000..f3e07e4 --- /dev/null +++ b/SOURCES/0001-ssh-Allow-the-remote-file-to-be-created.patch @@ -0,0 +1,293 @@ +From 6a2b0aac8be655524ea223e32cac0395fcc9f975 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 15 Apr 2022 12:08:37 +0100 +Subject: [PATCH] ssh: Allow the remote file to be created + +This adds new parameters, create=(true|false), create-size=SIZE and +create-mode=MODE to create and truncate the remote file. + +Reviewed-by: Laszlo Ersek +(cherry picked from commit 0793f30b1071753532362b2ebf9cb8156a88c3c3) +--- + plugins/ssh/nbdkit-ssh-plugin.pod | 34 ++++++++- + plugins/ssh/ssh.c | 112 +++++++++++++++++++++++++++--- + tests/test-ssh.sh | 13 +++- + 3 files changed, 146 insertions(+), 13 deletions(-) + +diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod +index 3f401c15..2bc2c4a7 100644 +--- a/plugins/ssh/nbdkit-ssh-plugin.pod ++++ b/plugins/ssh/nbdkit-ssh-plugin.pod +@@ -5,8 +5,10 @@ nbdkit-ssh-plugin - access disk images over the SSH protocol + =head1 SYNOPSIS + + nbdkit ssh host=HOST [path=]PATH +- [compression=true] [config=CONFIG_FILE] [identity=FILENAME] +- [known-hosts=FILENAME] [password=PASSWORD|-|+FILENAME] ++ [compression=true] [config=CONFIG_FILE] ++ [create=true] [create-mode=MODE] [create-size=SIZE] ++ [identity=FILENAME] [known-hosts=FILENAME] ++ [password=PASSWORD|-|+FILENAME] + [port=PORT] [timeout=SECS] [user=USER] + [verify-remote-host=false] + +@@ -62,6 +64,34 @@ The C parameter is optional. If it is I specified at all + then F<~/.ssh/config> and F are both read. + Missing or unreadable files are ignored. + ++=item B ++ ++(nbdkit E 1.32) ++ ++If set, the remote file will be created. The remote file is created ++on the first NBD connection to nbdkit, not when nbdkit starts up. If ++the file already exists, it will be replaced and any existing content ++lost. ++ ++If using this option, you must use C. C can ++be used to control the permissions of the new file. ++ ++=item BMODE ++ ++(nbdkit E 1.32) ++ ++If using C specify the default permissions of the new ++remote file. You can use octal modes like C or ++C. The default is C<0600>, ie. only readable and ++writable by the remote user. ++ ++=item BSIZE ++ ++(nbdkit E 1.32) ++ ++If using C, specify the virtual size of the new disk. ++C can use modifiers like C<100M> etc. ++ + =item BHOST + + Specify the name or IP address of the remote host. +diff --git a/plugins/ssh/ssh.c b/plugins/ssh/ssh.c +index 39d77e44..5e314cd7 100644 +--- a/plugins/ssh/ssh.c ++++ b/plugins/ssh/ssh.c +@@ -44,6 +44,8 @@ + #include + #include + ++#include ++ + #include + #include + #include +@@ -51,6 +53,7 @@ + #include + + #include "array-size.h" ++#include "cleanup.h" + #include "const-string-vector.h" + #include "minmax.h" + +@@ -64,6 +67,9 @@ static const char *known_hosts = NULL; + static const_string_vector identities = empty_vector; + static uint32_t timeout = 0; + static bool compression = false; ++static bool create = false; ++static int64_t create_size = -1; ++static unsigned create_mode = S_IRUSR | S_IWUSR /* 0600 */; + + /* config can be: + * NULL => parse options from default file +@@ -167,6 +173,27 @@ ssh_config (const char *key, const char *value) + return -1; + compression = r; + } ++ else if (strcmp (key, "create") == 0) { ++ r = nbdkit_parse_bool (value); ++ if (r == -1) ++ return -1; ++ create = r; ++ } ++ else if (strcmp (key, "create-size") == 0) { ++ create_size = nbdkit_parse_size (value); ++ if (create_size == -1) ++ return -1; ++ } ++ else if (strcmp (key, "create-mode") == 0) { ++ r = nbdkit_parse_unsigned (key, value, &create_mode); ++ if (r == -1) ++ return -1; ++ /* OpenSSH checks this too. */ ++ if (create_mode > 0777) { ++ nbdkit_error ("create-mode must be <= 0777"); ++ return -1; ++ } ++ } + + else { + nbdkit_error ("unknown parameter '%s'", key); +@@ -186,6 +213,13 @@ ssh_config_complete (void) + return -1; + } + ++ /* If create=true, create-size must be supplied. */ ++ if (create && create_size == -1) { ++ nbdkit_error ("if using create=true, you must specify the size " ++ "of the new remote file using create-size=SIZE"); ++ return -1; ++ } ++ + return 0; + } + +@@ -200,7 +234,10 @@ ssh_config_complete (void) + "identity= Prepend private key (identity) file.\n" \ + "timeout=SECS Set SSH connection timeout.\n" \ + "verify-remote-host=false Ignore known_hosts.\n" \ +- "compression=true Enable compression." ++ "compression=true Enable compression.\n" \ ++ "create=true Create the remote file.\n" \ ++ "create-mode=MODE Set the permissions of the remote file.\n" \ ++ "create-size=SIZE Set the size of the remote file." + + /* Since we must simulate atomic pread and pwrite using seek + + * read/write, calls on each handle must be serialized. +@@ -329,6 +366,65 @@ authenticate (struct ssh_handle *h) + return -1; + } + ++/* This function opens or creates the remote file (depending on ++ * create=false|true). Parallel connections might call this function ++ * at the same time, and so we must hold a lock to ensure that the ++ * file is created at most once. ++ */ ++static pthread_mutex_t create_lock = PTHREAD_MUTEX_INITIALIZER; ++ ++static sftp_file ++open_or_create_path (ssh_session session, sftp_session sftp, int readonly) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&create_lock); ++ int access_type; ++ int r; ++ sftp_file file; ++ ++ access_type = readonly ? O_RDONLY : O_RDWR; ++ if (create) access_type |= O_CREAT | O_TRUNC; ++ ++ file = sftp_open (sftp, path, access_type, S_IRWXU); ++ if (!file) { ++ nbdkit_error ("cannot %s file for %s: %s", ++ create ? "create" : "open", ++ readonly ? "reading" : "writing", ++ ssh_get_error (session)); ++ return NULL; ++ } ++ ++ if (create) { ++ /* There's no sftp_truncate call. However OpenSSH lets you call ++ * SSH_FXP_SETSTAT + SSH_FILEXFER_ATTR_SIZE which invokes ++ * truncate(2) on the server. Libssh doesn't provide a binding ++ * for SSH_FXP_FSETSTAT so we have to pass the session + path. ++ */ ++ struct sftp_attributes_struct attrs = { ++ .flags = SSH_FILEXFER_ATTR_SIZE | ++ SSH_FILEXFER_ATTR_PERMISSIONS, ++ .size = create_size, ++ .permissions = create_mode, ++ }; ++ ++ r = sftp_setstat (sftp, path, &attrs); ++ if (r != SSH_OK) { ++ nbdkit_error ("setstat failed: %s", ssh_get_error (session)); ++ ++ /* Best-effort attempt to delete the remote file on failure. */ ++ r = sftp_unlink (sftp, path); ++ if (r != SSH_OK) ++ nbdkit_debug ("unlink failed: %s", ssh_get_error (session)); ++ ++ return NULL; ++ } ++ } ++ ++ /* On the next connection, don't create or truncate the file. */ ++ create = false; ++ ++ return file; ++} ++ + /* Create the per-connection handle. */ + static void * + ssh_open (int readonly) +@@ -337,7 +433,6 @@ ssh_open (int readonly) + const int set = 1; + size_t i; + int r; +- int access_type; + + h = calloc (1, sizeof *h); + if (h == NULL) { +@@ -471,7 +566,7 @@ ssh_open (int readonly) + if (authenticate (h) == -1) + goto err; + +- /* Open the SFTP connection and file. */ ++ /* Open the SFTP connection. */ + h->sftp = sftp_new (h->session); + if (!h->sftp) { + nbdkit_error ("failed to allocate sftp session: %s", +@@ -484,14 +579,11 @@ ssh_open (int readonly) + ssh_get_error (h->session)); + goto err; + } +- access_type = readonly ? O_RDONLY : O_RDWR; +- h->file = sftp_open (h->sftp, path, access_type, S_IRWXU); +- if (!h->file) { +- nbdkit_error ("cannot open file for %s: %s", +- readonly ? "reading" : "writing", +- ssh_get_error (h->session)); ++ ++ /* Open or create the remote file. */ ++ h->file = open_or_create_path (h->session, h->sftp, readonly); ++ if (!h->file) + goto err; +- } + + nbdkit_debug ("opened libssh handle"); + +diff --git a/tests/test-ssh.sh b/tests/test-ssh.sh +index 6c0ce410..f04b4488 100755 +--- a/tests/test-ssh.sh ++++ b/tests/test-ssh.sh +@@ -36,6 +36,7 @@ set -x + + requires test -f disk + requires nbdcopy --version ++requires stat --version + + # Check that ssh to localhost will work without any passwords or phrases. + # +@@ -48,7 +49,7 @@ then + exit 77 + fi + +-files="ssh.img" ++files="ssh.img ssh2.img" + rm -f $files + cleanup_fn rm -f $files + +@@ -59,3 +60,13 @@ nbdkit -v -D ssh.log=2 -U - \ + + # The output should be identical. + cmp disk ssh.img ++ ++# Copy local file 'ssh.img' to newly created "remote" 'ssh2.img' ++size="$(stat -c %s disk)" ++nbdkit -v -D ssh.log=2 -U - \ ++ ssh host=localhost $PWD/ssh2.img \ ++ create=true create-size=$size \ ++ --run 'nbdcopy ssh.img "$uri"' ++ ++# The output should be identical. ++cmp disk ssh2.img +-- +2.31.1 + diff --git a/SOURCES/0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch b/SOURCES/0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch deleted file mode 100644 index 066e913..0000000 --- a/SOURCES/0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch +++ /dev/null @@ -1,117 +0,0 @@ -From 4f2f557b349ad621e502e304c87280835cf13146 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 21 Oct 2021 14:49:52 +0100 -Subject: [PATCH] vddk: Refactor how -D vddk.stats=1 is collected - -In order to allow us to collect more per-API stats, introduce a global -struct per API for storing these stats. - -(cherry picked from commit 3d8657f3d9a2c1b59284333566428b4c7ce32a74) ---- - plugins/vddk/vddk.c | 36 ++++++++++++++++++------------------ - 1 file changed, 18 insertions(+), 18 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 80f5870e..3d751544 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -1,5 +1,5 @@ - /* nbdkit -- * Copyright (C) 2013-2020 Red Hat Inc. -+ * Copyright (C) 2013-2021 Red Hat Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are -@@ -103,14 +103,23 @@ static bool is_remote; - /* For each VDDK API define a variable to store the time taken (used - * to implement -D vddk.stats=1). - */ -+struct vddk_stat { -+ const char *name; /* function name */ -+ int64_t usecs; /* total number of usecs consumed */ -+}; - static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; - static void display_stats (void); --#define STUB(fn,ret,args) static int64_t stats_##fn; --#define OPTIONAL_STUB(fn,ret,args) static int64_t stats_##fn; -+#define STUB(fn,ret,args) \ -+ static struct vddk_stat stats_##fn = { .name = #fn } -+#define OPTIONAL_STUB(fn,ret,args) \ -+ static struct vddk_stat stats_##fn = { .name = #fn } - #include "vddk-stubs.h" - #undef STUB - #undef OPTIONAL_STUB - -+/* Macros to bracket each VDDK API call, for printing debugging -+ * information and collecting statistics. -+ */ - #define VDDK_CALL_START(fn, fs, ...) \ - do { \ - struct timeval start_t, end_t; \ -@@ -131,10 +140,11 @@ static void display_stats (void); - if (vddk_debug_stats) { \ - gettimeofday (&end_t, NULL); \ - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ -- stats_##fn += tvdiff_usec (&start_t, &end_t); \ -+ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ - } \ - } while (0) - -+/* Print VDDK errors. */ - #define VDDK_ERROR(err, fs, ...) \ - do { \ - char *vddk_err_msg; \ -@@ -167,10 +177,6 @@ vddk_unload (void) - free (password); - } - --struct vddk_stat { -- const char *fn; -- int64_t usecs; --}; - DEFINE_VECTOR_TYPE(statlist, struct vddk_stat) - - static int -@@ -179,7 +185,7 @@ stat_compare (const void *vp1, const void *vp2) - const struct vddk_stat *st1 = vp1; - const struct vddk_stat *st2 = vp2; - -- /* Note: sorts in reverse order. */ -+ /* Note: sorts in reverse order of time spent in each API call. */ - if (st1->usecs < st2->usecs) return 1; - else if (st1->usecs > st2->usecs) return -1; - else return 0; -@@ -189,19 +195,13 @@ static void - display_stats (void) - { - statlist stats = empty_vector; -- struct vddk_stat st; - size_t i; - --#define ADD_ONE_STAT(fn_, usecs_) \ -- st.fn = fn_; \ -- st.usecs = usecs_; \ -- statlist_append (&stats, st) --#define STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn); --#define OPTIONAL_STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn); -+#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn) -+#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn) - #include "vddk-stubs.h" - #undef STUB - #undef OPTIONAL_STUB --#undef ADD_ONE_STAT - - qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); - -@@ -209,7 +209,7 @@ display_stats (void) - nbdkit_debug ("%-40s %9s", "", "µs"); - for (i = 0; i < stats.size; ++i) { - if (stats.ptr[i].usecs) -- nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].fn, stats.ptr[i].usecs); -+ nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].name, stats.ptr[i].usecs); - } - statlist_reset (&stats); - } --- -2.31.1 - diff --git a/SOURCES/0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch b/SOURCES/0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch new file mode 100644 index 0000000..69d5efa --- /dev/null +++ b/SOURCES/0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch @@ -0,0 +1,794 @@ +From ac40ae11bc9983e11185749b23e793568cb366cc Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 16 Apr 2022 18:39:13 +0100 +Subject: [PATCH] readahead: Rewrite this filter so it prefetches using .cache + +The previous readahead filter did not work well and we have stopped +using it in virt-v2v. However the concept is a good one if we can +make it work. This commit completely rethinks the filter. + +Now, in parallel with the ordinary pread command, we issue a prefetch +(ie. .cache) to the underlying plugin for the data immediately +following the pread. For example a simple sequence of operations: + + t=1 pread (offset=0, count=65536) + t=2 pread (offset=65536, count=65536) + t=3 pread (offset=131072, count=65536) + +would become: + + t=1 pread (offset=0, count=65536) <--\ issued + cache (offset=65536, count=65536) <--/ in parallel + t=2 pread (offset=65536, count=65536) + cache (offset=131072, count=65536) + t=3 pread (offset=131072, count=65536) + cache (offset=196608, count=65536) + +This requires that the underlying filter(s) and plugin chain can +actually do something with the .cache request. If this is not the +case then the filter does nothing (but it will print a warning). For +plugins which don't have native support for prefetching, it is +sufficient to insert nbdkit-cache-filter after this filter. +(nbdkit-cow-filter can also be used with cow-on-cache=true, but that +is more useful for advanced users who are already using the cow +filter). + +The implementation creates a background thread per connection to issue +the parallel .cache requests. This is safer than the alternative (one +background thread in total) since we don't have to deal with the +problem of cache requests being issued with the wrong export name or +stale next pointer. The background thread is controlled by a queue of +commands, with the only possible commands being "cache" or "quit". + +Because the background thread issues parallel requests on the same +connection, the underlying plugin must support the parallel thread +model, otherwise we would be violating the plugin's thread model. It +may be possible in future to open a new connection from the background +thread (but with the same exportname), which would lift this +restriction to at least serialize_requests. Because of the current +limitation, nbdkit-curl-plugin cannot use prefetch. + +(cherry picked from commit 2ff548d66ad3eae87868402ec5b3319edd12090f) +--- + TODO | 22 +- + filters/readahead/Makefile.am | 2 + + filters/readahead/bgthread.c | 76 ++++ + filters/readahead/nbdkit-readahead-filter.pod | 70 +++- + filters/readahead/readahead.c | 338 +++++++++--------- + filters/readahead/readahead.h | 60 ++++ + tests/test-readahead.sh | 2 +- + 7 files changed, 367 insertions(+), 203 deletions(-) + create mode 100644 filters/readahead/bgthread.c + create mode 100644 filters/readahead/readahead.h + +diff --git a/TODO b/TODO +index 5ae21db5..4d2a9796 100644 +--- a/TODO ++++ b/TODO +@@ -62,16 +62,6 @@ General ideas for improvements + continue to keep their non-standard handshake while utilizing nbdkit + to prototype new behaviors in serving the kernel. + +-* Background thread for filters. Some filters (readahead, cache and +- proposed scan filter - see below) could be more effective if they +- were able to defer work to a background thread. We finally have +- nbdkit_next_context_open and friends for allowing a background +- thread to have access into the plugin, but still need to worry about +- thread-safety (how much must the filter do vs. nbdkit, to avoid +- calling into the plugin too many times at once) and cleanup +- (spawning the thread during .after_fork is viable, but cleaning it +- up during .unload is too late). +- + * "nbdkit.so": nbdkit as a loadable shared library. The aim of nbdkit + is to make it reusable from other programs (see nbdkit-captive(1)). + If it was a loadable shared library it would be even more reusable. +@@ -228,6 +218,8 @@ Suggestions for filters + * nbdkit-cache-filter should handle ENOSPC errors automatically by + reclaiming blocks from the cache + ++* nbdkit-cache-filter could use a background thread for reclaiming. ++ + * zstd filter was requested as a way to do what we currently do with + xz but saving many hours on compression (at the cost of hundreds of + MBs of extra data) +@@ -240,6 +232,16 @@ Suggestions for filters + could inject a flush after pausing. However this requires that + filter background threads have access to the plugin (see above). + ++nbdkit-readahead-filter: ++ ++* The filter should open a new connection to the plugin per background ++ thread so it is able to work with plugins that use the ++ serialize_requests thread model (like curl). At the moment it makes ++ requests on the same connection, so it requires plugins to use the ++ parallel thread model. ++ ++* It should combine (or avoid) overlapping cache requests. ++ + nbdkit-rate-filter: + + * allow other kinds of traffic shaping such as VBR +diff --git a/filters/readahead/Makefile.am b/filters/readahead/Makefile.am +index ee5bb3fb..187993ae 100644 +--- a/filters/readahead/Makefile.am ++++ b/filters/readahead/Makefile.am +@@ -37,6 +37,8 @@ filter_LTLIBRARIES = nbdkit-readahead-filter.la + + nbdkit_readahead_filter_la_SOURCES = \ + readahead.c \ ++ readahead.h \ ++ bgthread.c \ + $(top_srcdir)/include/nbdkit-filter.h \ + $(NULL) + +diff --git a/filters/readahead/bgthread.c b/filters/readahead/bgthread.c +new file mode 100644 +index 00000000..5894bb5f +--- /dev/null ++++ b/filters/readahead/bgthread.c +@@ -0,0 +1,76 @@ ++/* nbdkit ++ * Copyright (C) 2019-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "readahead.h" ++ ++#include "cleanup.h" ++ ++void * ++readahead_thread (void *vp) ++{ ++ struct bgthread_ctrl *ctrl = vp; ++ ++ for (;;) { ++ struct command cmd; ++ ++ /* Wait until we are sent at least one command. */ ++ { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); ++ while (ctrl->cmds.len == 0) ++ pthread_cond_wait (&ctrl->cond, &ctrl->lock); ++ cmd = ctrl->cmds.ptr[0]; ++ command_queue_remove (&ctrl->cmds, 0); ++ } ++ ++ switch (cmd.type) { ++ case CMD_QUIT: ++ /* Finish processing and exit the thread. */ ++ return NULL; ++ ++ case CMD_CACHE: ++ /* Issue .cache (readahead) to underlying plugin. We ignore any ++ * errors because there's no way to communicate that back to the ++ * client, and readahead is only advisory. ++ */ ++ cmd.next->cache (cmd.next, cmd.count, cmd.offset, 0, NULL); ++ } ++ } ++} +diff --git a/filters/readahead/nbdkit-readahead-filter.pod b/filters/readahead/nbdkit-readahead-filter.pod +index c220d379..630e5924 100644 +--- a/filters/readahead/nbdkit-readahead-filter.pod ++++ b/filters/readahead/nbdkit-readahead-filter.pod +@@ -1,28 +1,66 @@ + =head1 NAME + +-nbdkit-readahead-filter - prefetch data when reading sequentially ++nbdkit-readahead-filter - prefetch data ahead of sequential reads + + =head1 SYNOPSIS + +- nbdkit --filter=readahead plugin ++ nbdkit --filter=readahead PLUGIN ++ ++ nbdkit --filter=readahead --filter=cache PLUGIN ++ ++ nbdkit --filter=readahead --filter=cow PLUGIN cow-on-cache=true + + =head1 DESCRIPTION + + C is a filter that prefetches data when the +-client is reading sequentially. ++client is reading. + +-A common use for this filter is to accelerate sequential copy +-operations (like S>) when plugin requests have a +-high overhead (like L). For example: +- +- nbdkit -U - --filter=readahead curl https://example.com/disk.img \ +- --run 'qemu-img convert $nbd disk.img' ++When the client issues a read, this filter issues a parallel prefetch ++(C<.cache>) for subsequent data. Plugins which support this command ++will prefetch the data, making sequential reads faster. For plugins ++which do not support this command, you can inject ++L below (after) this filter, giving ++approximately the same effect. L can be used ++instead of nbdkit-cache-filter, if you add the C ++option. + + The filter uses a simple adaptive algorithm which accelerates +-sequential reads, but has a small penalty if the client does random +-reads. If the client mixes reads with writes or write-like operations +-(trimming, zeroing) then it will work but there can be a large +-performance penalty. ++sequential reads and requires no further configuration. ++ ++=head2 Limitations ++ ++In a number of significant cases this filter will do nothing. The ++filter will print a warning message if this happens. ++ ++=over 4 ++ ++=item Thread model must be parallel ++ ++For example L only supports ++C, and so this filter cannot perform prefetches in ++parallel with the read requests. ++ ++We may be able to lift this restriction in future. ++ ++=item Underlying filters or plugin must support C<.cache> (prefetch) ++ ++Very many plugins do not have the concept of prefetching and/or ++do not implement the C<.cache> callback, and so there is no ++way for this filter to issue prefetches. ++ ++You can usually get around this by adding I<--filter=cache> after this ++filter as explained above. It may be necessary to limit the total ++size of the cache (see L). ++ ++=item Clients and kernels may do readahead already ++ ++It may be the case that NBD clients are already issuing ++C (NBD prefetch) commands. It may also be the case ++that your plugin is using local file functions where the kernel is ++doing readahead. In such cases this filter is not necessary and may ++be pessimal. ++ ++=back + + =head1 PARAMETERS + +@@ -50,9 +88,9 @@ C first appeared in nbdkit 1.12. + + L, + L, +-L, ++L, ++L, + L, +-L, + L, + L, + L, +@@ -64,4 +102,4 @@ Richard W.M. Jones + + =head1 COPYRIGHT + +-Copyright (C) 2019 Red Hat Inc. ++Copyright (C) 2019-2022 Red Hat Inc. +diff --git a/filters/readahead/readahead.c b/filters/readahead/readahead.c +index f5552d4c..1d7ae111 100644 +--- a/filters/readahead/readahead.c ++++ b/filters/readahead/readahead.c +@@ -1,5 +1,5 @@ + /* nbdkit +- * Copyright (C) 2019-2021 Red Hat Inc. ++ * Copyright (C) 2019-2022 Red Hat Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are +@@ -34,232 +34,218 @@ + + #include + #include ++#include + #include + #include + #include +- + #include + + #include + ++#include "readahead.h" ++ + #include "cleanup.h" + #include "minmax.h" +- +-/* Copied from server/plugins.c. */ +-#define MAX_REQUEST_SIZE (64 * 1024 * 1024) ++#include "vector.h" + + /* These could be made configurable in future. */ +-#define READAHEAD_MIN 65536 +-#define READAHEAD_MAX MAX_REQUEST_SIZE +- +-/* This lock protects the global state. */ +-static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; +- +-/* The real size of the underlying plugin. */ +-static uint64_t size; ++#define READAHEAD_MIN 32768 ++#define READAHEAD_MAX (4*1024*1024) + + /* Size of the readahead window. */ ++static pthread_mutex_t window_lock = PTHREAD_MUTEX_INITIALIZER; + static uint64_t window = READAHEAD_MIN; ++static uint64_t last_offset = 0, last_readahead = 0; + +-/* The single prefetch buffer shared by all threads, and its virtual +- * location in the virtual disk. The prefetch buffer grows +- * dynamically as required, but never shrinks. ++static int thread_model = -1; /* Thread model of the underlying plugin. */ ++ ++/* Per-connection data. */ ++struct readahead_handle { ++ int can_cache; /* Can the underlying plugin cache? */ ++ pthread_t thread; /* The background thread, one per connection. */ ++ struct bgthread_ctrl ctrl; ++}; ++ ++/* We have various requirements of the underlying filter(s) + plugin: ++ * - They must support NBDKIT_CACHE_NATIVE (otherwise our requests ++ * would not do anything useful). ++ * - They must use the PARALLEL thread model (otherwise we could ++ * violate their thread model). ++ */ ++static bool ++filter_working (struct readahead_handle *h) ++{ ++ return ++ h->can_cache == NBDKIT_CACHE_NATIVE && ++ thread_model == NBDKIT_THREAD_MODEL_PARALLEL; ++} ++ ++static bool ++suggest_cache_filter (struct readahead_handle *h) ++{ ++ return ++ h->can_cache != NBDKIT_CACHE_NATIVE && ++ thread_model == NBDKIT_THREAD_MODEL_PARALLEL; ++} ++ ++/* We need to hook into .get_ready() so we can read the final thread ++ * model (of the whole server). + */ +-static char *buffer = NULL; +-static size_t bufsize = 0; +-static uint64_t position; +-static uint32_t length = 0; ++static int ++readahead_get_ready (int final_thread_model) ++{ ++ thread_model = final_thread_model; ++ return 0; ++} ++ ++static int ++send_command_to_background_thread (struct bgthread_ctrl *ctrl, ++ const struct command cmd) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); ++ if (command_queue_append (&ctrl->cmds, cmd) == -1) ++ return -1; ++ /* Signal the thread if it could be sleeping on an empty queue. */ ++ if (ctrl->cmds.len == 1) ++ pthread_cond_signal (&ctrl->cond); ++ return 0; ++} ++ ++static void * ++readahead_open (nbdkit_next_open *next, nbdkit_context *nxdata, ++ int readonly, const char *exportname, int is_tls) ++{ ++ struct readahead_handle *h; ++ int err; ++ ++ if (next (nxdata, readonly, exportname) == -1) ++ return NULL; ++ ++ h = malloc (sizeof *h); ++ if (h == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return NULL; ++ } ++ ++ h->ctrl.cmds = (command_queue) empty_vector; ++ pthread_mutex_init (&h->ctrl.lock, NULL); ++ pthread_cond_init (&h->ctrl.cond, NULL); ++ ++ /* Create the background thread. */ ++ err = pthread_create (&h->thread, NULL, readahead_thread, &h->ctrl); ++ if (err != 0) { ++ errno = err; ++ nbdkit_error ("pthread_create: %m"); ++ pthread_cond_destroy (&h->ctrl.cond); ++ pthread_mutex_destroy (&h->ctrl.lock); ++ free (h); ++ return NULL; ++ } ++ ++ return h; ++} + + static void +-readahead_unload (void) ++readahead_close (void *handle) + { +- free (buffer); ++ struct readahead_handle *h = handle; ++ const struct command quit_cmd = { .type = CMD_QUIT }; ++ ++ send_command_to_background_thread (&h->ctrl, quit_cmd); ++ pthread_join (h->thread, NULL); ++ pthread_cond_destroy (&h->ctrl.cond); ++ pthread_mutex_destroy (&h->ctrl.lock); ++ command_queue_reset (&h->ctrl.cmds); ++ free (h); + } + +-static int64_t readahead_get_size (nbdkit_next *next, void *handle); +- +-/* In prepare, force a call to get_size which sets the size global. */ + static int +-readahead_prepare (nbdkit_next *next, void *handle, int readonly) ++readahead_can_cache (nbdkit_next *next, void *handle) + { +- int64_t r; ++ struct readahead_handle *h = handle; ++ int r; + +- r = readahead_get_size (next, handle); +- return r >= 0 ? 0 : -1; +-} +- +-/* Get the size. */ +-static int64_t +-readahead_get_size (nbdkit_next *next, void *handle) +-{ +- int64_t r; +- +- r = next->get_size (next); ++ /* Call next->can_cache to read the underlying 'can_cache'. */ ++ r = next->can_cache (next); + if (r == -1) + return -1; ++ h->can_cache = r; + +- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); +- size = r; ++ if (!filter_working (h)) { ++ nbdkit_error ("readahead: warning: underlying plugin does not support " ++ "NBD_CMD_CACHE or PARALLEL thread model, so the filter " ++ "won't do anything"); ++ if (suggest_cache_filter (h)) ++ nbdkit_error ("readahead: try adding --filter=cache " ++ "after this filter"); ++ /* This is an error, but that's just to ensure that the warning ++ * above is seen. We don't need to return -1 here. ++ */ ++ } + + return r; + } + +-/* Cache */ +-static int +-readahead_can_cache (nbdkit_next *next, void *handle) +-{ +- /* We are already operating as a cache regardless of the plugin's +- * underlying .can_cache, but it's easiest to just rely on nbdkit's +- * behavior of calling .pread for caching. +- */ +- return NBDKIT_CACHE_EMULATE; +-} +- + /* Read data. */ +- +-static int +-fill_readahead (nbdkit_next *next, +- uint32_t count, uint64_t offset, uint32_t flags, int *err) +-{ +- position = offset; +- +- /* Read at least window bytes, but if count is larger read that. +- * Note that the count cannot be bigger than the buffer size. +- */ +- length = MAX (count, window); +- +- /* Don't go beyond the end of the underlying file. */ +- length = MIN (length, size - position); +- +- /* Grow the buffer if necessary. */ +- if (bufsize < length) { +- char *new_buffer = realloc (buffer, length); +- if (new_buffer == NULL) { +- *err = errno; +- nbdkit_error ("realloc: %m"); +- return -1; +- } +- buffer = new_buffer; +- bufsize = length; +- } +- +- if (next->pread (next, buffer, length, offset, flags, err) == -1) { +- length = 0; /* failed to fill the prefetch buffer */ +- return -1; +- } +- +- return 0; +-} +- + static int + readahead_pread (nbdkit_next *next, + void *handle, void *buf, uint32_t count, uint64_t offset, + uint32_t flags, int *err) + { +- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); ++ struct readahead_handle *h = handle; + +- while (count > 0) { +- if (length == 0) { +- /* We don't have a prefetch buffer at all. This could be the +- * first request or reset after a miss. +- */ +- window = READAHEAD_MIN; +- if (fill_readahead (next, count, offset, flags, err) == -1) +- return -1; +- } ++ /* If the underlying plugin doesn't support caching then skip that ++ * step completely. The filter will do nothing. ++ */ ++ if (filter_working (h)) { ++ struct command ra_cmd = { .type = CMD_CACHE, .next = NULL }; ++ int64_t size; + +- /* Can we satisfy this request partly or entirely from the prefetch +- * buffer? +- */ +- else if (position <= offset && offset < position + length) { +- uint32_t n = MIN (position - offset + length, count); +- memcpy (buf, &buffer[offset-position], n); +- buf += n; +- offset += n; +- count -= n; +- } ++ size = next->get_size (next); ++ if (size >= 0) { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&window_lock); + +- /* Does the request start immediately after the prefetch buffer? +- * This is a “hit” allowing us to double the window size. +- */ +- else if (offset == position + length) { +- window = MIN (window * 2, READAHEAD_MAX); +- if (fill_readahead (next, count, offset, flags, err) == -1) +- return -1; ++ /* Generate the asynchronous (background) cache command for ++ * the readahead window. ++ */ ++ ra_cmd.offset = offset + count; ++ if (ra_cmd.offset < size) { ++ ra_cmd.count = MIN (window, size - ra_cmd.offset); ++ ra_cmd.next = next; /* If .next is non-NULL, we'll send it below. */ ++ } ++ ++ /* Should we change the window size? ++ * If the last readahead < current offset, double the window. ++ * If not, but we're still making forward progress, keep the window. ++ * If we're not making forward progress, reduce the window to minimum. ++ */ ++ if (last_readahead < offset) ++ window = MIN (window * 2, READAHEAD_MAX); ++ else if (last_offset < offset) ++ /* leave window unchanged */ ; ++ else ++ window = READAHEAD_MIN; ++ last_offset = offset; ++ last_readahead = ra_cmd.offset + ra_cmd.count; + } + +- /* Else it's a “miss”. Reset everything and start again. */ +- else +- length = 0; ++ if (ra_cmd.next && ++ send_command_to_background_thread (&h->ctrl, ra_cmd) == -1) ++ return -1; + } + +- return 0; +-} +- +-/* Any writes or write-like operations kill the prefetch buffer. +- * +- * We could do better here, but for the current use case of this +- * filter it doesn't matter. XXX +- */ +- +-static void +-kill_readahead (void) +-{ +- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); +- window = READAHEAD_MIN; +- length = 0; +-} +- +-static int +-readahead_pwrite (nbdkit_next *next, +- void *handle, +- const void *buf, uint32_t count, uint64_t offset, +- uint32_t flags, int *err) +-{ +- kill_readahead (); +- return next->pwrite (next, buf, count, offset, flags, err); +-} +- +-static int +-readahead_trim (nbdkit_next *next, +- void *handle, +- uint32_t count, uint64_t offset, uint32_t flags, +- int *err) +-{ +- kill_readahead (); +- return next->trim (next, count, offset, flags, err); +-} +- +-static int +-readahead_zero (nbdkit_next *next, +- void *handle, +- uint32_t count, uint64_t offset, uint32_t flags, +- int *err) +-{ +- kill_readahead (); +- return next->zero (next, count, offset, flags, err); +-} +- +-static int +-readahead_flush (nbdkit_next *next, +- void *handle, uint32_t flags, int *err) +-{ +- kill_readahead (); +- return next->flush (next, flags, err); ++ /* Issue the synchronous read. */ ++ return next->pread (next, buf, count, offset, flags, err); + } + + static struct nbdkit_filter filter = { + .name = "readahead", + .longname = "nbdkit readahead filter", +- .unload = readahead_unload, +- .prepare = readahead_prepare, +- .get_size = readahead_get_size, ++ .get_ready = readahead_get_ready, ++ .open = readahead_open, ++ .close = readahead_close, + .can_cache = readahead_can_cache, + .pread = readahead_pread, +- .pwrite = readahead_pwrite, +- .trim = readahead_trim, +- .zero = readahead_zero, +- .flush = readahead_flush, + }; + + NBDKIT_REGISTER_FILTER(filter) +diff --git a/filters/readahead/readahead.h b/filters/readahead/readahead.h +new file mode 100644 +index 00000000..a68204d5 +--- /dev/null ++++ b/filters/readahead/readahead.h +@@ -0,0 +1,60 @@ ++/* nbdkit ++ * Copyright (C) 2019-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#ifndef NBDKIT_READAHEAD_H ++#define NBDKIT_READAHEAD_H ++ ++#include ++ ++#include ++ ++#include "vector.h" ++ ++/* List of commands issued to the background thread. */ ++struct command { ++ enum { CMD_QUIT, CMD_CACHE } type; ++ nbdkit_next *next; ++ uint64_t offset; ++ uint32_t count; ++}; ++DEFINE_VECTOR_TYPE(command_queue, struct command); ++ ++struct bgthread_ctrl { ++ command_queue cmds; /* Command queue. */ ++ pthread_mutex_t lock; /* Lock for queue. */ ++ pthread_cond_t cond; /* Condition queue size 0 -> 1. */ ++}; ++ ++/* Start background thread (one per connection). */ ++extern void *readahead_thread (void *vp); ++ ++#endif /* NBDKIT_READAHEAD_H */ +diff --git a/tests/test-readahead.sh b/tests/test-readahead.sh +index 7ec7f8e9..17126e5a 100755 +--- a/tests/test-readahead.sh ++++ b/tests/test-readahead.sh +@@ -59,7 +59,7 @@ for i in range(0, 512*10, 512): + echo $((end_t - start_t)) + } + +-t1=$(test --filter=readahead) ++t1=$(test --filter=readahead --filter=cache) + t2=$(test) + + # In the t1 case we should make only 1 request into the plugin, +-- +2.31.1 + diff --git a/SOURCES/0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch b/SOURCES/0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch deleted file mode 100644 index 498acf6..0000000 --- a/SOURCES/0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch +++ /dev/null @@ -1,140 +0,0 @@ -From edfdfff0dae54a41bbfca30fa60f4fa6438d45b9 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 21 Oct 2021 15:10:00 +0100 -Subject: [PATCH] vddk: Extend -D vddk.stats=1 to show number of calls and - bytes transferred -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -The new output looks like this: - -nbdkit: debug: VDDK function stats (-D vddk.stats=1): -nbdkit: debug: VixDiskLib_... µs calls bytes -nbdkit: debug: Exit 1000854 1 -nbdkit: debug: InitEx 79304 1 -nbdkit: debug: Flush 13577 1 -nbdkit: debug: Write 12534 21 10485760 -nbdkit: debug: Open 4753 3 -nbdkit: debug: Read 966 20 5242880 -nbdkit: debug: Close 574 3 -nbdkit: debug: QueryAllocatedBlocks 116 4 -nbdkit: debug: ConnectEx 103 3 -nbdkit: debug: Disconnect 88 3 -nbdkit: debug: GetTransportMode 68 3 -nbdkit: debug: GetInfo 46 3 -nbdkit: debug: FreeConnectParams 36 3 -nbdkit: debug: FreeInfo 36 3 -nbdkit: debug: FreeBlockList 22 4 -nbdkit: debug: AllocateConnectParams 22 3 -(cherry picked from commit 5c80f0d290db45a679d55baf37ff39bacb8ce7ec) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 3 +-- - plugins/vddk/vddk.c | 41 +++++++++++++++++++++++++---- - 2 files changed, 37 insertions(+), 7 deletions(-) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index 078badcc..e53d3286 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -517,8 +517,7 @@ Suppress debugging of datapath calls (C and C). - - =item B<-D vddk.stats=1> - --When the plugin exits print some statistics about the amount of time --spent waiting on each VDDK call. -+When the plugin exits print some statistics about each VDDK call. - - =back - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 3d751544..5f1d223b 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -106,6 +106,8 @@ static bool is_remote; - struct vddk_stat { - const char *name; /* function name */ - int64_t usecs; /* total number of usecs consumed */ -+ uint64_t calls; /* number of times called */ -+ uint64_t bytes; /* bytes transferred, datapath calls only */ - }; - static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; - static void display_stats (void); -@@ -141,6 +143,17 @@ static void display_stats (void); - gettimeofday (&end_t, NULL); \ - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ - stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ -+ stats_##fn.calls++; \ -+ } \ -+ } while (0) -+#define VDDK_CALL_END_DATAPATH(fn, bytes_) \ -+ while (0); \ -+ if (vddk_debug_stats) { \ -+ gettimeofday (&end_t, NULL); \ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ -+ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ -+ stats_##fn.calls++; \ -+ stats_##fn.bytes += bytes_; \ - } \ - } while (0) - -@@ -191,6 +204,12 @@ stat_compare (const void *vp1, const void *vp2) - else return 0; - } - -+static const char * -+api_name_without_prefix (const char *name) -+{ -+ return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name; -+} -+ - static void - display_stats (void) - { -@@ -206,10 +225,22 @@ display_stats (void) - qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); - - nbdkit_debug ("VDDK function stats (-D vddk.stats=1):"); -- nbdkit_debug ("%-40s %9s", "", "µs"); -+ nbdkit_debug ("%-24s %15s %5s %15s", -+ "VixDiskLib_...", "µs", "calls", "bytes"); - for (i = 0; i < stats.size; ++i) { -- if (stats.ptr[i].usecs) -- nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].name, stats.ptr[i].usecs); -+ if (stats.ptr[i].usecs) { -+ if (stats.ptr[i].bytes > 0) -+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64, -+ api_name_without_prefix (stats.ptr[i].name), -+ stats.ptr[i].usecs, -+ stats.ptr[i].calls, -+ stats.ptr[i].bytes); -+ else -+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64, -+ api_name_without_prefix (stats.ptr[i].name), -+ stats.ptr[i].usecs, -+ stats.ptr[i].calls); -+ } - } - statlist_reset (&stats); - } -@@ -831,7 +862,7 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset, - "%" PRIu32 " sectors, buffer", - offset, count) { - err = VixDiskLib_Read (h->handle, offset, count, buf); -- } VDDK_CALL_END (VixDiskLib_Read); -+ } VDDK_CALL_END_DATAPATH (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_Read"); - return -1; -@@ -871,7 +902,7 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, - "%" PRIu32 " sectors, buffer", - offset, count) { - err = VixDiskLib_Write (h->handle, offset, count, buf); -- } VDDK_CALL_END (VixDiskLib_Write); -+ } VDDK_CALL_END_DATAPATH (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_Write"); - return -1; --- -2.31.1 - diff --git a/SOURCES/0003-readahead-Fix-test.patch b/SOURCES/0003-readahead-Fix-test.patch new file mode 100644 index 0000000..52962b7 --- /dev/null +++ b/SOURCES/0003-readahead-Fix-test.patch @@ -0,0 +1,120 @@ +From b41b7d7ddf6d3fba23ac7978c8b272f2ff84265d Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Apr 2022 16:14:46 +0100 +Subject: [PATCH] readahead: Fix test + +The previous test turned out to be pretty bad at testing the new +filter. A specific problem is that the filter starts a background +thread which issues .cache requests, while on the main connection +.pread requests are being passed through. The test used +--filter=readahead --filter=cache with the cache filter only caching +on .cache requests (since cache-on-read defaults to false), so only +caching requests made by the background thread. + + main thread + client ---- .pread ----- delay-filter -------> plugin + \ + \ background thread + .cache --- cache-filter + +Under very high load, the background thread could be starved. This +means no requests were being cached at all, and all requests were +passing through the delay filter. It would appear that readahead was +failing (which it was, in a way). + +It's not very easy to fix this since readahead is best-effort, but we +can go back to using a simpler plugin that logs reads and caches and +check that they look valid. + +Update: commit 2ff548d66ad3eae87868402ec5b3319edd12090f +(cherry picked from commit db1e3311727c6ecab3264a1811d33db1aa45a4d0) +--- + tests/test-readahead.sh | 61 +++++++++++++++++++++++------------------ + 1 file changed, 35 insertions(+), 26 deletions(-) + +diff --git a/tests/test-readahead.sh b/tests/test-readahead.sh +index 17126e5a..37f4a06f 100755 +--- a/tests/test-readahead.sh ++++ b/tests/test-readahead.sh +@@ -30,43 +30,52 @@ + # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + +-# Is the readahead filter faster? Copy a blank disk with a custom +-# plugin that sleeps on every request. Because the readahead filter +-# should result in fewer requests it should run faster. +- + source ./functions.sh + set -e + set -x + +-requires_filter delay ++requires_plugin sh + requires nbdsh --version + requires dd iflag=count_bytes > readahead.out ++ ;; ++ pread) ++ echo "$@" >> readahead.out ++ dd if=/dev/zero count=$3 iflag=count_bytes ++ ;; ++ *) ++ exit 2 ++ ;; ++esac ++EOF + +- end_t=$SECONDS +- echo $((end_t - start_t)) +-} ++cat readahead.out + +-t1=$(test --filter=readahead --filter=cache) +-t2=$(test) +- +-# In the t1 case we should make only 1 request into the plugin, +-# resulting in around 1 sleep period (5 seconds). In the t2 case we +-# make 10 requests so sleep for around 50 seconds. t1 should be < t2 +-# is every reasonable scenario. +-if [ $t1 -ge $t2 ]; then +- echo "$0: readahead filter took longer, should be shorter" +- exit 1 +-fi ++# We should see the pread requests, and additional cache requests for ++# the 32K region following each pread request. ++for i in `seq 0 512 $((512*10 - 512))` ; do ++ grep "pread 512 $i" readahead.out ++ grep "cache 32768 $((i+512))" readahead.out ++done +-- +2.31.1 + diff --git a/SOURCES/0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch b/SOURCES/0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch deleted file mode 100644 index f7ed950..0000000 --- a/SOURCES/0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch +++ /dev/null @@ -1,320 +0,0 @@ -From cbcf2a2f158a9889bd597b31159ab357dea05cd6 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 21 Oct 2021 22:55:17 +0100 -Subject: [PATCH] vddk: Simplify and consolidate VDDK_CALL_START/END macros - -We don't need the VDDK_CALL_*_DATAPATH versions of these macros -because the compiler is able to optimize some static strcmps. -Furthermore we can remove extra { .. } when the macros are applied. - -(cherry picked from commit 3ea0ed6582faa8f800b7a2a15d58032917a21bd5) ---- - plugins/vddk/vddk.c | 124 ++++++++++++++++++++------------------------ - 1 file changed, 56 insertions(+), 68 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 5f1d223b..993f2d76 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -125,28 +125,16 @@ static void display_stats (void); - #define VDDK_CALL_START(fn, fs, ...) \ - do { \ - struct timeval start_t, end_t; \ -+ /* GCC can optimize this away at compile time: */ \ -+ const bool datapath = \ -+ strcmp (#fn, "VixDiskLib_Read") == 0 || \ -+ strcmp (#fn, "VixDiskLib_Write") == 0; \ - if (vddk_debug_stats) \ - gettimeofday (&start_t, NULL); \ -- nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ -- do --#define VDDK_CALL_START_DATAPATH(fn, fs, ...) \ -- do { \ -- struct timeval start_t, end_t; \ -- if (vddk_debug_stats) \ -- gettimeofday (&start_t, NULL); \ -- if (vddk_debug_datapath) \ -+ if (!datapath || vddk_debug_datapath) \ - nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ - do --#define VDDK_CALL_END(fn) \ -- while (0); \ -- if (vddk_debug_stats) { \ -- gettimeofday (&end_t, NULL); \ -- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ -- stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ -- stats_##fn.calls++; \ -- } \ -- } while (0) --#define VDDK_CALL_END_DATAPATH(fn, bytes_) \ -+#define VDDK_CALL_END(fn, bytes_) \ - while (0); \ - if (vddk_debug_stats) { \ - gettimeofday (&end_t, NULL); \ -@@ -161,13 +149,13 @@ static void display_stats (void); - #define VDDK_ERROR(err, fs, ...) \ - do { \ - char *vddk_err_msg; \ -- VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) { \ -+ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \ - vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \ -- } VDDK_CALL_END (VixDiskLib_GetErrorText); \ -+ VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \ - nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \ -- VDDK_CALL_START (VixDiskLib_FreeErrorText, "") { \ -+ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \ - VixDiskLib_FreeErrorText (vddk_err_msg); \ -- } VDDK_CALL_END (VixDiskLib_FreeErrorText); \ -+ VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \ - } while (0) - - /* Unload the plugin. */ -@@ -175,9 +163,9 @@ static void - vddk_unload (void) - { - if (init_called) { -- VDDK_CALL_START (VixDiskLib_Exit, "") { -+ VDDK_CALL_START (VixDiskLib_Exit, "") - VixDiskLib_Exit (); -- } VDDK_CALL_END (VixDiskLib_Exit); -+ VDDK_CALL_END (VixDiskLib_Exit, 0); - } - if (dl) - dlclose (dl); -@@ -572,13 +560,13 @@ vddk_after_fork (void) - VDDK_CALL_START (VixDiskLib_InitEx, - "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s", - VDDK_MAJOR, VDDK_MINOR, -- libdir, config ? : "NULL") { -+ libdir, config ? : "NULL") - err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR, - &debug_function, /* log function */ - &error_function, /* warn function */ - &error_function, /* panic function */ - libdir, config); -- } VDDK_CALL_END (VixDiskLib_InitEx); -+ VDDK_CALL_END (VixDiskLib_InitEx, 0); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_InitEx"); - exit (EXIT_FAILURE); -@@ -640,9 +628,9 @@ allocate_connect_params (void) - VixDiskLibConnectParams *ret; - - if (VixDiskLib_AllocateConnectParams != NULL) { -- VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "") { -+ VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "") - ret = VixDiskLib_AllocateConnectParams (); -- } VDDK_CALL_END (VixDiskLib_AllocateConnectParams); -+ VDDK_CALL_END (VixDiskLib_AllocateConnectParams, 0); - } - else - ret = calloc (1, sizeof (VixDiskLibConnectParams)); -@@ -657,9 +645,9 @@ free_connect_params (VixDiskLibConnectParams *params) - * originally called. Otherwise use free. - */ - if (VixDiskLib_AllocateConnectParams != NULL) { -- VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params") { -+ VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params") - VixDiskLib_FreeConnectParams (params); -- } VDDK_CALL_END (VixDiskLib_FreeConnectParams); -+ VDDK_CALL_END (VixDiskLib_FreeConnectParams, 0); - } - else - free (params); -@@ -716,13 +704,13 @@ vddk_open (int readonly) - "h->params, %d, %s, %s, &connection", - readonly, - snapshot_moref ? : "NULL", -- transport_modes ? : "NULL") { -+ transport_modes ? : "NULL") - err = VixDiskLib_ConnectEx (h->params, - readonly, - snapshot_moref, - transport_modes, - &h->connection); -- } VDDK_CALL_END (VixDiskLib_ConnectEx); -+ VDDK_CALL_END (VixDiskLib_ConnectEx, 0); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_ConnectEx"); - goto err1; -@@ -743,25 +731,25 @@ vddk_open (int readonly) - } - - VDDK_CALL_START (VixDiskLib_Open, -- "connection, %s, %d, &handle", filename, flags) { -+ "connection, %s, %d, &handle", filename, flags) - err = VixDiskLib_Open (h->connection, filename, flags, &h->handle); -- } VDDK_CALL_END (VixDiskLib_Open); -+ VDDK_CALL_END (VixDiskLib_Open, 0); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_Open: %s", filename); - goto err2; - } - -- VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle") { -+ VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle") - transport_mode = VixDiskLib_GetTransportMode (h->handle); -- } VDDK_CALL_END (VixDiskLib_GetTransportMode); -+ VDDK_CALL_END (VixDiskLib_GetTransportMode, 0); - nbdkit_debug ("transport mode: %s", transport_mode); - - return h; - - err2: -- VDDK_CALL_START (VixDiskLib_Disconnect, "connection") { -+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") - VixDiskLib_Disconnect (h->connection); -- } VDDK_CALL_END (VixDiskLib_Disconnect); -+ VDDK_CALL_END (VixDiskLib_Disconnect, 0); - err1: - free_connect_params (h->params); - err0: -@@ -776,12 +764,12 @@ vddk_close (void *handle) - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&open_close_lock); - struct vddk_handle *h = handle; - -- VDDK_CALL_START (VixDiskLib_Close, "handle") { -+ VDDK_CALL_START (VixDiskLib_Close, "handle") - VixDiskLib_Close (h->handle); -- } VDDK_CALL_END (VixDiskLib_Close); -- VDDK_CALL_START (VixDiskLib_Disconnect, "connection") { -+ VDDK_CALL_END (VixDiskLib_Close, 0); -+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") - VixDiskLib_Disconnect (h->connection); -- } VDDK_CALL_END (VixDiskLib_Disconnect); -+ VDDK_CALL_END (VixDiskLib_Disconnect, 0); - - free_connect_params (h->params); - free (h); -@@ -796,9 +784,9 @@ vddk_get_size (void *handle) - VixError err; - uint64_t size; - -- VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") { -+ VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") - err = VixDiskLib_GetInfo (h->handle, &info); -- } VDDK_CALL_END (VixDiskLib_GetInfo); -+ VDDK_CALL_END (VixDiskLib_GetInfo, 0); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_GetInfo"); - return -1; -@@ -827,9 +815,9 @@ vddk_get_size (void *handle) - info->uuid ? : "NULL"); - } - -- VDDK_CALL_START (VixDiskLib_FreeInfo, "info") { -+ VDDK_CALL_START (VixDiskLib_FreeInfo, "info") - VixDiskLib_FreeInfo (info); -- } VDDK_CALL_END (VixDiskLib_FreeInfo); -+ VDDK_CALL_END (VixDiskLib_FreeInfo, 0); - - return (int64_t) size; - } -@@ -857,12 +845,12 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset, - offset /= VIXDISKLIB_SECTOR_SIZE; - count /= VIXDISKLIB_SECTOR_SIZE; - -- VDDK_CALL_START_DATAPATH (VixDiskLib_Read, -- "handle, %" PRIu64 " sectors, " -- "%" PRIu32 " sectors, buffer", -- offset, count) { -+ VDDK_CALL_START (VixDiskLib_Read, -+ "handle, %" PRIu64 " sectors, " -+ "%" PRIu32 " sectors, buffer", -+ offset, count) - err = VixDiskLib_Read (h->handle, offset, count, buf); -- } VDDK_CALL_END_DATAPATH (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE); -+ VDDK_CALL_END (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_Read"); - return -1; -@@ -897,12 +885,12 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, - offset /= VIXDISKLIB_SECTOR_SIZE; - count /= VIXDISKLIB_SECTOR_SIZE; - -- VDDK_CALL_START_DATAPATH (VixDiskLib_Write, -- "handle, %" PRIu64 " sectors, " -- "%" PRIu32 " sectors, buffer", -- offset, count) { -+ VDDK_CALL_START (VixDiskLib_Write, -+ "handle, %" PRIu64 " sectors, " -+ "%" PRIu32 " sectors, buffer", -+ offset, count) - err = VixDiskLib_Write (h->handle, offset, count, buf); -- } VDDK_CALL_END_DATAPATH (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE); -+ VDDK_CALL_END (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_Write"); - return -1; -@@ -945,9 +933,9 @@ vddk_flush (void *handle, uint32_t flags) - * file so it appears to be the correct call to use here. - */ - -- VDDK_CALL_START (VixDiskLib_Flush, "handle") { -+ VDDK_CALL_START (VixDiskLib_Flush, "handle") - err = VixDiskLib_Flush (h->handle); -- } VDDK_CALL_END (VixDiskLib_Flush); -+ VDDK_CALL_END (VixDiskLib_Flush, 0); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_Flush"); - return -1; -@@ -983,17 +971,17 @@ vddk_can_extents (void *handle) - */ - VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, - "handle, 0, %d sectors, %d sectors", -- VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) { -+ VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) - err = VixDiskLib_QueryAllocatedBlocks (h->handle, - 0, VIXDISKLIB_MIN_CHUNK_SIZE, - VIXDISKLIB_MIN_CHUNK_SIZE, - &block_list); -- } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks); -+ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); - error_suppression = 0; - if (err == VIX_OK) { -- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { -+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") - VixDiskLib_FreeBlockList (block_list); -- } VDDK_CALL_END (VixDiskLib_FreeBlockList); -+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); - } - if (err != VIX_OK) { - char *errmsg = VixDiskLib_GetErrorText (err, NULL); -@@ -1073,12 +1061,12 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags, - VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, - "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, " - "%d sectors", -- start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) { -+ start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) - err = VixDiskLib_QueryAllocatedBlocks (h->handle, - start_sector, nr_sectors, - VIXDISKLIB_MIN_CHUNK_SIZE, - &block_list); -- } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks); -+ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks"); - return -1; -@@ -1097,15 +1085,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags, - add_extent (extents, &position, blk_offset, true) == -1) || - (add_extent (extents, - &position, blk_offset + blk_length, false) == -1)) { -- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { -+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") - VixDiskLib_FreeBlockList (block_list); -- } VDDK_CALL_END (VixDiskLib_FreeBlockList); -+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); - return -1; - } - } -- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { -+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") - VixDiskLib_FreeBlockList (block_list); -- } VDDK_CALL_END (VixDiskLib_FreeBlockList); -+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); - - /* There's an implicit hole after the returned list of blocks, up - * to the end of the QueryAllocatedBlocks request. --- -2.31.1 - diff --git a/SOURCES/0004-New-filter-luks.patch b/SOURCES/0004-New-filter-luks.patch new file mode 100644 index 0000000..5d14e25 --- /dev/null +++ b/SOURCES/0004-New-filter-luks.patch @@ -0,0 +1,1816 @@ +From c19936170cf8b385687cf40f5a9507d87ae08267 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 30 Apr 2022 12:35:07 +0100 +Subject: [PATCH] New filter: luks + +This filter allows you to open, read and write LUKSv1 disk images, +compatible with the ones used by dm-crypt and qemu. + +(cherry picked from commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c) +--- + TODO | 11 +- + configure.ac | 6 +- + docs/nbdkit-tls.pod | 1 + + filters/luks/Makefile.am | 77 ++ + filters/luks/luks.c | 1263 +++++++++++++++++++++++++++ + filters/luks/nbdkit-luks-filter.pod | 120 +++ + plugins/file/nbdkit-file-plugin.pod | 1 + + tests/Makefile.am | 12 + + tests/test-luks-copy.sh | 125 +++ + tests/test-luks-info.sh | 56 ++ + 10 files changed, 1668 insertions(+), 4 deletions(-) + create mode 100644 filters/luks/Makefile.am + create mode 100644 filters/luks/luks.c + create mode 100644 filters/luks/nbdkit-luks-filter.pod + create mode 100755 tests/test-luks-copy.sh + create mode 100755 tests/test-luks-info.sh + +diff --git a/TODO b/TODO +index 4d2a9796..0f5dc41d 100644 +--- a/TODO ++++ b/TODO +@@ -195,9 +195,6 @@ Suggestions for filters + connections. This may even allow a filter to offer a more parallel + threading model than the underlying plugin. + +-* LUKS encrypt/decrypt filter, bonus points if compatible with qemu +- LUKS-encrypted disk images +- + * CBT filter to track dirty blocks. See these links for inspiration: + https://www.cloudandheat.com/block-level-data-tracking-using-davice-mappers-dm-era/ + https://github.com/qemu/qemu/blob/master/docs/interop/bitmaps.rst +@@ -232,6 +229,14 @@ Suggestions for filters + could inject a flush after pausing. However this requires that + filter background threads have access to the plugin (see above). + ++nbdkit-luks-filter: ++ ++* This filter should also support LUKSv2 (and so should qemu). ++ ++* There are some missing features: ESSIV, more ciphers. ++ ++* Implement trim and zero if possible. ++ + nbdkit-readahead-filter: + + * The filter should open a new connection to the plugin per background +diff --git a/configure.ac b/configure.ac +index a402921b..de85b4da 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -127,6 +127,7 @@ filters="\ + ip \ + limit \ + log \ ++ luks \ + multi-conn \ + nocache \ + noextents \ +@@ -614,8 +615,9 @@ PKG_CHECK_MODULES([GNUTLS], [gnutls >= 3.3.0], [ + ], [ + AC_MSG_WARN([gnutls not found or < 3.3.0, TLS support will be disabled.]) + ]) ++AM_CONDITIONAL([HAVE_GNUTLS], [test "x$GNUTLS_LIBS" != "x"]) + +-AS_IF([test "$GNUTLS_LIBS" != ""],[ ++AS_IF([test "x$GNUTLS_LIBS" != "x"],[ + AC_MSG_CHECKING([for default TLS session priority string]) + AC_ARG_WITH([tls-priority], + [AS_HELP_STRING([--with-tls-priority=...], +@@ -1383,6 +1385,7 @@ AC_CONFIG_FILES([Makefile + filters/ip/Makefile + filters/limit/Makefile + filters/log/Makefile ++ filters/luks/Makefile + filters/multi-conn/Makefile + filters/nocache/Makefile + filters/noextents/Makefile +@@ -1481,6 +1484,7 @@ echo "Optional filters:" + echo + feature "ext2" test "x$HAVE_EXT2_TRUE" = "x" + feature "gzip" test "x$HAVE_ZLIB_TRUE" = "x" ++feature "LUKS" test "x$HAVE_GNUTLS_TRUE" != "x" + feature "xz" test "x$HAVE_LIBLZMA_TRUE" = "x" + + echo +diff --git a/docs/nbdkit-tls.pod b/docs/nbdkit-tls.pod +index 86f5f984..4d0dc14c 100644 +--- a/docs/nbdkit-tls.pod ++++ b/docs/nbdkit-tls.pod +@@ -364,6 +364,7 @@ More information can be found in L. + =head1 SEE ALSO + + L, ++L, + L, + L, + L, +diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am +new file mode 100644 +index 00000000..30089621 +--- /dev/null ++++ b/filters/luks/Makefile.am +@@ -0,0 +1,77 @@ ++# nbdkit ++# Copyright (C) 2019-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++include $(top_srcdir)/common-rules.mk ++ ++EXTRA_DIST = nbdkit-luks-filter.pod ++ ++if HAVE_GNUTLS ++ ++filter_LTLIBRARIES = nbdkit-luks-filter.la ++ ++nbdkit_luks_filter_la_SOURCES = \ ++ luks.c \ ++ $(top_srcdir)/include/nbdkit-filter.h \ ++ $(NULL) ++ ++nbdkit_luks_filter_la_CPPFLAGS = \ ++ -I$(top_srcdir)/include \ ++ -I$(top_srcdir)/common/include \ ++ -I$(top_srcdir)/common/utils \ ++ $(NULL) ++nbdkit_luks_filter_la_CFLAGS = \ ++ $(WARNINGS_CFLAGS) \ ++ $(GNUTLS_CFLAGS) \ ++ $(NULL) ++nbdkit_luks_filter_la_LIBADD = \ ++ $(top_builddir)/common/utils/libutils.la \ ++ $(IMPORT_LIBRARY_ON_WINDOWS) \ ++ $(GNUTLS_LIBS) \ ++ $(NULL) ++nbdkit_luks_filter_la_LDFLAGS = \ ++ -module -avoid-version -shared $(NO_UNDEFINED_ON_WINDOWS) \ ++ -Wl,--version-script=$(top_srcdir)/filters/filters.syms \ ++ $(NULL) ++ ++if HAVE_POD ++ ++man_MANS = nbdkit-luks-filter.1 ++CLEANFILES += $(man_MANS) ++ ++nbdkit-luks-filter.1: nbdkit-luks-filter.pod \ ++ $(top_builddir)/podwrapper.pl ++ $(PODWRAPPER) --section=1 --man $@ \ ++ --html $(top_builddir)/html/$@.html \ ++ $< ++ ++endif HAVE_POD ++ ++endif +diff --git a/filters/luks/luks.c b/filters/luks/luks.c +new file mode 100644 +index 00000000..706a9bd2 +--- /dev/null ++++ b/filters/luks/luks.c +@@ -0,0 +1,1263 @@ ++/* nbdkit ++ * Copyright (C) 2018-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include ++ ++#include "byte-swapping.h" ++#include "cleanup.h" ++#include "isaligned.h" ++#include "minmax.h" ++#include "rounding.h" ++ ++/* LUKSv1 constants. */ ++#define LUKS_MAGIC { 'L', 'U', 'K', 'S', 0xBA, 0xBE } ++#define LUKS_MAGIC_LEN 6 ++#define LUKS_DIGESTSIZE 20 ++#define LUKS_SALTSIZE 32 ++#define LUKS_NUMKEYS 8 ++#define LUKS_KEY_DISABLED 0x0000DEAD ++#define LUKS_KEY_ENABLED 0x00AC71F3 ++#define LUKS_STRIPES 4000 ++#define LUKS_ALIGN_KEYSLOTS 4096 ++#define LUKS_SECTOR_SIZE 512 ++ ++/* Key slot. */ ++struct luks_keyslot { ++ uint32_t active; /* LUKS_KEY_DISABLED|LUKS_KEY_ENABLED */ ++ uint32_t password_iterations; ++ char password_salt[LUKS_SALTSIZE]; ++ uint32_t key_material_offset; ++ uint32_t stripes; ++} __attribute__((__packed__)); ++ ++/* LUKS superblock. */ ++struct luks_phdr { ++ char magic[LUKS_MAGIC_LEN]; /* LUKS_MAGIC */ ++ uint16_t version; /* Only 1 is supported. */ ++ char cipher_name[32]; ++ char cipher_mode[32]; ++ char hash_spec[32]; ++ uint32_t payload_offset; ++ uint32_t master_key_len; ++ uint8_t master_key_digest[LUKS_DIGESTSIZE]; ++ uint8_t master_key_salt[LUKS_SALTSIZE]; ++ uint32_t master_key_digest_iterations; ++ uint8_t uuid[40]; ++ ++ struct luks_keyslot keyslot[LUKS_NUMKEYS]; /* Key slots. */ ++} __attribute__((__packed__)); ++ ++static char *passphrase = NULL; ++ ++static void ++luks_unload (void) ++{ ++ /* XXX We should really store the passphrase (and master key) ++ * in mlock-ed memory. ++ */ ++ if (passphrase) { ++ memset (passphrase, 0, strlen (passphrase)); ++ free (passphrase); ++ } ++} ++ ++static int ++luks_thread_model (void) ++{ ++ return NBDKIT_THREAD_MODEL_PARALLEL; ++} ++ ++static int ++luks_config (nbdkit_next_config *next, nbdkit_backend *nxdata, ++ const char *key, const char *value) ++{ ++ if (strcmp (key, "passphrase") == 0) { ++ if (nbdkit_read_password (value, &passphrase) == -1) ++ return -1; ++ return 0; ++ } ++ ++ return next (nxdata, key, value); ++} ++ ++static int ++luks_config_complete (nbdkit_next_config_complete *next, nbdkit_backend *nxdata) ++{ ++ if (passphrase == NULL) { ++ nbdkit_error ("LUKS \"passphrase\" parameter is missing"); ++ return -1; ++ } ++ return next (nxdata); ++} ++ ++#define luks_config_help \ ++ "passphrase= Secret passphrase." ++ ++enum cipher_mode { ++ CIPHER_MODE_ECB, CIPHER_MODE_CBC, CIPHER_MODE_XTS, CIPHER_MODE_CTR, ++}; ++ ++static enum cipher_mode ++lookup_cipher_mode (const char *str) ++{ ++ if (strcmp (str, "ecb") == 0) ++ return CIPHER_MODE_ECB; ++ if (strcmp (str, "cbc") == 0) ++ return CIPHER_MODE_CBC; ++ if (strcmp (str, "xts") == 0) ++ return CIPHER_MODE_XTS; ++ if (strcmp (str, "ctr") == 0) ++ return CIPHER_MODE_CTR; ++ nbdkit_error ("unknown cipher mode: %s " ++ "(expecting \"ecb\", \"cbc\", \"xts\" or \"ctr\")", str); ++ return -1; ++} ++ ++static const char * ++cipher_mode_to_string (enum cipher_mode v) ++{ ++ switch (v) { ++ case CIPHER_MODE_ECB: return "ecb"; ++ case CIPHER_MODE_CBC: return "cbc"; ++ case CIPHER_MODE_XTS: return "xts"; ++ case CIPHER_MODE_CTR: return "ctr"; ++ default: abort (); ++ } ++} ++ ++enum ivgen { ++ IVGEN_PLAIN, IVGEN_PLAIN64, /* IVGEN_ESSIV, */ ++}; ++ ++static enum ivgen ++lookup_ivgen (const char *str) ++{ ++ if (strcmp (str, "plain") == 0) ++ return IVGEN_PLAIN; ++ if (strcmp (str, "plain64") == 0) ++ return IVGEN_PLAIN64; ++/* ++ if (strcmp (str, "essiv") == 0) ++ return IVGEN_ESSIV; ++*/ ++ nbdkit_error ("unknown IV generation algorithm: %s " ++ "(expecting \"plain\", \"plain64\" etc)", str); ++ return -1; ++} ++ ++static const char * ++ivgen_to_string (enum ivgen v) ++{ ++ switch (v) { ++ case IVGEN_PLAIN: return "plain"; ++ case IVGEN_PLAIN64: return "plain64"; ++ /*case IVGEN_ESSIV: return "essiv";*/ ++ default: abort (); ++ } ++} ++ ++static void ++calculate_iv (enum ivgen v, uint8_t *iv, size_t ivlen, uint64_t sector) ++{ ++ size_t prefixlen; ++ uint32_t sector32; ++ ++ switch (v) { ++ case IVGEN_PLAIN: ++ prefixlen = 4; /* 32 bits */ ++ if (prefixlen > ivlen) ++ prefixlen = ivlen; ++ sector32 = (uint32_t) sector; /* truncate to only lower bits */ ++ sector32 = htole32 (sector32); ++ memcpy (iv, §or32, prefixlen); ++ memset (iv + prefixlen, 0, ivlen - prefixlen); ++ break; ++ ++ case IVGEN_PLAIN64: ++ prefixlen = 8; /* 64 bits */ ++ if (prefixlen > ivlen) ++ prefixlen = ivlen; ++ sector = htole64 (sector); ++ memcpy (iv, §or, prefixlen); ++ memset (iv + prefixlen, 0, ivlen - prefixlen); ++ break; ++ ++ /*case IVGEN_ESSIV:*/ ++ default: abort (); ++ } ++} ++ ++enum cipher_alg { ++ CIPHER_ALG_AES_128, CIPHER_ALG_AES_192, CIPHER_ALG_AES_256, ++}; ++ ++static enum cipher_alg ++lookup_cipher_alg (const char *str, enum cipher_mode mode, int key_bytes) ++{ ++ if (mode == CIPHER_MODE_XTS) ++ key_bytes /= 2; ++ ++ if (strcmp (str, "aes") == 0) { ++ if (key_bytes == 16) ++ return CIPHER_ALG_AES_128; ++ if (key_bytes == 24) ++ return CIPHER_ALG_AES_192; ++ if (key_bytes == 32) ++ return CIPHER_ALG_AES_256; ++ } ++ nbdkit_error ("unknown cipher algorithm: %s (expecting \"aes\", etc)", str); ++ return -1; ++} ++ ++static const char * ++cipher_alg_to_string (enum cipher_alg v) ++{ ++ switch (v) { ++ case CIPHER_ALG_AES_128: return "aes-128"; ++ case CIPHER_ALG_AES_192: return "aes-192"; ++ case CIPHER_ALG_AES_256: return "aes-256"; ++ default: abort (); ++ } ++} ++ ++#if 0 ++static int ++cipher_alg_key_bytes (enum cipher_alg v) ++{ ++ switch (v) { ++ case CIPHER_ALG_AES_128: return 16; ++ case CIPHER_ALG_AES_192: return 24; ++ case CIPHER_ALG_AES_256: return 32; ++ default: abort (); ++ } ++} ++#endif ++ ++static int ++cipher_alg_iv_len (enum cipher_alg v, enum cipher_mode mode) ++{ ++ if (CIPHER_MODE_ECB) ++ return 0; /* Don't need an IV in this mode. */ ++ ++ switch (v) { ++ case CIPHER_ALG_AES_128: ++ case CIPHER_ALG_AES_192: ++ case CIPHER_ALG_AES_256: ++ return 16; ++ default: abort (); ++ } ++} ++ ++static gnutls_digest_algorithm_t ++lookup_hash (const char *str) ++{ ++ if (strcmp (str, "md5") == 0) ++ return GNUTLS_DIG_MD5; ++ if (strcmp (str, "sha1") == 0) ++ return GNUTLS_DIG_SHA1; ++ if (strcmp (str, "sha224") == 0) ++ return GNUTLS_DIG_SHA224; ++ if (strcmp (str, "sha256") == 0) ++ return GNUTLS_DIG_SHA256; ++ if (strcmp (str, "sha384") == 0) ++ return GNUTLS_DIG_SHA384; ++ if (strcmp (str, "sha512") == 0) ++ return GNUTLS_DIG_SHA512; ++ if (strcmp (str, "ripemd160") == 0) ++ return GNUTLS_DIG_RMD160; ++ nbdkit_error ("unknown hash algorithm: %s " ++ "(expecting \"md5\", \"sha1\", \"sha224\", etc)", str); ++ return -1; ++} ++ ++static const char * ++hash_to_string (gnutls_digest_algorithm_t v) ++{ ++ switch (v) { ++ case GNUTLS_DIG_UNKNOWN: return "unknown"; ++ case GNUTLS_DIG_MD5: return "md5"; ++ case GNUTLS_DIG_SHA1: return "sha1"; ++ case GNUTLS_DIG_SHA224: return "sha224"; ++ case GNUTLS_DIG_SHA256: return "sha256"; ++ case GNUTLS_DIG_SHA384: return "sha384"; ++ case GNUTLS_DIG_SHA512: return "sha512"; ++ case GNUTLS_DIG_RMD160: return "ripemd160"; ++ default: abort (); ++ } ++} ++ ++#if 0 ++/* See qemu & dm-crypt implementations for an explanation of what's ++ * going on here. ++ */ ++static enum cipher_alg ++lookup_essiv_cipher (enum cipher_alg cipher_alg, ++ gnutls_digest_algorithm_t ivgen_hash_alg) ++{ ++ int digest_bytes = gnutls_hash_get_len (ivgen_hash_alg); ++ int key_bytes = cipher_alg_key_bytes (cipher_alg); ++ ++ if (digest_bytes == key_bytes) ++ return cipher_alg; ++ ++ switch (cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ case CIPHER_ALG_AES_192: ++ case CIPHER_ALG_AES_256: ++ if (digest_bytes == 16) return CIPHER_ALG_AES_128; ++ if (digest_bytes == 24) return CIPHER_ALG_AES_192; ++ if (digest_bytes == 32) return CIPHER_ALG_AES_256; ++ nbdkit_error ("no %s cipher available with key size %d", ++ "AES", digest_bytes); ++ return -1; ++ default: ++ nbdkit_error ("ESSIV does not support cipher %s", ++ cipher_alg_to_string (cipher_alg)); ++ return -1; ++ } ++} ++#endif ++ ++/* Per-connection handle. */ ++struct handle { ++ /* LUKS header, if necessary byte-swapped into host order. */ ++ struct luks_phdr phdr; ++ ++ /* Decoded algorithm etc. */ ++ enum cipher_alg cipher_alg; ++ enum cipher_mode cipher_mode; ++ gnutls_digest_algorithm_t hash_alg; ++ enum ivgen ivgen_alg; ++ gnutls_digest_algorithm_t ivgen_hash_alg; ++ enum cipher_alg ivgen_cipher_alg; ++ ++ /* GnuTLS algorithm. */ ++ gnutls_cipher_algorithm_t gnutls_cipher; ++ ++ /* If we managed to decrypt one of the keyslots using the passphrase ++ * then this contains the master key, otherwise NULL. ++ */ ++ uint8_t *masterkey; ++}; ++ ++static void * ++luks_open (nbdkit_next_open *next, nbdkit_context *nxdata, ++ int readonly, const char *exportname, int is_tls) ++{ ++ struct handle *h; ++ ++ if (next (nxdata, readonly, exportname) == -1) ++ return NULL; ++ ++ h = calloc (1, sizeof *h); ++ if (h == NULL) { ++ nbdkit_error ("calloc: %m"); ++ return NULL; ++ } ++ ++ return h; ++} ++ ++static void ++luks_close (void *handle) ++{ ++ struct handle *h = handle; ++ ++ if (h->masterkey) { ++ memset (h->masterkey, 0, h->phdr.master_key_len); ++ free (h->masterkey); ++ } ++ free (h); ++} ++ ++/* Perform decryption of a block of data in memory. */ ++static int ++do_decrypt (struct handle *h, gnutls_cipher_hd_t cipher, ++ uint64_t offset, uint8_t *buf, size_t len) ++{ ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ uint64_t sector = offset / LUKS_SECTOR_SIZE; ++ CLEANUP_FREE uint8_t *iv = malloc (ivlen); ++ int r; ++ ++ assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); ++ assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); ++ ++ while (len) { ++ calculate_iv (h->ivgen_alg, iv, ivlen, sector); ++ gnutls_cipher_set_iv (cipher, iv, ivlen); ++ r = gnutls_cipher_decrypt2 (cipher, ++ buf, LUKS_SECTOR_SIZE, /* ciphertext */ ++ buf, LUKS_SECTOR_SIZE /* plaintext */); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ buf += LUKS_SECTOR_SIZE; ++ offset += LUKS_SECTOR_SIZE; ++ len -= LUKS_SECTOR_SIZE; ++ sector++; ++ } ++ ++ return 0; ++} ++ ++/* Perform encryption of a block of data in memory. */ ++static int ++do_encrypt (struct handle *h, gnutls_cipher_hd_t cipher, ++ uint64_t offset, uint8_t *buf, size_t len) ++{ ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ uint64_t sector = offset / LUKS_SECTOR_SIZE; ++ CLEANUP_FREE uint8_t *iv = malloc (ivlen); ++ int r; ++ ++ assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); ++ assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); ++ ++ while (len) { ++ calculate_iv (h->ivgen_alg, iv, ivlen, sector); ++ gnutls_cipher_set_iv (cipher, iv, ivlen); ++ r = gnutls_cipher_encrypt2 (cipher, ++ buf, LUKS_SECTOR_SIZE, /* plaintext */ ++ buf, LUKS_SECTOR_SIZE /* ciphertext */); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ buf += LUKS_SECTOR_SIZE; ++ offset += LUKS_SECTOR_SIZE; ++ len -= LUKS_SECTOR_SIZE; ++ sector++; ++ } ++ ++ return 0; ++} ++ ++/* Parse the header fields containing cipher algorithm, mode, etc. */ ++static int ++parse_cipher_strings (struct handle *h) ++{ ++ char cipher_name[33], cipher_mode[33], hash_spec[33]; ++ char *ivgen, *ivhash; ++ ++ /* Copy the header fields locally and ensure they are \0 terminated. */ ++ memcpy (cipher_name, h->phdr.cipher_name, 32); ++ cipher_name[32] = 0; ++ memcpy (cipher_mode, h->phdr.cipher_mode, 32); ++ cipher_mode[32] = 0; ++ memcpy (hash_spec, h->phdr.hash_spec, 32); ++ hash_spec[32] = 0; ++ ++ nbdkit_debug ("LUKS v%" PRIu16 " cipher: %s mode: %s hash: %s " ++ "master key: %" PRIu32 " bits", ++ h->phdr.version, cipher_name, cipher_mode, hash_spec, ++ h->phdr.master_key_len * 8); ++ ++ /* The cipher_mode header has the form: "ciphermode-ivgen[:ivhash]" ++ * QEmu writes: "xts-plain64" ++ */ ++ ivgen = strchr (cipher_mode, '-'); ++ if (!ivgen) { ++ nbdkit_error ("incorrect cipher_mode header, " ++ "expecting mode-ivgenerator but got \"%s\"", cipher_mode); ++ return -1; ++ } ++ *ivgen = '\0'; ++ ivgen++; ++ ++ ivhash = strchr (ivgen, ':'); ++ if (!ivhash) ++ h->ivgen_hash_alg = GNUTLS_DIG_UNKNOWN; ++ else { ++ *ivhash = '\0'; ++ ivhash++; ++ ++ h->ivgen_hash_alg = lookup_hash (ivhash); ++ if (h->ivgen_hash_alg == -1) ++ return -1; ++ } ++ ++ h->cipher_mode = lookup_cipher_mode (cipher_mode); ++ if (h->cipher_mode == -1) ++ return -1; ++ ++ h->cipher_alg = lookup_cipher_alg (cipher_name, h->cipher_mode, ++ h->phdr.master_key_len); ++ if (h->cipher_alg == -1) ++ return -1; ++ ++ h->hash_alg = lookup_hash (hash_spec); ++ if (h->hash_alg == -1) ++ return -1; ++ ++ h->ivgen_alg = lookup_ivgen (ivgen); ++ if (h->ivgen_alg == -1) ++ return -1; ++ ++#if 0 ++ if (h->ivgen_alg == IVGEN_ESSIV) { ++ if (!ivhash) { ++ nbdkit_error ("incorrect IV generator hash specification"); ++ return -1; ++ } ++ h->ivgen_cipher_alg = lookup_essiv_cipher (h->cipher_alg, ++ h->ivgen_hash_alg); ++ if (h->ivgen_cipher_alg == -1) ++ return -1; ++ } ++ else ++#endif ++ h->ivgen_cipher_alg = h->cipher_alg; ++ ++ nbdkit_debug ("LUKS parsed ciphers: %s %s %s %s %s %s", ++ cipher_alg_to_string (h->cipher_alg), ++ cipher_mode_to_string (h->cipher_mode), ++ hash_to_string (h->hash_alg), ++ ivgen_to_string (h->ivgen_alg), ++ hash_to_string (h->ivgen_hash_alg), ++ cipher_alg_to_string (h->ivgen_cipher_alg)); ++ ++ /* GnuTLS combines cipher and block mode into a single value. Not ++ * all possible combinations are available in GnuTLS. See: ++ * https://www.gnutls.org/manual/html_node/Supported-ciphersuites.html ++ */ ++ h->gnutls_cipher = GNUTLS_CIPHER_NULL; ++ switch (h->cipher_mode) { ++ case CIPHER_MODE_XTS: ++ switch (h->cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; ++ break; ++ case CIPHER_ALG_AES_256: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; ++ break; ++ default: break; ++ } ++ break; ++ case CIPHER_MODE_CBC: ++ switch (h->cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_CBC; ++ break; ++ case CIPHER_ALG_AES_192: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_192_CBC; ++ break; ++ case CIPHER_ALG_AES_256: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_CBC; ++ break; ++ default: break; ++ } ++ default: break; ++ } ++ if (h->gnutls_cipher == GNUTLS_CIPHER_NULL) { ++ nbdkit_error ("cipher algorithm %s in mode %s is not supported by GnuTLS", ++ cipher_alg_to_string (h->cipher_alg), ++ cipher_mode_to_string (h->cipher_mode)); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++/* Anti-Forensic merge operation. */ ++static void ++xor (const uint8_t *in1, const uint8_t *in2, uint8_t *out, size_t len) ++{ ++ size_t i; ++ ++ for (i = 0; i < len; ++i) ++ out[i] = in1[i] ^ in2[i]; ++} ++ ++static int ++af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) ++{ ++ size_t digest_bytes = gnutls_hash_get_len (hash_alg); ++ size_t nr_blocks, last_block_len; ++ size_t i; ++ CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); ++ int r; ++ gnutls_hash_hd_t hash; ++ ++ nr_blocks = len / digest_bytes; ++ last_block_len = len % digest_bytes; ++ if (last_block_len != 0) ++ nr_blocks++; ++ else ++ last_block_len = digest_bytes; ++ ++ for (i = 0; i < nr_blocks; ++i) { ++ const uint32_t iv = htobe32 (i); ++ const size_t blen = i < nr_blocks - 1 ? digest_bytes : last_block_len; ++ ++ /* Hash iv + i'th block into temp. */ ++ r = gnutls_hash_init (&hash, hash_alg); ++ if (r != 0) { ++ nbdkit_error ("gnutls_hash_init: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ gnutls_hash (hash, &iv, sizeof iv); ++ gnutls_hash (hash, &block[i*digest_bytes], blen); ++ gnutls_hash_deinit (hash, temp); ++ ++ memcpy (&block[i*digest_bytes], temp, blen); ++ } ++ ++ return 0; ++} ++ ++static int ++afmerge (gnutls_digest_algorithm_t hash_alg, uint32_t stripes, ++ const uint8_t *in, uint8_t *out, size_t outlen) ++{ ++ CLEANUP_FREE uint8_t *block = calloc (1, outlen); ++ size_t i; ++ ++ /* NB: input size is stripes * master_key_len where ++ * master_key_len == outlen ++ */ ++ for (i = 0; i < stripes-1; ++i) { ++ xor (&in[i*outlen], block, block, outlen); ++ if (af_hash (hash_alg, block, outlen) == -1) ++ return -1; ++ } ++ xor (&in[i*outlen], block, out, outlen); ++ return 0; ++} ++ ++/* Length of key material in key slot i (sectors). ++ * ++ * This is basically copied from qemu because the spec description is ++ * unintelligible and apparently doesn't match reality. ++ */ ++static uint64_t ++key_material_length_in_sectors (struct handle *h, size_t i) ++{ ++ uint64_t len, r; ++ ++ len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; ++ r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); ++ r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); ++ return r; ++} ++ ++/* Try the passphrase in key slot i. If this returns true then the ++ * passphrase was able to decrypt the master key, and the master key ++ * has been stored in h->masterkey. ++ */ ++static int ++try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) ++{ ++ struct luks_keyslot *ks = &h->phdr.keyslot[i]; ++ size_t split_key_len; ++ CLEANUP_FREE uint8_t *split_key = NULL; ++ CLEANUP_FREE uint8_t *masterkey = NULL; ++ const gnutls_datum_t key = ++ { (unsigned char *) passphrase, strlen (passphrase) }; ++ const gnutls_datum_t salt = ++ { (unsigned char *) ks->password_salt, LUKS_SALTSIZE }; ++ const gnutls_datum_t msalt = ++ { (unsigned char *) h->phdr.master_key_salt, LUKS_SALTSIZE }; ++ gnutls_datum_t mkey; ++ gnutls_cipher_hd_t cipher; ++ int r, err = 0; ++ uint64_t start; ++ uint8_t key_digest[LUKS_DIGESTSIZE]; ++ ++ if (ks->active != LUKS_KEY_ENABLED) ++ return 0; ++ ++ split_key_len = h->phdr.master_key_len * ks->stripes; ++ split_key = malloc (split_key_len); ++ if (split_key == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ masterkey = malloc (h->phdr.master_key_len); ++ if (masterkey == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ ++ /* Hash the passphrase to make a possible masterkey. */ ++ r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, ++ masterkey, h->phdr.master_key_len); ++ if (r != 0) { ++ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ /* Read master key material from plugin. */ ++ start = ks->key_material_offset * LUKS_SECTOR_SIZE; ++ if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { ++ errno = err; ++ return -1; ++ } ++ ++ /* Decrypt the (still AFsplit) master key material. */ ++ mkey.data = (unsigned char *) masterkey; ++ mkey.size = h->phdr.master_key_len; ++ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ r = do_decrypt (h, cipher, 0, split_key, split_key_len); ++ gnutls_cipher_deinit (cipher); ++ if (r == -1) ++ return -1; ++ ++ /* Decode AFsplit key to a possible masterkey. */ ++ if (afmerge (h->hash_alg, ks->stripes, split_key, ++ masterkey, h->phdr.master_key_len) == -1) ++ return -1; ++ ++ /* Check if the masterkey is correct by comparing hash of the ++ * masterkey with LUKS header. ++ */ ++ r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt, ++ h->phdr.master_key_digest_iterations, ++ key_digest, LUKS_DIGESTSIZE); ++ if (r != 0) { ++ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ if (memcmp (key_digest, h->phdr.master_key_digest, LUKS_DIGESTSIZE) == 0) { ++ /* The passphrase is correct so save the master key in the handle. */ ++ h->masterkey = malloc (h->phdr.master_key_len); ++ if (h->masterkey == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ memcpy (h->masterkey, masterkey, h->phdr.master_key_len); ++ return 1; ++ } ++ ++ return 0; ++} ++ ++static int ++luks_prepare (nbdkit_next *next, void *handle, int readonly) ++{ ++ static const char expected_magic[] = LUKS_MAGIC; ++ struct handle *h = handle; ++ int64_t size; ++ int err = 0, r; ++ size_t i; ++ struct luks_keyslot *ks; ++ char uuid[41]; ++ ++ /* Check we haven't been called before, this should never happen. */ ++ assert (h->phdr.version == 0); ++ ++ /* Check the struct size matches the documentation. */ ++ assert (sizeof (struct luks_phdr) == 592); ++ ++ /* Check this is a LUKSv1 disk. */ ++ size = next->get_size (next); ++ if (size == -1) ++ return -1; ++ if (size < 16384) { ++ nbdkit_error ("disk is too small to be LUKS-encrypted"); ++ return -1; ++ } ++ ++ /* Read the phdr. */ ++ if (next->pread (next, &h->phdr, sizeof h->phdr, 0, 0, &err) == -1) { ++ errno = err; ++ return -1; ++ } ++ ++ if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { ++ nbdkit_error ("this disk does not contain a LUKS header"); ++ return -1; ++ } ++ h->phdr.version = be16toh (h->phdr.version); ++ if (h->phdr.version != 1) { ++ nbdkit_error ("this disk contains a LUKS version %" PRIu16 " header, " ++ "but this filter only supports LUKSv1", ++ h->phdr.version); ++ return -1; ++ } ++ ++ /* Byte-swap the rest of the header. */ ++ h->phdr.payload_offset = be32toh (h->phdr.payload_offset); ++ h->phdr.master_key_len = be32toh (h->phdr.master_key_len); ++ h->phdr.master_key_digest_iterations = ++ be32toh (h->phdr.master_key_digest_iterations); ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ ks = &h->phdr.keyslot[i]; ++ ks->active = be32toh (ks->active); ++ ks->password_iterations = be32toh (ks->password_iterations); ++ ks->key_material_offset = be32toh (ks->key_material_offset); ++ ks->stripes = be32toh (ks->stripes); ++ } ++ ++ /* Sanity check some fields. */ ++ if (h->phdr.payload_offset >= size / LUKS_SECTOR_SIZE) { ++ nbdkit_error ("bad LUKSv1 header: payload offset points beyond " ++ "the end of the disk"); ++ return -1; ++ } ++ ++ /* We derive several allocations from master_key_len so make sure ++ * it's not insane. ++ */ ++ if (h->phdr.master_key_len > 1024) { ++ nbdkit_error ("bad LUKSv1 header: master key is too long"); ++ return -1; ++ } ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ uint64_t start, len; ++ ++ ks = &h->phdr.keyslot[i]; ++ switch (ks->active) { ++ case LUKS_KEY_ENABLED: ++ if (!ks->stripes) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu is corrupted", i); ++ return -1; ++ } ++ if (ks->stripes >= 10000) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu stripes too large", i); ++ return -1; ++ } ++ start = ks->key_material_offset; ++ len = key_material_length_in_sectors (h, i); ++ if (len > 4096) /* bound it at something reasonable */ { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu key material length " ++ "is too large", i); ++ return -1; ++ } ++ if (start * LUKS_SECTOR_SIZE >= size || ++ (start + len) * LUKS_SECTOR_SIZE >= size) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu key material offset " ++ "points beyond the end of the disk", i); ++ return -1; ++ } ++ if (ks->password_iterations > ULONG_MAX) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu " ++ "iterations too large", i); ++ return -1; ++ } ++ /*FALLTHROUGH*/ ++ case LUKS_KEY_DISABLED: ++ break; ++ ++ default: ++ nbdkit_error ("bad LUKSv1 header: key slot %zu has " ++ "an invalid active flag", i); ++ return -1; ++ } ++ } ++ ++ /* Decode the ciphers. */ ++ if (parse_cipher_strings (h) == -1) ++ return -1; ++ ++ /* Dump some information about the header. */ ++ memcpy (uuid, h->phdr.uuid, 40); ++ uuid[40] = 0; ++ nbdkit_debug ("LUKS UUID: %s", uuid); ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ uint64_t start, len; ++ ++ ks = &h->phdr.keyslot[i]; ++ if (ks->active == LUKS_KEY_ENABLED) { ++ start = ks->key_material_offset; ++ len = key_material_length_in_sectors (h, i); ++ nbdkit_debug ("LUKS key slot %zu: key material in sectors %" PRIu64 ++ "..%" PRIu64, ++ i, start, start+len-1); ++ } ++ } ++ ++ /* Now try to unlock the master key. */ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ r = try_passphrase_in_keyslot (next, h, i); ++ if (r == -1) ++ return -1; ++ if (r > 0) ++ goto unlocked; ++ } ++ nbdkit_error ("LUKS passphrase is not correct, " ++ "no key slot could be unlocked"); ++ return -1; ++ ++ unlocked: ++ assert (h->masterkey != NULL); ++ nbdkit_debug ("LUKS unlocked block device with passphrase"); ++ ++ return 0; ++} ++ ++static int64_t ++luks_get_size (nbdkit_next *next, void *handle) ++{ ++ struct handle *h = handle; ++ int64_t size; ++ ++ /* Check that prepare has been called already. */ ++ assert (h->phdr.version > 0); ++ ++ size = next->get_size (next); ++ if (size == -1) ++ return -1; ++ ++ if (size < h->phdr.payload_offset * LUKS_SECTOR_SIZE) { ++ nbdkit_error ("disk too small, or contains an incomplete LUKS partition"); ++ return -1; ++ } ++ ++ size -= h->phdr.payload_offset * LUKS_SECTOR_SIZE; ++ return size; ++} ++ ++/* Whatever the plugin says, several operations are not supported by ++ * this filter: ++ * ++ * - extents ++ * - trim ++ * - zero ++ */ ++static int ++luks_can_extents (nbdkit_next *next, void *handle) ++{ ++ return 0; ++} ++ ++static int ++luks_can_trim (nbdkit_next *next, void *handle) ++{ ++ return 0; ++} ++ ++static int ++luks_can_zero (nbdkit_next *next, void *handle) ++{ ++ return NBDKIT_ZERO_EMULATE; ++} ++ ++static int ++luks_can_fast_zero (nbdkit_next *next, void *handle) ++{ ++ return 0; ++} ++ ++/* Rely on nbdkit to call .pread to emulate .cache calls. We will ++ * respond by decrypting the block which could be stored by the cache ++ * filter or similar on top. ++ */ ++static int ++luks_can_cache (nbdkit_next *next, void *handle) ++{ ++ return NBDKIT_CACHE_EMULATE; ++} ++ ++/* Advertise minimum/preferred sector-sized blocks, although we can in ++ * fact handle any read or write. ++ */ ++static int ++luks_block_size (nbdkit_next *next, void *handle, ++ uint32_t *minimum, uint32_t *preferred, uint32_t *maximum) ++{ ++ if (next->block_size (next, minimum, preferred, maximum) == -1) ++ return -1; ++ ++ if (*minimum == 0) { /* No constraints set by the plugin. */ ++ *minimum = LUKS_SECTOR_SIZE; ++ *preferred = LUKS_SECTOR_SIZE; ++ *maximum = 0xffffffff; ++ } ++ else { ++ *minimum = MAX (*minimum, LUKS_SECTOR_SIZE); ++ *preferred = MAX (*minimum, MAX (*preferred, LUKS_SECTOR_SIZE)); ++ } ++ return 0; ++} ++ ++/* Decrypt data. */ ++static int ++luks_pread (nbdkit_next *next, void *handle, ++ void *buf, uint32_t count, uint64_t offset, ++ uint32_t flags, int *err) ++{ ++ struct handle *h = handle; ++ const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; ++ CLEANUP_FREE uint8_t *sector = NULL; ++ uint64_t sectnum, sectoffs; ++ const gnutls_datum_t mkey = ++ { (unsigned char *) h->masterkey, h->phdr.master_key_len }; ++ gnutls_cipher_hd_t cipher; ++ int r; ++ ++ if (!h->masterkey) { ++ *err = EIO; ++ return -1; ++ } ++ ++ if (!IS_ALIGNED (count | offset, LUKS_SECTOR_SIZE)) { ++ sector = malloc (LUKS_SECTOR_SIZE); ++ if (sector == NULL) { ++ *err = errno; ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ } ++ ++ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); ++ *err = EIO; ++ return -1; ++ } ++ ++ sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ ++ sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ ++ ++ /* Unaligned head */ ++ if (sectoffs) { ++ uint64_t n = MIN (LUKS_SECTOR_SIZE - sectoffs, count); ++ ++ assert (sector); ++ if (next->pread (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ if (do_decrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, ++ sector, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ memcpy (buf, §or[sectoffs], n); ++ ++ buf += n; ++ count -= n; ++ offset += n; ++ sectnum++; ++ } ++ ++ /* Aligned body */ ++ while (count >= LUKS_SECTOR_SIZE) { ++ if (next->pread (next, buf, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ if (do_decrypt (h, cipher, offset, buf, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ buf += LUKS_SECTOR_SIZE; ++ count -= LUKS_SECTOR_SIZE; ++ offset += LUKS_SECTOR_SIZE; ++ sectnum++; ++ } ++ ++ /* Unaligned tail */ ++ if (count) { ++ assert (sector); ++ if (next->pread (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ if (do_decrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ memcpy (buf, sector, count); ++ } ++ ++ gnutls_cipher_deinit (cipher); ++ return 0; ++ ++ err: ++ gnutls_cipher_deinit (cipher); ++ return -1; ++} ++ ++/* Lock preventing read-modify-write cycles from overlapping. */ ++static pthread_mutex_t read_modify_write_lock = PTHREAD_MUTEX_INITIALIZER; ++ ++/* Encrypt data. */ ++static int ++luks_pwrite (nbdkit_next *next, void *handle, ++ const void *buf, uint32_t count, uint64_t offset, ++ uint32_t flags, int *err) ++{ ++ struct handle *h = handle; ++ const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; ++ CLEANUP_FREE uint8_t *sector = NULL; ++ uint64_t sectnum, sectoffs; ++ const gnutls_datum_t mkey = ++ { (unsigned char *) h->masterkey, h->phdr.master_key_len }; ++ gnutls_cipher_hd_t cipher; ++ int r; ++ ++ if (!h->masterkey) { ++ *err = EIO; ++ return -1; ++ } ++ ++ sector = malloc (LUKS_SECTOR_SIZE); ++ if (sector == NULL) { ++ *err = errno; ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ ++ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); ++ *err = EIO; ++ return -1; ++ } ++ ++ sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ ++ sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ ++ ++ /* Unaligned head */ ++ if (sectoffs) { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&read_modify_write_lock); ++ ++ uint64_t n = MIN (LUKS_SECTOR_SIZE - sectoffs, count); ++ ++ if (next->pread (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ memcpy (§or[sectoffs], buf, n); ++ ++ if (do_encrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, ++ sector, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ buf += n; ++ count -= n; ++ offset += n; ++ sectnum++; ++ } ++ ++ /* Aligned body */ ++ while (count >= LUKS_SECTOR_SIZE) { ++ memcpy (sector, buf, LUKS_SECTOR_SIZE); ++ ++ if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ buf += LUKS_SECTOR_SIZE; ++ count -= LUKS_SECTOR_SIZE; ++ offset += LUKS_SECTOR_SIZE; ++ sectnum++; ++ } ++ ++ /* Unaligned tail */ ++ if (count) { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&read_modify_write_lock); ++ ++ if (next->pread (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ memcpy (sector, buf, count); ++ ++ if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ } ++ ++ gnutls_cipher_deinit (cipher); ++ return 0; ++ ++ err: ++ gnutls_cipher_deinit (cipher); ++ return -1; ++} ++ ++static struct nbdkit_filter filter = { ++ .name = "luks", ++ .longname = "nbdkit luks filter", ++ .unload = luks_unload, ++ .thread_model = luks_thread_model, ++ .config = luks_config, ++ .config_complete = luks_config_complete, ++ .config_help = luks_config_help, ++ .open = luks_open, ++ .close = luks_close, ++ .prepare = luks_prepare, ++ .get_size = luks_get_size, ++ .can_extents = luks_can_extents, ++ .can_trim = luks_can_trim, ++ .can_zero = luks_can_zero, ++ .can_fast_zero = luks_can_fast_zero, ++ .can_cache = luks_can_cache, ++ .block_size = luks_block_size, ++ .pread = luks_pread, ++ .pwrite = luks_pwrite, ++}; ++ ++NBDKIT_REGISTER_FILTER(filter) +diff --git a/filters/luks/nbdkit-luks-filter.pod b/filters/luks/nbdkit-luks-filter.pod +new file mode 100644 +index 00000000..56e51561 +--- /dev/null ++++ b/filters/luks/nbdkit-luks-filter.pod +@@ -0,0 +1,120 @@ ++=head1 NAME ++ ++nbdkit-luks-filter - read and write LUKS-encrypted disks and partitions ++ ++=head1 SYNOPSIS ++ ++ nbdkit file encrypted-disk.img --filter=luks passphrase=+/tmp/secret ++ ++=head1 DESCRIPTION ++ ++C is a filter for L which transparently ++opens a LUKS-encrypted disk image. LUKS ("Linux Unified Key Setup") ++is the Full Disk Encryption (FDE) system commonly used by Linux ++systems. This filter is compatible with LUKSv1 as implemented by the ++Linux kernel (dm_crypt), and by qemu. ++ ++You can place this filter on top of L to ++decrypt a local file: ++ ++ nbdkit file encrypted-disk.img --filter=luks passphrase=+/tmp/secret ++ ++If LUKS is present inside a partition in the disk image then you will ++have to combine this filter with L. The ++order of the filters is important: ++ ++ nbdkit file encrypted-disk.img \ ++ --filter=luks passphrase=+/tmp/secret \ ++ --filter=partition partition=1 ++ ++This filter also works on top of other plugins such as ++L: ++ ++ nbdkit curl https://example.com/encrypted-disk.img \ ++ --filter=luks passphrase=+/tmp/secret ++ ++The web server sees only the encrypted data. Without knowing the ++passphrase, the web server cannot access the decrypted disk. Only ++encrypted data is sent over the HTTP connection. nbdkit itself will ++serve I disk data over the NBD connection (if this is a ++problem see L, or use a Unix domain socket I<-U>). ++ ++The passphrase can be stored in a file (as shown), passed directly on ++the command line (insecure), entered interactively, or passed to ++nbdkit over a file descriptor. ++ ++This filter can read and write LUKSv1. It cannot create disks, change ++passphrases, add keyslots, etc. To do that, you can use ordinary ++Linux tools like L. Note you must force LUKSv1 ++(eg. using cryptsetup I<--type luks1>). L can also ++create compatible disk images: ++ ++ qemu-img create -f luks \ ++ --object secret,data=SECRET,id=sec0 \ ++ -o key-secret=sec0 \ ++ encrypted-disk.img 1G ++ ++=head1 PARAMETERS ++ ++=over 4 ++ ++=item BSECRET ++ ++Use the secret passphrase when decrypting the disk. ++ ++Note that passing this on the command line is not secure on shared ++machines. ++ ++=item B ++ ++Ask for the passphrase (interactively) when nbdkit starts up. ++ ++=item BFILENAME ++ ++Read the passphrase from the named file. This is a secure method to ++supply a passphrase, as long as you set the permissions on the file ++appropriately. ++ ++=item BFD ++ ++Read the passphrase from file descriptor number C, inherited from ++the parent process when nbdkit starts up. This is also a secure ++method to supply a passphrase. ++ ++=back ++ ++=head1 FILES ++ ++=over 4 ++ ++=item F<$filterdir/nbdkit-luks-filter.so> ++ ++The plugin. ++ ++Use C to find the location of C<$filterdir>. ++ ++=back ++ ++=head1 VERSION ++ ++C first appeared in nbdkit 1.32. ++ ++=head1 SEE ALSO ++ ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L. ++ ++=head1 AUTHORS ++ ++Richard W.M. Jones ++ ++=head1 COPYRIGHT ++ ++Copyright (C) 2013-2022 Red Hat Inc. +diff --git a/plugins/file/nbdkit-file-plugin.pod b/plugins/file/nbdkit-file-plugin.pod +index f8f0e198..b95e7349 100644 +--- a/plugins/file/nbdkit-file-plugin.pod ++++ b/plugins/file/nbdkit-file-plugin.pod +@@ -223,6 +223,7 @@ L, + L, + L, + L, ++L, + L. + + =head1 AUTHORS +diff --git a/tests/Makefile.am b/tests/Makefile.am +index b310e8a2..c29453ba 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1596,6 +1596,18 @@ EXTRA_DIST += \ + test-log-script-info.sh \ + $(NULL) + ++# luks filter test. ++if HAVE_GNUTLS ++TESTS += \ ++ test-luks-info.sh \ ++ test-luks-copy.sh \ ++ $(NULL) ++endif ++EXTRA_DIST += \ ++ test-luks-info.sh \ ++ test-luks-copy.sh \ ++ $(NULL) ++ + # multi-conn filter test. + TESTS += \ + test-multi-conn.sh \ +diff --git a/tests/test-luks-copy.sh b/tests/test-luks-copy.sh +new file mode 100755 +index 00000000..99f300d0 +--- /dev/null ++++ b/tests/test-luks-copy.sh +@@ -0,0 +1,125 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires nbdcopy --version ++requires nbdsh --version ++requires_nbdsh_uri ++requires qemu-img --version ++requires bash -c 'qemu-img --help | grep -- --target-image-opts' ++requires hexdump --version ++requires truncate --version ++requires_filter luks ++ ++encrypt_disk=luks-copy1.img ++plain_disk=luks-copy2.img ++pid=luks-copy.pid ++sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) ++cleanup_fn rm -f $encrypt_disk $plain_disk $pid $sock ++rm -f $encrypt_disk $plain_disk $pid $sock ++ ++# Create an empty encrypted disk container. ++# ++# NB: This is complicated because qemu doesn't create an all-zeroes ++# plaintext disk for some reason when you use create -f luks. It ++# starts with random plaintext. ++# ++# https://stackoverflow.com/a/44669936 ++qemu-img create -f luks \ ++ --object secret,data=123456,id=sec0 \ ++ -o key-secret=sec0 \ ++ $encrypt_disk 10M ++truncate -s 10M $plain_disk ++qemu-img convert --target-image-opts -n \ ++ --object secret,data=123456,id=sec0 \ ++ $plain_disk \ ++ driver=luks,file.filename=$encrypt_disk,key-secret=sec0 ++rm $plain_disk ++ ++# Start nbdkit on the encrypted disk. ++start_nbdkit -P $pid -U $sock \ ++ file $encrypt_disk --filter=luks passphrase=123456 ++uri="nbd+unix:///?socket=$sock" ++ ++# Copy the whole disk out. It should be empty. ++nbdcopy "$uri" $plain_disk ++ ++if [ "$(hexdump -C $plain_disk)" != '00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++00a00000' ]; then ++ echo "$0: expected plaintext disk to be empty" ++ exit 1 ++fi ++ ++# Use nbdsh to overwrite with some known data and check we can read ++# back what we wrote. ++nbdsh -u "$uri" \ ++ -c 'h.pwrite(b"1"*65536, 0)' \ ++ -c 'h.pwrite(b"2"*65536, 128*1024)' \ ++ -c 'h.pwrite(b"3"*65536, 9*1024*1024)' \ ++ -c 'buf = h.pread(65536, 0)' \ ++ -c 'assert buf == b"1"*65536' \ ++ -c 'buf = h.pread(65536, 65536)' \ ++ -c 'assert buf == bytearray(65536)' \ ++ -c 'buf = h.pread(65536, 128*1024)' \ ++ -c 'assert buf == b"2"*65536' \ ++ -c 'buf = h.pread(65536, 9*1024*1024)' \ ++ -c 'assert buf == b"3"*65536' \ ++ -c 'h.flush()' ++ ++# Use qemu to copy out the whole disk. Note we called flush() above ++# so the disk should be synchronised. ++qemu-img convert --image-opts \ ++ --object secret,data=123456,id=sec0 \ ++ driver=luks,file.filename=$encrypt_disk,key-secret=sec0 \ ++ $plain_disk ++ ++# Check the contents are expected. ++if [ "$(hexdump -C $plain_disk)" != '00000000 31 31 31 31 31 31 31 31 31 31 31 31 31 31 31 31 |1111111111111111| ++* ++00010000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++00020000 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 |2222222222222222| ++* ++00030000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++00900000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333| ++* ++00910000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++00a00000' ]; then ++ echo "$0: unexpected content" ++ exit 1 ++fi +diff --git a/tests/test-luks-info.sh b/tests/test-luks-info.sh +new file mode 100755 +index 00000000..3eff657b +--- /dev/null ++++ b/tests/test-luks-info.sh +@@ -0,0 +1,56 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires nbdinfo --version ++requires qemu-img --version ++requires_filter luks ++ ++disk=luks-info.img ++info=luks-info.log ++cleanup_fn rm -f $disk $info ++rm -f $disk $info ++ ++qemu-img create -f luks \ ++ --object secret,data=123456,id=sec0 \ ++ -o key-secret=sec0 \ ++ $disk 10M ++ ++nbdkit -U - file $disk --filter=luks passphrase=123456 \ ++ --run 'nbdinfo $uri' > $info ++cat $info ++ ++# Check the size is 10M (so it doesn't include the LUKS header). ++grep "10485760" $info +-- +2.31.1 + diff --git a/SOURCES/0004-vddk-Document-troubleshooting-performance-problems.patch b/SOURCES/0004-vddk-Document-troubleshooting-performance-problems.patch deleted file mode 100644 index b1d94a0..0000000 --- a/SOURCES/0004-vddk-Document-troubleshooting-performance-problems.patch +++ /dev/null @@ -1,84 +0,0 @@ -From 8353ab55b8c6e7f1dc9ea27260fd7ec90b9d75af Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Fri, 22 Oct 2021 18:00:27 +0100 -Subject: [PATCH] vddk: Document troubleshooting performance problems - -Document how to use -D vddk.stats=1 to diagnose performance problems -with VDDK. - -(cherry picked from commit e491978c193f49010cc28ad344d0fb3c1b5ede35) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 57 +++++++++++++++++++++++++++++ - 1 file changed, 57 insertions(+) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index e53d3286..5a426135 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -475,6 +475,63 @@ and restarting the C service: - - For more information see L. - -+=head2 Troubleshooting performance problems -+ -+VDDK has very uneven performance with some operations being very slow. -+This plugin has options to allow you to debug performance issues. If -+your application has a debug or diagnostic setting, add the following -+nbdkit command line options: -+ -+ -v -D nbdkit.backend.datapath=0 -D vddk.datapath=0 -D vddk.stats=1 -+ -+C<-v> enables verbose messages and the two datapath options I -+the very verbose per-read/-write messages. C<-D vddk.stats=1> enables -+a summary when nbdkit exits of the cumulative time taken in each VDDK -+function, the number of times each function was called, and (for read -+and write) the number of bytes transferred. An example of what those -+stats look like can be found here: -+L -+ -+You can interpret the stats as follows: -+ -+=over 4 -+ -+=item C -+ -+The cumulative time spent waiting for VDDK to return from -+C calls, the number of times this function was -+called, and the total bytes read. You can use this to determine the -+read bandwidth to the VMware server. -+ -+=item C -+ -+=item C -+ -+Same as above, but for writing and flushing writes. -+ -+=item C -+ -+This call is used to query information about the sparseness of the -+remote disk. It is only available in VDDK E 6.7. The call is -+notably very slow in all versions of VMware we have tested. -+ -+=item C -+ -+=item C -+ -+=item C -+ -+=item C -+ -+=item C -+ -+=item C -+ -+The cumulative time spent connecting and disconnecting from the VMware -+server, which can also be very slow. -+ -+=back -+ - =head1 SUPPORTED VERSIONS OF VDDK - - This plugin requires VDDK E 5.5.5, which in turn means that it --- -2.31.1 - diff --git a/SOURCES/0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch b/SOURCES/0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch new file mode 100644 index 0000000..2ff4e07 --- /dev/null +++ b/SOURCES/0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch @@ -0,0 +1,95 @@ +From 66daae1a7daf680e06f884e9af6a14830263c932 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 8 May 2022 12:13:39 +0100 +Subject: [PATCH] luks: Disable filter with old GnuTLS in Debian 10 +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +On Debian 10: + +luks.c: In function ‘parse_cipher_strings’: +luks.c:574:26: error: ‘GNUTLS_CIPHER_AES_128_XTS’ undeclared (first use in this function); did you mean ‘GNUTLS_CIPHER_AES_128_CCM’? + h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; + ^~~~~~~~~~~~~~~~~~~~~~~~~ + GNUTLS_CIPHER_AES_128_CCM +luks.c:574:26: note: each undeclared identifier is reported only once for each function it appears in +luks.c:577:26: error: ‘GNUTLS_CIPHER_AES_256_XTS’ undeclared (first use in this function); did you mean ‘GNUTLS_CIPHER_AES_256_CCM’? + h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; + ^~~~~~~~~~~~~~~~~~~~~~~~~ + GNUTLS_CIPHER_AES_256_CCM +luks.c: In function ‘try_passphrase_in_keyslot’: +luks.c:728:7: error: implicit declaration of function ‘gnutls_pbkdf2’; did you mean ‘gnutls_prf’? [-Werror=implicit-function-declaration] + r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, + ^~~~~~~~~~~~~ + gnutls_prf + +Because gnutls_pbkdf2 is missing there's no chance of making this +filter work on this platform so it's best to compile it out. + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit f9f67e483f4aad19ad6101163d32562f13504ca7) +--- + configure.ac | 5 ++++- + filters/luks/Makefile.am | 2 +- + tests/Makefile.am | 2 +- + 3 files changed, 6 insertions(+), 3 deletions(-) + +diff --git a/configure.ac b/configure.ac +index de85b4da..1d209f67 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -636,12 +636,15 @@ AS_IF([test "x$GNUTLS_LIBS" != "x"],[ + gnutls_certificate_set_known_dh_params \ + gnutls_group_get \ + gnutls_group_get_name \ ++ gnutls_pbkdf2 \ + gnutls_session_set_verify_cert \ + gnutls_srp_server_get_username \ + gnutls_transport_is_ktls_enabled \ + ]) + LIBS="$old_LIBS" + ]) ++AM_CONDITIONAL([HAVE_GNUTLS_PBKDF2], ++ [test "x$GNUTLS_LIBS" != "x" && test "x$ac_cv_func_gnutls_pbkdf2" = xyes]) + + AC_ARG_ENABLE([linuxdisk], + [AS_HELP_STRING([--disable-linuxdisk], +@@ -1484,7 +1487,7 @@ echo "Optional filters:" + echo + feature "ext2" test "x$HAVE_EXT2_TRUE" = "x" + feature "gzip" test "x$HAVE_ZLIB_TRUE" = "x" +-feature "LUKS" test "x$HAVE_GNUTLS_TRUE" != "x" ++feature "luks" test "x$HAVE_GNUTLS_PBKDF2_TRUE" = "x" + feature "xz" test "x$HAVE_LIBLZMA_TRUE" = "x" + + echo +diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am +index 30089621..622e5c3d 100644 +--- a/filters/luks/Makefile.am ++++ b/filters/luks/Makefile.am +@@ -33,7 +33,7 @@ include $(top_srcdir)/common-rules.mk + + EXTRA_DIST = nbdkit-luks-filter.pod + +-if HAVE_GNUTLS ++if HAVE_GNUTLS_PBKDF2 + + filter_LTLIBRARIES = nbdkit-luks-filter.la + +diff --git a/tests/Makefile.am b/tests/Makefile.am +index c29453ba..5585b3b7 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1597,7 +1597,7 @@ EXTRA_DIST += \ + $(NULL) + + # luks filter test. +-if HAVE_GNUTLS ++if HAVE_GNUTLS_PBKDF2 + TESTS += \ + test-luks-info.sh \ + test-luks-copy.sh \ +-- +2.31.1 + diff --git a/SOURCES/0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch b/SOURCES/0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch deleted file mode 100644 index 85b3f1c..0000000 --- a/SOURCES/0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch +++ /dev/null @@ -1,141 +0,0 @@ -From d994773724266dd5f0a8b4282cc604f6b75e077c Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 23 Oct 2021 16:16:39 +0100 -Subject: [PATCH] vddk: Include VDDK major library version in --dump-plugin - output - -Although it doesn't seem to be possible to get the precise VDDK -version, With a relatively simple change we can at least return the -VDDK major version. Currently this can be 5, 6 or 7. - -(cherry picked from commit 8700649d147948897f3b97810a1dff37924bdd6e) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 4 ++++ - plugins/vddk/vddk.c | 29 +++++++++++++++++++---------- - tests/test-vddk-real-dump-plugin.sh | 2 ++ - 3 files changed, 25 insertions(+), 10 deletions(-) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index 5a426135..bc3c3c94 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -422,6 +422,10 @@ at runtime. - If this is printed then the C parameter is supported - by this build. - -+=item C -+ -+The VDDK major library version: 5, 6, 7, ... -+ - =item C - - Prints the full path to the VDDK shared library. Since this requires -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 993f2d76..d74a484d 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -81,6 +81,7 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats; - static void *dl; /* dlopen handle */ - static bool init_called; /* was InitEx called */ - static __thread int error_suppression; /* threadlocal error suppression */ -+static int library_version; /* VDDK major: 5, 6, 7, ... */ - - static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */ - static char *config; /* config */ -@@ -405,7 +406,10 @@ vddk_config (const char *key, const char *value) - static void - load_library (bool load_error_is_fatal) - { -- static const char *sonames[] = { -+ static struct { -+ const char *soname; -+ int library_version; -+ } libs[] = { - /* Prefer the newest library in case multiple exist. Check two - * possible directories: the usual VDDK installation puts .so - * files in an arch-specific subdirectory of $libdir (our minimum -@@ -413,12 +417,13 @@ load_library (bool load_error_is_fatal) - * but our testsuite is easier to write if we point libdir - * directly to a stub .so. - */ -- "lib64/libvixDiskLib.so.7", -- "libvixDiskLib.so.7", -- "lib64/libvixDiskLib.so.6", -- "libvixDiskLib.so.6", -- "lib64/libvixDiskLib.so.5", -- "libvixDiskLib.so.5", -+ { "lib64/libvixDiskLib.so.7", 7 }, -+ { "libvixDiskLib.so.7", 7 }, -+ { "lib64/libvixDiskLib.so.6", 6 }, -+ { "libvixDiskLib.so.6", 6 }, -+ { "lib64/libvixDiskLib.so.5", 5 }, -+ { "libvixDiskLib.so.5", 5 }, -+ { NULL } - }; - size_t i; - CLEANUP_FREE char *orig_error = NULL; -@@ -431,19 +436,20 @@ load_library (bool load_error_is_fatal) - } - } - -- for (i = 0; i < sizeof sonames / sizeof sonames[0]; ++i) { -+ for (i = 0; libs[i].soname != NULL; ++i) { - CLEANUP_FREE char *path; - - /* Set the full path so that dlopen will preferentially load the - * system libraries from the same directory. - */ -- if (asprintf (&path, "%s/%s", libdir, sonames[i]) == -1) { -+ if (asprintf (&path, "%s/%s", libdir, libs[i].soname) == -1) { - nbdkit_error ("asprintf: %m"); - exit (EXIT_FAILURE); - } - - dl = dlopen (path, RTLD_NOW); - if (dl != NULL) { -+ library_version = libs[i].library_version; - /* Now that we found the library, ensure that LD_LIBRARY_PATH - * includes its directory for all future loads. This may modify - * path in-place and/or re-exec nbdkit, but that's okay. -@@ -464,10 +470,12 @@ load_library (bool load_error_is_fatal) - "If '%s' is located on a non-standard path you may need to\n" - "set libdir=/path/to/vmware-vix-disklib-distrib.\n\n" - "See nbdkit-vddk-plugin(1) man page section \"LIBRARY LOCATION\" for details.", -- orig_error ? : "(unknown error)", sonames[0]); -+ orig_error ? : "(unknown error)", libs[0].soname); - exit (EXIT_FAILURE); - } - -+ assert (library_version >= 5); -+ - /* Load symbols. */ - #define STUB(fn,ret,args) \ - do { \ -@@ -583,6 +591,7 @@ vddk_dump_plugin (void) - - printf ("vddk_default_libdir=%s\n", VDDK_LIBDIR); - printf ("vddk_has_nfchostport=1\n"); -+ printf ("vddk_library_version=%d\n", library_version); - - #if defined(HAVE_DLADDR) - /* It would be nice to print the version of VDDK from the shared -diff --git a/tests/test-vddk-real-dump-plugin.sh b/tests/test-vddk-real-dump-plugin.sh -index 2cb7724e..0a079c6c 100755 ---- a/tests/test-vddk-real-dump-plugin.sh -+++ b/tests/test-vddk-real-dump-plugin.sh -@@ -58,10 +58,12 @@ rm -f $files - cleanup_fn rm -f $files - - nbdkit -f -v vddk libdir="$vddkdir" --dump-plugin > $out -+cat $out - - # Check the vddk_* entries are set. - grep ^vddk_default_libdir= $out - grep ^vddk_has_nfchostport= $out -+grep ^vddk_library_version= $out - grep ^vddk_dll= $out - - dll="$(grep ^vddk_dll $out | cut -d= -f2)" --- -2.31.1 - diff --git a/SOURCES/0006-luks-Various-fixes-for-Clang.patch b/SOURCES/0006-luks-Various-fixes-for-Clang.patch new file mode 100644 index 0000000..c0f7f2f --- /dev/null +++ b/SOURCES/0006-luks-Various-fixes-for-Clang.patch @@ -0,0 +1,71 @@ +From b3c05065801c723966a3e8d93c9b84e808ff38b9 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 8 May 2022 12:30:09 +0100 +Subject: [PATCH] luks: Various fixes for Clang + +With Clang: + +luks.c:728:25: error: implicit conversion from enumeration type 'gnutls_digest_algorithm_t' to different enumeration type 'gnutls_mac_algorithm_t' [-Werror,-Wenum-conversion] + r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, + ~~~~~~~~~~~~~ ~~~^~~~~~~~ +luks.c:764:25: error: implicit conversion from enumeration type 'gnutls_digest_algorithm_t' to different enumeration type 'gnutls_mac_algorithm_t' [-Werror,-Wenum-conversion] + r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt, + ~~~~~~~~~~~~~ ~~~^~~~~~~~ +luks.c:886:35: error: result of comparison of constant 18446744073709551615 with expression of type 'uint32_t' (aka 'unsigned int') is always false [-Werror,-Wtautological-constant-out-of-range-compare] + if (ks->password_iterations > ULONG_MAX) { + ~~~~~~~~~~~~~~~~~~~~~~~ ^ ~~~~~~~~~ + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit 87d488ede9101a2effc71cd1851bf4a4caa521d2) +--- + filters/luks/luks.c | 13 ++++++------- + 1 file changed, 6 insertions(+), 7 deletions(-) + +diff --git a/filters/luks/luks.c b/filters/luks/luks.c +index 706a9bd2..cc619698 100644 +--- a/filters/luks/luks.c ++++ b/filters/luks/luks.c +@@ -693,6 +693,10 @@ key_material_length_in_sectors (struct handle *h, size_t i) + static int + try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) + { ++ /* I believe this is supposed to be safe, looking at the GnuTLS ++ * header file. ++ */ ++ const gnutls_mac_algorithm_t mac = (gnutls_mac_algorithm_t) h->hash_alg; + struct luks_keyslot *ks = &h->phdr.keyslot[i]; + size_t split_key_len; + CLEANUP_FREE uint8_t *split_key = NULL; +@@ -725,7 +729,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) + } + + /* Hash the passphrase to make a possible masterkey. */ +- r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, ++ r = gnutls_pbkdf2 (mac, &key, &salt, ks->password_iterations, + masterkey, h->phdr.master_key_len); + if (r != 0) { + nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); +@@ -761,7 +765,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) + /* Check if the masterkey is correct by comparing hash of the + * masterkey with LUKS header. + */ +- r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt, ++ r = gnutls_pbkdf2 (mac, &mkey, &msalt, + h->phdr.master_key_digest_iterations, + key_digest, LUKS_DIGESTSIZE); + if (r != 0) { +@@ -883,11 +887,6 @@ luks_prepare (nbdkit_next *next, void *handle, int readonly) + "points beyond the end of the disk", i); + return -1; + } +- if (ks->password_iterations > ULONG_MAX) { +- nbdkit_error ("bad LUKSv1 header: key slot %zu " +- "iterations too large", i); +- return -1; +- } + /*FALLTHROUGH*/ + case LUKS_KEY_DISABLED: + break; +-- +2.31.1 + diff --git a/SOURCES/0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch b/SOURCES/0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch deleted file mode 100644 index dd44d14..0000000 --- a/SOURCES/0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch +++ /dev/null @@ -1,52 +0,0 @@ -From 4c80b474a2c2a552e5bdfcaabfa2981540afe8d8 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 23 Oct 2021 16:24:27 +0100 -Subject: [PATCH] vddk: Add logical and physical sector size to -D - vddk.diskinfo output - -In VDDK >= 7 it is possible to display the logical and physical sector -size in debug output. - -This commit also extends the test since this flag was not tested -before. - -(cherry picked from commit 5bb8f0586e1faabcbf4f43d722a3b3cb5b352e33) ---- - plugins/vddk/vddk.c | 6 ++++++ - tests/test-vddk-real.sh | 3 ++- - 2 files changed, 8 insertions(+), 1 deletion(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index d74a484d..50bdde26 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -822,6 +822,12 @@ vddk_get_size (void *handle) - info->parentFileNameHint ? : "NULL"); - nbdkit_debug ("disk info: uuid: %s", - info->uuid ? : "NULL"); -+ if (library_version >= 7) { -+ nbdkit_debug ("disk info: sectory size: " -+ "logical %" PRIu32 " physical %" PRIu32, -+ info->logicalSectorSize, -+ info->physicalSectorSize); -+ } - } - - VDDK_CALL_START (VixDiskLib_FreeInfo, "info") -diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh -index a6aceac9..ae965245 100755 ---- a/tests/test-vddk-real.sh -+++ b/tests/test-vddk-real.sh -@@ -89,7 +89,8 @@ if grep 'cannot open shared object file' $log; then - fi - - # Now run nbdkit for the test. --start_nbdkit -P $pid -U $sock -D vddk.stats=1 vddk libdir="$vddkdir" $vmdk -+start_nbdkit -P $pid -U $sock -D vddk.stats=1 -D vddk.diskinfo=1 \ -+ vddk libdir="$vddkdir" $vmdk - uri="nbd+unix:///?socket=$sock" - - # VDDK < 6.0 did not support flush, so disable flush test there. Also --- -2.31.1 - diff --git a/SOURCES/0007-luks-Link-with-libcompat-on-Windows.patch b/SOURCES/0007-luks-Link-with-libcompat-on-Windows.patch new file mode 100644 index 0000000..f93465b --- /dev/null +++ b/SOURCES/0007-luks-Link-with-libcompat-on-Windows.patch @@ -0,0 +1,43 @@ +From 9416effd73a5cb2e1c929449fca88fd7152aa1be Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 8 May 2022 12:38:00 +0100 +Subject: [PATCH] luks: Link with libcompat on Windows + +/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-full-rw.o): in function `full_pread': +/builds/nbdkit/nbdkit/common/utils/full-rw.c:53: undefined reference to `pread' +/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-full-rw.o): in function `full_pwrite': +/builds/nbdkit/nbdkit/common/utils/full-rw.c:76: undefined reference to `pwrite' +/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-vector.o): in function `generic_vector_reserve_page_aligned': +/builds/nbdkit/nbdkit/common/utils/vector.c:112: undefined reference to `sysconf' +/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: /builds/nbdkit/nbdkit/common/utils/vector.c:134: undefined reference to `posix_memalign' +collect2: error: ld returned 1 exit status + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit 4a28c4c46aedf270929a62a1c5ecf2c1129cd456) +--- + filters/luks/Makefile.am | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am +index 622e5c3d..2688f696 100644 +--- a/filters/luks/Makefile.am ++++ b/filters/luks/Makefile.am +@@ -45,6 +45,7 @@ nbdkit_luks_filter_la_SOURCES = \ + nbdkit_luks_filter_la_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/common/include \ ++ -I$(top_srcdir)/common/replacements \ + -I$(top_srcdir)/common/utils \ + $(NULL) + nbdkit_luks_filter_la_CFLAGS = \ +@@ -53,6 +54,7 @@ nbdkit_luks_filter_la_CFLAGS = \ + $(NULL) + nbdkit_luks_filter_la_LIBADD = \ + $(top_builddir)/common/utils/libutils.la \ ++ $(top_builddir)/common/replacements/libcompat.la \ + $(IMPORT_LIBRARY_ON_WINDOWS) \ + $(GNUTLS_LIBS) \ + $(NULL) +-- +2.31.1 + diff --git a/SOURCES/0007-vddk-Fix-typo-in-debug-message.patch b/SOURCES/0007-vddk-Fix-typo-in-debug-message.patch deleted file mode 100644 index 8ef9f10..0000000 --- a/SOURCES/0007-vddk-Fix-typo-in-debug-message.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4b0d278f3851baf37affa26d34e52963dc8c7c04 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 23 Oct 2021 19:41:07 +0100 -Subject: [PATCH] vddk: Fix typo in debug message - -Fixes: commit 5bb8f0586e1faabcbf4f43d722a3b3cb5b352e33 -(cherry picked from commit 343dadeb7340d7b8c5730e2bbab33c829b569122) ---- - plugins/vddk/vddk.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 50bdde26..65399a91 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -823,7 +823,7 @@ vddk_get_size (void *handle) - nbdkit_debug ("disk info: uuid: %s", - info->uuid ? : "NULL"); - if (library_version >= 7) { -- nbdkit_debug ("disk info: sectory size: " -+ nbdkit_debug ("disk info: sector size: " - "logical %" PRIu32 " physical %" PRIu32, - info->logicalSectorSize, - info->physicalSectorSize); --- -2.31.1 - diff --git a/SOURCES/0008-luks-Refactor-the-filter.patch b/SOURCES/0008-luks-Refactor-the-filter.patch new file mode 100644 index 0000000..feaf986 --- /dev/null +++ b/SOURCES/0008-luks-Refactor-the-filter.patch @@ -0,0 +1,2096 @@ +From e8279107801bb93303b22e1b927929ce18279dc5 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 8 May 2022 16:13:13 +0100 +Subject: [PATCH] luks: Refactor the filter + +Move the LUKS-specific code out into a separate file. This is mostly +pure refactoring to tidy things up. + +(cherry picked from commit 2159d85e0bed1943542da58b43c91f2caa096d6c) +--- + filters/luks/Makefile.am | 2 + + filters/luks/luks-encryption.c | 926 +++++++++++++++++++++++++++++++++ + filters/luks/luks-encryption.h | 78 +++ + filters/luks/luks.c | 874 ++----------------------------- + 4 files changed, 1037 insertions(+), 843 deletions(-) + create mode 100644 filters/luks/luks-encryption.c + create mode 100644 filters/luks/luks-encryption.h + +diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am +index 2688f696..0894ac8b 100644 +--- a/filters/luks/Makefile.am ++++ b/filters/luks/Makefile.am +@@ -38,6 +38,8 @@ if HAVE_GNUTLS_PBKDF2 + filter_LTLIBRARIES = nbdkit-luks-filter.la + + nbdkit_luks_filter_la_SOURCES = \ ++ luks-encryption.c \ ++ luks-encryption.h \ + luks.c \ + $(top_srcdir)/include/nbdkit-filter.h \ + $(NULL) +diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c +new file mode 100644 +index 00000000..8ee0eb35 +--- /dev/null ++++ b/filters/luks/luks-encryption.c +@@ -0,0 +1,926 @@ ++/* nbdkit ++ * Copyright (C) 2018-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include ++ ++#include "luks-encryption.h" ++ ++#include "byte-swapping.h" ++#include "cleanup.h" ++#include "isaligned.h" ++#include "rounding.h" ++ ++/* LUKSv1 constants. */ ++#define LUKS_MAGIC { 'L', 'U', 'K', 'S', 0xBA, 0xBE } ++#define LUKS_MAGIC_LEN 6 ++#define LUKS_DIGESTSIZE 20 ++#define LUKS_SALTSIZE 32 ++#define LUKS_NUMKEYS 8 ++#define LUKS_KEY_DISABLED 0x0000DEAD ++#define LUKS_KEY_ENABLED 0x00AC71F3 ++#define LUKS_STRIPES 4000 ++#define LUKS_ALIGN_KEYSLOTS 4096 ++ ++/* Key slot. */ ++struct luks_keyslot { ++ uint32_t active; /* LUKS_KEY_DISABLED|LUKS_KEY_ENABLED */ ++ uint32_t password_iterations; ++ char password_salt[LUKS_SALTSIZE]; ++ uint32_t key_material_offset; ++ uint32_t stripes; ++} __attribute__((__packed__)); ++ ++/* LUKS superblock. */ ++struct luks_phdr { ++ char magic[LUKS_MAGIC_LEN]; /* LUKS_MAGIC */ ++ uint16_t version; /* Only 1 is supported. */ ++ char cipher_name[32]; ++ char cipher_mode[32]; ++ char hash_spec[32]; ++ uint32_t payload_offset; ++ uint32_t master_key_len; ++ uint8_t master_key_digest[LUKS_DIGESTSIZE]; ++ uint8_t master_key_salt[LUKS_SALTSIZE]; ++ uint32_t master_key_digest_iterations; ++ uint8_t uuid[40]; ++ ++ struct luks_keyslot keyslot[LUKS_NUMKEYS]; /* Key slots. */ ++} __attribute__((__packed__)); ++ ++/* Block cipher mode of operation. ++ * https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation ++ */ ++enum cipher_mode { ++ CIPHER_MODE_ECB, CIPHER_MODE_CBC, CIPHER_MODE_XTS, CIPHER_MODE_CTR, ++}; ++ ++static enum cipher_mode ++lookup_cipher_mode (const char *str) ++{ ++ if (strcmp (str, "ecb") == 0) ++ return CIPHER_MODE_ECB; ++ if (strcmp (str, "cbc") == 0) ++ return CIPHER_MODE_CBC; ++ if (strcmp (str, "xts") == 0) ++ return CIPHER_MODE_XTS; ++ if (strcmp (str, "ctr") == 0) ++ return CIPHER_MODE_CTR; ++ nbdkit_error ("unknown cipher mode: %s " ++ "(expecting \"ecb\", \"cbc\", \"xts\" or \"ctr\")", str); ++ return -1; ++} ++ ++static const char * ++cipher_mode_to_string (enum cipher_mode v) ++{ ++ switch (v) { ++ case CIPHER_MODE_ECB: return "ecb"; ++ case CIPHER_MODE_CBC: return "cbc"; ++ case CIPHER_MODE_XTS: return "xts"; ++ case CIPHER_MODE_CTR: return "ctr"; ++ default: abort (); ++ } ++} ++ ++/* Methods used by LUKS to generate initial vectors. ++ * ++ * ESSIV is a bit more complicated to implement. It is supported by ++ * qemu but not by us. ++ */ ++enum ivgen { ++ IVGEN_PLAIN, IVGEN_PLAIN64, /* IVGEN_ESSIV, */ ++}; ++ ++static enum ivgen ++lookup_ivgen (const char *str) ++{ ++ if (strcmp (str, "plain") == 0) ++ return IVGEN_PLAIN; ++ if (strcmp (str, "plain64") == 0) ++ return IVGEN_PLAIN64; ++/* ++ if (strcmp (str, "essiv") == 0) ++ return IVGEN_ESSIV; ++*/ ++ nbdkit_error ("unknown IV generation algorithm: %s " ++ "(expecting \"plain\", \"plain64\" etc)", str); ++ return -1; ++} ++ ++static const char * ++ivgen_to_string (enum ivgen v) ++{ ++ switch (v) { ++ case IVGEN_PLAIN: return "plain"; ++ case IVGEN_PLAIN64: return "plain64"; ++ /*case IVGEN_ESSIV: return "essiv";*/ ++ default: abort (); ++ } ++} ++ ++static void ++calculate_iv (enum ivgen v, uint8_t *iv, size_t ivlen, uint64_t sector) ++{ ++ size_t prefixlen; ++ uint32_t sector32; ++ ++ switch (v) { ++ case IVGEN_PLAIN: ++ prefixlen = 4; /* 32 bits */ ++ if (prefixlen > ivlen) ++ prefixlen = ivlen; ++ sector32 = (uint32_t) sector; /* truncate to only lower bits */ ++ sector32 = htole32 (sector32); ++ memcpy (iv, §or32, prefixlen); ++ memset (iv + prefixlen, 0, ivlen - prefixlen); ++ break; ++ ++ case IVGEN_PLAIN64: ++ prefixlen = 8; /* 64 bits */ ++ if (prefixlen > ivlen) ++ prefixlen = ivlen; ++ sector = htole64 (sector); ++ memcpy (iv, §or, prefixlen); ++ memset (iv + prefixlen, 0, ivlen - prefixlen); ++ break; ++ ++ /*case IVGEN_ESSIV:*/ ++ default: abort (); ++ } ++} ++ ++/* Cipher algorithm. ++ * ++ * qemu in theory supports many more, but with the GnuTLS backend only ++ * AES is supported. The kernel seems to only support AES for LUKSv1. ++ */ ++enum cipher_alg { ++ CIPHER_ALG_AES_128, CIPHER_ALG_AES_192, CIPHER_ALG_AES_256, ++}; ++ ++static enum cipher_alg ++lookup_cipher_alg (const char *str, enum cipher_mode mode, int key_bytes) ++{ ++ if (mode == CIPHER_MODE_XTS) ++ key_bytes /= 2; ++ ++ if (strcmp (str, "aes") == 0) { ++ if (key_bytes == 16) ++ return CIPHER_ALG_AES_128; ++ if (key_bytes == 24) ++ return CIPHER_ALG_AES_192; ++ if (key_bytes == 32) ++ return CIPHER_ALG_AES_256; ++ } ++ nbdkit_error ("unknown cipher algorithm: %s (expecting \"aes\", etc)", str); ++ return -1; ++} ++ ++static const char * ++cipher_alg_to_string (enum cipher_alg v) ++{ ++ switch (v) { ++ case CIPHER_ALG_AES_128: return "aes-128"; ++ case CIPHER_ALG_AES_192: return "aes-192"; ++ case CIPHER_ALG_AES_256: return "aes-256"; ++ default: abort (); ++ } ++} ++ ++#if 0 ++static int ++cipher_alg_key_bytes (enum cipher_alg v) ++{ ++ switch (v) { ++ case CIPHER_ALG_AES_128: return 16; ++ case CIPHER_ALG_AES_192: return 24; ++ case CIPHER_ALG_AES_256: return 32; ++ default: abort (); ++ } ++} ++#endif ++ ++static int ++cipher_alg_iv_len (enum cipher_alg v, enum cipher_mode mode) ++{ ++ if (CIPHER_MODE_ECB) ++ return 0; /* Don't need an IV in this mode. */ ++ ++ switch (v) { ++ case CIPHER_ALG_AES_128: ++ case CIPHER_ALG_AES_192: ++ case CIPHER_ALG_AES_256: ++ return 16; ++ default: abort (); ++ } ++} ++ ++/* Hash, eg.MD5, SHA1 etc. ++ * ++ * We reuse the GnuTLS digest algorithm enum here since it supports at ++ * least all the ones that LUKSv1 does. ++ */ ++static gnutls_digest_algorithm_t ++lookup_hash (const char *str) ++{ ++ if (strcmp (str, "md5") == 0) ++ return GNUTLS_DIG_MD5; ++ if (strcmp (str, "sha1") == 0) ++ return GNUTLS_DIG_SHA1; ++ if (strcmp (str, "sha224") == 0) ++ return GNUTLS_DIG_SHA224; ++ if (strcmp (str, "sha256") == 0) ++ return GNUTLS_DIG_SHA256; ++ if (strcmp (str, "sha384") == 0) ++ return GNUTLS_DIG_SHA384; ++ if (strcmp (str, "sha512") == 0) ++ return GNUTLS_DIG_SHA512; ++ if (strcmp (str, "ripemd160") == 0) ++ return GNUTLS_DIG_RMD160; ++ nbdkit_error ("unknown hash algorithm: %s " ++ "(expecting \"md5\", \"sha1\", \"sha224\", etc)", str); ++ return -1; ++} ++ ++static const char * ++hash_to_string (gnutls_digest_algorithm_t v) ++{ ++ switch (v) { ++ case GNUTLS_DIG_UNKNOWN: return "unknown"; ++ case GNUTLS_DIG_MD5: return "md5"; ++ case GNUTLS_DIG_SHA1: return "sha1"; ++ case GNUTLS_DIG_SHA224: return "sha224"; ++ case GNUTLS_DIG_SHA256: return "sha256"; ++ case GNUTLS_DIG_SHA384: return "sha384"; ++ case GNUTLS_DIG_SHA512: return "sha512"; ++ case GNUTLS_DIG_RMD160: return "ripemd160"; ++ default: abort (); ++ } ++} ++ ++#if 0 ++/* See qemu & dm-crypt implementations for an explanation of what's ++ * going on here. ++ */ ++enum cipher_alg ++lookup_essiv_cipher (enum cipher_alg cipher_alg, ++ gnutls_digest_algorithm_t ivgen_hash_alg) ++{ ++ int digest_bytes = gnutls_hash_get_len (ivgen_hash_alg); ++ int key_bytes = cipher_alg_key_bytes (cipher_alg); ++ ++ if (digest_bytes == key_bytes) ++ return cipher_alg; ++ ++ switch (cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ case CIPHER_ALG_AES_192: ++ case CIPHER_ALG_AES_256: ++ if (digest_bytes == 16) return CIPHER_ALG_AES_128; ++ if (digest_bytes == 24) return CIPHER_ALG_AES_192; ++ if (digest_bytes == 32) return CIPHER_ALG_AES_256; ++ nbdkit_error ("no %s cipher available with key size %d", ++ "AES", digest_bytes); ++ return -1; ++ default: ++ nbdkit_error ("ESSIV does not support cipher %s", ++ cipher_alg_to_string (cipher_alg)); ++ return -1; ++ } ++} ++#endif ++ ++/* Per-connection data. */ ++struct luks_data { ++ /* LUKS header, if necessary byte-swapped into host order. */ ++ struct luks_phdr phdr; ++ ++ /* Decoded algorithm etc. */ ++ enum cipher_alg cipher_alg; ++ enum cipher_mode cipher_mode; ++ gnutls_digest_algorithm_t hash_alg; ++ enum ivgen ivgen_alg; ++ gnutls_digest_algorithm_t ivgen_hash_alg; ++ enum cipher_alg ivgen_cipher_alg; ++ ++ /* GnuTLS algorithm. */ ++ gnutls_cipher_algorithm_t gnutls_cipher; ++ ++ /* If we managed to decrypt one of the keyslots using the passphrase ++ * then this contains the master key, otherwise NULL. ++ */ ++ uint8_t *masterkey; ++}; ++ ++/* Parse the header fields containing cipher algorithm, mode, etc. */ ++static int ++parse_cipher_strings (struct luks_data *h) ++{ ++ char cipher_name[33], cipher_mode[33], hash_spec[33]; ++ char *ivgen, *ivhash; ++ ++ /* Copy the header fields locally and ensure they are \0 terminated. */ ++ memcpy (cipher_name, h->phdr.cipher_name, 32); ++ cipher_name[32] = 0; ++ memcpy (cipher_mode, h->phdr.cipher_mode, 32); ++ cipher_mode[32] = 0; ++ memcpy (hash_spec, h->phdr.hash_spec, 32); ++ hash_spec[32] = 0; ++ ++ nbdkit_debug ("LUKS v%" PRIu16 " cipher: %s mode: %s hash: %s " ++ "master key: %" PRIu32 " bits", ++ h->phdr.version, cipher_name, cipher_mode, hash_spec, ++ h->phdr.master_key_len * 8); ++ ++ /* The cipher_mode header has the form: "ciphermode-ivgen[:ivhash]" ++ * QEmu writes: "xts-plain64" ++ */ ++ ivgen = strchr (cipher_mode, '-'); ++ if (!ivgen) { ++ nbdkit_error ("incorrect cipher_mode header, " ++ "expecting mode-ivgenerator but got \"%s\"", cipher_mode); ++ return -1; ++ } ++ *ivgen = '\0'; ++ ivgen++; ++ ++ ivhash = strchr (ivgen, ':'); ++ if (!ivhash) ++ h->ivgen_hash_alg = GNUTLS_DIG_UNKNOWN; ++ else { ++ *ivhash = '\0'; ++ ivhash++; ++ ++ h->ivgen_hash_alg = lookup_hash (ivhash); ++ if (h->ivgen_hash_alg == -1) ++ return -1; ++ } ++ ++ h->cipher_mode = lookup_cipher_mode (cipher_mode); ++ if (h->cipher_mode == -1) ++ return -1; ++ ++ h->cipher_alg = lookup_cipher_alg (cipher_name, h->cipher_mode, ++ h->phdr.master_key_len); ++ if (h->cipher_alg == -1) ++ return -1; ++ ++ h->hash_alg = lookup_hash (hash_spec); ++ if (h->hash_alg == -1) ++ return -1; ++ ++ h->ivgen_alg = lookup_ivgen (ivgen); ++ if (h->ivgen_alg == -1) ++ return -1; ++ ++#if 0 ++ if (h->ivgen_alg == IVGEN_ESSIV) { ++ if (!ivhash) { ++ nbdkit_error ("incorrect IV generator hash specification"); ++ return -1; ++ } ++ h->ivgen_cipher_alg = lookup_essiv_cipher (h->cipher_alg, ++ h->ivgen_hash_alg); ++ if (h->ivgen_cipher_alg == -1) ++ return -1; ++ } ++ else ++#endif ++ h->ivgen_cipher_alg = h->cipher_alg; ++ ++ nbdkit_debug ("LUKS parsed ciphers: %s %s %s %s %s %s", ++ cipher_alg_to_string (h->cipher_alg), ++ cipher_mode_to_string (h->cipher_mode), ++ hash_to_string (h->hash_alg), ++ ivgen_to_string (h->ivgen_alg), ++ hash_to_string (h->ivgen_hash_alg), ++ cipher_alg_to_string (h->ivgen_cipher_alg)); ++ ++ /* GnuTLS combines cipher and block mode into a single value. Not ++ * all possible combinations are available in GnuTLS. See: ++ * https://www.gnutls.org/manual/html_node/Supported-ciphersuites.html ++ */ ++ h->gnutls_cipher = GNUTLS_CIPHER_NULL; ++ switch (h->cipher_mode) { ++ case CIPHER_MODE_XTS: ++ switch (h->cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; ++ break; ++ case CIPHER_ALG_AES_256: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; ++ break; ++ default: break; ++ } ++ break; ++ case CIPHER_MODE_CBC: ++ switch (h->cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_CBC; ++ break; ++ case CIPHER_ALG_AES_192: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_192_CBC; ++ break; ++ case CIPHER_ALG_AES_256: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_CBC; ++ break; ++ default: break; ++ } ++ default: break; ++ } ++ if (h->gnutls_cipher == GNUTLS_CIPHER_NULL) { ++ nbdkit_error ("cipher algorithm %s in mode %s is not supported by GnuTLS", ++ cipher_alg_to_string (h->cipher_alg), ++ cipher_mode_to_string (h->cipher_mode)); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++/* Anti-Forensic merge operation. */ ++static void ++xor (const uint8_t *in1, const uint8_t *in2, uint8_t *out, size_t len) ++{ ++ size_t i; ++ ++ for (i = 0; i < len; ++i) ++ out[i] = in1[i] ^ in2[i]; ++} ++ ++static int ++af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) ++{ ++ size_t digest_bytes = gnutls_hash_get_len (hash_alg); ++ size_t nr_blocks, last_block_len; ++ size_t i; ++ CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); ++ int r; ++ gnutls_hash_hd_t hash; ++ ++ nr_blocks = len / digest_bytes; ++ last_block_len = len % digest_bytes; ++ if (last_block_len != 0) ++ nr_blocks++; ++ else ++ last_block_len = digest_bytes; ++ ++ for (i = 0; i < nr_blocks; ++i) { ++ const uint32_t iv = htobe32 (i); ++ const size_t blen = i < nr_blocks - 1 ? digest_bytes : last_block_len; ++ ++ /* Hash iv + i'th block into temp. */ ++ r = gnutls_hash_init (&hash, hash_alg); ++ if (r != 0) { ++ nbdkit_error ("gnutls_hash_init: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ gnutls_hash (hash, &iv, sizeof iv); ++ gnutls_hash (hash, &block[i*digest_bytes], blen); ++ gnutls_hash_deinit (hash, temp); ++ ++ memcpy (&block[i*digest_bytes], temp, blen); ++ } ++ ++ return 0; ++} ++ ++static int ++afmerge (gnutls_digest_algorithm_t hash_alg, uint32_t stripes, ++ const uint8_t *in, uint8_t *out, size_t outlen) ++{ ++ CLEANUP_FREE uint8_t *block = calloc (1, outlen); ++ size_t i; ++ ++ /* NB: input size is stripes * master_key_len where ++ * master_key_len == outlen ++ */ ++ for (i = 0; i < stripes-1; ++i) { ++ xor (&in[i*outlen], block, block, outlen); ++ if (af_hash (hash_alg, block, outlen) == -1) ++ return -1; ++ } ++ xor (&in[i*outlen], block, out, outlen); ++ return 0; ++} ++ ++/* Length of key material in key slot i (sectors). ++ * ++ * This is basically copied from qemu because the spec description is ++ * unintelligible and apparently doesn't match reality. ++ */ ++static uint64_t ++key_material_length_in_sectors (struct luks_data *h, size_t i) ++{ ++ uint64_t len, r; ++ ++ len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; ++ r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); ++ r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); ++ return r; ++} ++ ++/* Try the passphrase in key slot i. If this returns true then the ++ * passphrase was able to decrypt the master key, and the master key ++ * has been stored in h->masterkey. ++ */ ++static int ++try_passphrase_in_keyslot (nbdkit_next *next, struct luks_data *h, ++ size_t i, const char *passphrase) ++{ ++ /* I believe this is supposed to be safe, looking at the GnuTLS ++ * header file. ++ */ ++ const gnutls_mac_algorithm_t mac = (gnutls_mac_algorithm_t) h->hash_alg; ++ struct luks_keyslot *ks = &h->phdr.keyslot[i]; ++ size_t split_key_len; ++ CLEANUP_FREE uint8_t *split_key = NULL; ++ CLEANUP_FREE uint8_t *masterkey = NULL; ++ const gnutls_datum_t key = ++ { (unsigned char *) passphrase, strlen (passphrase) }; ++ const gnutls_datum_t salt = ++ { (unsigned char *) ks->password_salt, LUKS_SALTSIZE }; ++ const gnutls_datum_t msalt = ++ { (unsigned char *) h->phdr.master_key_salt, LUKS_SALTSIZE }; ++ gnutls_datum_t mkey; ++ gnutls_cipher_hd_t cipher; ++ int r, err = 0; ++ uint64_t start; ++ uint8_t key_digest[LUKS_DIGESTSIZE]; ++ ++ if (ks->active != LUKS_KEY_ENABLED) ++ return 0; ++ ++ split_key_len = h->phdr.master_key_len * ks->stripes; ++ split_key = malloc (split_key_len); ++ if (split_key == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ masterkey = malloc (h->phdr.master_key_len); ++ if (masterkey == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ ++ /* Hash the passphrase to make a possible masterkey. */ ++ r = gnutls_pbkdf2 (mac, &key, &salt, ks->password_iterations, ++ masterkey, h->phdr.master_key_len); ++ if (r != 0) { ++ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ /* Read master key material from plugin. */ ++ start = ks->key_material_offset * LUKS_SECTOR_SIZE; ++ if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { ++ errno = err; ++ return -1; ++ } ++ ++ /* Decrypt the (still AFsplit) master key material. */ ++ mkey.data = (unsigned char *) masterkey; ++ mkey.size = h->phdr.master_key_len; ++ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ r = do_decrypt (h, cipher, 0, split_key, split_key_len / LUKS_SECTOR_SIZE); ++ gnutls_cipher_deinit (cipher); ++ if (r == -1) ++ return -1; ++ ++ /* Decode AFsplit key to a possible masterkey. */ ++ if (afmerge (h->hash_alg, ks->stripes, split_key, ++ masterkey, h->phdr.master_key_len) == -1) ++ return -1; ++ ++ /* Check if the masterkey is correct by comparing hash of the ++ * masterkey with LUKS header. ++ */ ++ r = gnutls_pbkdf2 (mac, &mkey, &msalt, ++ h->phdr.master_key_digest_iterations, ++ key_digest, LUKS_DIGESTSIZE); ++ if (r != 0) { ++ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ if (memcmp (key_digest, h->phdr.master_key_digest, LUKS_DIGESTSIZE) == 0) { ++ /* The passphrase is correct so save the master key in the handle. */ ++ h->masterkey = malloc (h->phdr.master_key_len); ++ if (h->masterkey == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ memcpy (h->masterkey, masterkey, h->phdr.master_key_len); ++ return 1; ++ } ++ ++ return 0; ++} ++ ++struct luks_data * ++load_header (nbdkit_next *next, const char *passphrase) ++{ ++ static const char expected_magic[] = LUKS_MAGIC; ++ struct luks_data *h; ++ int64_t size; ++ int err = 0, r; ++ size_t i; ++ struct luks_keyslot *ks; ++ char uuid[41]; ++ ++ h = calloc (1, sizeof *h); ++ if (h == NULL) { ++ nbdkit_error ("calloc: %m"); ++ return NULL; ++ } ++ ++ /* Check the struct size matches the documentation. */ ++ assert (sizeof (struct luks_phdr) == 592); ++ ++ /* Check this is a LUKSv1 disk. */ ++ size = next->get_size (next); ++ if (size == -1) { ++ free (h); ++ return NULL; ++ } ++ if (size < 16384) { ++ nbdkit_error ("disk is too small to be LUKS-encrypted"); ++ free (h); ++ return NULL; ++ } ++ ++ /* Read the phdr. */ ++ if (next->pread (next, &h->phdr, sizeof h->phdr, 0, 0, &err) == -1) { ++ free (h); ++ errno = err; ++ return NULL; ++ } ++ ++ if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { ++ nbdkit_error ("this disk does not contain a LUKS header"); ++ return NULL; ++ } ++ h->phdr.version = be16toh (h->phdr.version); ++ if (h->phdr.version != 1) { ++ nbdkit_error ("this disk contains a LUKS version %" PRIu16 " header, " ++ "but this filter only supports LUKSv1", ++ h->phdr.version); ++ free (h); ++ return NULL; ++ } ++ ++ /* Byte-swap the rest of the header. */ ++ h->phdr.payload_offset = be32toh (h->phdr.payload_offset); ++ h->phdr.master_key_len = be32toh (h->phdr.master_key_len); ++ h->phdr.master_key_digest_iterations = ++ be32toh (h->phdr.master_key_digest_iterations); ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ ks = &h->phdr.keyslot[i]; ++ ks->active = be32toh (ks->active); ++ ks->password_iterations = be32toh (ks->password_iterations); ++ ks->key_material_offset = be32toh (ks->key_material_offset); ++ ks->stripes = be32toh (ks->stripes); ++ } ++ ++ /* Sanity check some fields. */ ++ if (h->phdr.payload_offset >= size / LUKS_SECTOR_SIZE) { ++ nbdkit_error ("bad LUKSv1 header: payload offset points beyond " ++ "the end of the disk"); ++ free (h); ++ return NULL; ++ } ++ ++ /* We derive several allocations from master_key_len so make sure ++ * it's not insane. ++ */ ++ if (h->phdr.master_key_len > 1024) { ++ nbdkit_error ("bad LUKSv1 header: master key is too long"); ++ free (h); ++ return NULL; ++ } ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ uint64_t start, len; ++ ++ ks = &h->phdr.keyslot[i]; ++ switch (ks->active) { ++ case LUKS_KEY_ENABLED: ++ if (!ks->stripes) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu is corrupted", i); ++ free (h); ++ return NULL; ++ } ++ if (ks->stripes >= 10000) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu stripes too large", i); ++ return NULL; ++ } ++ start = ks->key_material_offset; ++ len = key_material_length_in_sectors (h, i); ++ if (len > 4096) /* bound it at something reasonable */ { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu key material length " ++ "is too large", i); ++ free (h); ++ return NULL; ++ } ++ if (start * LUKS_SECTOR_SIZE >= size || ++ (start + len) * LUKS_SECTOR_SIZE >= size) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu key material offset " ++ "points beyond the end of the disk", i); ++ free (h); ++ return NULL; ++ } ++ /*FALLTHROUGH*/ ++ case LUKS_KEY_DISABLED: ++ break; ++ ++ default: ++ nbdkit_error ("bad LUKSv1 header: key slot %zu has " ++ "an invalid active flag", i); ++ return NULL; ++ } ++ } ++ ++ /* Decode the ciphers. */ ++ if (parse_cipher_strings (h) == -1) { ++ free (h); ++ return NULL; ++ } ++ ++ /* Dump some information about the header. */ ++ memcpy (uuid, h->phdr.uuid, 40); ++ uuid[40] = 0; ++ nbdkit_debug ("LUKS UUID: %s", uuid); ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ uint64_t start, len; ++ ++ ks = &h->phdr.keyslot[i]; ++ if (ks->active == LUKS_KEY_ENABLED) { ++ start = ks->key_material_offset; ++ len = key_material_length_in_sectors (h, i); ++ nbdkit_debug ("LUKS key slot %zu: key material in sectors %" PRIu64 ++ "..%" PRIu64, ++ i, start, start+len-1); ++ } ++ } ++ ++ /* Now try to unlock the master key. */ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ r = try_passphrase_in_keyslot (next, h, i, passphrase); ++ if (r == -1) { ++ free (h); ++ return NULL; ++ } ++ if (r > 0) ++ goto unlocked; ++ } ++ nbdkit_error ("LUKS passphrase is not correct, " ++ "no key slot could be unlocked"); ++ free (h); ++ return NULL; ++ ++ unlocked: ++ assert (h->masterkey != NULL); ++ nbdkit_debug ("LUKS unlocked block device with passphrase"); ++ ++ return h; ++} ++ ++/* Free fields in the handle that might have been set by load_header. */ ++void ++free_luks_data (struct luks_data *h) ++{ ++ if (h->masterkey) { ++ memset (h->masterkey, 0, h->phdr.master_key_len); ++ free (h->masterkey); ++ } ++ free (h); ++} ++ ++uint64_t ++get_payload_offset (struct luks_data *h) ++{ ++ return h->phdr.payload_offset; ++} ++ ++gnutls_cipher_hd_t ++create_cipher (struct luks_data *h) ++{ ++ gnutls_datum_t mkey; ++ gnutls_cipher_hd_t cipher; ++ int r; ++ ++ assert (h->masterkey != NULL); ++ ++ mkey.data = (unsigned char *) h->masterkey; ++ mkey.size = h->phdr.master_key_len; ++ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); ++ return NULL; ++ } ++ return cipher; ++} ++ ++/* Perform decryption of a block of data in memory. */ ++int ++do_decrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, ++ uint64_t sector, uint8_t *buf, size_t nr_sectors) ++{ ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ CLEANUP_FREE uint8_t *iv = malloc (ivlen); ++ int r; ++ ++ while (nr_sectors) { ++ calculate_iv (h->ivgen_alg, iv, ivlen, sector); ++ gnutls_cipher_set_iv (cipher, iv, ivlen); ++ r = gnutls_cipher_decrypt2 (cipher, ++ buf, LUKS_SECTOR_SIZE, /* ciphertext */ ++ buf, LUKS_SECTOR_SIZE /* plaintext */); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ buf += LUKS_SECTOR_SIZE; ++ nr_sectors--; ++ sector++; ++ } ++ ++ return 0; ++} ++ ++/* Perform encryption of a block of data in memory. */ ++int ++do_encrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, ++ uint64_t sector, uint8_t *buf, size_t nr_sectors) ++{ ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ CLEANUP_FREE uint8_t *iv = malloc (ivlen); ++ int r; ++ ++ while (nr_sectors) { ++ calculate_iv (h->ivgen_alg, iv, ivlen, sector); ++ gnutls_cipher_set_iv (cipher, iv, ivlen); ++ r = gnutls_cipher_encrypt2 (cipher, ++ buf, LUKS_SECTOR_SIZE, /* plaintext */ ++ buf, LUKS_SECTOR_SIZE /* ciphertext */); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ buf += LUKS_SECTOR_SIZE; ++ nr_sectors--; ++ sector++; ++ } ++ ++ return 0; ++} +diff --git a/filters/luks/luks-encryption.h b/filters/luks/luks-encryption.h +new file mode 100644 +index 00000000..3f8b9c9e +--- /dev/null ++++ b/filters/luks/luks-encryption.h +@@ -0,0 +1,78 @@ ++/* nbdkit ++ * Copyright (C) 2013-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++/* This header file defines the file format used by LUKSv1. See also: ++ * https://gitlab.com/cryptsetup/cryptsetup/-/wikis/LUKS-standard/on-disk-format.pdf ++ * Note we do not yet support LUKSv2. ++ */ ++ ++#ifndef NBDKIT_LUKS_ENCRYPTION_H ++#define NBDKIT_LUKS_ENCRYPTION_H ++ ++#include ++#include ++ ++#define LUKS_SECTOR_SIZE 512 ++ ++/* Per-connection data. */ ++struct luks_data; ++ ++/* Load the LUKS header, parse the algorithms, unlock the masterkey ++ * using the passphrase, initialize all the fields in the handle. ++ * ++ * This function may call next->pread (many times). ++ */ ++extern struct luks_data *load_header (nbdkit_next *next, ++ const char *passphrase); ++ ++/* Free the handle and all fields inside it. */ ++extern void free_luks_data (struct luks_data *h); ++ ++/* Get the offset where the encrypted data starts (in sectors). */ ++extern uint64_t get_payload_offset (struct luks_data *h); ++ ++/* Create an GnuTLS cipher, initialized with the master key. Must be ++ * freed by the caller using gnutls_cipher_deinit. ++ */ ++extern gnutls_cipher_hd_t create_cipher (struct luks_data *h); ++ ++/* Perform decryption/encryption of a block of memory in-place. ++ * ++ * 'sector' is the sector number on disk, used to calculate IVs. (The ++ * keyslots also use these functions, but sector must be 0). ++ */ ++extern int do_decrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, ++ uint64_t sector, uint8_t *buf, size_t nr_sectors); ++extern int do_encrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, ++ uint64_t sector, uint8_t *buf, size_t nr_sectors); ++ ++#endif /* NBDKIT_LUKS_ENCRYPTION_H */ +diff --git a/filters/luks/luks.c b/filters/luks/luks.c +index cc619698..8ad3f4ec 100644 +--- a/filters/luks/luks.c ++++ b/filters/luks/luks.c +@@ -45,49 +45,11 @@ + + #include + +-#include "byte-swapping.h" ++#include "luks-encryption.h" ++ + #include "cleanup.h" + #include "isaligned.h" + #include "minmax.h" +-#include "rounding.h" +- +-/* LUKSv1 constants. */ +-#define LUKS_MAGIC { 'L', 'U', 'K', 'S', 0xBA, 0xBE } +-#define LUKS_MAGIC_LEN 6 +-#define LUKS_DIGESTSIZE 20 +-#define LUKS_SALTSIZE 32 +-#define LUKS_NUMKEYS 8 +-#define LUKS_KEY_DISABLED 0x0000DEAD +-#define LUKS_KEY_ENABLED 0x00AC71F3 +-#define LUKS_STRIPES 4000 +-#define LUKS_ALIGN_KEYSLOTS 4096 +-#define LUKS_SECTOR_SIZE 512 +- +-/* Key slot. */ +-struct luks_keyslot { +- uint32_t active; /* LUKS_KEY_DISABLED|LUKS_KEY_ENABLED */ +- uint32_t password_iterations; +- char password_salt[LUKS_SALTSIZE]; +- uint32_t key_material_offset; +- uint32_t stripes; +-} __attribute__((__packed__)); +- +-/* LUKS superblock. */ +-struct luks_phdr { +- char magic[LUKS_MAGIC_LEN]; /* LUKS_MAGIC */ +- uint16_t version; /* Only 1 is supported. */ +- char cipher_name[32]; +- char cipher_mode[32]; +- char hash_spec[32]; +- uint32_t payload_offset; +- uint32_t master_key_len; +- uint8_t master_key_digest[LUKS_DIGESTSIZE]; +- uint8_t master_key_salt[LUKS_SALTSIZE]; +- uint32_t master_key_digest_iterations; +- uint8_t uuid[40]; +- +- struct luks_keyslot keyslot[LUKS_NUMKEYS]; /* Key slots. */ +-} __attribute__((__packed__)); + + static char *passphrase = NULL; + +@@ -135,251 +97,9 @@ luks_config_complete (nbdkit_next_config_complete *next, nbdkit_backend *nxdata) + #define luks_config_help \ + "passphrase= Secret passphrase." + +-enum cipher_mode { +- CIPHER_MODE_ECB, CIPHER_MODE_CBC, CIPHER_MODE_XTS, CIPHER_MODE_CTR, +-}; +- +-static enum cipher_mode +-lookup_cipher_mode (const char *str) +-{ +- if (strcmp (str, "ecb") == 0) +- return CIPHER_MODE_ECB; +- if (strcmp (str, "cbc") == 0) +- return CIPHER_MODE_CBC; +- if (strcmp (str, "xts") == 0) +- return CIPHER_MODE_XTS; +- if (strcmp (str, "ctr") == 0) +- return CIPHER_MODE_CTR; +- nbdkit_error ("unknown cipher mode: %s " +- "(expecting \"ecb\", \"cbc\", \"xts\" or \"ctr\")", str); +- return -1; +-} +- +-static const char * +-cipher_mode_to_string (enum cipher_mode v) +-{ +- switch (v) { +- case CIPHER_MODE_ECB: return "ecb"; +- case CIPHER_MODE_CBC: return "cbc"; +- case CIPHER_MODE_XTS: return "xts"; +- case CIPHER_MODE_CTR: return "ctr"; +- default: abort (); +- } +-} +- +-enum ivgen { +- IVGEN_PLAIN, IVGEN_PLAIN64, /* IVGEN_ESSIV, */ +-}; +- +-static enum ivgen +-lookup_ivgen (const char *str) +-{ +- if (strcmp (str, "plain") == 0) +- return IVGEN_PLAIN; +- if (strcmp (str, "plain64") == 0) +- return IVGEN_PLAIN64; +-/* +- if (strcmp (str, "essiv") == 0) +- return IVGEN_ESSIV; +-*/ +- nbdkit_error ("unknown IV generation algorithm: %s " +- "(expecting \"plain\", \"plain64\" etc)", str); +- return -1; +-} +- +-static const char * +-ivgen_to_string (enum ivgen v) +-{ +- switch (v) { +- case IVGEN_PLAIN: return "plain"; +- case IVGEN_PLAIN64: return "plain64"; +- /*case IVGEN_ESSIV: return "essiv";*/ +- default: abort (); +- } +-} +- +-static void +-calculate_iv (enum ivgen v, uint8_t *iv, size_t ivlen, uint64_t sector) +-{ +- size_t prefixlen; +- uint32_t sector32; +- +- switch (v) { +- case IVGEN_PLAIN: +- prefixlen = 4; /* 32 bits */ +- if (prefixlen > ivlen) +- prefixlen = ivlen; +- sector32 = (uint32_t) sector; /* truncate to only lower bits */ +- sector32 = htole32 (sector32); +- memcpy (iv, §or32, prefixlen); +- memset (iv + prefixlen, 0, ivlen - prefixlen); +- break; +- +- case IVGEN_PLAIN64: +- prefixlen = 8; /* 64 bits */ +- if (prefixlen > ivlen) +- prefixlen = ivlen; +- sector = htole64 (sector); +- memcpy (iv, §or, prefixlen); +- memset (iv + prefixlen, 0, ivlen - prefixlen); +- break; +- +- /*case IVGEN_ESSIV:*/ +- default: abort (); +- } +-} +- +-enum cipher_alg { +- CIPHER_ALG_AES_128, CIPHER_ALG_AES_192, CIPHER_ALG_AES_256, +-}; +- +-static enum cipher_alg +-lookup_cipher_alg (const char *str, enum cipher_mode mode, int key_bytes) +-{ +- if (mode == CIPHER_MODE_XTS) +- key_bytes /= 2; +- +- if (strcmp (str, "aes") == 0) { +- if (key_bytes == 16) +- return CIPHER_ALG_AES_128; +- if (key_bytes == 24) +- return CIPHER_ALG_AES_192; +- if (key_bytes == 32) +- return CIPHER_ALG_AES_256; +- } +- nbdkit_error ("unknown cipher algorithm: %s (expecting \"aes\", etc)", str); +- return -1; +-} +- +-static const char * +-cipher_alg_to_string (enum cipher_alg v) +-{ +- switch (v) { +- case CIPHER_ALG_AES_128: return "aes-128"; +- case CIPHER_ALG_AES_192: return "aes-192"; +- case CIPHER_ALG_AES_256: return "aes-256"; +- default: abort (); +- } +-} +- +-#if 0 +-static int +-cipher_alg_key_bytes (enum cipher_alg v) +-{ +- switch (v) { +- case CIPHER_ALG_AES_128: return 16; +- case CIPHER_ALG_AES_192: return 24; +- case CIPHER_ALG_AES_256: return 32; +- default: abort (); +- } +-} +-#endif +- +-static int +-cipher_alg_iv_len (enum cipher_alg v, enum cipher_mode mode) +-{ +- if (CIPHER_MODE_ECB) +- return 0; /* Don't need an IV in this mode. */ +- +- switch (v) { +- case CIPHER_ALG_AES_128: +- case CIPHER_ALG_AES_192: +- case CIPHER_ALG_AES_256: +- return 16; +- default: abort (); +- } +-} +- +-static gnutls_digest_algorithm_t +-lookup_hash (const char *str) +-{ +- if (strcmp (str, "md5") == 0) +- return GNUTLS_DIG_MD5; +- if (strcmp (str, "sha1") == 0) +- return GNUTLS_DIG_SHA1; +- if (strcmp (str, "sha224") == 0) +- return GNUTLS_DIG_SHA224; +- if (strcmp (str, "sha256") == 0) +- return GNUTLS_DIG_SHA256; +- if (strcmp (str, "sha384") == 0) +- return GNUTLS_DIG_SHA384; +- if (strcmp (str, "sha512") == 0) +- return GNUTLS_DIG_SHA512; +- if (strcmp (str, "ripemd160") == 0) +- return GNUTLS_DIG_RMD160; +- nbdkit_error ("unknown hash algorithm: %s " +- "(expecting \"md5\", \"sha1\", \"sha224\", etc)", str); +- return -1; +-} +- +-static const char * +-hash_to_string (gnutls_digest_algorithm_t v) +-{ +- switch (v) { +- case GNUTLS_DIG_UNKNOWN: return "unknown"; +- case GNUTLS_DIG_MD5: return "md5"; +- case GNUTLS_DIG_SHA1: return "sha1"; +- case GNUTLS_DIG_SHA224: return "sha224"; +- case GNUTLS_DIG_SHA256: return "sha256"; +- case GNUTLS_DIG_SHA384: return "sha384"; +- case GNUTLS_DIG_SHA512: return "sha512"; +- case GNUTLS_DIG_RMD160: return "ripemd160"; +- default: abort (); +- } +-} +- +-#if 0 +-/* See qemu & dm-crypt implementations for an explanation of what's +- * going on here. +- */ +-static enum cipher_alg +-lookup_essiv_cipher (enum cipher_alg cipher_alg, +- gnutls_digest_algorithm_t ivgen_hash_alg) +-{ +- int digest_bytes = gnutls_hash_get_len (ivgen_hash_alg); +- int key_bytes = cipher_alg_key_bytes (cipher_alg); +- +- if (digest_bytes == key_bytes) +- return cipher_alg; +- +- switch (cipher_alg) { +- case CIPHER_ALG_AES_128: +- case CIPHER_ALG_AES_192: +- case CIPHER_ALG_AES_256: +- if (digest_bytes == 16) return CIPHER_ALG_AES_128; +- if (digest_bytes == 24) return CIPHER_ALG_AES_192; +- if (digest_bytes == 32) return CIPHER_ALG_AES_256; +- nbdkit_error ("no %s cipher available with key size %d", +- "AES", digest_bytes); +- return -1; +- default: +- nbdkit_error ("ESSIV does not support cipher %s", +- cipher_alg_to_string (cipher_alg)); +- return -1; +- } +-} +-#endif +- + /* Per-connection handle. */ + struct handle { +- /* LUKS header, if necessary byte-swapped into host order. */ +- struct luks_phdr phdr; +- +- /* Decoded algorithm etc. */ +- enum cipher_alg cipher_alg; +- enum cipher_mode cipher_mode; +- gnutls_digest_algorithm_t hash_alg; +- enum ivgen ivgen_alg; +- gnutls_digest_algorithm_t ivgen_hash_alg; +- enum cipher_alg ivgen_cipher_alg; +- +- /* GnuTLS algorithm. */ +- gnutls_cipher_algorithm_t gnutls_cipher; +- +- /* If we managed to decrypt one of the keyslots using the passphrase +- * then this contains the master key, otherwise NULL. +- */ +- uint8_t *masterkey; ++ struct luks_data *h; + }; + + static void * +@@ -405,536 +125,21 @@ luks_close (void *handle) + { + struct handle *h = handle; + +- if (h->masterkey) { +- memset (h->masterkey, 0, h->phdr.master_key_len); +- free (h->masterkey); +- } ++ free_luks_data (h->h); + free (h); + } + +-/* Perform decryption of a block of data in memory. */ +-static int +-do_decrypt (struct handle *h, gnutls_cipher_hd_t cipher, +- uint64_t offset, uint8_t *buf, size_t len) +-{ +- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); +- uint64_t sector = offset / LUKS_SECTOR_SIZE; +- CLEANUP_FREE uint8_t *iv = malloc (ivlen); +- int r; +- +- assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); +- assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); +- +- while (len) { +- calculate_iv (h->ivgen_alg, iv, ivlen, sector); +- gnutls_cipher_set_iv (cipher, iv, ivlen); +- r = gnutls_cipher_decrypt2 (cipher, +- buf, LUKS_SECTOR_SIZE, /* ciphertext */ +- buf, LUKS_SECTOR_SIZE /* plaintext */); +- if (r != 0) { +- nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); +- return -1; +- } +- +- buf += LUKS_SECTOR_SIZE; +- offset += LUKS_SECTOR_SIZE; +- len -= LUKS_SECTOR_SIZE; +- sector++; +- } +- +- return 0; +-} +- +-/* Perform encryption of a block of data in memory. */ +-static int +-do_encrypt (struct handle *h, gnutls_cipher_hd_t cipher, +- uint64_t offset, uint8_t *buf, size_t len) +-{ +- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); +- uint64_t sector = offset / LUKS_SECTOR_SIZE; +- CLEANUP_FREE uint8_t *iv = malloc (ivlen); +- int r; +- +- assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); +- assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); +- +- while (len) { +- calculate_iv (h->ivgen_alg, iv, ivlen, sector); +- gnutls_cipher_set_iv (cipher, iv, ivlen); +- r = gnutls_cipher_encrypt2 (cipher, +- buf, LUKS_SECTOR_SIZE, /* plaintext */ +- buf, LUKS_SECTOR_SIZE /* ciphertext */); +- if (r != 0) { +- nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); +- return -1; +- } +- +- buf += LUKS_SECTOR_SIZE; +- offset += LUKS_SECTOR_SIZE; +- len -= LUKS_SECTOR_SIZE; +- sector++; +- } +- +- return 0; +-} +- +-/* Parse the header fields containing cipher algorithm, mode, etc. */ +-static int +-parse_cipher_strings (struct handle *h) +-{ +- char cipher_name[33], cipher_mode[33], hash_spec[33]; +- char *ivgen, *ivhash; +- +- /* Copy the header fields locally and ensure they are \0 terminated. */ +- memcpy (cipher_name, h->phdr.cipher_name, 32); +- cipher_name[32] = 0; +- memcpy (cipher_mode, h->phdr.cipher_mode, 32); +- cipher_mode[32] = 0; +- memcpy (hash_spec, h->phdr.hash_spec, 32); +- hash_spec[32] = 0; +- +- nbdkit_debug ("LUKS v%" PRIu16 " cipher: %s mode: %s hash: %s " +- "master key: %" PRIu32 " bits", +- h->phdr.version, cipher_name, cipher_mode, hash_spec, +- h->phdr.master_key_len * 8); +- +- /* The cipher_mode header has the form: "ciphermode-ivgen[:ivhash]" +- * QEmu writes: "xts-plain64" +- */ +- ivgen = strchr (cipher_mode, '-'); +- if (!ivgen) { +- nbdkit_error ("incorrect cipher_mode header, " +- "expecting mode-ivgenerator but got \"%s\"", cipher_mode); +- return -1; +- } +- *ivgen = '\0'; +- ivgen++; +- +- ivhash = strchr (ivgen, ':'); +- if (!ivhash) +- h->ivgen_hash_alg = GNUTLS_DIG_UNKNOWN; +- else { +- *ivhash = '\0'; +- ivhash++; +- +- h->ivgen_hash_alg = lookup_hash (ivhash); +- if (h->ivgen_hash_alg == -1) +- return -1; +- } +- +- h->cipher_mode = lookup_cipher_mode (cipher_mode); +- if (h->cipher_mode == -1) +- return -1; +- +- h->cipher_alg = lookup_cipher_alg (cipher_name, h->cipher_mode, +- h->phdr.master_key_len); +- if (h->cipher_alg == -1) +- return -1; +- +- h->hash_alg = lookup_hash (hash_spec); +- if (h->hash_alg == -1) +- return -1; +- +- h->ivgen_alg = lookup_ivgen (ivgen); +- if (h->ivgen_alg == -1) +- return -1; +- +-#if 0 +- if (h->ivgen_alg == IVGEN_ESSIV) { +- if (!ivhash) { +- nbdkit_error ("incorrect IV generator hash specification"); +- return -1; +- } +- h->ivgen_cipher_alg = lookup_essiv_cipher (h->cipher_alg, +- h->ivgen_hash_alg); +- if (h->ivgen_cipher_alg == -1) +- return -1; +- } +- else +-#endif +- h->ivgen_cipher_alg = h->cipher_alg; +- +- nbdkit_debug ("LUKS parsed ciphers: %s %s %s %s %s %s", +- cipher_alg_to_string (h->cipher_alg), +- cipher_mode_to_string (h->cipher_mode), +- hash_to_string (h->hash_alg), +- ivgen_to_string (h->ivgen_alg), +- hash_to_string (h->ivgen_hash_alg), +- cipher_alg_to_string (h->ivgen_cipher_alg)); +- +- /* GnuTLS combines cipher and block mode into a single value. Not +- * all possible combinations are available in GnuTLS. See: +- * https://www.gnutls.org/manual/html_node/Supported-ciphersuites.html +- */ +- h->gnutls_cipher = GNUTLS_CIPHER_NULL; +- switch (h->cipher_mode) { +- case CIPHER_MODE_XTS: +- switch (h->cipher_alg) { +- case CIPHER_ALG_AES_128: +- h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; +- break; +- case CIPHER_ALG_AES_256: +- h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; +- break; +- default: break; +- } +- break; +- case CIPHER_MODE_CBC: +- switch (h->cipher_alg) { +- case CIPHER_ALG_AES_128: +- h->gnutls_cipher = GNUTLS_CIPHER_AES_128_CBC; +- break; +- case CIPHER_ALG_AES_192: +- h->gnutls_cipher = GNUTLS_CIPHER_AES_192_CBC; +- break; +- case CIPHER_ALG_AES_256: +- h->gnutls_cipher = GNUTLS_CIPHER_AES_256_CBC; +- break; +- default: break; +- } +- default: break; +- } +- if (h->gnutls_cipher == GNUTLS_CIPHER_NULL) { +- nbdkit_error ("cipher algorithm %s in mode %s is not supported by GnuTLS", +- cipher_alg_to_string (h->cipher_alg), +- cipher_mode_to_string (h->cipher_mode)); +- return -1; +- } +- +- return 0; +-} +- +-/* Anti-Forensic merge operation. */ +-static void +-xor (const uint8_t *in1, const uint8_t *in2, uint8_t *out, size_t len) +-{ +- size_t i; +- +- for (i = 0; i < len; ++i) +- out[i] = in1[i] ^ in2[i]; +-} +- +-static int +-af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) +-{ +- size_t digest_bytes = gnutls_hash_get_len (hash_alg); +- size_t nr_blocks, last_block_len; +- size_t i; +- CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); +- int r; +- gnutls_hash_hd_t hash; +- +- nr_blocks = len / digest_bytes; +- last_block_len = len % digest_bytes; +- if (last_block_len != 0) +- nr_blocks++; +- else +- last_block_len = digest_bytes; +- +- for (i = 0; i < nr_blocks; ++i) { +- const uint32_t iv = htobe32 (i); +- const size_t blen = i < nr_blocks - 1 ? digest_bytes : last_block_len; +- +- /* Hash iv + i'th block into temp. */ +- r = gnutls_hash_init (&hash, hash_alg); +- if (r != 0) { +- nbdkit_error ("gnutls_hash_init: %s", gnutls_strerror (r)); +- return -1; +- } +- gnutls_hash (hash, &iv, sizeof iv); +- gnutls_hash (hash, &block[i*digest_bytes], blen); +- gnutls_hash_deinit (hash, temp); +- +- memcpy (&block[i*digest_bytes], temp, blen); +- } +- +- return 0; +-} +- +-static int +-afmerge (gnutls_digest_algorithm_t hash_alg, uint32_t stripes, +- const uint8_t *in, uint8_t *out, size_t outlen) +-{ +- CLEANUP_FREE uint8_t *block = calloc (1, outlen); +- size_t i; +- +- /* NB: input size is stripes * master_key_len where +- * master_key_len == outlen +- */ +- for (i = 0; i < stripes-1; ++i) { +- xor (&in[i*outlen], block, block, outlen); +- if (af_hash (hash_alg, block, outlen) == -1) +- return -1; +- } +- xor (&in[i*outlen], block, out, outlen); +- return 0; +-} +- +-/* Length of key material in key slot i (sectors). +- * +- * This is basically copied from qemu because the spec description is +- * unintelligible and apparently doesn't match reality. +- */ +-static uint64_t +-key_material_length_in_sectors (struct handle *h, size_t i) +-{ +- uint64_t len, r; +- +- len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; +- r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); +- r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); +- return r; +-} +- +-/* Try the passphrase in key slot i. If this returns true then the +- * passphrase was able to decrypt the master key, and the master key +- * has been stored in h->masterkey. +- */ +-static int +-try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) +-{ +- /* I believe this is supposed to be safe, looking at the GnuTLS +- * header file. +- */ +- const gnutls_mac_algorithm_t mac = (gnutls_mac_algorithm_t) h->hash_alg; +- struct luks_keyslot *ks = &h->phdr.keyslot[i]; +- size_t split_key_len; +- CLEANUP_FREE uint8_t *split_key = NULL; +- CLEANUP_FREE uint8_t *masterkey = NULL; +- const gnutls_datum_t key = +- { (unsigned char *) passphrase, strlen (passphrase) }; +- const gnutls_datum_t salt = +- { (unsigned char *) ks->password_salt, LUKS_SALTSIZE }; +- const gnutls_datum_t msalt = +- { (unsigned char *) h->phdr.master_key_salt, LUKS_SALTSIZE }; +- gnutls_datum_t mkey; +- gnutls_cipher_hd_t cipher; +- int r, err = 0; +- uint64_t start; +- uint8_t key_digest[LUKS_DIGESTSIZE]; +- +- if (ks->active != LUKS_KEY_ENABLED) +- return 0; +- +- split_key_len = h->phdr.master_key_len * ks->stripes; +- split_key = malloc (split_key_len); +- if (split_key == NULL) { +- nbdkit_error ("malloc: %m"); +- return -1; +- } +- masterkey = malloc (h->phdr.master_key_len); +- if (masterkey == NULL) { +- nbdkit_error ("malloc: %m"); +- return -1; +- } +- +- /* Hash the passphrase to make a possible masterkey. */ +- r = gnutls_pbkdf2 (mac, &key, &salt, ks->password_iterations, +- masterkey, h->phdr.master_key_len); +- if (r != 0) { +- nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); +- return -1; +- } +- +- /* Read master key material from plugin. */ +- start = ks->key_material_offset * LUKS_SECTOR_SIZE; +- if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { +- errno = err; +- return -1; +- } +- +- /* Decrypt the (still AFsplit) master key material. */ +- mkey.data = (unsigned char *) masterkey; +- mkey.size = h->phdr.master_key_len; +- r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); +- if (r != 0) { +- nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); +- return -1; +- } +- +- r = do_decrypt (h, cipher, 0, split_key, split_key_len); +- gnutls_cipher_deinit (cipher); +- if (r == -1) +- return -1; +- +- /* Decode AFsplit key to a possible masterkey. */ +- if (afmerge (h->hash_alg, ks->stripes, split_key, +- masterkey, h->phdr.master_key_len) == -1) +- return -1; +- +- /* Check if the masterkey is correct by comparing hash of the +- * masterkey with LUKS header. +- */ +- r = gnutls_pbkdf2 (mac, &mkey, &msalt, +- h->phdr.master_key_digest_iterations, +- key_digest, LUKS_DIGESTSIZE); +- if (r != 0) { +- nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); +- return -1; +- } +- +- if (memcmp (key_digest, h->phdr.master_key_digest, LUKS_DIGESTSIZE) == 0) { +- /* The passphrase is correct so save the master key in the handle. */ +- h->masterkey = malloc (h->phdr.master_key_len); +- if (h->masterkey == NULL) { +- nbdkit_error ("malloc: %m"); +- return -1; +- } +- memcpy (h->masterkey, masterkey, h->phdr.master_key_len); +- return 1; +- } +- +- return 0; +-} +- + static int + luks_prepare (nbdkit_next *next, void *handle, int readonly) + { +- static const char expected_magic[] = LUKS_MAGIC; + struct handle *h = handle; +- int64_t size; +- int err = 0, r; +- size_t i; +- struct luks_keyslot *ks; +- char uuid[41]; + + /* Check we haven't been called before, this should never happen. */ +- assert (h->phdr.version == 0); ++ assert (h->h == NULL); + +- /* Check the struct size matches the documentation. */ +- assert (sizeof (struct luks_phdr) == 592); +- +- /* Check this is a LUKSv1 disk. */ +- size = next->get_size (next); +- if (size == -1) +- return -1; +- if (size < 16384) { +- nbdkit_error ("disk is too small to be LUKS-encrypted"); +- return -1; +- } +- +- /* Read the phdr. */ +- if (next->pread (next, &h->phdr, sizeof h->phdr, 0, 0, &err) == -1) { +- errno = err; +- return -1; +- } +- +- if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { +- nbdkit_error ("this disk does not contain a LUKS header"); ++ h->h = load_header (next, passphrase); ++ if (h->h == NULL) + return -1; +- } +- h->phdr.version = be16toh (h->phdr.version); +- if (h->phdr.version != 1) { +- nbdkit_error ("this disk contains a LUKS version %" PRIu16 " header, " +- "but this filter only supports LUKSv1", +- h->phdr.version); +- return -1; +- } +- +- /* Byte-swap the rest of the header. */ +- h->phdr.payload_offset = be32toh (h->phdr.payload_offset); +- h->phdr.master_key_len = be32toh (h->phdr.master_key_len); +- h->phdr.master_key_digest_iterations = +- be32toh (h->phdr.master_key_digest_iterations); +- +- for (i = 0; i < LUKS_NUMKEYS; ++i) { +- ks = &h->phdr.keyslot[i]; +- ks->active = be32toh (ks->active); +- ks->password_iterations = be32toh (ks->password_iterations); +- ks->key_material_offset = be32toh (ks->key_material_offset); +- ks->stripes = be32toh (ks->stripes); +- } +- +- /* Sanity check some fields. */ +- if (h->phdr.payload_offset >= size / LUKS_SECTOR_SIZE) { +- nbdkit_error ("bad LUKSv1 header: payload offset points beyond " +- "the end of the disk"); +- return -1; +- } +- +- /* We derive several allocations from master_key_len so make sure +- * it's not insane. +- */ +- if (h->phdr.master_key_len > 1024) { +- nbdkit_error ("bad LUKSv1 header: master key is too long"); +- return -1; +- } +- +- for (i = 0; i < LUKS_NUMKEYS; ++i) { +- uint64_t start, len; +- +- ks = &h->phdr.keyslot[i]; +- switch (ks->active) { +- case LUKS_KEY_ENABLED: +- if (!ks->stripes) { +- nbdkit_error ("bad LUKSv1 header: key slot %zu is corrupted", i); +- return -1; +- } +- if (ks->stripes >= 10000) { +- nbdkit_error ("bad LUKSv1 header: key slot %zu stripes too large", i); +- return -1; +- } +- start = ks->key_material_offset; +- len = key_material_length_in_sectors (h, i); +- if (len > 4096) /* bound it at something reasonable */ { +- nbdkit_error ("bad LUKSv1 header: key slot %zu key material length " +- "is too large", i); +- return -1; +- } +- if (start * LUKS_SECTOR_SIZE >= size || +- (start + len) * LUKS_SECTOR_SIZE >= size) { +- nbdkit_error ("bad LUKSv1 header: key slot %zu key material offset " +- "points beyond the end of the disk", i); +- return -1; +- } +- /*FALLTHROUGH*/ +- case LUKS_KEY_DISABLED: +- break; +- +- default: +- nbdkit_error ("bad LUKSv1 header: key slot %zu has " +- "an invalid active flag", i); +- return -1; +- } +- } +- +- /* Decode the ciphers. */ +- if (parse_cipher_strings (h) == -1) +- return -1; +- +- /* Dump some information about the header. */ +- memcpy (uuid, h->phdr.uuid, 40); +- uuid[40] = 0; +- nbdkit_debug ("LUKS UUID: %s", uuid); +- +- for (i = 0; i < LUKS_NUMKEYS; ++i) { +- uint64_t start, len; +- +- ks = &h->phdr.keyslot[i]; +- if (ks->active == LUKS_KEY_ENABLED) { +- start = ks->key_material_offset; +- len = key_material_length_in_sectors (h, i); +- nbdkit_debug ("LUKS key slot %zu: key material in sectors %" PRIu64 +- "..%" PRIu64, +- i, start, start+len-1); +- } +- } +- +- /* Now try to unlock the master key. */ +- for (i = 0; i < LUKS_NUMKEYS; ++i) { +- r = try_passphrase_in_keyslot (next, h, i); +- if (r == -1) +- return -1; +- if (r > 0) +- goto unlocked; +- } +- nbdkit_error ("LUKS passphrase is not correct, " +- "no key slot could be unlocked"); +- return -1; +- +- unlocked: +- assert (h->masterkey != NULL); +- nbdkit_debug ("LUKS unlocked block device with passphrase"); + + return 0; + } +@@ -946,19 +151,20 @@ luks_get_size (nbdkit_next *next, void *handle) + int64_t size; + + /* Check that prepare has been called already. */ +- assert (h->phdr.version > 0); ++ assert (h->h != NULL); ++ ++ const uint64_t payload_offset = get_payload_offset (h->h) * LUKS_SECTOR_SIZE; + + size = next->get_size (next); + if (size == -1) + return -1; + +- if (size < h->phdr.payload_offset * LUKS_SECTOR_SIZE) { ++ if (size < payload_offset) { + nbdkit_error ("disk too small, or contains an incomplete LUKS partition"); + return -1; + } + +- size -= h->phdr.payload_offset * LUKS_SECTOR_SIZE; +- return size; ++ return size - payload_offset; + } + + /* Whatever the plugin says, several operations are not supported by +@@ -1031,15 +237,12 @@ luks_pread (nbdkit_next *next, void *handle, + uint32_t flags, int *err) + { + struct handle *h = handle; +- const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; ++ const uint64_t payload_offset = get_payload_offset (h->h) * LUKS_SECTOR_SIZE; + CLEANUP_FREE uint8_t *sector = NULL; + uint64_t sectnum, sectoffs; +- const gnutls_datum_t mkey = +- { (unsigned char *) h->masterkey, h->phdr.master_key_len }; + gnutls_cipher_hd_t cipher; +- int r; + +- if (!h->masterkey) { ++ if (!h->h) { + *err = EIO; + return -1; + } +@@ -1053,16 +256,13 @@ luks_pread (nbdkit_next *next, void *handle, + } + } + +- r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); +- if (r != 0) { +- nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); +- *err = EIO; +- return -1; +- } +- + sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ + sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ + ++ cipher = create_cipher (h->h); ++ if (!cipher) ++ return -1; ++ + /* Unaligned head */ + if (sectoffs) { + uint64_t n = MIN (LUKS_SECTOR_SIZE - sectoffs, count); +@@ -1073,15 +273,13 @@ luks_pread (nbdkit_next *next, void *handle, + flags, err) == -1) + goto err; + +- if (do_decrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, +- sector, LUKS_SECTOR_SIZE) == -1) ++ if (do_decrypt (h->h, cipher, sectnum, sector, 1) == -1) + goto err; + + memcpy (buf, §or[sectoffs], n); + + buf += n; + count -= n; +- offset += n; + sectnum++; + } + +@@ -1092,12 +290,11 @@ luks_pread (nbdkit_next *next, void *handle, + flags, err) == -1) + goto err; + +- if (do_decrypt (h, cipher, offset, buf, LUKS_SECTOR_SIZE) == -1) ++ if (do_decrypt (h->h, cipher, sectnum, buf, 1) == -1) + goto err; + + buf += LUKS_SECTOR_SIZE; + count -= LUKS_SECTOR_SIZE; +- offset += LUKS_SECTOR_SIZE; + sectnum++; + } + +@@ -1109,7 +306,7 @@ luks_pread (nbdkit_next *next, void *handle, + flags, err) == -1) + goto err; + +- if (do_decrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ if (do_decrypt (h->h, cipher, sectnum, sector, 1) == -1) + goto err; + + memcpy (buf, sector, count); +@@ -1120,7 +317,7 @@ luks_pread (nbdkit_next *next, void *handle, + + err: + gnutls_cipher_deinit (cipher); +- return -1; ++ goto err; + } + + /* Lock preventing read-modify-write cycles from overlapping. */ +@@ -1133,15 +330,12 @@ luks_pwrite (nbdkit_next *next, void *handle, + uint32_t flags, int *err) + { + struct handle *h = handle; +- const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; ++ const uint64_t payload_offset = get_payload_offset (h->h) * LUKS_SECTOR_SIZE; + CLEANUP_FREE uint8_t *sector = NULL; + uint64_t sectnum, sectoffs; +- const gnutls_datum_t mkey = +- { (unsigned char *) h->masterkey, h->phdr.master_key_len }; + gnutls_cipher_hd_t cipher; +- int r; + +- if (!h->masterkey) { ++ if (!h->h) { + *err = EIO; + return -1; + } +@@ -1153,16 +347,13 @@ luks_pwrite (nbdkit_next *next, void *handle, + return -1; + } + +- r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); +- if (r != 0) { +- nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); +- *err = EIO; +- return -1; +- } +- + sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ + sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ + ++ cipher = create_cipher (h->h); ++ if (!cipher) ++ return -1; ++ + /* Unaligned head */ + if (sectoffs) { + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&read_modify_write_lock); +@@ -1176,8 +367,7 @@ luks_pwrite (nbdkit_next *next, void *handle, + + memcpy (§or[sectoffs], buf, n); + +- if (do_encrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, +- sector, LUKS_SECTOR_SIZE) == -1) ++ if (do_encrypt (h->h, cipher, sectnum, sector, 1) == -1) + goto err; + + if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, +@@ -1187,7 +377,6 @@ luks_pwrite (nbdkit_next *next, void *handle, + + buf += n; + count -= n; +- offset += n; + sectnum++; + } + +@@ -1195,7 +384,7 @@ luks_pwrite (nbdkit_next *next, void *handle, + while (count >= LUKS_SECTOR_SIZE) { + memcpy (sector, buf, LUKS_SECTOR_SIZE); + +- if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ if (do_encrypt (h->h, cipher, sectnum, sector, 1) == -1) + goto err; + + if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, +@@ -1205,7 +394,6 @@ luks_pwrite (nbdkit_next *next, void *handle, + + buf += LUKS_SECTOR_SIZE; + count -= LUKS_SECTOR_SIZE; +- offset += LUKS_SECTOR_SIZE; + sectnum++; + } + +@@ -1220,7 +408,7 @@ luks_pwrite (nbdkit_next *next, void *handle, + + memcpy (sector, buf, count); + +- if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ if (do_encrypt (h->h, cipher, sectnum, sector, 1) == -1) + goto err; + + if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, +-- +2.31.1 + diff --git a/SOURCES/0008-vddk-Only-print-vddk_library_version-when-we-managed.patch b/SOURCES/0008-vddk-Only-print-vddk_library_version-when-we-managed.patch deleted file mode 100644 index 9067eef..0000000 --- a/SOURCES/0008-vddk-Only-print-vddk_library_version-when-we-managed.patch +++ /dev/null @@ -1,55 +0,0 @@ -From 670c1ddb6591046256511a680605c5e2349746e8 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 23 Oct 2021 19:50:52 +0100 -Subject: [PATCH] vddk: Only print vddk_library_version when we managed to load - the library - -Because --dump-plugin calls load_library (false) it won't fail if we -didn't manage to load the library. This results in library_version -being 0, which we printed incorrectly. - -Resolve this problem by not printing the vddk_library_version entry in -this case. - -Fixes: commit 8700649d147948897f3b97810a1dff37924bdd6e -(cherry picked from commit a3fba12c3e9c2113009f556360ae0bd04c45f6bb) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 1 + - plugins/vddk/vddk.c | 9 ++++++++- - 2 files changed, 9 insertions(+), 1 deletion(-) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index bc3c3c94..49e3d75d 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -425,6 +425,7 @@ by this build. - =item C - - The VDDK major library version: 5, 6, 7, ... -+If this is omitted it means the library could not be loaded. - - =item C - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 65399a91..39a7d261 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -591,7 +591,14 @@ vddk_dump_plugin (void) - - printf ("vddk_default_libdir=%s\n", VDDK_LIBDIR); - printf ("vddk_has_nfchostport=1\n"); -- printf ("vddk_library_version=%d\n", library_version); -+ -+ /* Because load_library (false) we might not have loaded VDDK, in -+ * which case we didn't set library_version. Note this cannot -+ * happen in the normal (non-debug-plugin) path because there we use -+ * load_library (true). -+ */ -+ if (library_version > 0) -+ printf ("vddk_library_version=%d\n", library_version); - - #if defined(HAVE_DLADDR) - /* It would be nice to print the version of VDDK from the shared --- -2.31.1 - diff --git a/SOURCES/0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch b/SOURCES/0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch new file mode 100644 index 0000000..34419d8 --- /dev/null +++ b/SOURCES/0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch @@ -0,0 +1,101 @@ +From 387bd4c6fee8ab339fd04e0b841b0c67e6020c8a Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 8 May 2022 18:05:45 +0100 +Subject: [PATCH] tests: luks: Reduce time taken to run these tests + +Under valgrind they ran very slowly. Turns out valgrinding over +GnuTLS hashing code is not pretty. About half the time seems to be +taken opening the keyslot, and the rest copying the data. + +This change reduces the time (under valgrind) from 15 minutes 45 seconds +to about 6 mins 30 seconds. + +(cherry picked from commit 7320ae5dba476171a024ca44b889b3474302dc40) +--- + tests/test-luks-copy.sh | 18 +++++++++--------- + tests/test-luks-info.sh | 6 +++--- + 2 files changed, 12 insertions(+), 12 deletions(-) + +diff --git a/tests/test-luks-copy.sh b/tests/test-luks-copy.sh +index 99f300d0..01801811 100755 +--- a/tests/test-luks-copy.sh ++++ b/tests/test-luks-copy.sh +@@ -60,8 +60,8 @@ rm -f $encrypt_disk $plain_disk $pid $sock + qemu-img create -f luks \ + --object secret,data=123456,id=sec0 \ + -o key-secret=sec0 \ +- $encrypt_disk 10M +-truncate -s 10M $plain_disk ++ $encrypt_disk 1M ++truncate -s 1M $plain_disk + qemu-img convert --target-image-opts -n \ + --object secret,data=123456,id=sec0 \ + $plain_disk \ +@@ -74,11 +74,11 @@ start_nbdkit -P $pid -U $sock \ + uri="nbd+unix:///?socket=$sock" + + # Copy the whole disk out. It should be empty. +-nbdcopy "$uri" $plain_disk ++nbdcopy -C 1 "$uri" $plain_disk + + if [ "$(hexdump -C $plain_disk)" != '00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-00a00000' ]; then ++00100000' ]; then + echo "$0: expected plaintext disk to be empty" + exit 1 + fi +@@ -88,14 +88,14 @@ fi + nbdsh -u "$uri" \ + -c 'h.pwrite(b"1"*65536, 0)' \ + -c 'h.pwrite(b"2"*65536, 128*1024)' \ +- -c 'h.pwrite(b"3"*65536, 9*1024*1024)' \ ++ -c 'h.pwrite(b"3"*65536, 900*1024)' \ + -c 'buf = h.pread(65536, 0)' \ + -c 'assert buf == b"1"*65536' \ + -c 'buf = h.pread(65536, 65536)' \ + -c 'assert buf == bytearray(65536)' \ + -c 'buf = h.pread(65536, 128*1024)' \ + -c 'assert buf == b"2"*65536' \ +- -c 'buf = h.pread(65536, 9*1024*1024)' \ ++ -c 'buf = h.pread(65536, 900*1024)' \ + -c 'assert buf == b"3"*65536' \ + -c 'h.flush()' + +@@ -115,11 +115,11 @@ if [ "$(hexdump -C $plain_disk)" != '00000000 31 31 31 31 31 31 31 31 31 31 31 + * + 00030000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-00900000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333| ++000e1000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333| + * +-00910000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++000f1000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-00a00000' ]; then ++00100000' ]; then + echo "$0: unexpected content" + exit 1 + fi +diff --git a/tests/test-luks-info.sh b/tests/test-luks-info.sh +index 3eff657b..ef141ecd 100755 +--- a/tests/test-luks-info.sh ++++ b/tests/test-luks-info.sh +@@ -46,11 +46,11 @@ rm -f $disk $info + qemu-img create -f luks \ + --object secret,data=123456,id=sec0 \ + -o key-secret=sec0 \ +- $disk 10M ++ $disk 1M + + nbdkit -U - file $disk --filter=luks passphrase=123456 \ + --run 'nbdinfo $uri' > $info + cat $info + +-# Check the size is 10M (so it doesn't include the LUKS header). +-grep "10485760" $info ++# Check the size is 1M (so it doesn't include the LUKS header). ++grep "1048576" $info +-- +2.31.1 + diff --git a/SOURCES/0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch b/SOURCES/0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch deleted file mode 100644 index 1be4831..0000000 --- a/SOURCES/0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch +++ /dev/null @@ -1,72 +0,0 @@ -From 21d6c2f8f29f0d7f98852b72ee33751814be49fe Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 25 Oct 2021 08:36:53 +0100 -Subject: [PATCH] vddk: Print one line in --dump-plugin output for each VDDK - API - -Helps when detecting if certain optional features are being used, such -as flush and extents. - -(cherry picked from commit 4ee13559e46cf622410d0bdd7db29bb00908b40a) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 9 +++++++++ - plugins/vddk/vddk.c | 10 ++++++++++ - tests/test-vddk-real-dump-plugin.sh | 1 + - 3 files changed, 20 insertions(+) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index 49e3d75d..0702aa75 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -432,6 +432,15 @@ If this is omitted it means the library could not be loaded. - Prints the full path to the VDDK shared library. Since this requires - a glibc extension it may not be available in all builds of the plugin. - -+=item C -+ -+For each VDDK API that the plugin uses I which is present in the -+VDDK library that was loaded, we print the name of the API -+(eg. C). This lets you see which optional APIs are -+available, such as C and -+C. If the library could not be -+loaded then these lines are not printed. -+ - =back - - =head1 NOTES -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 39a7d261..096b04bf 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -616,6 +616,16 @@ vddk_dump_plugin (void) - printf ("vddk_dll=%s\n", p); - } - #endif -+ -+ /* Note we print all VDDK APIs found here, not just the optional -+ * ones. That is so if we update the baseline VDDK in future and -+ * make optional into required APIs, the output doesn't change. -+ */ -+#define STUB(fn,ret,args) if (fn != NULL) printf ("%s=1\n", #fn); -+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) -+#include "vddk-stubs.h" -+#undef STUB -+#undef OPTIONAL_STUB - } - - /* The rules on threads and VDDK are here: -diff --git a/tests/test-vddk-real-dump-plugin.sh b/tests/test-vddk-real-dump-plugin.sh -index 0a079c6c..e37c8b54 100755 ---- a/tests/test-vddk-real-dump-plugin.sh -+++ b/tests/test-vddk-real-dump-plugin.sh -@@ -65,6 +65,7 @@ grep ^vddk_default_libdir= $out - grep ^vddk_has_nfchostport= $out - grep ^vddk_library_version= $out - grep ^vddk_dll= $out -+grep ^VixDiskLib_Open=1 $out - - dll="$(grep ^vddk_dll $out | cut -d= -f2)" - test -f "$dll" --- -2.31.1 - diff --git a/SOURCES/0010-Add-nbdkit.parse_size-Python-function.patch b/SOURCES/0010-Add-nbdkit.parse_size-Python-function.patch new file mode 100644 index 0000000..5cb340b --- /dev/null +++ b/SOURCES/0010-Add-nbdkit.parse_size-Python-function.patch @@ -0,0 +1,112 @@ +From 52ee1dab95436128b44c37cc495022ff90108b2e Mon Sep 17 00:00:00 2001 +From: Nikolaus Rath +Date: Mon, 9 May 2022 10:04:30 +0100 +Subject: [PATCH] Add nbdkit.parse_size() Python function. + +This enables Python plugins to parse sizes the same way as C plugins. + +I'm not sure about the best way to test this - input is appreciated. + +I'm not too happy with the way this code is tested. It workes, but putting the tests into +test-python-plugin.py feels misplaced: this file is intended to support the unit tests in +test_python.py, not run its own unit tests. + +(cherry picked from commit 1b7d72542be68e254c1ef86ecb1a82b05c78ff63) +--- + plugins/python/modfunctions.c | 21 +++++++++++++++++++++ + plugins/python/nbdkit-python-plugin.pod | 5 +++++ + tests/test-python-plugin.py | 19 +++++++++++++++++++ + 3 files changed, 45 insertions(+) + +diff --git a/plugins/python/modfunctions.c b/plugins/python/modfunctions.c +index fffbaab2..46b0c904 100644 +--- a/plugins/python/modfunctions.c ++++ b/plugins/python/modfunctions.c +@@ -93,11 +93,32 @@ do_shutdown (PyObject *self, PyObject *args) + Py_RETURN_NONE; + } + ++/* nbdkit.parse_size */ ++static PyObject * ++parse_size (PyObject *self, PyObject *args) ++{ ++ const char *s; ++ if (!PyArg_ParseTuple (args, "s", &s)) { ++ PyErr_SetString (PyExc_TypeError, "Expected string, got something else"); ++ return NULL; ++ } ++ ++ int64_t size = nbdkit_parse_size(s); ++ if (size == -1) { ++ PyErr_SetString (PyExc_ValueError, "Unable to parse string as size"); ++ return NULL; ++ } ++ ++ return PyLong_FromSize_t((size_t)size); ++} ++ + static PyMethodDef NbdkitMethods[] = { + { "debug", debug, METH_VARARGS, + "Print a debug message" }, + { "export_name", export_name, METH_VARARGS, + "Return the optional export name negotiated with the client" }, ++ { "parse_size", parse_size, METH_VARARGS, ++ "Parse human-readable size strings into bytes" }, + { "set_error", set_error, METH_VARARGS, + "Store an errno value prior to throwing an exception" }, + { "shutdown", do_shutdown, METH_VARARGS, +diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod +index 051b0237..ccc9406f 100644 +--- a/plugins/python/nbdkit-python-plugin.pod ++++ b/plugins/python/nbdkit-python-plugin.pod +@@ -131,6 +131,11 @@ Record C as the reason you are about to throw an exception. C + should correspond to usual errno values, where it may help to + C. + ++=head3 C ++ ++Parse a string (such as "100M") into a size in bytes. Wraps the ++C C function. ++ + =head3 C + + Request asynchronous server shutdown. +diff --git a/tests/test-python-plugin.py b/tests/test-python-plugin.py +index 0b34d532..d4f379fc 100644 +--- a/tests/test-python-plugin.py ++++ b/tests/test-python-plugin.py +@@ -34,12 +34,31 @@ + import nbdkit + import pickle + import base64 ++import unittest + + API_VERSION = 2 + + cfg = {} + + ++# Not nice, but there doesn't seem to be a better way of putting this ++class TestAPI(unittest.TestCase): ++ ++ def test_parse_size(self): ++ self.assertEqual(nbdkit.parse_size('511'), 511) ++ self.assertEqual(nbdkit.parse_size('7k'), 7*1024) ++ self.assertEqual(nbdkit.parse_size('17M'), 17*1024*1024) ++ ++ with self.assertRaises(TypeError): ++ nbdkit.parse_size(17) ++ ++ with self.assertRaises(ValueError): ++ nbdkit.parse_size('foo') ++ ++ ++TestAPI().test_parse_size() ++ ++ + def config(k, v): + global cfg + if k == "cfg": +-- +2.31.1 + diff --git a/SOURCES/0010-vddk-Move-minimum-version-to-VDDK-6.5.patch b/SOURCES/0010-vddk-Move-minimum-version-to-VDDK-6.5.patch deleted file mode 100644 index d0a4222..0000000 --- a/SOURCES/0010-vddk-Move-minimum-version-to-VDDK-6.5.patch +++ /dev/null @@ -1,147 +0,0 @@ -From f4379f04ea27e25c00e98db2e60d0fdb647442e9 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 26 Oct 2021 19:46:32 +0100 -Subject: [PATCH] vddk: Move minimum version to VDDK 6.5 - -Drop support for VDDK 5.5.5 (released in 2015) and 6.0 (released the -same year). Move minimum supported version to 6.5 (released Nov -2016). This is so we can use asynchronous operations. - -Acked-by: Laszlo Ersek -(cherry picked from commit 5ed23616762a72e039531a9a7cd81353cd4f436e) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 10 +++------- - plugins/vddk/vddk-stubs.h | 3 +-- - plugins/vddk/vddk.c | 24 ++++++++++++++++-------- - tests/dummy-vddk.c | 6 ++++++ - 4 files changed, 26 insertions(+), 17 deletions(-) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index 0702aa75..1c16d096 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -424,7 +424,7 @@ by this build. - - =item C - --The VDDK major library version: 5, 6, 7, ... -+The VDDK major library version: 6, 7, ... - If this is omitted it means the library could not be loaded. - - =item C -@@ -548,16 +548,12 @@ server, which can also be very slow. - - =head1 SUPPORTED VERSIONS OF VDDK - --This plugin requires VDDK E 5.5.5, which in turn means that it --is only supported on x64-64 platforms. -+This plugin requires VDDK E 6.5 (released Nov 2016). It is only -+supported on the x64-64 archtecture. - - It has been tested with all versions up to 7.0.3 (but should work with - future versions). - --VDDK E 6.0 should be used if possible. This is the first version --which added Flush support which is crucial for data integrity when --writing. -- - VDDK 6.7 was the first version that supported the - C API, required to provide extent - information over NBD. -diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h -index 5e70238d..a94df9cd 100644 ---- a/plugins/vddk/vddk-stubs.h -+++ b/plugins/vddk/vddk-stubs.h -@@ -40,8 +40,7 @@ - */ - - /* Required stubs, present in all versions of VDDK that we support. I -- * have checked that all these exist in at least VDDK 5.5.5 (2015) -- * which is the earliest version of VDDK that we support. -+ * have checked that all these exist in at least VDDK 5.5.5 (2015). - */ - - STUB (VixDiskLib_InitEx, -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 096b04bf..babffc28 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -75,13 +75,13 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats; - #undef OPTIONAL_STUB - - /* Parameters passed to InitEx. */ --#define VDDK_MAJOR 5 -+#define VDDK_MAJOR 6 - #define VDDK_MINOR 5 - - static void *dl; /* dlopen handle */ - static bool init_called; /* was InitEx called */ - static __thread int error_suppression; /* threadlocal error suppression */ --static int library_version; /* VDDK major: 5, 6, 7, ... */ -+static int library_version; /* VDDK major: 6, 7, ... */ - - static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */ - static char *config; /* config */ -@@ -413,16 +413,14 @@ load_library (bool load_error_is_fatal) - /* Prefer the newest library in case multiple exist. Check two - * possible directories: the usual VDDK installation puts .so - * files in an arch-specific subdirectory of $libdir (our minimum -- * supported version is VDDK 5.5.5, which only supports x64-64); -- * but our testsuite is easier to write if we point libdir -- * directly to a stub .so. -+ * supported version is VDDK 6.5, which only supports x64-64); but -+ * our testsuite is easier to write if we point libdir directly to -+ * a stub .so. - */ - { "lib64/libvixDiskLib.so.7", 7 }, - { "libvixDiskLib.so.7", 7 }, - { "lib64/libvixDiskLib.so.6", 6 }, - { "libvixDiskLib.so.6", 6 }, -- { "lib64/libvixDiskLib.so.5", 5 }, -- { "libvixDiskLib.so.5", 5 }, - { NULL } - }; - size_t i; -@@ -474,7 +472,7 @@ load_library (bool load_error_is_fatal) - exit (EXIT_FAILURE); - } - -- assert (library_version >= 5); -+ assert (library_version >= 6); - - /* Load symbols. */ - #define STUB(fn,ret,args) \ -@@ -490,6 +488,16 @@ load_library (bool load_error_is_fatal) - #include "vddk-stubs.h" - #undef STUB - #undef OPTIONAL_STUB -+ -+ /* Additionally, VDDK version must be >= 6.5. This was the first -+ * version which introduced VixDiskLib_Wait symbol so we can check -+ * for that. -+ */ -+ if (VixDiskLib_Wait == NULL) { -+ nbdkit_error ("VDDK version must be >= 6.5. " -+ "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\"."); -+ exit (EXIT_FAILURE); -+ } - } - - static int -diff --git a/tests/dummy-vddk.c b/tests/dummy-vddk.c -index 9b5ae0a2..cb88380c 100644 ---- a/tests/dummy-vddk.c -+++ b/tests/dummy-vddk.c -@@ -198,3 +198,9 @@ VixDiskLib_Write (VixDiskLibHandle handle, - memcpy (disk + offset, buf, nr_sectors * VIXDISKLIB_SECTOR_SIZE); - return VIX_OK; - } -+ -+NBDKIT_DLL_PUBLIC VixError -+VixDiskLib_Wait (VixDiskLibHandle handle) -+{ -+ return VIX_OK; -+} --- -2.31.1 - diff --git a/SOURCES/0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch b/SOURCES/0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch new file mode 100644 index 0000000..ed4a4e3 --- /dev/null +++ b/SOURCES/0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch @@ -0,0 +1,34 @@ +From 644e0ed6333cf5fe2c1e39da157e8f1ce97267b9 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 13:47:19 +0100 +Subject: [PATCH] cache: Fix cross-reference nbdkit-readahead-filter + +After the readahead filter was reimplemented so that it only issues +cache requests, the two filters should be used together, not as +alternatives. Update the documentation of the cache filter to make +this clear. + +Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f +(cherry picked from commit 894771f39a8fd2632caad00e497146d69cac4bac) +--- + filters/cache/nbdkit-cache-filter.pod | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod +index d85fef09..f4234e1a 100644 +--- a/filters/cache/nbdkit-cache-filter.pod ++++ b/filters/cache/nbdkit-cache-filter.pod +@@ -28,8 +28,8 @@ loss, as the name suggests). + + This filter only caches image contents. To cache image metadata, use + L between this filter and the plugin. +-To accelerate sequential reads, use L +-instead. ++To accelerate sequential reads, use L on ++top of this filter. + + =head1 PARAMETERS + +-- +2.31.1 + diff --git a/SOURCES/0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch b/SOURCES/0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch deleted file mode 100644 index fc5b20c..0000000 --- a/SOURCES/0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch +++ /dev/null @@ -1,72 +0,0 @@ -From 90dc3311582784f8b078a30a7207c15c6298b1e2 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 27 Oct 2021 11:57:35 +0100 -Subject: [PATCH] vddk: Add read, write and wait asynchronous functions - -These functions added in VDDK 6.0 - 6.5 implement asynchronous read -and write. - -Acked-by: Laszlo Ersek -(cherry picked from commit ad53e7becafed6ca3573795a79c534281fe9c274) ---- - plugins/vddk/vddk-structs.h | 3 +++ - plugins/vddk/vddk-stubs.h | 19 ++++++++++++++++++- - 2 files changed, 21 insertions(+), 1 deletion(-) - -diff --git a/plugins/vddk/vddk-structs.h b/plugins/vddk/vddk-structs.h -index aeb5bfd0..e97f017c 100644 ---- a/plugins/vddk/vddk-structs.h -+++ b/plugins/vddk/vddk-structs.h -@@ -43,6 +43,7 @@ - - typedef uint64_t VixError; - #define VIX_OK 0 -+#define VIX_ASYNC 25000 - - #define VIXDISKLIB_FLAG_OPEN_UNBUFFERED 1 - #define VIXDISKLIB_FLAG_OPEN_SINGLE_LINK 2 -@@ -61,6 +62,8 @@ typedef void *VixDiskLibHandle; - - typedef void VixDiskLibGenericLogFunc (const char *fmt, va_list args); - -+typedef void (*VixDiskLibCompletionCB) (void *data, VixError result); -+ - enum VixDiskLibCredType { - VIXDISKLIB_CRED_UID = 1, - VIXDISKLIB_CRED_SESSIONID = 2, -diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h -index a94df9cd..66353691 100644 ---- a/plugins/vddk/vddk-stubs.h -+++ b/plugins/vddk/vddk-stubs.h -@@ -103,10 +103,27 @@ STUB (VixDiskLib_Write, - uint64_t start_sector, uint64_t nr_sectors, - const unsigned char *buf)); - --/* Added in VDDK 6.0, this will be NULL in earlier versions. */ -+/* Added in VDDK 6.0, these will be NULL in earlier versions. */ - OPTIONAL_STUB (VixDiskLib_Flush, - VixError, - (VixDiskLibHandle handle)); -+OPTIONAL_STUB (VixDiskLib_ReadAsync, -+ VixError, -+ (VixDiskLibHandle handle, -+ uint64_t start_sector, uint64_t nr_sectors, -+ unsigned char *buf, -+ VixDiskLibCompletionCB callback, void *data)); -+OPTIONAL_STUB (VixDiskLib_WriteAsync, -+ VixError, -+ (VixDiskLibHandle handle, -+ uint64_t start_sector, uint64_t nr_sectors, -+ const unsigned char *buf, -+ VixDiskLibCompletionCB callback, void *data)); -+ -+/* Added in VDDK 6.5, this will be NULL in earlier versions. */ -+OPTIONAL_STUB (VixDiskLib_Wait, -+ VixError, -+ (VixDiskLibHandle handle)); - - /* Added in VDDK 6.7, these will be NULL for earlier versions: */ - OPTIONAL_STUB (VixDiskLib_QueryAllocatedBlocks, --- -2.31.1 - diff --git a/SOURCES/0012-curl-Don-t-document-curl-plugin-readahead-filter.patch b/SOURCES/0012-curl-Don-t-document-curl-plugin-readahead-filter.patch new file mode 100644 index 0000000..4c8288e --- /dev/null +++ b/SOURCES/0012-curl-Don-t-document-curl-plugin-readahead-filter.patch @@ -0,0 +1,48 @@ +From 4a7e5169935c8850fddcea8da79639ded907c549 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 14:00:16 +0100 +Subject: [PATCH] curl: Don't document curl plugin + readahead filter + +nbdkit readahead filter does not support plugins which do not use the +parallel thread model. + +Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f +(cherry picked from commit 92fbb76d11b9f17c527debd803aa2505f3642783) +--- + docs/nbdkit-captive.pod | 7 ------- + plugins/curl/nbdkit-curl-plugin.pod | 1 - + 2 files changed, 8 deletions(-) + +diff --git a/docs/nbdkit-captive.pod b/docs/nbdkit-captive.pod +index eafe36d8..d41a824d 100644 +--- a/docs/nbdkit-captive.pod ++++ b/docs/nbdkit-captive.pod +@@ -110,13 +110,6 @@ an embedded disk image. To copy it out: + + nbdkit -U - example1 --run 'qemu-img convert $nbd disk.img' + +-If plugin requests have a high overhead (for example making HTTP +-requests to a remote server), adding L may +-help performance: +- +- nbdkit -U - --filter=readahead curl https://example.com/disk.img \ +- --run 'qemu-img convert $nbd disk.img' +- + If the source suffers from temporary network failures + L or L may + help. +diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod +index 54fce66c..fc422ca2 100644 +--- a/plugins/curl/nbdkit-curl-plugin.pod ++++ b/plugins/curl/nbdkit-curl-plugin.pod +@@ -509,7 +509,6 @@ L, + L, + L, + L, +-L, + L, + L, + L, +-- +2.31.1 + diff --git a/SOURCES/0012-vddk-Start-to-split-VDDK-over-several-files.patch b/SOURCES/0012-vddk-Start-to-split-VDDK-over-several-files.patch deleted file mode 100644 index 805b695..0000000 --- a/SOURCES/0012-vddk-Start-to-split-VDDK-over-several-files.patch +++ /dev/null @@ -1,259 +0,0 @@ -From c9e432e08e889d9e6edea52344b2452f0141f56b Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 27 Oct 2021 12:20:31 +0100 -Subject: [PATCH] vddk: Start to split VDDK over several files - -This change doesn't do anything except move some definitions into the -header file vddk.h, but it allows future commits to split up the very -large vddk.c file. - -Acked-by: Laszlo Ersek -(cherry picked from commit 117634dccf4e29394e8718a8d62e93a9edf0a39c) ---- - plugins/vddk/vddk.c | 91 +++++++++++++-------------------------------- - plugins/vddk/vddk.h | 89 +++++++++++++++++++++++++++++++++++++++++++- - 2 files changed, 112 insertions(+), 68 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index babffc28..041bff1a 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -50,14 +50,12 @@ - #include - - #include "cleanup.h" --#include "isaligned.h" - #include "minmax.h" - #include "rounding.h" - #include "tvdiff.h" - #include "vector.h" - - #include "vddk.h" --#include "vddk-structs.h" - - /* Debug flags. */ - NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo; -@@ -65,11 +63,11 @@ NBDKIT_DLL_PUBLIC int vddk_debug_extents; - NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1; - NBDKIT_DLL_PUBLIC int vddk_debug_stats; - --/* For each VDDK API define a static global variable. These globals -- * are initialized when the plugin is loaded (by vddk_get_ready). -+/* For each VDDK API define a global variable. These globals are -+ * initialized when the plugin is loaded (by vddk_get_ready). - */ --#define STUB(fn,ret,args) static ret (*fn) args --#define OPTIONAL_STUB(fn,ret,args) static ret (*fn) args -+#define STUB(fn,ret,args) ret (*fn) args -+#define OPTIONAL_STUB(fn,ret,args) ret (*fn) args - #include "vddk-stubs.h" - #undef STUB - #undef OPTIONAL_STUB -@@ -78,28 +76,28 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats; - #define VDDK_MAJOR 6 - #define VDDK_MINOR 5 - --static void *dl; /* dlopen handle */ --static bool init_called; /* was InitEx called */ --static __thread int error_suppression; /* threadlocal error suppression */ --static int library_version; /* VDDK major: 6, 7, ... */ -+void *dl; /* dlopen handle */ -+bool init_called; /* was InitEx called */ -+__thread int error_suppression; /* threadlocal error suppression */ -+int library_version; /* VDDK major: 6, 7, ... */ -+bool is_remote; /* true if remote connection */ - --static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */ --static char *config; /* config */ --static const char *cookie; /* cookie */ --static const char *filename; /* file */ --char *libdir; /* libdir */ --static uint16_t nfc_host_port; /* nfchostport */ --char *password; /* password */ --static uint16_t port; /* port */ --static const char *server_name; /* server */ --static bool single_link; /* single-link */ --static const char *snapshot_moref; /* snapshot */ --static const char *thumb_print; /* thumbprint */ --static const char *transport_modes; /* transports */ --static bool unbuffered; /* unbuffered */ --static const char *username; /* user */ --static const char *vmx_spec; /* vm */ --static bool is_remote; -+enum compression_type compression; /* compression */ -+char *config; /* config */ -+const char *cookie; /* cookie */ -+const char *filename; /* file */ -+char *libdir; /* libdir */ -+uint16_t nfc_host_port; /* nfchostport */ -+char *password; /* password */ -+uint16_t port; /* port */ -+const char *server_name; /* server */ -+bool single_link; /* single-link */ -+const char *snapshot_moref; /* snapshot */ -+const char *thumb_print; /* thumbprint */ -+const char *transport_modes; /* transports */ -+bool unbuffered; /* unbuffered */ -+const char *username; /* user */ -+const char *vmx_spec; /* vm */ - - /* For each VDDK API define a variable to store the time taken (used - * to implement -D vddk.stats=1). -@@ -120,45 +118,6 @@ static void display_stats (void); - #undef STUB - #undef OPTIONAL_STUB - --/* Macros to bracket each VDDK API call, for printing debugging -- * information and collecting statistics. -- */ --#define VDDK_CALL_START(fn, fs, ...) \ -- do { \ -- struct timeval start_t, end_t; \ -- /* GCC can optimize this away at compile time: */ \ -- const bool datapath = \ -- strcmp (#fn, "VixDiskLib_Read") == 0 || \ -- strcmp (#fn, "VixDiskLib_Write") == 0; \ -- if (vddk_debug_stats) \ -- gettimeofday (&start_t, NULL); \ -- if (!datapath || vddk_debug_datapath) \ -- nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ -- do --#define VDDK_CALL_END(fn, bytes_) \ -- while (0); \ -- if (vddk_debug_stats) { \ -- gettimeofday (&end_t, NULL); \ -- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ -- stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ -- stats_##fn.calls++; \ -- stats_##fn.bytes += bytes_; \ -- } \ -- } while (0) -- --/* Print VDDK errors. */ --#define VDDK_ERROR(err, fs, ...) \ -- do { \ -- char *vddk_err_msg; \ -- VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \ -- vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \ -- VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \ -- nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \ -- VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \ -- VixDiskLib_FreeErrorText (vddk_err_msg); \ -- VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \ -- } while (0) -- - /* Unload the plugin. */ - static void - vddk_unload (void) -diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h -index 8c63b4ee..29775eb4 100644 ---- a/plugins/vddk/vddk.h -+++ b/plugins/vddk/vddk.h -@@ -1,5 +1,5 @@ - /* nbdkit -- * Copyright (C) 2013-2020 Red Hat Inc. -+ * Copyright (C) 2013-2021 Red Hat Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are -@@ -33,11 +33,96 @@ - #ifndef NBDKIT_VDDK_H - #define NBDKIT_VDDK_H - -+#include -+#include -+#include -+ -+#include -+ -+#include "isaligned.h" -+#include "tvdiff.h" -+#include "vector.h" -+ -+#include "vddk-structs.h" -+ -+enum compression_type { NONE = 0, ZLIB, FASTLZ, SKIPZ }; -+ -+extern void *dl; -+extern bool init_called; -+extern __thread int error_suppression; -+extern int library_version; -+extern bool is_remote; -+ -+extern enum compression_type compression; -+extern char *config; -+extern const char *cookie; -+extern const char *filename; - extern char *libdir; -+extern uint16_t nfc_host_port; - extern char *password; -+extern uint16_t port; -+extern const char *server_name; -+extern bool single_link; -+extern const char *snapshot_moref; -+extern const char *thumb_print; -+extern const char *transport_modes; -+extern bool unbuffered; -+extern const char *username; -+extern const char *vmx_spec; -+ -+extern int vddk_debug_diskinfo; -+extern int vddk_debug_extents; -+extern int vddk_debug_datapath; -+extern int vddk_debug_stats; -+ -+#define STUB(fn,ret,args) extern ret (*fn) args -+#define OPTIONAL_STUB(fn,ret,args) extern ret (*fn) args -+#include "vddk-stubs.h" -+#undef STUB -+#undef OPTIONAL_STUB -+ -+/* Macros to bracket each VDDK API call, for printing debugging -+ * information and collecting statistics. -+ */ -+#define VDDK_CALL_START(fn, fs, ...) \ -+ do { \ -+ struct timeval start_t, end_t; \ -+ /* GCC can optimize this away at compile time: */ \ -+ const bool datapath = \ -+ strcmp (#fn, "VixDiskLib_Read") == 0 || \ -+ strcmp (#fn, "VixDiskLib_Write") == 0; \ -+ if (vddk_debug_stats) \ -+ gettimeofday (&start_t, NULL); \ -+ if (!datapath || vddk_debug_datapath) \ -+ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ -+ do -+#define VDDK_CALL_END(fn, bytes_) \ -+ while (0); \ -+ if (vddk_debug_stats) { \ -+ gettimeofday (&end_t, NULL); \ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ -+ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ -+ stats_##fn.calls++; \ -+ stats_##fn.bytes += bytes_; \ -+ } \ -+ } while (0) -+ -+/* Print VDDK errors. */ -+#define VDDK_ERROR(err, fs, ...) \ -+ do { \ -+ char *vddk_err_msg; \ -+ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \ -+ vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \ -+ VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \ -+ nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \ -+ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \ -+ VixDiskLib_FreeErrorText (vddk_err_msg); \ -+ VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \ -+ } while (0) -+ -+/* reexec.c */ - extern bool noreexec; - extern char *reexeced; -- - extern void reexec_if_needed (const char *prepend); - extern int restore_ld_library_path (void); - --- -2.31.1 - diff --git a/SOURCES/0013-New-filter-scan.patch b/SOURCES/0013-New-filter-scan.patch new file mode 100644 index 0000000..527a597 --- /dev/null +++ b/SOURCES/0013-New-filter-scan.patch @@ -0,0 +1,1021 @@ +From 8bfe6512d07caf778fd001425435b048c45513eb Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 13:46:56 +0100 +Subject: [PATCH] New filter: scan + +This filter will simply scan across the disk issuing a series of cache +requests to the underlying plugin. It is similar in scope and usage +to the new nbdkit-readahead-filter. + +(cherry picked from commit 65c20a09ceacb4431986a2982f2c2e746df63fcb) +--- + TODO | 8 - + configure.ac | 2 + + filters/cache/nbdkit-cache-filter.pod | 4 +- + .../nbdkit-cacheextents-filter.pod | 1 + + filters/readahead/nbdkit-readahead-filter.pod | 5 + + filters/scan/Makefile.am | 72 +++++ + filters/scan/bgthread.c | 131 ++++++++ + filters/scan/nbdkit-scan-filter.pod | 159 ++++++++++ + filters/scan/scan.c | 280 ++++++++++++++++++ + filters/scan/scan.h | 64 ++++ + plugins/ssh/nbdkit-ssh-plugin.pod | 1 + + plugins/torrent/nbdkit-torrent-plugin.pod | 1 + + plugins/vddk/nbdkit-vddk-plugin.pod | 1 + + tests/Makefile.am | 10 + + tests/test-scan-copy.sh | 42 +++ + tests/test-scan-info.sh | 46 +++ + 16 files changed, 817 insertions(+), 10 deletions(-) + create mode 100644 filters/scan/Makefile.am + create mode 100644 filters/scan/bgthread.c + create mode 100644 filters/scan/nbdkit-scan-filter.pod + create mode 100644 filters/scan/scan.c + create mode 100644 filters/scan/scan.h + create mode 100755 tests/test-scan-copy.sh + create mode 100755 tests/test-scan-info.sh + +diff --git a/TODO b/TODO +index 0f5dc41d..8600d9e4 100644 +--- a/TODO ++++ b/TODO +@@ -182,14 +182,6 @@ Python: + Suggestions for filters + ----------------------- + +-* Add scan filter. This would be placed on top of cache filters and +- would scan (read) the whole disk in the background, ensuring it is +- copied into the cache. Useful if you have a slow plugin, limited +- size device, and lots of local disk space, especially if you know +- that the NBD clients will eventually read all of the device. RWMJ +- wrote an implementation of this but it doesn't work well without a +- background thread. +- + * Add shared filter. Take advantage of filter context APIs to open a + single context into the backend shared among multiple client + connections. This may even allow a filter to offer a more parallel +diff --git a/configure.ac b/configure.ac +index 1d209f67..466dbd9b 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -142,6 +142,7 @@ filters="\ + readahead \ + retry \ + retry-request \ ++ scan \ + stats \ + swab \ + tar \ +@@ -1403,6 +1404,7 @@ AC_CONFIG_FILES([Makefile + filters/readahead/Makefile + filters/retry/Makefile + filters/retry-request/Makefile ++ filters/scan/Makefile + filters/stats/Makefile + filters/swab/Makefile + filters/tar/Makefile +diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod +index f4234e1a..935804b5 100644 +--- a/filters/cache/nbdkit-cache-filter.pod ++++ b/filters/cache/nbdkit-cache-filter.pod +@@ -28,8 +28,8 @@ loss, as the name suggests). + + This filter only caches image contents. To cache image metadata, use + L between this filter and the plugin. +-To accelerate sequential reads, use L on +-top of this filter. ++To accelerate sequential reads, use L or ++L on top of this filter. + + =head1 PARAMETERS + +diff --git a/filters/cacheextents/nbdkit-cacheextents-filter.pod b/filters/cacheextents/nbdkit-cacheextents-filter.pod +index bb2514a4..6464eac2 100644 +--- a/filters/cacheextents/nbdkit-cacheextents-filter.pod ++++ b/filters/cacheextents/nbdkit-cacheextents-filter.pod +@@ -54,6 +54,7 @@ L, + L, + L, + L, ++L, + L, + L, + L. +diff --git a/filters/readahead/nbdkit-readahead-filter.pod b/filters/readahead/nbdkit-readahead-filter.pod +index 630e5924..99d64dfb 100644 +--- a/filters/readahead/nbdkit-readahead-filter.pod ++++ b/filters/readahead/nbdkit-readahead-filter.pod +@@ -27,6 +27,10 @@ option. + The filter uses a simple adaptive algorithm which accelerates + sequential reads and requires no further configuration. + ++A similar filter is L which reads ahead over ++the whole disk, useful if you know that the client will be reading ++sequentially across most or all of the disk. ++ + =head2 Limitations + + In a number of significant cases this filter will do nothing. The +@@ -91,6 +95,7 @@ L, + L, + L, + L, ++L, + L, + L, + L, +diff --git a/filters/scan/Makefile.am b/filters/scan/Makefile.am +new file mode 100644 +index 00000000..d4aabfc6 +--- /dev/null ++++ b/filters/scan/Makefile.am +@@ -0,0 +1,72 @@ ++# nbdkit ++# Copyright (C) 2019-2021 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++include $(top_srcdir)/common-rules.mk ++ ++EXTRA_DIST = nbdkit-scan-filter.pod ++ ++filter_LTLIBRARIES = nbdkit-scan-filter.la ++ ++nbdkit_scan_filter_la_SOURCES = \ ++ scan.c \ ++ scan.h \ ++ bgthread.c \ ++ $(top_srcdir)/include/nbdkit-filter.h \ ++ $(NULL) ++ ++nbdkit_scan_filter_la_CPPFLAGS = \ ++ -I$(top_srcdir)/include \ ++ -I$(top_srcdir)/common/include \ ++ -I$(top_srcdir)/common/utils \ ++ $(NULL) ++nbdkit_scan_filter_la_CFLAGS = $(WARNINGS_CFLAGS) ++nbdkit_scan_filter_la_LDFLAGS = \ ++ -module -avoid-version -shared $(NO_UNDEFINED_ON_WINDOWS) \ ++ -Wl,--version-script=$(top_srcdir)/filters/filters.syms \ ++ $(NULL) ++nbdkit_scan_filter_la_LIBADD = \ ++ $(top_builddir)/common/utils/libutils.la \ ++ $(top_builddir)/common/replacements/libcompat.la \ ++ $(IMPORT_LIBRARY_ON_WINDOWS) \ ++ $(NULL) ++ ++if HAVE_POD ++ ++man_MANS = nbdkit-scan-filter.1 ++CLEANFILES += $(man_MANS) ++ ++nbdkit-scan-filter.1: nbdkit-scan-filter.pod \ ++ $(top_builddir)/podwrapper.pl ++ $(PODWRAPPER) --section=1 --man $@ \ ++ --html $(top_builddir)/html/$@.html \ ++ $< ++ ++endif HAVE_POD +diff --git a/filters/scan/bgthread.c b/filters/scan/bgthread.c +new file mode 100644 +index 00000000..384e79b6 +--- /dev/null ++++ b/filters/scan/bgthread.c +@@ -0,0 +1,131 @@ ++/* nbdkit ++ * Copyright (C) 2019-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "scan.h" ++ ++#include "cleanup.h" ++#include "minmax.h" ++ ++static pthread_mutex_t clock_lock; ++static uint64_t clock_ = 0; ++ ++static void ++adjust_clock (uint64_t offset) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&clock_lock); ++ if (clock_ < offset) ++ clock_ = offset; ++} ++ ++static void ++reset_clock (uint64_t offset) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&clock_lock); ++ clock_ = 0; ++} ++ ++static uint64_t ++get_starting_offset (void) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&clock_lock); ++ return scan_clock ? clock_ : 0; ++} ++ ++void * ++scan_thread (void *vp) ++{ ++ struct bgthread_ctrl *ctrl = vp; ++ uint64_t offset, size; ++ int64_t r; ++ ++ assert (ctrl->next != NULL); ++ ++ /* Get the size of the underlying plugin. Exit the thread on error ++ * because there's not much we can do without knowing the size. ++ */ ++ r = ctrl->next->get_size (ctrl->next); ++ if (r == -1) ++ return NULL; ++ size = r; ++ ++ /* Start scanning. */ ++ start: ++ for (offset = get_starting_offset (); offset < size; offset += scan_size) { ++ uint64_t n; ++ ++ /* Execute any commands in the queue. */ ++ { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); ++ struct command cmd; ++ ++ while (ctrl->cmds.len) { ++ cmd = ctrl->cmds.ptr[0]; ++ command_queue_remove (&ctrl->cmds, 0); ++ ++ switch (cmd.type) { ++ case CMD_QUIT: ++ nbdkit_debug ("scan: exiting background thread on connection close"); ++ return NULL; ++ ++ case CMD_NOTIFY_PREAD: ++ if (offset < cmd.offset) ++ offset = cmd.offset; ++ } ++ } ++ } ++ ++ adjust_clock (offset); ++ if (offset > size) ++ continue; ++ ++ /* Issue the next prefetch. */ ++ n = MIN (scan_size, size - offset); ++ ctrl->next->cache (ctrl->next, n, offset, 0, NULL); ++ } ++ ++ if (scan_forever) { ++ reset_clock (offset); ++ goto start; ++ } ++ ++ nbdkit_debug ("scan: finished scanning the plugin"); ++ return NULL; ++} +diff --git a/filters/scan/nbdkit-scan-filter.pod b/filters/scan/nbdkit-scan-filter.pod +new file mode 100644 +index 00000000..4a8d0ef9 +--- /dev/null ++++ b/filters/scan/nbdkit-scan-filter.pod +@@ -0,0 +1,159 @@ ++=head1 NAME ++ ++nbdkit-scan-filter - scan disk prefetching data ahead of sequential reads ++ ++=head1 SYNOPSIS ++ ++ nbdkit --filter=scan PLUGIN [scan-ahead=false] [scan-clock=false] ++ [scan-forever=true] [scan-size=]NN ++ ++ nbdkit --filter=scan --filter=cache PLUGIN ++ ++ nbdkit --filter=scan --filter=cow PLUGIN cow-on-cache=true ++ ++=head1 DESCRIPTION ++ ++C is a filter that scans the disk prefetching ++data. It is sometimes useful if you expect that the client will read ++the disk sequentially. ++ ++The basic operation of the filter is that when a client connects, the ++filter will start issuing C<.cache> (prefetch) requests to the plugin ++across the whole disk. Plugins which support this command will ++prefetch the data, making subsequent reads faster. For plugins which ++do not support this command, you can inject L ++below (after) this filter, giving approximately the same effect. ++L can be used instead of nbdkit-cache-filter, if ++you add the C option. ++ ++Various C parameters can be used to tune scanning, although ++the defaults should be suitable in most cases. ++ ++A similar filter is L. ++ ++=head2 Limitations ++ ++In a number of significant cases this filter will do nothing. The ++filter will print a warning message if this happens. ++ ++=over 4 ++ ++=item Thread model must be parallel * ++ ++For example L only supports ++C, and so this filter cannot perform prefetches in ++parallel with the read requests. ++ ++=item Only scans while clients are connected * ++ ++The current filter only scans while there is at least one client ++connected. ++ ++=item Only scans the default export * ++ ++The current filter only scans the default export and ignores all ++clients connecting to the non-default export name. ++ ++* We may be able to lift these restrictions in future. ++ ++=item Underlying filters or plugin must support C<.cache> (prefetch) ++ ++Very many plugins do not have the concept of prefetching and/or ++do not implement the C<.cache> callback, and so there is no ++way for this filter to issue prefetches. ++ ++You can usually get around this by adding I<--filter=cache> after this ++filter as explained above. ++ ++=item Prefetching the whole disk may load it all into cache ++ ++In particular if you use this filter together with ++L or L, they will cache ++the whole content of the plugin into a temporary file. This may be ++many gigabytes of data, consuming all space in F. Of course ++this is the whole point of using this filter, but you should be aware ++of it. ++ ++If using the cache filter, the total size of the cache can be limited ++(see L). ++ ++=back ++ ++=head1 PARAMETERS ++ ++=over 4 ++ ++=item B ++ ++By default the filter tries to stay ahead of incoming read requests. ++That is to say, it starts prefetching at the beginning of the disk and ++continues incrementally, but if the client issues a read beyond the ++current prefetch point then the filter skips forward and begins ++prefetching after the read. ++ ++However if you set this parameter to false, then this behaviour is ++disabled. The filter simply prefetches sequentially regardless of ++client requests. ++ ++=item B ++ ++By default, if all clients disconnect and then another client ++connects, prefetching resumes at the same place in the disk. (Like ++stopping and starting a clock.) ++ ++If you set this parameter to false, then the filter starts prefetching ++from the beginning of the disk again. ++ ++=item B ++ ++By default the filter scans over the disk once and then stops. ++ ++If you set this parameter to true, then after the disk has been ++prefetched completely, the filter goes back to the beginning and ++starts over, repeating this for as long as nbdkit is running and there ++are clients connected. ++ ++=item BNN ++ ++This parameter controls the prefetch block size. The default is ++C<2M>. This must be a power of 2 and most plugins will have their own ++limits on the amount of data they can prefetch in a single request. ++ ++=back ++ ++=head1 FILES ++ ++=over 4 ++ ++=item F<$filterdir/nbdkit-scan-filter.so> ++ ++The filter. ++ ++Use C to find the location of C<$filterdir>. ++ ++=back ++ ++=head1 VERSION ++ ++C first appeared in nbdkit 1.32. ++ ++=head1 SEE ALSO ++ ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L. ++ ++=head1 AUTHORS ++ ++Richard W.M. Jones ++ ++=head1 COPYRIGHT ++ ++Copyright (C) 2019-2022 Red Hat Inc. +diff --git a/filters/scan/scan.c b/filters/scan/scan.c +new file mode 100644 +index 00000000..ac5b18d2 +--- /dev/null ++++ b/filters/scan/scan.c +@@ -0,0 +1,280 @@ ++/* nbdkit ++ * Copyright (C) 2019-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "scan.h" ++ ++#include "cleanup.h" ++#include "ispowerof2.h" ++#include "vector.h" ++ ++static bool scan_ahead = true; ++bool scan_clock = true; ++bool scan_forever = false; ++unsigned scan_size = 2*1024*1024; ++ ++static int thread_model = -1; /* Thread model of the underlying plugin. */ ++ ++/* Per-connection data. */ ++struct scan_handle { ++ bool is_default_export; /* If exportname == "". */ ++ bool running; /* True if background thread is running. */ ++ pthread_t thread; /* The background thread, one per connection. */ ++ struct bgthread_ctrl ctrl; ++}; ++ ++static int ++scan_config (nbdkit_next_config *next, nbdkit_backend *nxdata, ++ const char *key, const char *value) ++{ ++ int r; ++ ++ if (strcmp (key, "scan-ahead") == 0) { ++ r = nbdkit_parse_bool (value); ++ if (r == -1) ++ return -1; ++ scan_ahead = r; ++ return 0; ++ } ++ else if (strcmp (key, "scan-clock") == 0) { ++ r = nbdkit_parse_bool (value); ++ if (r == -1) ++ return -1; ++ scan_clock = r; ++ return 0; ++ } ++ else if (strcmp (key, "scan-forever") == 0) { ++ r = nbdkit_parse_bool (value); ++ if (r == -1) ++ return -1; ++ scan_forever = r; ++ return 0; ++ } ++ else if (strcmp (key, "scan-size") == 0) { ++ scan_size = nbdkit_parse_size (value); ++ if (scan_size == -1) ++ return -1; ++ return 0; ++ } ++ ++ return next (nxdata, key, value); ++} ++ ++static int ++scan_config_complete (nbdkit_next_config_complete *next, nbdkit_backend *nxdata) ++{ ++ if (scan_size < 512 || scan_size > 32*1024*1024 || ++ !is_power_of_2 (scan_size)) { ++ nbdkit_error ("scan-size parameter should be [512..32M] " ++ "and a power of two"); ++ return -1; ++ } ++ ++ return next (nxdata); ++} ++ ++#define scan_config_help \ ++ "scan-ahead=false Skip ahead when client reads faster.\n" \ ++ "scan-clock=false Always start prefetching from beginning.\n" \ ++ "scan-forever=true Scan in a loop while clients connected.\n" \ ++ "scan-size=NN Set scan block size." ++ ++/* We need to hook into .get_ready() so we can read the final thread ++ * model (of the whole server). ++ */ ++static int ++scan_get_ready (int final_thread_model) ++{ ++ thread_model = final_thread_model; ++ return 0; ++} ++ ++static int ++send_command_to_background_thread (struct bgthread_ctrl *ctrl, ++ const struct command cmd) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); ++ if (command_queue_append (&ctrl->cmds, cmd) == -1) ++ return -1; ++ /* Signal the thread if it could be sleeping on an empty queue. */ ++ if (ctrl->cmds.len == 1) ++ pthread_cond_signal (&ctrl->cond); ++ return 0; ++} ++ ++static void * ++scan_open (nbdkit_next_open *next, nbdkit_context *nxdata, ++ int readonly, const char *exportname, int is_tls) ++{ ++ struct scan_handle *h; ++ ++ if (next (nxdata, readonly, exportname) == -1) ++ return NULL; ++ ++ h = calloc (1, sizeof *h); ++ if (h == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return NULL; ++ } ++ ++ h->is_default_export = strcmp (exportname, "") == 0; ++ return h; ++} ++ ++/* In prepare we check if it's possible to support the scan filter on ++ * this connection (or print a warning), and start the background ++ * thread. ++ */ ++static int ++scan_prepare (nbdkit_next *next, void *handle, int readonly) ++{ ++ struct scan_handle *h = handle; ++ int r, err; ++ ++ if (!h->is_default_export) { ++ nbdkit_error ("scan: warning: not the default export, not scanning"); ++ return 0; ++ } ++ ++ if (thread_model != NBDKIT_THREAD_MODEL_PARALLEL) { ++ nbdkit_error ("scan: warning: underlying plugin does not support " ++ "the PARALLEL thread model, not scanning"); ++ return 0; ++ } ++ ++ /* Call next->can_cache to read the underlying 'can_cache'. */ ++ r = next->can_cache (next); ++ if (r == -1) ++ return -1; ++ if (r != NBDKIT_CACHE_NATIVE) { ++ nbdkit_error ("scan: warning: underlying plugin does not support " ++ "NBD_CMD_CACHE, not scanning; try adding --filter=cache " ++ "after this filter"); ++ return 0; ++ } ++ ++ /* Save the connection in the handle, for the background thread to use. */ ++ h->ctrl.next = next; ++ ++ /* Create the background thread. */ ++ h->ctrl.cmds = (command_queue) empty_vector; ++ pthread_mutex_init (&h->ctrl.lock, NULL); ++ pthread_cond_init (&h->ctrl.cond, NULL); ++ ++ err = pthread_create (&h->thread, NULL, scan_thread, &h->ctrl); ++ if (err != 0) { ++ errno = err; ++ nbdkit_error ("pthread_create: %m"); ++ pthread_cond_destroy (&h->ctrl.cond); ++ pthread_mutex_destroy (&h->ctrl.lock); ++ return -1; ++ } ++ ++ h->running = true; ++ ++ return 0; ++} ++ ++/* Finalize cleans up the thread if it is running. */ ++static int ++scan_finalize (nbdkit_next *next, void *handle) ++{ ++ struct scan_handle *h = handle; ++ const struct command quit_cmd = { .type = CMD_QUIT }; ++ ++ if (!h->running) ++ return 0; ++ ++ send_command_to_background_thread (&h->ctrl, quit_cmd); ++ pthread_join (h->thread, NULL); ++ pthread_cond_destroy (&h->ctrl.cond); ++ pthread_mutex_destroy (&h->ctrl.lock); ++ command_queue_reset (&h->ctrl.cmds); ++ h->running = false; ++ ++ return 0; ++} ++ ++static void ++scan_close (void *handle) ++{ ++ struct scan_handle *h = handle; ++ ++ free (h); ++} ++ ++/* Read data. */ ++static int ++scan_pread (nbdkit_next *next, ++ void *handle, void *buf, uint32_t count, uint64_t offset, ++ uint32_t flags, int *err) ++{ ++ struct scan_handle *h = handle; ++ ++ if (scan_ahead && h->running) { ++ const struct command cmd = ++ { .type = CMD_NOTIFY_PREAD, .offset = offset + count }; ++ ++ if (send_command_to_background_thread (&h->ctrl, cmd) == -1) ++ return -1; ++ } ++ ++ /* Issue the normal read. */ ++ return next->pread (next, buf, count, offset, flags, err); ++} ++ ++static struct nbdkit_filter filter = { ++ .name = "scan", ++ .longname = "nbdkit scan filter", ++ .get_ready = scan_get_ready, ++ .config = scan_config, ++ .config_complete = scan_config_complete, ++ .config_help = scan_config_help, ++ .open = scan_open, ++ .prepare = scan_prepare, ++ .finalize = scan_finalize, ++ .close = scan_close, ++ .pread = scan_pread, ++}; ++ ++NBDKIT_REGISTER_FILTER(filter) +diff --git a/filters/scan/scan.h b/filters/scan/scan.h +new file mode 100644 +index 00000000..7ff39310 +--- /dev/null ++++ b/filters/scan/scan.h +@@ -0,0 +1,64 @@ ++/* nbdkit ++ * Copyright (C) 2019-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#ifndef NBDKIT_SCAN_H ++#define NBDKIT_SCAN_H ++ ++#include ++#include ++ ++#include ++ ++#include "vector.h" ++ ++extern bool scan_clock; ++extern bool scan_forever; ++extern unsigned scan_size; ++ ++/* List of commands issued to the background thread. */ ++struct command { ++ enum { CMD_QUIT, CMD_NOTIFY_PREAD } type; ++ uint64_t offset; ++}; ++DEFINE_VECTOR_TYPE(command_queue, struct command); ++ ++struct bgthread_ctrl { ++ command_queue cmds; /* Command queue. */ ++ pthread_mutex_t lock; /* Lock for queue. */ ++ pthread_cond_t cond; /* Condition queue size 0 -> 1. */ ++ nbdkit_next *next; /* For sending cache operations. */ ++}; ++ ++/* Start background thread (one per connection). */ ++extern void *scan_thread (void *vp); ++ ++#endif /* NBDKIT_SCAN_H */ +diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod +index 2bc2c4a7..214957d6 100644 +--- a/plugins/ssh/nbdkit-ssh-plugin.pod ++++ b/plugins/ssh/nbdkit-ssh-plugin.pod +@@ -349,6 +349,7 @@ L, + L, + L, + L, ++L, + L, + L, + L, +diff --git a/plugins/torrent/nbdkit-torrent-plugin.pod b/plugins/torrent/nbdkit-torrent-plugin.pod +index 196ce4e9..f09ac3d2 100644 +--- a/plugins/torrent/nbdkit-torrent-plugin.pod ++++ b/plugins/torrent/nbdkit-torrent-plugin.pod +@@ -175,6 +175,7 @@ L, + L, + L, + L, ++L, + L, + L, + L. +diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod +index ea5899dc..3991e86b 100644 +--- a/plugins/vddk/nbdkit-vddk-plugin.pod ++++ b/plugins/vddk/nbdkit-vddk-plugin.pod +@@ -733,6 +733,7 @@ L, + L, + L, + L, ++L, + L, + L, + L, +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 5585b3b7..799aa6c2 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1754,6 +1754,16 @@ test_retry_request_mirror_LDADD = \ + $(LIBNBD_LIBS) \ + $(NULL) + ++# scan filter test. ++TESTS += \ ++ test-scan-copy.sh \ ++ test-scan-info.sh \ ++ $(NULL) ++EXTRA_DIST += \ ++ test-scan-copy.sh \ ++ test-scan-info.sh \ ++ $(NULL) ++ + # swab filter test. + TESTS += \ + test-swab-8.sh \ +diff --git a/tests/test-scan-copy.sh b/tests/test-scan-copy.sh +new file mode 100755 +index 00000000..227ad7b2 +--- /dev/null ++++ b/tests/test-scan-copy.sh +@@ -0,0 +1,42 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires nbdcopy --version ++requires_plugin sparse-random ++requires_filter scan ++ ++nbdkit -fv -U - sparse-random 1M --filter=scan --run 'nbdcopy "$uri" "$uri"' ++nbdkit -fv -U - sparse-random 1G --filter=scan --run 'nbdcopy "$uri" "$uri"' +diff --git a/tests/test-scan-info.sh b/tests/test-scan-info.sh +new file mode 100755 +index 00000000..6b109ca8 +--- /dev/null ++++ b/tests/test-scan-info.sh +@@ -0,0 +1,46 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires nbdinfo --version ++requires_filter scan ++ ++# We're just testing that there are no problematic races with the ++# background thread. ++ ++nbdkit -fv -U - memory 1 --filter=scan --run 'nbdinfo $uri' ++nbdkit -fv -U - memory 1M --filter=scan --run 'nbdinfo $uri' ++nbdkit -fv -U - memory 1G --filter=scan --run 'nbdinfo $uri' ++nbdkit -fv -U - memory 1G --filter=scan -e test --run 'nbdinfo $uri' +-- +2.31.1 + diff --git a/SOURCES/0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch b/SOURCES/0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch deleted file mode 100644 index d5f3813..0000000 --- a/SOURCES/0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch +++ /dev/null @@ -1,287 +0,0 @@ -From 66945d24e9192a67af421eecbb1835d42636ab93 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 27 Oct 2021 12:30:41 +0100 -Subject: [PATCH] vddk: Refactor -D vddk.stats=1 into a new file - -Acked-by: Laszlo Ersek -(cherry picked from commit dcd5bc51ed7710c32d956345ea8da14ba15ef8f5) ---- - plugins/vddk/Makefile.am | 1 + - plugins/vddk/stats.c | 118 +++++++++++++++++++++++++++++++++++++++ - plugins/vddk/vddk.c | 78 +------------------------- - plugins/vddk/vddk.h | 15 +++++ - 4 files changed, 135 insertions(+), 77 deletions(-) - create mode 100644 plugins/vddk/stats.c - -diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am -index 232aaedd..4f470ff9 100644 ---- a/plugins/vddk/Makefile.am -+++ b/plugins/vddk/Makefile.am -@@ -46,6 +46,7 @@ nbdkit_vddk_plugin_la_SOURCES = \ - vddk.c \ - vddk.h \ - reexec.c \ -+ stats.c \ - vddk-structs.h \ - vddk-stubs.h \ - $(top_srcdir)/include/nbdkit-plugin.h \ -diff --git a/plugins/vddk/stats.c b/plugins/vddk/stats.c -new file mode 100644 -index 00000000..18a42714 ---- /dev/null -+++ b/plugins/vddk/stats.c -@@ -0,0 +1,118 @@ -+/* nbdkit -+ * Copyright (C) 2013-2021 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+ -+#include -+ -+#define NBDKIT_API_VERSION 2 -+#include -+ -+#include "vector.h" -+ -+#include "vddk.h" -+ -+/* Debug flags. */ -+NBDKIT_DLL_PUBLIC int vddk_debug_stats; -+ -+pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; -+ -+/* For each VDDK API define a variable to store the time taken (used -+ * to implement -D vddk.stats=1). -+ */ -+#define STUB(fn,ret,args) struct vddk_stat stats_##fn = { .name = #fn } -+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) -+#include "vddk-stubs.h" -+#undef STUB -+#undef OPTIONAL_STUB -+ -+DEFINE_VECTOR_TYPE(statlist, struct vddk_stat) -+ -+static int -+stat_compare (const void *vp1, const void *vp2) -+{ -+ const struct vddk_stat *st1 = vp1; -+ const struct vddk_stat *st2 = vp2; -+ -+ /* Note: sorts in reverse order of time spent in each API call. */ -+ if (st1->usecs < st2->usecs) return 1; -+ else if (st1->usecs > st2->usecs) return -1; -+ else return 0; -+} -+ -+static const char * -+api_name_without_prefix (const char *name) -+{ -+ return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name; -+} -+ -+void -+display_stats (void) -+{ -+ statlist stats = empty_vector; -+ size_t i; -+ -+ if (!vddk_debug_stats) return; -+ -+#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn) -+#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn) -+#include "vddk-stubs.h" -+#undef STUB -+#undef OPTIONAL_STUB -+ -+ qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); -+ -+ nbdkit_debug ("VDDK function stats (-D vddk.stats=1):"); -+ nbdkit_debug ("%-24s %15s %5s %15s", -+ "VixDiskLib_...", "µs", "calls", "bytes"); -+ for (i = 0; i < stats.size; ++i) { -+ if (stats.ptr[i].usecs) { -+ if (stats.ptr[i].bytes > 0) -+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64, -+ api_name_without_prefix (stats.ptr[i].name), -+ stats.ptr[i].usecs, -+ stats.ptr[i].calls, -+ stats.ptr[i].bytes); -+ else -+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64, -+ api_name_without_prefix (stats.ptr[i].name), -+ stats.ptr[i].usecs, -+ stats.ptr[i].calls); -+ } -+ } -+ statlist_reset (&stats); -+} -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 041bff1a..67ac775c 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -61,7 +61,6 @@ - NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo; - NBDKIT_DLL_PUBLIC int vddk_debug_extents; - NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1; --NBDKIT_DLL_PUBLIC int vddk_debug_stats; - - /* For each VDDK API define a global variable. These globals are - * initialized when the plugin is loaded (by vddk_get_ready). -@@ -99,25 +98,6 @@ bool unbuffered; /* unbuffered */ - const char *username; /* user */ - const char *vmx_spec; /* vm */ - --/* For each VDDK API define a variable to store the time taken (used -- * to implement -D vddk.stats=1). -- */ --struct vddk_stat { -- const char *name; /* function name */ -- int64_t usecs; /* total number of usecs consumed */ -- uint64_t calls; /* number of times called */ -- uint64_t bytes; /* bytes transferred, datapath calls only */ --}; --static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; --static void display_stats (void); --#define STUB(fn,ret,args) \ -- static struct vddk_stat stats_##fn = { .name = #fn } --#define OPTIONAL_STUB(fn,ret,args) \ -- static struct vddk_stat stats_##fn = { .name = #fn } --#include "vddk-stubs.h" --#undef STUB --#undef OPTIONAL_STUB -- - /* Unload the plugin. */ - static void - vddk_unload (void) -@@ -130,69 +110,13 @@ vddk_unload (void) - if (dl) - dlclose (dl); - -- if (vddk_debug_stats) -- display_stats (); -+ display_stats (); - - free (config); - free (libdir); - free (password); - } - --DEFINE_VECTOR_TYPE(statlist, struct vddk_stat) -- --static int --stat_compare (const void *vp1, const void *vp2) --{ -- const struct vddk_stat *st1 = vp1; -- const struct vddk_stat *st2 = vp2; -- -- /* Note: sorts in reverse order of time spent in each API call. */ -- if (st1->usecs < st2->usecs) return 1; -- else if (st1->usecs > st2->usecs) return -1; -- else return 0; --} -- --static const char * --api_name_without_prefix (const char *name) --{ -- return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name; --} -- --static void --display_stats (void) --{ -- statlist stats = empty_vector; -- size_t i; -- --#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn) --#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn) --#include "vddk-stubs.h" --#undef STUB --#undef OPTIONAL_STUB -- -- qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); -- -- nbdkit_debug ("VDDK function stats (-D vddk.stats=1):"); -- nbdkit_debug ("%-24s %15s %5s %15s", -- "VixDiskLib_...", "µs", "calls", "bytes"); -- for (i = 0; i < stats.size; ++i) { -- if (stats.ptr[i].usecs) { -- if (stats.ptr[i].bytes > 0) -- nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64, -- api_name_without_prefix (stats.ptr[i].name), -- stats.ptr[i].usecs, -- stats.ptr[i].calls, -- stats.ptr[i].bytes); -- else -- nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64, -- api_name_without_prefix (stats.ptr[i].name), -- stats.ptr[i].usecs, -- stats.ptr[i].calls); -- } -- } -- statlist_reset (&stats); --} -- - static void - trim (char *str) - { -diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h -index 29775eb4..1400589d 100644 ---- a/plugins/vddk/vddk.h -+++ b/plugins/vddk/vddk.h -@@ -126,4 +126,19 @@ extern char *reexeced; - extern void reexec_if_needed (const char *prepend); - extern int restore_ld_library_path (void); - -+/* stats.c */ -+struct vddk_stat { -+ const char *name; /* function name */ -+ int64_t usecs; /* total number of usecs consumed */ -+ uint64_t calls; /* number of times called */ -+ uint64_t bytes; /* bytes transferred, datapath calls only */ -+}; -+extern pthread_mutex_t stats_lock; -+#define STUB(fn,ret,args) extern struct vddk_stat stats_##fn; -+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) -+#include "vddk-stubs.h" -+#undef STUB -+#undef OPTIONAL_STUB -+extern void display_stats (void); -+ - #endif /* NBDKIT_VDDK_H */ --- -2.31.1 - diff --git a/SOURCES/0014-scan-Remove-condition-variable.patch b/SOURCES/0014-scan-Remove-condition-variable.patch new file mode 100644 index 0000000..7500254 --- /dev/null +++ b/SOURCES/0014-scan-Remove-condition-variable.patch @@ -0,0 +1,67 @@ +From 91677241184ab1aa77adadd612fa069d084863ec Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 18:54:32 +0100 +Subject: [PATCH] scan: Remove condition variable + +This was copied in from the readahead filter code, but is not actually +needed in this filter because it never has to sleep waiting for a +command. + +Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb +(cherry picked from commit 43ad586698347997cdfa1bd56bfed0292f89f134) +--- + filters/scan/scan.c | 6 ------ + filters/scan/scan.h | 1 - + 2 files changed, 7 deletions(-) + +diff --git a/filters/scan/scan.c b/filters/scan/scan.c +index ac5b18d2..8a966577 100644 +--- a/filters/scan/scan.c ++++ b/filters/scan/scan.c +@@ -136,9 +136,6 @@ send_command_to_background_thread (struct bgthread_ctrl *ctrl, + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); + if (command_queue_append (&ctrl->cmds, cmd) == -1) + return -1; +- /* Signal the thread if it could be sleeping on an empty queue. */ +- if (ctrl->cmds.len == 1) +- pthread_cond_signal (&ctrl->cond); + return 0; + } + +@@ -199,13 +196,11 @@ scan_prepare (nbdkit_next *next, void *handle, int readonly) + /* Create the background thread. */ + h->ctrl.cmds = (command_queue) empty_vector; + pthread_mutex_init (&h->ctrl.lock, NULL); +- pthread_cond_init (&h->ctrl.cond, NULL); + + err = pthread_create (&h->thread, NULL, scan_thread, &h->ctrl); + if (err != 0) { + errno = err; + nbdkit_error ("pthread_create: %m"); +- pthread_cond_destroy (&h->ctrl.cond); + pthread_mutex_destroy (&h->ctrl.lock); + return -1; + } +@@ -227,7 +222,6 @@ scan_finalize (nbdkit_next *next, void *handle) + + send_command_to_background_thread (&h->ctrl, quit_cmd); + pthread_join (h->thread, NULL); +- pthread_cond_destroy (&h->ctrl.cond); + pthread_mutex_destroy (&h->ctrl.lock); + command_queue_reset (&h->ctrl.cmds); + h->running = false; +diff --git a/filters/scan/scan.h b/filters/scan/scan.h +index 7ff39310..98c0228b 100644 +--- a/filters/scan/scan.h ++++ b/filters/scan/scan.h +@@ -54,7 +54,6 @@ DEFINE_VECTOR_TYPE(command_queue, struct command); + struct bgthread_ctrl { + command_queue cmds; /* Command queue. */ + pthread_mutex_t lock; /* Lock for queue. */ +- pthread_cond_t cond; /* Condition queue size 0 -> 1. */ + nbdkit_next *next; /* For sending cache operations. */ + }; + +-- +2.31.1 + diff --git a/SOURCES/0014-vddk-Implement-parallel-thread-model.patch b/SOURCES/0014-vddk-Implement-parallel-thread-model.patch deleted file mode 100644 index 0deb820..0000000 --- a/SOURCES/0014-vddk-Implement-parallel-thread-model.patch +++ /dev/null @@ -1,1287 +0,0 @@ -From 11a40792fde602861b987dc5a2c91a0539abfe78 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 27 Oct 2021 10:17:22 +0100 -Subject: [PATCH] vddk: Implement parallel thread model - -Since VDDK 6.0, asynchronous read and write operations are available. -This commit makes use of these, allowing us to use the parallel thread -model for increased performance. - -Note that at least VDDK 6.5 is required because VDDK 6.0 had a -different and incompatible signature for VixDiskLibCompletionCB. - -Also note at least vSphere 6.7 is required for asynch calls to make -any performance difference. In older versions they work -synchronously. - -In the parallel thread model, nbdkit will be calling us in parallel -from multiple nbdkit threads. VDDK does not allow multiple threads to -simultaneously call VDDK operations on the same handle. So we create -a background thread per handle (== connection). - -Only the background thread makes VDDK calls[1]. The background thread -handles a mix of synchronous (like extents, flush) and asynchronous -(like read, write) operations, but all from one thread. - -Parallel nbdkit threads issue commands to the background thread -associated with each handle, and wait until they are retired. - -[1] All VDDK calls except for connecting and disconnecting which for -different reasons are protected by a global lock, so I did not need to -change those. - -(cherry picked from commit 1eecf15fc3d8ea253ccec4f5883fdbb9aa6f8c2b) ---- - plugins/vddk/Makefile.am | 1 + - plugins/vddk/nbdkit-vddk-plugin.pod | 11 +- - plugins/vddk/vddk.c | 380 +++++-------------- - plugins/vddk/vddk.h | 49 ++- - plugins/vddk/worker.c | 567 ++++++++++++++++++++++++++++ - tests/dummy-vddk.c | 32 ++ - 6 files changed, 745 insertions(+), 295 deletions(-) - create mode 100644 plugins/vddk/worker.c - -diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am -index 4f470ff9..f8382fc9 100644 ---- a/plugins/vddk/Makefile.am -+++ b/plugins/vddk/Makefile.am -@@ -49,6 +49,7 @@ nbdkit_vddk_plugin_la_SOURCES = \ - stats.c \ - vddk-structs.h \ - vddk-stubs.h \ -+ worker.c \ - $(top_srcdir)/include/nbdkit-plugin.h \ - $(NULL) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index 1c16d096..ce82a734 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -523,6 +523,14 @@ read bandwidth to the VMware server. - - Same as above, but for writing and flushing writes. - -+=item C -+ -+=item C -+ -+Same as above, but for asynchronous read and write calls introduced in -+nbdkit 1.30. Unfortunately at the moment the amount of time spent in -+these calls is not accounted for correctly. -+ - =item C - - This call is used to query information about the sparseness of the -@@ -580,7 +588,8 @@ Debug extents returned by C. - - =item B<-D vddk.datapath=0> - --Suppress debugging of datapath calls (C and C). -+Suppress debugging of datapath calls (C, C, C -+and C). - - =item B<-D vddk.stats=1> - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 67ac775c..9f223db0 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -50,9 +50,6 @@ - #include - - #include "cleanup.h" --#include "minmax.h" --#include "rounding.h" --#include "tvdiff.h" - #include "vector.h" - - #include "vddk.h" -@@ -522,23 +519,18 @@ vddk_dump_plugin (void) - /* The rules on threads and VDDK are here: - * https://code.vmware.com/docs/11750/virtual-disk-development-kit-programming-guide/GUID-6BE903E8-DC70-46D9-98E4-E34A2002C2AD.html - * -- * Before nbdkit 1.22 we used SERIALIZE_ALL_REQUESTS. Since nbdkit -- * 1.22 we changed this to SERIALIZE_REQUESTS and added a mutex around -- * calls to VixDiskLib_Open and VixDiskLib_Close. This is not quite -- * within the letter of the rules, but is within the spirit. -+ * Before nbdkit 1.22 we used SERIALIZE_ALL_REQUESTS. In nbdkit -+ * 1.22-1.28 we changed this to SERIALIZE_REQUESTS and added a mutex -+ * around calls to VixDiskLib_Open and VixDiskLib_Close. In nbdkit -+ * 1.30 and above we assign a background thread per connection to do -+ * asynch operations and use the PARALLEL model. We still need the -+ * lock around Open and Close. - */ --#define THREAD_MODEL NBDKIT_THREAD_MODEL_SERIALIZE_REQUESTS -+#define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL - - /* Lock protecting open/close calls - see above. */ - static pthread_mutex_t open_close_lock = PTHREAD_MUTEX_INITIALIZER; - --/* The per-connection handle. */ --struct vddk_handle { -- VixDiskLibConnectParams *params; /* connection parameters */ -- VixDiskLibConnection connection; /* connection */ -- VixDiskLibHandle handle; /* disk handle */ --}; -- - static inline VixDiskLibConnectParams * - allocate_connect_params (void) - { -@@ -579,12 +571,16 @@ vddk_open (int readonly) - VixError err; - uint32_t flags; - const char *transport_mode; -+ int pterr; - -- h = malloc (sizeof *h); -+ h = calloc (1, sizeof *h); - if (h == NULL) { -- nbdkit_error ("malloc: %m"); -+ nbdkit_error ("calloc: %m"); - return NULL; - } -+ h->commands = (command_queue) empty_vector; -+ pthread_mutex_init (&h->commands_lock, NULL); -+ pthread_cond_init (&h->commands_cond, NULL); - - h->params = allocate_connect_params (); - if (h->params == NULL) { -@@ -661,8 +657,22 @@ vddk_open (int readonly) - VDDK_CALL_END (VixDiskLib_GetTransportMode, 0); - nbdkit_debug ("transport mode: %s", transport_mode); - -+ /* Start the background thread which actually does the asynchronous -+ * work. -+ */ -+ pterr = pthread_create (&h->thread, NULL, vddk_worker_thread, h); -+ if (pterr != 0) { -+ errno = pterr; -+ nbdkit_error ("pthread_create: %m"); -+ goto err3; -+ } -+ - return h; - -+ err3: -+ VDDK_CALL_START (VixDiskLib_Close, "handle") -+ VixDiskLib_Close (h->handle); -+ VDDK_CALL_END (VixDiskLib_Close, 0); - err2: - VDDK_CALL_START (VixDiskLib_Disconnect, "connection") - VixDiskLib_Disconnect (h->connection); -@@ -670,6 +680,8 @@ vddk_open (int readonly) - err1: - free_connect_params (h->params); - err0: -+ pthread_mutex_destroy (&h->commands_lock); -+ pthread_cond_destroy (&h->commands_cond); - free (h); - return NULL; - } -@@ -680,6 +692,10 @@ vddk_close (void *handle) - { - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&open_close_lock); - struct vddk_handle *h = handle; -+ struct command stop_cmd = { .type = STOP }; -+ -+ send_command_and_wait (h, &stop_cmd); -+ pthread_join (h->thread, NULL); - - VDDK_CALL_START (VixDiskLib_Close, "handle") - VixDiskLib_Close (h->handle); -@@ -689,6 +705,9 @@ vddk_close (void *handle) - VDDK_CALL_END (VixDiskLib_Disconnect, 0); - - free_connect_params (h->params); -+ pthread_mutex_destroy (&h->commands_lock); -+ pthread_cond_destroy (&h->commands_cond); -+ command_queue_reset (&h->commands); - free (h); - } - -@@ -697,54 +716,29 @@ static int64_t - vddk_get_size (void *handle) - { - struct vddk_handle *h = handle; -- VixDiskLibInfo *info; -- VixError err; - uint64_t size; -+ struct command get_size_cmd = { .type = GET_SIZE, .ptr = &size }; - -- VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") -- err = VixDiskLib_GetInfo (h->handle, &info); -- VDDK_CALL_END (VixDiskLib_GetInfo, 0); -- if (err != VIX_OK) { -- VDDK_ERROR (err, "VixDiskLib_GetInfo"); -+ if (send_command_and_wait (h, &get_size_cmd) == -1) - return -1; -- } -- -- size = info->capacity * (uint64_t)VIXDISKLIB_SECTOR_SIZE; -- -- if (vddk_debug_diskinfo) { -- nbdkit_debug ("disk info: capacity: %" PRIu64 " sectors " -- "(%" PRIi64 " bytes)", -- info->capacity, size); -- nbdkit_debug ("disk info: biosGeo: C:%" PRIu32 " H:%" PRIu32 " S:%" PRIu32, -- info->biosGeo.cylinders, -- info->biosGeo.heads, -- info->biosGeo.sectors); -- nbdkit_debug ("disk info: physGeo: C:%" PRIu32 " H:%" PRIu32 " S:%" PRIu32, -- info->physGeo.cylinders, -- info->physGeo.heads, -- info->physGeo.sectors); -- nbdkit_debug ("disk info: adapter type: %d", -- (int) info->adapterType); -- nbdkit_debug ("disk info: num links: %d", info->numLinks); -- nbdkit_debug ("disk info: parent filename hint: %s", -- info->parentFileNameHint ? : "NULL"); -- nbdkit_debug ("disk info: uuid: %s", -- info->uuid ? : "NULL"); -- if (library_version >= 7) { -- nbdkit_debug ("disk info: sector size: " -- "logical %" PRIu32 " physical %" PRIu32, -- info->logicalSectorSize, -- info->physicalSectorSize); -- } -- } -- -- VDDK_CALL_START (VixDiskLib_FreeInfo, "info") -- VixDiskLib_FreeInfo (info); -- VDDK_CALL_END (VixDiskLib_FreeInfo, 0); - - return (int64_t) size; - } - -+static int -+vddk_can_fua (void *handle) -+{ -+ /* The Flush call was not available in VDDK < 6.0. */ -+ return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE; -+} -+ -+static int -+vddk_can_flush (void *handle) -+{ -+ /* The Flush call was not available in VDDK < 6.0. */ -+ return VixDiskLib_Flush != NULL; -+} -+ - /* Read data from the file. - * - * Note that reads have to be aligned to sectors (XXX). -@@ -754,32 +748,14 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset, - uint32_t flags) - { - struct vddk_handle *h = handle; -- VixError err; -+ struct command read_cmd = { -+ .type = READ, -+ .ptr = buf, -+ .count = count, -+ .offset = offset, -+ }; - -- /* Align to sectors. */ -- if (!IS_ALIGNED (offset, VIXDISKLIB_SECTOR_SIZE)) { -- nbdkit_error ("%s is not aligned to sectors", "read"); -- return -1; -- } -- if (!IS_ALIGNED (count, VIXDISKLIB_SECTOR_SIZE)) { -- nbdkit_error ("%s is not aligned to sectors", "read"); -- return -1; -- } -- offset /= VIXDISKLIB_SECTOR_SIZE; -- count /= VIXDISKLIB_SECTOR_SIZE; -- -- VDDK_CALL_START (VixDiskLib_Read, -- "handle, %" PRIu64 " sectors, " -- "%" PRIu32 " sectors, buffer", -- offset, count) -- err = VixDiskLib_Read (h->handle, offset, count, buf); -- VDDK_CALL_END (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE); -- if (err != VIX_OK) { -- VDDK_ERROR (err, "VixDiskLib_Read"); -- return -1; -- } -- -- return 0; -+ return send_command_and_wait (h, &read_cmd); - } - - static int vddk_flush (void *handle, uint32_t flags); -@@ -792,32 +768,17 @@ static int - vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, - uint32_t flags) - { -+ struct vddk_handle *h = handle; - const bool fua = flags & NBDKIT_FLAG_FUA; -- struct vddk_handle *h = handle; -- VixError err; -+ struct command write_cmd = { -+ .type = WRITE, -+ .ptr = (void *) buf, -+ .count = count, -+ .offset = offset, -+ }; - -- /* Align to sectors. */ -- if (!IS_ALIGNED (offset, VIXDISKLIB_SECTOR_SIZE)) { -- nbdkit_error ("%s is not aligned to sectors", "write"); -+ if (send_command_and_wait (h, &write_cmd) == -1) - return -1; -- } -- if (!IS_ALIGNED (count, VIXDISKLIB_SECTOR_SIZE)) { -- nbdkit_error ("%s is not aligned to sectors", "write"); -- return -1; -- } -- offset /= VIXDISKLIB_SECTOR_SIZE; -- count /= VIXDISKLIB_SECTOR_SIZE; -- -- VDDK_CALL_START (VixDiskLib_Write, -- "handle, %" PRIu64 " sectors, " -- "%" PRIu32 " sectors, buffer", -- offset, count) -- err = VixDiskLib_Write (h->handle, offset, count, buf); -- VDDK_CALL_END (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE); -- if (err != VIX_OK) { -- VDDK_ERROR (err, "VixDiskLib_Write"); -- return -1; -- } - - if (fua) { - if (vddk_flush (handle, 0) == -1) -@@ -827,126 +788,32 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, - return 0; - } - --static int --vddk_can_fua (void *handle) --{ -- /* The Flush call was not available in VDDK < 6.0. */ -- return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE; --} -- --static int --vddk_can_flush (void *handle) --{ -- /* The Flush call was not available in VDDK < 6.0. */ -- return VixDiskLib_Flush != NULL; --} -- - /* Flush data to the file. */ - static int - vddk_flush (void *handle, uint32_t flags) - { - struct vddk_handle *h = handle; -- VixError err; -+ struct command flush_cmd = { -+ .type = FLUSH, -+ }; - -- /* The documentation for Flush is missing, but the comment in the -- * header file seems to indicate that it waits for WriteAsync -- * commands to finish. We don't use WriteAsync, and in any case -- * there's a new function Wait to wait for those. However I -- * verified using strace that in fact Flush does call fsync on the -- * file so it appears to be the correct call to use here. -- */ -- -- VDDK_CALL_START (VixDiskLib_Flush, "handle") -- err = VixDiskLib_Flush (h->handle); -- VDDK_CALL_END (VixDiskLib_Flush, 0); -- if (err != VIX_OK) { -- VDDK_ERROR (err, "VixDiskLib_Flush"); -- return -1; -- } -- -- return 0; -+ return send_command_and_wait (h, &flush_cmd); - } - - static int - vddk_can_extents (void *handle) - { - struct vddk_handle *h = handle; -- VixError err; -- VixDiskLibBlockList *block_list; -+ int ret; -+ struct command can_extents_cmd = { -+ .type = CAN_EXTENTS, -+ .ptr = &ret, -+ }; - -- /* This call was added in VDDK 6.7. In earlier versions the -- * function pointer will be NULL and we cannot query extents. -- */ -- if (VixDiskLib_QueryAllocatedBlocks == NULL) { -- nbdkit_debug ("can_extents: VixDiskLib_QueryAllocatedBlocks == NULL, " -- "probably this is VDDK < 6.7"); -- return 0; -- } -- -- /* Suppress errors around this call. See: -- * https://bugzilla.redhat.com/show_bug.cgi?id=1709211#c7 -- */ -- error_suppression = 1; -- -- /* However even when the call is available it rarely works well so -- * the best thing we can do here is to try the call and if it's -- * non-functional return false. -- */ -- VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, -- "handle, 0, %d sectors, %d sectors", -- VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) -- err = VixDiskLib_QueryAllocatedBlocks (h->handle, -- 0, VIXDISKLIB_MIN_CHUNK_SIZE, -- VIXDISKLIB_MIN_CHUNK_SIZE, -- &block_list); -- VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); -- error_suppression = 0; -- if (err == VIX_OK) { -- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") -- VixDiskLib_FreeBlockList (block_list); -- VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); -- } -- if (err != VIX_OK) { -- char *errmsg = VixDiskLib_GetErrorText (err, NULL); -- nbdkit_debug ("can_extents: VixDiskLib_QueryAllocatedBlocks test failed, " -- "extents support will be disabled: " -- "original error: %s", -- errmsg); -- VixDiskLib_FreeErrorText (errmsg); -- return 0; -- } -- -- return 1; --} -- --static int --add_extent (struct nbdkit_extents *extents, -- uint64_t *position, uint64_t next_position, bool is_hole) --{ -- uint32_t type = 0; -- const uint64_t length = next_position - *position; -- -- if (is_hole) { -- type = NBDKIT_EXTENT_HOLE; -- /* Images opened as single link might be backed by another file in the -- chain, so the holes are not guaranteed to be zeroes. */ -- if (!single_link) -- type |= NBDKIT_EXTENT_ZERO; -- } -- -- assert (*position <= next_position); -- if (*position == next_position) -- return 0; -- -- if (vddk_debug_extents) -- nbdkit_debug ("adding extent type %s at [%" PRIu64 "...%" PRIu64 "]", -- is_hole ? "hole" : "allocated data", -- *position, next_position-1); -- if (nbdkit_add_extent (extents, *position, length, type) == -1) -+ if (send_command_and_wait (h, &can_extents_cmd) == -1) - return -1; - -- *position = next_position; -- return 0; -+ return ret; - } - - static int -@@ -955,88 +822,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags, - { - struct vddk_handle *h = handle; - bool req_one = flags & NBDKIT_FLAG_REQ_ONE; -- uint64_t position, end, start_sector; -- -- position = offset; -- end = offset + count; -- -- /* We can only query whole chunks. Therefore start with the first -- * chunk before offset. -- */ -- start_sector = -- ROUND_DOWN (offset, VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE) -- / VIXDISKLIB_SECTOR_SIZE; -- while (start_sector * VIXDISKLIB_SECTOR_SIZE < end) { -- VixError err; -- uint32_t i; -- uint64_t nr_chunks, nr_sectors; -- VixDiskLibBlockList *block_list; -- -- assert (IS_ALIGNED (start_sector, VIXDISKLIB_MIN_CHUNK_SIZE)); -- -- nr_chunks = -- ROUND_UP (end - start_sector * VIXDISKLIB_SECTOR_SIZE, -- VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE) -- / (VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE); -- nr_chunks = MIN (nr_chunks, VIXDISKLIB_MAX_CHUNK_NUMBER); -- nr_sectors = nr_chunks * VIXDISKLIB_MIN_CHUNK_SIZE; -- -- VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, -- "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, " -- "%d sectors", -- start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) -- err = VixDiskLib_QueryAllocatedBlocks (h->handle, -- start_sector, nr_sectors, -- VIXDISKLIB_MIN_CHUNK_SIZE, -- &block_list); -- VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); -- if (err != VIX_OK) { -- VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks"); -- return -1; -- } -- -- for (i = 0; i < block_list->numBlocks; ++i) { -- uint64_t blk_offset, blk_length; -- -- blk_offset = block_list->blocks[i].offset * VIXDISKLIB_SECTOR_SIZE; -- blk_length = block_list->blocks[i].length * VIXDISKLIB_SECTOR_SIZE; -- -- /* The query returns allocated blocks. We must insert holes -- * between the blocks as necessary. -- */ -- if ((position < blk_offset && -- add_extent (extents, &position, blk_offset, true) == -1) || -- (add_extent (extents, -- &position, blk_offset + blk_length, false) == -1)) { -- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") -- VixDiskLib_FreeBlockList (block_list); -- VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); -- return -1; -- } -- } -- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") -- VixDiskLib_FreeBlockList (block_list); -- VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); -- -- /* There's an implicit hole after the returned list of blocks, up -- * to the end of the QueryAllocatedBlocks request. -- */ -- if (add_extent (extents, -- &position, -- (start_sector + nr_sectors) * VIXDISKLIB_SECTOR_SIZE, -- true) == -1) -- return -1; -- -- start_sector += nr_sectors; -- -- /* If one extent was requested, as long as we've added an extent -- * overlapping the original offset we're done. -- */ -- if (req_one && position > offset) -- break; -- } -- -- return 0; -+ struct command extents_cmd = { -+ .type = EXTENTS, -+ .ptr = extents, -+ .count = count, -+ .offset = offset, -+ .req_one = req_one, -+ }; -+ -+ return send_command_and_wait (h, &extents_cmd); - } - - static struct nbdkit_plugin plugin = { -diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h -index 1400589d..be0b3492 100644 ---- a/plugins/vddk/vddk.h -+++ b/plugins/vddk/vddk.h -@@ -90,7 +90,9 @@ extern int vddk_debug_stats; - /* GCC can optimize this away at compile time: */ \ - const bool datapath = \ - strcmp (#fn, "VixDiskLib_Read") == 0 || \ -- strcmp (#fn, "VixDiskLib_Write") == 0; \ -+ strcmp (#fn, "VixDiskLib_ReadAsync") == 0 || \ -+ strcmp (#fn, "VixDiskLib_Write") == 0 || \ -+ strcmp (#fn, "VixDiskLib_WriteAsync") == 0; \ - if (vddk_debug_stats) \ - gettimeofday (&start_t, NULL); \ - if (!datapath || vddk_debug_datapath) \ -@@ -120,6 +122,46 @@ extern int vddk_debug_stats; - VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \ - } while (0) - -+/* Queue of asynchronous commands sent to the background thread. */ -+enum command_type { GET_SIZE, READ, WRITE, FLUSH, CAN_EXTENTS, EXTENTS, STOP }; -+struct command { -+ /* These fields are set by the caller. */ -+ enum command_type type; /* command */ -+ void *ptr; /* buffer, extents list, return values */ -+ uint32_t count; /* READ, WRITE, EXTENTS */ -+ uint64_t offset; /* READ, WRITE, EXTENTS */ -+ bool req_one; /* EXTENTS NBDKIT_FLAG_REQ_ONE */ -+ -+ /* This field is set to a unique value by send_command_and_wait. */ -+ uint64_t id; /* serial number */ -+ -+ /* These fields are used by the internal implementation. */ -+ pthread_mutex_t mutex; /* completion mutex */ -+ pthread_cond_t cond; /* completion condition */ -+ enum { SUBMITTED, SUCCEEDED, FAILED } status; -+}; -+ -+DEFINE_VECTOR_TYPE(command_queue, struct command *) -+ -+/* The per-connection handle. */ -+struct vddk_handle { -+ VixDiskLibConnectParams *params; /* connection parameters */ -+ VixDiskLibConnection connection; /* connection */ -+ VixDiskLibHandle handle; /* disk handle */ -+ -+ pthread_t thread; /* background thread for asynch work */ -+ -+ /* Command queue of commands sent to the background thread. Use -+ * send_command_and_wait to add a command. Only the background -+ * thread must make VDDK API calls (apart from opening and closing). -+ * The lock protects all of these fields. -+ */ -+ pthread_mutex_t commands_lock; /* lock */ -+ command_queue commands; /* command queue */ -+ pthread_cond_t commands_cond; /* condition (queue size 0 -> 1) */ -+ uint64_t id; /* next command ID */ -+}; -+ - /* reexec.c */ - extern bool noreexec; - extern char *reexeced; -@@ -141,4 +183,9 @@ extern pthread_mutex_t stats_lock; - #undef OPTIONAL_STUB - extern void display_stats (void); - -+/* worker.c */ -+extern const char *command_type_string (enum command_type type); -+extern int send_command_and_wait (struct vddk_handle *h, struct command *cmd); -+extern void *vddk_worker_thread (void *handle); -+ - #endif /* NBDKIT_VDDK_H */ -diff --git a/plugins/vddk/worker.c b/plugins/vddk/worker.c -new file mode 100644 -index 00000000..2a1d4f26 ---- /dev/null -+++ b/plugins/vddk/worker.c -@@ -0,0 +1,567 @@ -+/* nbdkit -+ * Copyright (C) 2013-2021 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+ -+#include -+ -+#define NBDKIT_API_VERSION 2 -+#include -+ -+#include "cleanup.h" -+#include "minmax.h" -+#include "rounding.h" -+#include "vector.h" -+ -+#include "vddk.h" -+ -+const char * -+command_type_string (enum command_type type) -+{ -+ switch (type) { -+ case GET_SIZE: return "get_size"; -+ case READ: return "read"; -+ case WRITE: return "write"; -+ case FLUSH: return "flush"; -+ case CAN_EXTENTS: return "can_extents"; -+ case EXTENTS: return "extents"; -+ case STOP: return "stop"; -+ default: abort (); -+ } -+} -+ -+/* Send command to the background thread and wait for completion. -+ * -+ * Returns 0 for OK -+ * On error, calls nbdkit_error and returns -1. -+ */ -+int -+send_command_and_wait (struct vddk_handle *h, struct command *cmd) -+{ -+ /* Add the command to the command queue. */ -+ { -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&h->commands_lock); -+ cmd->id = h->id++; -+ -+ if (command_queue_append (&h->commands, cmd) == -1) -+ /* On error command_queue_append will call nbdkit_error. */ -+ return -1; -+ -+ /* Signal the caller if it could be sleeping on an empty queue. */ -+ if (h->commands.size == 1) -+ pthread_cond_signal (&h->commands_cond); -+ -+ /* This will be used to signal command completion back to us. */ -+ pthread_mutex_init (&cmd->mutex, NULL); -+ pthread_cond_init (&cmd->cond, NULL); -+ } -+ -+ /* Wait for the command to be completed by the background thread. */ -+ { -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&cmd->mutex); -+ while (cmd->status == SUBMITTED) -+ pthread_cond_wait (&cmd->cond, &cmd->mutex); -+ } -+ -+ pthread_mutex_destroy (&cmd->mutex); -+ pthread_cond_destroy (&cmd->cond); -+ -+ /* On error the background thread will call nbdkit_error. */ -+ switch (cmd->status) { -+ case SUCCEEDED: return 0; -+ case FAILED: return -1; -+ default: abort (); -+ } -+} -+ -+/* Asynchronous commands are completed when this function is called. */ -+static void -+complete_command (void *vp, VixError result) -+{ -+ struct command *cmd = vp; -+ -+ if (vddk_debug_datapath) -+ nbdkit_debug ("command %" PRIu64 " completed", cmd->id); -+ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&cmd->mutex); -+ -+ if (result == VIX_OK) { -+ cmd->status = SUCCEEDED; -+ } else { -+ VDDK_ERROR (result, "command %" PRIu64 ": asynchronous %s failed", -+ cmd->id, command_type_string (cmd->type)); -+ cmd->status = FAILED; -+ } -+ -+ pthread_cond_signal (&cmd->cond); -+} -+ -+/* Wait for any asynchronous commands to complete. */ -+static int -+do_stop (struct command *cmd, struct vddk_handle *h) -+{ -+ VixError err; -+ -+ /* Because we assume VDDK >= 6.5, VixDiskLib_Wait must exist. */ -+ VDDK_CALL_START (VixDiskLib_Wait, "handle") -+ err = VixDiskLib_Wait (h->handle); -+ VDDK_CALL_END (VixDiskLib_Wait, 0); -+ if (err != VIX_OK) { -+ VDDK_ERROR (err, "VixDiskLib_Wait"); -+ /* In the end this error indication is ignored because it only -+ * happens on the close path when we cannot handle errors. -+ */ -+ return -1; -+ } -+ return 0; -+} -+ -+/* Get size command. */ -+static int64_t -+do_get_size (struct command *cmd, struct vddk_handle *h) -+{ -+ VixError err; -+ VixDiskLibInfo *info; -+ uint64_t size; -+ -+ VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") -+ err = VixDiskLib_GetInfo (h->handle, &info); -+ VDDK_CALL_END (VixDiskLib_GetInfo, 0); -+ if (err != VIX_OK) { -+ VDDK_ERROR (err, "VixDiskLib_GetInfo"); -+ return -1; -+ } -+ -+ size = info->capacity * (uint64_t)VIXDISKLIB_SECTOR_SIZE; -+ -+ if (vddk_debug_diskinfo) { -+ nbdkit_debug ("disk info: capacity: %" PRIu64 " sectors " -+ "(%" PRIi64 " bytes)", -+ info->capacity, size); -+ nbdkit_debug ("disk info: biosGeo: C:%" PRIu32 " H:%" PRIu32 " S:%" PRIu32, -+ info->biosGeo.cylinders, -+ info->biosGeo.heads, -+ info->biosGeo.sectors); -+ nbdkit_debug ("disk info: physGeo: C:%" PRIu32 " H:%" PRIu32 " S:%" PRIu32, -+ info->physGeo.cylinders, -+ info->physGeo.heads, -+ info->physGeo.sectors); -+ nbdkit_debug ("disk info: adapter type: %d", -+ (int) info->adapterType); -+ nbdkit_debug ("disk info: num links: %d", info->numLinks); -+ nbdkit_debug ("disk info: parent filename hint: %s", -+ info->parentFileNameHint ? : "NULL"); -+ nbdkit_debug ("disk info: uuid: %s", -+ info->uuid ? : "NULL"); -+ if (library_version >= 7) { -+ nbdkit_debug ("disk info: sector size: " -+ "logical %" PRIu32 " physical %" PRIu32, -+ info->logicalSectorSize, -+ info->physicalSectorSize); -+ } -+ } -+ -+ VDDK_CALL_START (VixDiskLib_FreeInfo, "info") -+ VixDiskLib_FreeInfo (info); -+ VDDK_CALL_END (VixDiskLib_FreeInfo, 0); -+ -+ return (int64_t) size; -+} -+ -+static int -+do_read (struct command *cmd, struct vddk_handle *h) -+{ -+ VixError err; -+ uint32_t count = cmd->count; -+ uint64_t offset = cmd->offset; -+ void *buf = cmd->ptr; -+ -+ /* Align to sectors. */ -+ if (!IS_ALIGNED (offset, VIXDISKLIB_SECTOR_SIZE)) { -+ nbdkit_error ("%s is not aligned to sectors", "read"); -+ return -1; -+ } -+ if (!IS_ALIGNED (count, VIXDISKLIB_SECTOR_SIZE)) { -+ nbdkit_error ("%s is not aligned to sectors", "read"); -+ return -1; -+ } -+ offset /= VIXDISKLIB_SECTOR_SIZE; -+ count /= VIXDISKLIB_SECTOR_SIZE; -+ -+ VDDK_CALL_START (VixDiskLib_ReadAsync, -+ "handle, %" PRIu64 " sectors, " -+ "%" PRIu32 " sectors, buffer, callback, %" PRIu64, -+ offset, count, cmd->id) -+ err = VixDiskLib_ReadAsync (h->handle, offset, count, buf, -+ complete_command, cmd); -+ VDDK_CALL_END (VixDiskLib_ReadAsync, count * VIXDISKLIB_SECTOR_SIZE); -+ if (err != VIX_ASYNC) { -+ VDDK_ERROR (err, "VixDiskLib_ReadAsync"); -+ return -1; -+ } -+ -+ return 0; -+} -+ -+static int -+do_write (struct command *cmd, struct vddk_handle *h) -+{ -+ VixError err; -+ uint32_t count = cmd->count; -+ uint64_t offset = cmd->offset; -+ const void *buf = cmd->ptr; -+ -+ /* Align to sectors. */ -+ if (!IS_ALIGNED (offset, VIXDISKLIB_SECTOR_SIZE)) { -+ nbdkit_error ("%s is not aligned to sectors", "write"); -+ return -1; -+ } -+ if (!IS_ALIGNED (count, VIXDISKLIB_SECTOR_SIZE)) { -+ nbdkit_error ("%s is not aligned to sectors", "write"); -+ return -1; -+ } -+ offset /= VIXDISKLIB_SECTOR_SIZE; -+ count /= VIXDISKLIB_SECTOR_SIZE; -+ -+ VDDK_CALL_START (VixDiskLib_WriteAsync, -+ "handle, %" PRIu64 " sectors, " -+ "%" PRIu32 " sectors, buffer, callback, %" PRIu64, -+ offset, count, cmd->id) -+ err = VixDiskLib_WriteAsync (h->handle, offset, count, buf, -+ complete_command, cmd); -+ VDDK_CALL_END (VixDiskLib_WriteAsync, count * VIXDISKLIB_SECTOR_SIZE); -+ if (err != VIX_ASYNC) { -+ VDDK_ERROR (err, "VixDiskLib_WriteAsync"); -+ return -1; -+ } -+ -+ return 0; -+} -+ -+static int -+do_flush (struct command *cmd, struct vddk_handle *h) -+{ -+ VixError err; -+ -+ /* It seems safer to wait for outstanding asynchronous commands to -+ * complete before doing a flush, so do this but ignore errors -+ * except to print them. -+ */ -+ VDDK_CALL_START (VixDiskLib_Wait, "handle") -+ err = VixDiskLib_Wait (h->handle); -+ VDDK_CALL_END (VixDiskLib_Wait, 0); -+ if (err != VIX_OK) -+ VDDK_ERROR (err, "VixDiskLib_Wait"); -+ -+ /* The documentation for Flush is missing, but the comment in the -+ * header file seems to indicate that it waits for WriteAsync -+ * commands to finish. There's a new function Wait to wait for -+ * those. However I verified using strace that in fact Flush calls -+ * fsync on the file so it appears to be the correct call to use -+ * here. -+ */ -+ VDDK_CALL_START (VixDiskLib_Flush, "handle") -+ err = VixDiskLib_Flush (h->handle); -+ VDDK_CALL_END (VixDiskLib_Flush, 0); -+ if (err != VIX_OK) { -+ VDDK_ERROR (err, "VixDiskLib_Flush"); -+ return -1; -+ } -+ -+ return 0; -+} -+ -+static int -+do_can_extents (struct command *cmd, struct vddk_handle *h) -+{ -+ VixError err; -+ VixDiskLibBlockList *block_list; -+ -+ /* This call was added in VDDK 6.7. In earlier versions the -+ * function pointer will be NULL and we cannot query extents. -+ */ -+ if (VixDiskLib_QueryAllocatedBlocks == NULL) { -+ nbdkit_debug ("can_extents: VixDiskLib_QueryAllocatedBlocks == NULL, " -+ "probably this is VDDK < 6.7"); -+ return 0; -+ } -+ -+ /* Suppress errors around this call. See: -+ * https://bugzilla.redhat.com/show_bug.cgi?id=1709211#c7 -+ */ -+ error_suppression = 1; -+ -+ /* However even when the call is available it rarely works well so -+ * the best thing we can do here is to try the call and if it's -+ * non-functional return false. -+ */ -+ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, -+ "handle, 0, %d sectors, %d sectors", -+ VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) -+ err = VixDiskLib_QueryAllocatedBlocks (h->handle, -+ 0, VIXDISKLIB_MIN_CHUNK_SIZE, -+ VIXDISKLIB_MIN_CHUNK_SIZE, -+ &block_list); -+ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); -+ error_suppression = 0; -+ if (err == VIX_OK) { -+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") -+ VixDiskLib_FreeBlockList (block_list); -+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); -+ } -+ if (err != VIX_OK) { -+ char *errmsg = VixDiskLib_GetErrorText (err, NULL); -+ nbdkit_debug ("can_extents: " -+ "VixDiskLib_QueryAllocatedBlocks test failed, " -+ "extents support will be disabled: " -+ "original error: %s", -+ errmsg); -+ VixDiskLib_FreeErrorText (errmsg); -+ return 0; -+ } -+ -+ return 1; -+} -+ -+/* Add an extent to the list of extents. */ -+static int -+add_extent (struct nbdkit_extents *extents, -+ uint64_t *position, uint64_t next_position, bool is_hole) -+{ -+ uint32_t type = 0; -+ const uint64_t length = next_position - *position; -+ -+ if (is_hole) { -+ type = NBDKIT_EXTENT_HOLE; -+ /* Images opened as single link might be backed by another file in the -+ chain, so the holes are not guaranteed to be zeroes. */ -+ if (!single_link) -+ type |= NBDKIT_EXTENT_ZERO; -+ } -+ -+ assert (*position <= next_position); -+ if (*position == next_position) -+ return 0; -+ -+ if (vddk_debug_extents) -+ nbdkit_debug ("adding extent type %s at [%" PRIu64 "...%" PRIu64 "]", -+ is_hole ? "hole" : "allocated data", -+ *position, next_position-1); -+ if (nbdkit_add_extent (extents, *position, length, type) == -1) -+ return -1; -+ -+ *position = next_position; -+ return 0; -+} -+ -+static int -+do_extents (struct command *cmd, struct vddk_handle *h) -+{ -+ uint32_t count = cmd->count; -+ uint64_t offset = cmd->offset; -+ bool req_one = cmd->req_one; -+ struct nbdkit_extents *extents = cmd->ptr; -+ uint64_t position, end, start_sector; -+ -+ position = offset; -+ end = offset + count; -+ -+ /* We can only query whole chunks. Therefore start with the -+ * first chunk before offset. -+ */ -+ start_sector = -+ ROUND_DOWN (offset, VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE) -+ / VIXDISKLIB_SECTOR_SIZE; -+ while (start_sector * VIXDISKLIB_SECTOR_SIZE < end) { -+ VixError err; -+ uint32_t i; -+ uint64_t nr_chunks, nr_sectors; -+ VixDiskLibBlockList *block_list; -+ -+ assert (IS_ALIGNED (start_sector, VIXDISKLIB_MIN_CHUNK_SIZE)); -+ -+ nr_chunks = -+ ROUND_UP (end - start_sector * VIXDISKLIB_SECTOR_SIZE, -+ VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE) -+ / (VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE); -+ nr_chunks = MIN (nr_chunks, VIXDISKLIB_MAX_CHUNK_NUMBER); -+ nr_sectors = nr_chunks * VIXDISKLIB_MIN_CHUNK_SIZE; -+ -+ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, -+ "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, " -+ "%d sectors", -+ start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) -+ err = VixDiskLib_QueryAllocatedBlocks (h->handle, -+ start_sector, nr_sectors, -+ VIXDISKLIB_MIN_CHUNK_SIZE, -+ &block_list); -+ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); -+ if (err != VIX_OK) { -+ VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks"); -+ return -1; -+ } -+ -+ for (i = 0; i < block_list->numBlocks; ++i) { -+ uint64_t blk_offset, blk_length; -+ -+ blk_offset = block_list->blocks[i].offset * VIXDISKLIB_SECTOR_SIZE; -+ blk_length = block_list->blocks[i].length * VIXDISKLIB_SECTOR_SIZE; -+ -+ /* The query returns allocated blocks. We must insert holes -+ * between the blocks as necessary. -+ */ -+ if ((position < blk_offset && -+ add_extent (extents, &position, blk_offset, true) == -1) || -+ (add_extent (extents, -+ &position, blk_offset + blk_length, false) == -1)) { -+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") -+ VixDiskLib_FreeBlockList (block_list); -+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); -+ return -1; -+ } -+ } -+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") -+ VixDiskLib_FreeBlockList (block_list); -+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); -+ -+ /* There's an implicit hole after the returned list of blocks, -+ * up to the end of the QueryAllocatedBlocks request. -+ */ -+ if (add_extent (extents, -+ &position, -+ (start_sector + nr_sectors) * VIXDISKLIB_SECTOR_SIZE, -+ true) == -1) { -+ return -1; -+ } -+ -+ start_sector += nr_sectors; -+ -+ /* If one extent was requested, as long as we've added an extent -+ * overlapping the original offset we're done. -+ */ -+ if (req_one && position > offset) -+ break; -+ } -+ -+ return 0; -+} -+ -+/* Background worker thread, one per connection, which is where the -+ * VDDK commands are issued. -+ */ -+void * -+vddk_worker_thread (void *handle) -+{ -+ struct vddk_handle *h = handle; -+ bool stop = false; -+ -+ while (!stop) { -+ struct command *cmd; -+ int r; -+ bool async = false; -+ -+ /* Wait until we are sent at least one command. */ -+ { -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&h->commands_lock); -+ while (h->commands.size == 0) -+ pthread_cond_wait (&h->commands_cond, &h->commands_lock); -+ cmd = h->commands.ptr[0]; -+ command_queue_remove (&h->commands, 0); -+ } -+ -+ switch (cmd->type) { -+ case STOP: -+ r = do_stop (cmd, h); -+ stop = true; -+ break; -+ -+ case GET_SIZE: { -+ int64_t size = do_get_size (cmd, h); -+ if (size == -1) -+ r = -1; -+ else { -+ r = 0; -+ *(uint64_t *)cmd->ptr = size; -+ } -+ break; -+ } -+ -+ case READ: -+ r = do_read (cmd, h); -+ /* If async is true, don't retire this command now. */ -+ async = r == 0; -+ break; -+ -+ case WRITE: -+ r = do_write (cmd, h); -+ /* If async is true, don't retire this command now. */ -+ async = r == 0; -+ break; -+ -+ case FLUSH: -+ r = do_flush (cmd, h); -+ break; -+ -+ case CAN_EXTENTS: -+ r = do_can_extents (cmd, h); -+ if (r >= 0) -+ *(int *)cmd->ptr = r; -+ break; -+ -+ case EXTENTS: -+ r = do_extents (cmd, h); -+ break; -+ -+ default: abort (); /* impossible, but keeps GCC happy */ -+ } /* switch */ -+ -+ if (!async) { -+ /* Update the command status. */ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&cmd->mutex); -+ cmd->status = r >= 0 ? SUCCEEDED : FAILED; -+ -+ /* For synchronous commands signal the caller thread that the -+ * command has completed. (Asynchronous commands are completed in -+ * the callback handler). -+ */ -+ pthread_cond_signal (&cmd->cond); -+ } -+ } /* while (!stop) */ -+ -+ /* Exit the worker thread. */ -+ return NULL; -+} -diff --git a/tests/dummy-vddk.c b/tests/dummy-vddk.c -index cb88380c..b6f12042 100644 ---- a/tests/dummy-vddk.c -+++ b/tests/dummy-vddk.c -@@ -188,6 +188,19 @@ VixDiskLib_Read (VixDiskLibHandle handle, - return VIX_OK; - } - -+NBDKIT_DLL_PUBLIC VixError -+VixDiskLib_ReadAsync (VixDiskLibHandle handle, -+ uint64_t start_sector, uint64_t nr_sectors, -+ unsigned char *buf, -+ VixDiskLibCompletionCB callback, void *data) -+{ -+ size_t offset = start_sector * VIXDISKLIB_SECTOR_SIZE; -+ -+ memcpy (buf, disk + offset, nr_sectors * VIXDISKLIB_SECTOR_SIZE); -+ callback (data, VIX_OK); -+ return VIX_ASYNC; -+} -+ - NBDKIT_DLL_PUBLIC VixError - VixDiskLib_Write (VixDiskLibHandle handle, - uint64_t start_sector, uint64_t nr_sectors, -@@ -199,6 +212,25 @@ VixDiskLib_Write (VixDiskLibHandle handle, - return VIX_OK; - } - -+NBDKIT_DLL_PUBLIC VixError -+VixDiskLib_WriteAsync (VixDiskLibHandle handle, -+ uint64_t start_sector, uint64_t nr_sectors, -+ const unsigned char *buf, -+ VixDiskLibCompletionCB callback, void *data) -+{ -+ size_t offset = start_sector * VIXDISKLIB_SECTOR_SIZE; -+ -+ memcpy (disk + offset, buf, nr_sectors * VIXDISKLIB_SECTOR_SIZE); -+ callback (data, VIX_OK); -+ return VIX_ASYNC; -+} -+ -+NBDKIT_DLL_PUBLIC VixError -+VixDiskLib_Flush (VixDiskLibHandle handle) -+{ -+ return VIX_OK; -+} -+ - NBDKIT_DLL_PUBLIC VixError - VixDiskLib_Wait (VixDiskLibHandle handle) - { --- -2.31.1 - diff --git a/SOURCES/0015-scan-Small-typographical-fix-in-manual.patch b/SOURCES/0015-scan-Small-typographical-fix-in-manual.patch new file mode 100644 index 0000000..c2c6ae0 --- /dev/null +++ b/SOURCES/0015-scan-Small-typographical-fix-in-manual.patch @@ -0,0 +1,57 @@ +From c191f45530d4dd7f978803c0bfa402ca0fc950df Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 19:02:48 +0100 +Subject: [PATCH] scan: Small typographical fix in manual + +Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb +(cherry picked from commit 67d4e3437d2e28fa3ce1c4b3818d2b1e7939c5ec) +--- + filters/scan/nbdkit-scan-filter.pod | 12 ++++++------ + 1 file changed, 6 insertions(+), 6 deletions(-) + +diff --git a/filters/scan/nbdkit-scan-filter.pod b/filters/scan/nbdkit-scan-filter.pod +index 4a8d0ef9..2fe9bb80 100644 +--- a/filters/scan/nbdkit-scan-filter.pod ++++ b/filters/scan/nbdkit-scan-filter.pod +@@ -26,8 +26,8 @@ below (after) this filter, giving approximately the same effect. + L can be used instead of nbdkit-cache-filter, if + you add the C option. + +-Various C parameters can be used to tune scanning, although +-the defaults should be suitable in most cases. ++Various parameters can be used to tune scanning, although the defaults ++should be suitable in most cases. + + A similar filter is L. + +@@ -38,23 +38,23 @@ filter will print a warning message if this happens. + + =over 4 + +-=item Thread model must be parallel * ++=item Thread model must be parallel* + + For example L only supports + C, and so this filter cannot perform prefetches in + parallel with the read requests. + +-=item Only scans while clients are connected * ++=item Only scans while clients are connected* + + The current filter only scans while there is at least one client + connected. + +-=item Only scans the default export * ++=item Only scans the default export* + + The current filter only scans the default export and ignores all + clients connecting to the non-default export name. + +-* We may be able to lift these restrictions in future. ++*We may be able to lift these restrictions in future. + + =item Underlying filters or plugin must support C<.cache> (prefetch) + +-- +2.31.1 + diff --git a/SOURCES/0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch b/SOURCES/0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch deleted file mode 100644 index 8741479..0000000 --- a/SOURCES/0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch +++ /dev/null @@ -1,57 +0,0 @@ -From c91ac233f6474b07ef181a08093c5d0f2f4ec4c3 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Fri, 29 Oct 2021 20:56:55 +0100 -Subject: [PATCH] vddk: Assume that VixDiskLib_Flush is available - -Since we now require and check that VDDK >= 6.5, we can assume that -VixDiskLib_Flush is always available. - -(cherry picked from commit e3685e6f0d0b71ab24b96fe85430a3b75da58736) ---- - plugins/vddk/vddk.c | 15 +++++++++++---- - 1 file changed, 11 insertions(+), 4 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 9f223db0..f967e2d9 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -378,6 +378,12 @@ load_library (bool load_error_is_fatal) - "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\"."); - exit (EXIT_FAILURE); - } -+ -+ /* Added in VDDK 6.0 so it must always be present. Since we are -+ * going to call this function unconditionally, fail early and hard -+ * if for some reason it's not present. -+ */ -+ assert (VixDiskLib_Flush != NULL); - } - - static int -@@ -725,18 +731,19 @@ vddk_get_size (void *handle) - return (int64_t) size; - } - -+/* The Flush call was added in VDDK 6.0, since we support minimum 6.5 -+ * we are always able to do FUA / flush. -+ */ - static int - vddk_can_fua (void *handle) - { -- /* The Flush call was not available in VDDK < 6.0. */ -- return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE; -+ return NBDKIT_FUA_NATIVE; - } - - static int - vddk_can_flush (void *handle) - { -- /* The Flush call was not available in VDDK < 6.0. */ -- return VixDiskLib_Flush != NULL; -+ return 1; - } - - /* Read data from the file. --- -2.31.1 - diff --git a/SOURCES/0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch b/SOURCES/0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch new file mode 100644 index 0000000..d6e387b --- /dev/null +++ b/SOURCES/0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch @@ -0,0 +1,34 @@ +From 651045d703804d7dafab04a0387ca92573f52467 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 20:57:38 +0100 +Subject: [PATCH] ssh: Don't reference readahead or scan filters from this + plugin + +These filters do not support plugins which do not use the parallel +thread model. + +Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f +Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb +See-also: commit 92fbb76d11b9f17c527debd803aa2505f3642783 +(cherry picked from commit 7eb356719376c4d0b2379cea5d39c81602d2d304) +--- + plugins/ssh/nbdkit-ssh-plugin.pod | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod +index 214957d6..bb922d37 100644 +--- a/plugins/ssh/nbdkit-ssh-plugin.pod ++++ b/plugins/ssh/nbdkit-ssh-plugin.pod +@@ -347,9 +347,7 @@ C first appeared in nbdkit 1.12. + L, + L, + L, +-L, + L, +-L, + L, + L, + L, +-- +2.31.1 + diff --git a/SOURCES/0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch b/SOURCES/0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch deleted file mode 100644 index 0421316..0000000 --- a/SOURCES/0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch +++ /dev/null @@ -1,186 +0,0 @@ -From 984e95fcbdb19c2495851322a4c33f34291ecfab Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Fri, 29 Oct 2021 21:02:54 +0100 -Subject: [PATCH] vddk: Simplify detection of VDDK symbols and baseline 6.5 - -Make all symbols from VDDK 6.5 into required symbols and use a single -error message function if one of these is missing. The new error is: - - nbdkit: error: required VDDK symbol "VixDiskLib_Wait" is - missing. VDDK version must be >= 6.5. See nbdkit-vddk-plugin(1) man - page section "SUPPORTED VERSIONS OF VDDK". Original dlopen error: - vmware-vix-disklib-distrib/lib64/libvixDiskLib.so.6: undefined - symbol: VixDiskLib_Wait - -Remove the extra check and assert. - -Be more consistent about #define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) -when we want the optional and required stubs to do the same thing. - -(cherry picked from commit ec0d22e61881efa39a69d02ccb9e4ede8bf95e75) ---- - plugins/vddk/stats.c | 2 +- - plugins/vddk/vddk-stubs.h | 45 ++++++++++++++++++--------------------- - plugins/vddk/vddk.c | 36 ++++++++++++------------------- - plugins/vddk/vddk.h | 2 +- - 4 files changed, 37 insertions(+), 48 deletions(-) - -diff --git a/plugins/vddk/stats.c b/plugins/vddk/stats.c -index 18a42714..76e0c244 100644 ---- a/plugins/vddk/stats.c -+++ b/plugins/vddk/stats.c -@@ -89,7 +89,7 @@ display_stats (void) - if (!vddk_debug_stats) return; - - #define STUB(fn,ret,args) statlist_append (&stats, stats_##fn) --#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn) -+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) - #include "vddk-stubs.h" - #undef STUB - #undef OPTIONAL_STUB -diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h -index 66353691..7d8644c3 100644 ---- a/plugins/vddk/vddk-stubs.h -+++ b/plugins/vddk/vddk-stubs.h -@@ -39,10 +39,7 @@ - * function name, return value, arguments. - */ - --/* Required stubs, present in all versions of VDDK that we support. I -- * have checked that all these exist in at least VDDK 5.5.5 (2015). -- */ -- -+/* Required stubs, present in all versions of VDDK since 6.5 (Nov 2016). */ - STUB (VixDiskLib_InitEx, - VixError, - (uint32_t major, uint32_t minor, -@@ -103,27 +100,27 @@ STUB (VixDiskLib_Write, - uint64_t start_sector, uint64_t nr_sectors, - const unsigned char *buf)); - --/* Added in VDDK 6.0, these will be NULL in earlier versions. */ --OPTIONAL_STUB (VixDiskLib_Flush, -- VixError, -- (VixDiskLibHandle handle)); --OPTIONAL_STUB (VixDiskLib_ReadAsync, -- VixError, -- (VixDiskLibHandle handle, -- uint64_t start_sector, uint64_t nr_sectors, -- unsigned char *buf, -- VixDiskLibCompletionCB callback, void *data)); --OPTIONAL_STUB (VixDiskLib_WriteAsync, -- VixError, -- (VixDiskLibHandle handle, -- uint64_t start_sector, uint64_t nr_sectors, -- const unsigned char *buf, -- VixDiskLibCompletionCB callback, void *data)); -+/* Added in VDDK 6.0. */ -+STUB (VixDiskLib_Flush, -+ VixError, -+ (VixDiskLibHandle handle)); -+STUB (VixDiskLib_ReadAsync, -+ VixError, -+ (VixDiskLibHandle handle, -+ uint64_t start_sector, uint64_t nr_sectors, -+ unsigned char *buf, -+ VixDiskLibCompletionCB callback, void *data)); -+STUB (VixDiskLib_WriteAsync, -+ VixError, -+ (VixDiskLibHandle handle, -+ uint64_t start_sector, uint64_t nr_sectors, -+ const unsigned char *buf, -+ VixDiskLibCompletionCB callback, void *data)); - --/* Added in VDDK 6.5, this will be NULL in earlier versions. */ --OPTIONAL_STUB (VixDiskLib_Wait, -- VixError, -- (VixDiskLibHandle handle)); -+/* Added in VDDK 6.5. */ -+STUB (VixDiskLib_Wait, -+ VixError, -+ (VixDiskLibHandle handle)); - - /* Added in VDDK 6.7, these will be NULL for earlier versions: */ - OPTIONAL_STUB (VixDiskLib_QueryAllocatedBlocks, -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index f967e2d9..271b5ee0 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -63,7 +63,7 @@ NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1; - * initialized when the plugin is loaded (by vddk_get_ready). - */ - #define STUB(fn,ret,args) ret (*fn) args --#define OPTIONAL_STUB(fn,ret,args) ret (*fn) args -+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) - #include "vddk-stubs.h" - #undef STUB - #undef OPTIONAL_STUB -@@ -282,6 +282,17 @@ vddk_config (const char *key, const char *value) - return 0; - } - -+static void -+missing_required_symbol (const char *fn) -+{ -+ nbdkit_error ("required VDDK symbol \"%s\" is missing. " -+ "VDDK version must be >= 6.5. " -+ "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\". " -+ "Original dlopen error: %s\n", -+ fn, dlerror ()); -+ exit (EXIT_FAILURE); -+} -+ - /* Load the VDDK library. */ - static void - load_library (bool load_error_is_fatal) -@@ -358,32 +369,13 @@ load_library (bool load_error_is_fatal) - #define STUB(fn,ret,args) \ - do { \ - fn = dlsym (dl, #fn); \ -- if (fn == NULL) { \ -- nbdkit_error ("required VDDK symbol \"%s\" is missing: %s", \ -- #fn, dlerror ()); \ -- exit (EXIT_FAILURE); \ -- } \ -+ if (fn == NULL) \ -+ missing_required_symbol (#fn); \ - } while (0) - #define OPTIONAL_STUB(fn,ret,args) fn = dlsym (dl, #fn) - #include "vddk-stubs.h" - #undef STUB - #undef OPTIONAL_STUB -- -- /* Additionally, VDDK version must be >= 6.5. This was the first -- * version which introduced VixDiskLib_Wait symbol so we can check -- * for that. -- */ -- if (VixDiskLib_Wait == NULL) { -- nbdkit_error ("VDDK version must be >= 6.5. " -- "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\"."); -- exit (EXIT_FAILURE); -- } -- -- /* Added in VDDK 6.0 so it must always be present. Since we are -- * going to call this function unconditionally, fail early and hard -- * if for some reason it's not present. -- */ -- assert (VixDiskLib_Flush != NULL); - } - - static int -diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h -index be0b3492..0e3dd79e 100644 ---- a/plugins/vddk/vddk.h -+++ b/plugins/vddk/vddk.h -@@ -76,7 +76,7 @@ extern int vddk_debug_datapath; - extern int vddk_debug_stats; - - #define STUB(fn,ret,args) extern ret (*fn) args --#define OPTIONAL_STUB(fn,ret,args) extern ret (*fn) args -+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) - #include "vddk-stubs.h" - #undef STUB - #undef OPTIONAL_STUB --- -2.31.1 - diff --git a/SOURCES/0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch b/SOURCES/0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch new file mode 100644 index 0000000..f78049c --- /dev/null +++ b/SOURCES/0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch @@ -0,0 +1,56 @@ +From f58d2a04338edc647e2334ff58b49508424e3f3b Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 17 May 2022 13:20:17 +0100 +Subject: [PATCH] scan: Fix bound so we don't try to prefetch beyond end of + disk + +An off-by-one error in the bound could cause the filter to try to +prefetch beyond the end of the underlying plugin. This would cause +nbdkit to crash with this assertion failure: + +nbdkit: backend.c:782: backend_cache: Assertion `backend_valid_range (c, offset, count)' failed. + +The sequence of events was: + + - scan filter background thread started + + - client reads to the end of the disk + + - background thread skips ahead to end of disk (offset == size) + + - background thread tries to prefetch from this point + +In the final step the calculations caused to the background thread to +prefetch a scan-size block beyond the end of the plugin. + +Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb +(cherry picked from commit 953643429b8c57b4dd20a6c0e5b83704ae9a0e88) +--- + filters/scan/bgthread.c | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/filters/scan/bgthread.c b/filters/scan/bgthread.c +index 384e79b6..5fa5f27f 100644 +--- a/filters/scan/bgthread.c ++++ b/filters/scan/bgthread.c +@@ -113,12 +113,12 @@ scan_thread (void *vp) + } + + adjust_clock (offset); +- if (offset > size) +- continue; + +- /* Issue the next prefetch. */ +- n = MIN (scan_size, size - offset); +- ctrl->next->cache (ctrl->next, n, offset, 0, NULL); ++ if (offset < size) { ++ /* Issue the next prefetch. */ ++ n = MIN (scan_size, size - offset); ++ ctrl->next->cache (ctrl->next, n, offset, 0, NULL); ++ } + } + + if (scan_forever) { +-- +2.31.1 + diff --git a/SOURCES/0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch b/SOURCES/0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch deleted file mode 100644 index f9c1a1d..0000000 --- a/SOURCES/0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch +++ /dev/null @@ -1,40 +0,0 @@ -From 342efed6bb9f8f0c8d2cb4aa2b09da64ed2e7ed4 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 30 Oct 2021 08:34:28 +0100 -Subject: [PATCH] vddk: Remove some whitespace from a couple of functions - -(cherry picked from commit 974dce2c2ef84fc096ee319f340054234a29df91) ---- - plugins/vddk/vddk.c | 9 ++------- - 1 file changed, 2 insertions(+), 7 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 271b5ee0..184f1a9c 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -792,9 +792,7 @@ static int - vddk_flush (void *handle, uint32_t flags) - { - struct vddk_handle *h = handle; -- struct command flush_cmd = { -- .type = FLUSH, -- }; -+ struct command flush_cmd = { .type = FLUSH }; - - return send_command_and_wait (h, &flush_cmd); - } -@@ -804,10 +802,7 @@ vddk_can_extents (void *handle) - { - struct vddk_handle *h = handle; - int ret; -- struct command can_extents_cmd = { -- .type = CAN_EXTENTS, -- .ptr = &ret, -- }; -+ struct command can_extents_cmd = { .type = CAN_EXTENTS, .ptr = &ret }; - - if (send_command_and_wait (h, &can_extents_cmd) == -1) - return -1; --- -2.31.1 - diff --git a/SOURCES/0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch b/SOURCES/0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch new file mode 100644 index 0000000..123c091 --- /dev/null +++ b/SOURCES/0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch @@ -0,0 +1,110 @@ +From d1d2f43223bcda062d10c8e68776590956892f71 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 10 Jun 2022 22:11:44 +0100 +Subject: [PATCH] tests: Add a regression test for LUKS zeroing crash + +https://listman.redhat.com/archives/libguestfs/2022-June/029188.html +(cherry picked from commit 7ab2ef96803bfc385f786be82ebfdd4cc977d504) +--- + tests/Makefile.am | 2 ++ + tests/test-luks-copy-zero.sh | 70 ++++++++++++++++++++++++++++++++++++ + 2 files changed, 72 insertions(+) + create mode 100755 tests/test-luks-copy-zero.sh + +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 799aa6c2..0f4b0746 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1601,11 +1601,13 @@ if HAVE_GNUTLS_PBKDF2 + TESTS += \ + test-luks-info.sh \ + test-luks-copy.sh \ ++ test-luks-copy-zero.sh \ + $(NULL) + endif + EXTRA_DIST += \ + test-luks-info.sh \ + test-luks-copy.sh \ ++ test-luks-copy-zero.sh \ + $(NULL) + + # multi-conn filter test. +diff --git a/tests/test-luks-copy-zero.sh b/tests/test-luks-copy-zero.sh +new file mode 100755 +index 00000000..6ff560e3 +--- /dev/null ++++ b/tests/test-luks-copy-zero.sh +@@ -0,0 +1,70 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++# Regression test for: ++# https://listman.redhat.com/archives/libguestfs/2022-June/029188.html ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires qemu-img --version ++requires nbdcopy --version ++requires truncate --version ++requires file --version ++requires_filter luks ++ ++encrypt_disk=luks-copy-zero1.img ++zero_disk=luks-copy-zero2.img ++cleanup_fn rm -f $encrypt_disk $zero_disk ++rm -f $encrypt_disk $zero_disk ++ ++# Create an empty encrypted disk container. ++qemu-img create -f luks \ ++ --object secret,data=123456,id=sec0 \ ++ -o key-secret=sec0 \ ++ $encrypt_disk 100M ++ ++# Create an all zeroes disk of the same size. ++truncate -s 100M $zero_disk ++ ++# Using nbdkit-luks-filter, write the zero disk into the encrypted ++# disk. nbdcopy will do this using NBD_CMD_ZERO operations. ++nbdkit -U - -fv \ ++ file $encrypt_disk --filter=luks passphrase=123456 \ ++ --run "nbdcopy -C 1 $zero_disk \$nbd" ++ ++# Check that the encrypted disk is still a LUKS disk. If zeroing is ++# wrong in the filter it's possible that it writes through to the ++# underlying disk, erasing the container. ++file $encrypt_disk ++file $encrypt_disk | grep "LUKS encrypted file" +-- +2.31.1 + diff --git a/SOURCES/0018-vddk-Move-config-debug-error-and-utility-functions-a.patch b/SOURCES/0018-vddk-Move-config-debug-error-and-utility-functions-a.patch deleted file mode 100644 index bb03b19..0000000 --- a/SOURCES/0018-vddk-Move-config-debug-error-and-utility-functions-a.patch +++ /dev/null @@ -1,338 +0,0 @@ -From edbded52b144ce3c8c45c7ef352f8969a1f5d1bb Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 30 Oct 2021 08:27:39 +0100 -Subject: [PATCH] vddk: Move config, debug/error and utility functions around - -Move the functions so they are nearer to where they are used. -Introduce a utils.c file for utility functions. - -This is just code rearrangement with no other effects. - -(cherry picked from commit c59be086210a06688b9195e0b91f8603a668654a) ---- - plugins/vddk/Makefile.am | 1 + - plugins/vddk/utils.c | 51 ++++++++++ - plugins/vddk/vddk.c | 201 +++++++++++++++++++-------------------- - plugins/vddk/vddk.h | 3 + - 4 files changed, 151 insertions(+), 105 deletions(-) - create mode 100644 plugins/vddk/utils.c - -diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am -index f8382fc9..02113da0 100644 ---- a/plugins/vddk/Makefile.am -+++ b/plugins/vddk/Makefile.am -@@ -47,6 +47,7 @@ nbdkit_vddk_plugin_la_SOURCES = \ - vddk.h \ - reexec.c \ - stats.c \ -+ utils.c \ - vddk-structs.h \ - vddk-stubs.h \ - worker.c \ -diff --git a/plugins/vddk/utils.c b/plugins/vddk/utils.c -new file mode 100644 -index 00000000..f0c19950 ---- /dev/null -+++ b/plugins/vddk/utils.c -@@ -0,0 +1,51 @@ -+/* nbdkit -+ * Copyright (C) 2013-2021 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+ -+#define NBDKIT_API_VERSION 2 -+#include -+ -+#include "vddk.h" -+ -+void -+trim (char *str) -+{ -+ size_t len = strlen (str); -+ -+ if (len > 0 && str[len-1] == '\n') -+ str[len-1] = '\0'; -+} -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 184f1a9c..31e5e23b 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -114,61 +114,6 @@ vddk_unload (void) - free (password); - } - --static void --trim (char *str) --{ -- size_t len = strlen (str); -- -- if (len > 0 && str[len-1] == '\n') -- str[len-1] = '\0'; --} -- --/* Turn log messages from the library into nbdkit_debug. */ --static void --debug_function (const char *fs, va_list args) --{ -- CLEANUP_FREE char *str = NULL; -- -- if (vasprintf (&str, fs, args) == -1) { -- nbdkit_debug ("lost debug message: %s", fs); -- return; -- } -- -- trim (str); -- -- nbdkit_debug ("%s", str); --} -- --/* Turn error messages from the library into nbdkit_error. */ --static void --error_function (const char *fs, va_list args) --{ -- CLEANUP_FREE char *str = NULL; -- -- /* If the thread-local error_suppression flag is non-zero then we -- * will suppress error messages from VDDK in this thread. -- */ -- if (error_suppression) return; -- -- if (vasprintf (&str, fs, args) == -1) { -- nbdkit_error ("lost error message: %s", fs); -- return; -- } -- -- trim (str); -- -- /* VDDK 7 added a useless error message about their "phone home" -- * system called CEIP which only panics users. Demote it to a debug -- * statement. https://bugzilla.redhat.com/show_bug.cgi?id=1834267 -- */ -- if (strstr (str, "Get CEIP status failed") != NULL) { -- nbdkit_debug ("%s", str); -- return; -- } -- -- nbdkit_error ("%s", str); --} -- - /* Configuration. */ - static int - vddk_config (const char *key, const char *value) -@@ -282,6 +227,56 @@ vddk_config (const char *key, const char *value) - return 0; - } - -+static int -+vddk_config_complete (void) -+{ -+ if (filename == NULL) { -+ nbdkit_error ("you must supply the file= parameter " -+ "after the plugin name on the command line"); -+ return -1; -+ } -+ -+ /* For remote connections, check all the parameters have been -+ * passed. Note that VDDK will segfault if parameters that it -+ * expects are NULL (and there's no real way to tell what parameters -+ * it is expecting). This implements the same test that the VDDK -+ * sample program does. -+ */ -+ is_remote = -+ vmx_spec || -+ server_name || -+ username || -+ password || -+ cookie || -+ thumb_print || -+ port || -+ nfc_host_port; -+ -+ if (is_remote) { -+#define missing(test, param) \ -+ if (test) { \ -+ nbdkit_error ("remote connection requested, missing parameter: %s", \ -+ param); \ -+ return -1; \ -+ } -+ missing (!server_name, "server"); -+ missing (!username, "user"); -+ missing (!password, "password"); -+ missing (!vmx_spec, "vm"); -+#undef missing -+ } -+ -+ /* Restore original LD_LIBRARY_PATH after reexec. */ -+ if (restore_ld_library_path () == -1) -+ return -1; -+ -+ return 0; -+} -+ -+#define vddk_config_help \ -+ "[file=] (required) The filename (eg. VMDK file) to serve.\n" \ -+ "Many optional parameters are supported, see nbdkit-vddk-plugin(1)." -+ - static void - missing_required_symbol (const char *fn) - { -@@ -378,56 +373,6 @@ load_library (bool load_error_is_fatal) - #undef OPTIONAL_STUB - } - --static int --vddk_config_complete (void) --{ -- if (filename == NULL) { -- nbdkit_error ("you must supply the file= parameter " -- "after the plugin name on the command line"); -- return -1; -- } -- -- /* For remote connections, check all the parameters have been -- * passed. Note that VDDK will segfault if parameters that it -- * expects are NULL (and there's no real way to tell what parameters -- * it is expecting). This implements the same test that the VDDK -- * sample program does. -- */ -- is_remote = -- vmx_spec || -- server_name || -- username || -- password || -- cookie || -- thumb_print || -- port || -- nfc_host_port; -- -- if (is_remote) { --#define missing(test, param) \ -- if (test) { \ -- nbdkit_error ("remote connection requested, missing parameter: %s", \ -- param); \ -- return -1; \ -- } -- missing (!server_name, "server"); -- missing (!username, "user"); -- missing (!password, "password"); -- missing (!vmx_spec, "vm"); --#undef missing -- } -- -- /* Restore original LD_LIBRARY_PATH after reexec. */ -- if (restore_ld_library_path () == -1) -- return -1; -- -- return 0; --} -- --#define vddk_config_help \ -- "[file=] (required) The filename (eg. VMDK file) to serve.\n" \ -- "Many optional parameters are supported, see nbdkit-vddk-plugin(1)." -- - static int - vddk_get_ready (void) - { -@@ -435,6 +380,52 @@ vddk_get_ready (void) - return 0; - } - -+/* Turn log messages from the library into nbdkit_debug. */ -+static void -+debug_function (const char *fs, va_list args) -+{ -+ CLEANUP_FREE char *str = NULL; -+ -+ if (vasprintf (&str, fs, args) == -1) { -+ nbdkit_debug ("lost debug message: %s", fs); -+ return; -+ } -+ -+ trim (str); -+ -+ nbdkit_debug ("%s", str); -+} -+ -+/* Turn error messages from the library into nbdkit_error. */ -+static void -+error_function (const char *fs, va_list args) -+{ -+ CLEANUP_FREE char *str = NULL; -+ -+ /* If the thread-local error_suppression flag is non-zero then we -+ * will suppress error messages from VDDK in this thread. -+ */ -+ if (error_suppression) return; -+ -+ if (vasprintf (&str, fs, args) == -1) { -+ nbdkit_error ("lost error message: %s", fs); -+ return; -+ } -+ -+ trim (str); -+ -+ /* VDDK 7 added a useless error message about their "phone home" -+ * system called CEIP which only panics users. Demote it to a debug -+ * statement. https://bugzilla.redhat.com/show_bug.cgi?id=1834267 -+ */ -+ if (strstr (str, "Get CEIP status failed") != NULL) { -+ nbdkit_debug ("%s", str); -+ return; -+ } -+ -+ nbdkit_error ("%s", str); -+} -+ - /* Defer VDDK initialization until after fork because it is known to - * create background threads from VixDiskLib_InitEx. Unfortunately - * error reporting from this callback is difficult, but we have -diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h -index 0e3dd79e..d99b6f4b 100644 ---- a/plugins/vddk/vddk.h -+++ b/plugins/vddk/vddk.h -@@ -183,6 +183,9 @@ extern pthread_mutex_t stats_lock; - #undef OPTIONAL_STUB - extern void display_stats (void); - -+/* utils.c */ -+extern void trim (char *str); -+ - /* worker.c */ - extern const char *command_type_string (enum command_type type); - extern int send_command_and_wait (struct vddk_handle *h, struct command *cmd); --- -2.31.1 - diff --git a/SOURCES/0019-common-utils-test-vector.c-Add-vector-benchmarks.patch b/SOURCES/0019-common-utils-test-vector.c-Add-vector-benchmarks.patch deleted file mode 100644 index 8fd61b1..0000000 --- a/SOURCES/0019-common-utils-test-vector.c-Add-vector-benchmarks.patch +++ /dev/null @@ -1,245 +0,0 @@ -From 239df6ee9583bc520e9a3e18f0c0d8e58602fb5c Mon Sep 17 00:00:00 2001 -From: Nir Soffer -Date: Fri, 5 Nov 2021 20:36:42 +0200 -Subject: [PATCH] common/utils/test-vector.c: Add vector benchmarks - -The generic vector reallocs on every append. Add benchmarks to measure -the cost with uint32 vector (used for copying extents) and the effect of -reserving space upfront. - -The tests show that realloc is pretty efficient, but calling reserve -before the appends speeds the appends up significantly. - - NBDKIT_BENCH=1 ./test-vector - bench_reserve: 1000000 appends in 0.004503 s - bench_append: 1000000 appends in 0.014986 s - -The new benchmarks do not run by default to avoid trouble in CI on -overloaded machines or under qemu emulation. - -A new target added to run all benchmaks: - - make bench - -Ported from libnbd: -- commit dc9ae0174ab1384081a57a8d54b10f8147ea6430 -- commit f6c06a3b4d87fe976a96ea04f8da1f22b2531dbd - -(cherry picked from commit a227af7921c9a51c4f1ab699a3b9f06a9a645126) ---- - Makefile.am | 5 +++ - README | 7 ++++ - common/utils/Makefile.am | 5 ++- - common/utils/bench.h | 72 ++++++++++++++++++++++++++++++++++++++ - common/utils/test-vector.c | 55 +++++++++++++++++++++++++++-- - 5 files changed, 141 insertions(+), 3 deletions(-) - create mode 100644 common/utils/bench.h - -diff --git a/Makefile.am b/Makefile.am -index b21d69ed..49f5d91c 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -102,6 +102,11 @@ check-root: - check-vddk: - $(MAKE) -C tests check-vddk - -+bench: all -+ @for d in common/utils; do \ -+ $(MAKE) -C $$d bench || exit 1; \ -+ done -+ - #---------------------------------------------------------------------- - # Maintainers only! - -diff --git a/README b/README -index a04325be..b001620c 100644 ---- a/README -+++ b/README -@@ -274,6 +274,13 @@ nbdkit-vddk-plugin against the library like this: - - make check-vddk vddkdir=vmware-vix-disklib-distrib - -+Running the benchmarks -+---------------------- -+ -+To run benchmarks: -+ -+ make bench -+ - DOWNLOAD TARBALLS - ================= - -diff --git a/common/utils/Makefile.am b/common/utils/Makefile.am -index c33811fc..b2f08cb4 100644 ---- a/common/utils/Makefile.am -+++ b/common/utils/Makefile.am -@@ -101,6 +101,9 @@ test_quotes_SOURCES = test-quotes.c quote.c utils.h - test_quotes_CPPFLAGS = -I$(srcdir) - test_quotes_CFLAGS = $(WARNINGS_CFLAGS) - --test_vector_SOURCES = test-vector.c vector.c vector.h -+test_vector_SOURCES = test-vector.c vector.c vector.h bench.h - test_vector_CPPFLAGS = -I$(srcdir) - test_vector_CFLAGS = $(WARNINGS_CFLAGS) -+ -+bench: test-vector -+ NBDKIT_BENCH=1 ./test-vector -diff --git a/common/utils/bench.h b/common/utils/bench.h -new file mode 100644 -index 00000000..496a3614 ---- /dev/null -+++ b/common/utils/bench.h -@@ -0,0 +1,72 @@ -+/* libnbd -+ * Copyright (C) 2021 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#ifndef LIBNBD_BENCH_H -+#define LIBNBD_BENCH_H -+ -+#include -+ -+#define MICROSECONDS 1000000 -+ -+struct bench { -+ struct timeval start, stop; -+}; -+ -+static inline void -+bench_start(struct bench *b) -+{ -+ gettimeofday (&b->start, NULL); -+} -+ -+static inline void -+bench_stop(struct bench *b) -+{ -+ gettimeofday (&b->stop, NULL); -+} -+ -+static inline double -+bench_sec(struct bench *b) -+{ -+ struct timeval dt; -+ -+ dt.tv_sec = b->stop.tv_sec - b->start.tv_sec; -+ dt.tv_usec = b->stop.tv_usec - b->start.tv_usec; -+ -+ if (dt.tv_usec < 0) { -+ dt.tv_sec -= 1; -+ dt.tv_usec += MICROSECONDS; -+ } -+ -+ return ((double)dt.tv_sec * MICROSECONDS + dt.tv_usec) / MICROSECONDS; -+} -+ -+#endif /* LIBNBD_BENCH_H */ -diff --git a/common/utils/test-vector.c b/common/utils/test-vector.c -index 94b2aeb7..28af59b8 100644 ---- a/common/utils/test-vector.c -+++ b/common/utils/test-vector.c -@@ -38,9 +38,13 @@ - #undef NDEBUG /* Keep test strong even for nbdkit built without assertions */ - #include - -+#include "bench.h" - #include "vector.h" - -+#define APPENDS 1000000 -+ - DEFINE_VECTOR_TYPE(int64_vector, int64_t); -+DEFINE_VECTOR_TYPE(uint32_vector, uint32_t); - DEFINE_VECTOR_TYPE(string_vector, char *); - - static int -@@ -113,10 +117,57 @@ test_string_vector (void) - free (v.ptr); - } - -+static void -+bench_reserve (void) -+{ -+ uint32_vector v = empty_vector; -+ struct bench b; -+ -+ bench_start(&b); -+ -+ uint32_vector_reserve(&v, APPENDS); -+ -+ for (uint32_t i = 0; i < APPENDS; i++) { -+ uint32_vector_append (&v, i); -+ } -+ -+ bench_stop(&b); -+ -+ assert (v.ptr[APPENDS - 1] == APPENDS - 1); -+ free (v.ptr); -+ -+ printf ("bench_reserve: %d appends in %.6f s\n", APPENDS, bench_sec (&b)); -+} -+ -+static void -+bench_append (void) -+{ -+ uint32_vector v = empty_vector; -+ struct bench b; -+ -+ bench_start(&b); -+ -+ for (uint32_t i = 0; i < APPENDS; i++) { -+ uint32_vector_append (&v, i); -+ } -+ -+ bench_stop(&b); -+ -+ assert (v.ptr[APPENDS - 1] == APPENDS - 1); -+ free (v.ptr); -+ -+ printf ("bench_append: %d appends in %.6f s\n", APPENDS, bench_sec (&b)); -+} -+ - int - main (int argc, char *argv[]) - { -- test_int64_vector (); -- test_string_vector (); -+ if (getenv("NBDKIT_BENCH")) { -+ bench_reserve (); -+ bench_append (); -+ } else { -+ test_int64_vector (); -+ test_string_vector (); -+ } - return 0; - } --- -2.31.1 - diff --git a/SOURCES/0019-rate-Allow-burstiness-to-be-controlled.patch b/SOURCES/0019-rate-Allow-burstiness-to-be-controlled.patch new file mode 100644 index 0000000..3453169 --- /dev/null +++ b/SOURCES/0019-rate-Allow-burstiness-to-be-controlled.patch @@ -0,0 +1,121 @@ +From c1a7c87fb9710fb29d699d1f39d0da19caf98da0 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 11 Jun 2022 12:34:02 +0100 +Subject: [PATCH] rate: Allow burstiness to be controlled + +Previously it was fixed at 2.0 seconds. Allowing it to be adjusted +upwards could help with large, lumpy requests. + +(cherry picked from commit f79e951c20510381d5cd83c203c670874a4978f4) +--- + filters/rate/nbdkit-rate-filter.pod | 12 ++++++++++-- + filters/rate/rate.c | 20 +++++++++++++------- + tests/test-rate.sh | 2 +- + 3 files changed, 24 insertions(+), 10 deletions(-) + +diff --git a/filters/rate/nbdkit-rate-filter.pod b/filters/rate/nbdkit-rate-filter.pod +index 8956e641..09ce7dbc 100644 +--- a/filters/rate/nbdkit-rate-filter.pod ++++ b/filters/rate/nbdkit-rate-filter.pod +@@ -9,6 +9,7 @@ nbdkit-rate-filter - limit bandwidth by connection or server + [connection-rate=BITSPERSEC] + [rate-file=FILENAME] + [connection-rate-file=FILENAME] ++ [burstiness=SECS] + + =head1 DESCRIPTION + +@@ -63,6 +64,13 @@ Limit total bandwidth across all connections to C. + Adjust the per-connection or total bandwidth dynamically by writing + C into C. See L below. + ++=item BSECS ++ ++Control the bucket capacity, expressed as a length of time in ++"rate-equivalent seconds" that the client is allowed to burst for ++after a period of inactivity. The default is 2.0 seconds. It's not ++recommended to set this smaller than the default. ++ + =back + + C can be specified as a simple number, or you can use a +@@ -105,8 +113,8 @@ If the size of requests made by your client is much larger than the + rate limit then you can see long, lumpy sleeps in this filter. In the + future we may modify the filter to break up large requests + automatically in order to limit the length of sleeps. Placing the +-L in front of this filter may help in the +-meantime. ++L in front of this filter, or adjusting ++C upwards may help. + + =head1 FILES + +diff --git a/filters/rate/rate.c b/filters/rate/rate.c +index 1a70d212..26082f8c 100644 +--- a/filters/rate/rate.c ++++ b/filters/rate/rate.c +@@ -68,10 +68,9 @@ static char *rate_file = NULL; + + /* Bucket capacity controls the burst rate. It is expressed as the + * length of time in "rate-equivalent seconds" that the client can +- * burst for after a period of inactivity. This could be adjustable +- * in future. ++ * burst for after a period of inactivity. + */ +-#define BUCKET_CAPACITY 2.0 ++static double bucket_capacity = 2.0 /* seconds */; + + /* Global read and write buckets. */ + static struct bucket read_bucket; +@@ -142,6 +141,13 @@ rate_config (nbdkit_next_config *next, nbdkit_backend *nxdata, + return -1; + return 0; + } ++ else if (strcmp (key, "burstiness") == 0) { ++ if (sscanf (value, "%lg", &bucket_capacity) != 1) { ++ nbdkit_error ("burstiness must be a floating point number (seconds)"); ++ return -1; ++ } ++ return 0; ++ } + else + return next (nxdata, key, value); + } +@@ -150,8 +156,8 @@ static int + rate_get_ready (int thread_model) + { + /* Initialize the global buckets. */ +- bucket_init (&read_bucket, rate, BUCKET_CAPACITY); +- bucket_init (&write_bucket, rate, BUCKET_CAPACITY); ++ bucket_init (&read_bucket, rate, bucket_capacity); ++ bucket_init (&write_bucket, rate, bucket_capacity); + + return 0; + } +@@ -178,8 +184,8 @@ rate_open (nbdkit_next_open *next, nbdkit_context *nxdata, + return NULL; + } + +- bucket_init (&h->read_bucket, connection_rate, BUCKET_CAPACITY); +- bucket_init (&h->write_bucket, connection_rate, BUCKET_CAPACITY); ++ bucket_init (&h->read_bucket, connection_rate, bucket_capacity); ++ bucket_init (&h->write_bucket, connection_rate, bucket_capacity); + pthread_mutex_init (&h->read_bucket_lock, NULL); + pthread_mutex_init (&h->write_bucket_lock, NULL); + +diff --git a/tests/test-rate.sh b/tests/test-rate.sh +index 7305c928..ff781c21 100755 +--- a/tests/test-rate.sh ++++ b/tests/test-rate.sh +@@ -56,7 +56,7 @@ nbdkit -U - \ + --filter=blocksize --filter=rate \ + pattern 25M \ + maxdata=65536 \ +- rate=10M \ ++ rate=10M burstiness=2.0 \ + --run 'nbdcopy "$uri" rate.img' + end_t=$SECONDS + +-- +2.31.1 + diff --git a/SOURCES/0020-common-urils-vector.c-Optimize-vector-append.patch b/SOURCES/0020-common-urils-vector.c-Optimize-vector-append.patch deleted file mode 100644 index bc4c8f9..0000000 --- a/SOURCES/0020-common-urils-vector.c-Optimize-vector-append.patch +++ /dev/null @@ -1,54 +0,0 @@ -From e544d86c797edec613673c7272f8d4f8b05d87f8 Mon Sep 17 00:00:00 2001 -From: Nir Soffer -Date: Fri, 5 Nov 2021 22:16:26 +0200 -Subject: [PATCH] common/urils/vector.c: Optimize vector append - -Minimize reallocs by growing the backing array by factor of 1.5. - -Testing show that now append() is fast without calling reserve() -upfront, simplifying code using vector. - - NBDKIT_BENCH=1 ./test-vector - bench_reserve: 1000000 appends in 0.004496 s - bench_append: 1000000 appends in 0.004180 s - -This can make a difference in code appending millions of items. - -Ported from libnbd commit 985dfa72ae2e41901f0af21e7205ef85428cd4bd. - -(cherry picked from commit 12356fa97a840de19bb61e0abedd6e7c7e578e5a) ---- - common/utils/vector.c | 14 ++++++++++++-- - 1 file changed, 12 insertions(+), 2 deletions(-) - -diff --git a/common/utils/vector.c b/common/utils/vector.c -index 00cd2546..7df17e1b 100644 ---- a/common/utils/vector.c -+++ b/common/utils/vector.c -@@ -41,11 +41,21 @@ int - generic_vector_reserve (struct generic_vector *v, size_t n, size_t itemsize) - { - void *newptr; -+ size_t reqalloc, newalloc; - -- newptr = realloc (v->ptr, (n + v->alloc) * itemsize); -+ reqalloc = v->alloc + n; -+ if (reqalloc < v->alloc) -+ return -1; /* overflow */ -+ -+ newalloc = (v->alloc * 3 + 1) / 2; -+ -+ if (newalloc < reqalloc) -+ newalloc = reqalloc; -+ -+ newptr = realloc (v->ptr, newalloc * itemsize); - if (newptr == NULL) - return -1; - v->ptr = newptr; -- v->alloc += n; -+ v->alloc = newalloc; - return 0; - } --- -2.31.1 - diff --git a/SOURCES/0020-luks-Check-return-values-from-malloc-more-carefully.patch b/SOURCES/0020-luks-Check-return-values-from-malloc-more-carefully.patch new file mode 100644 index 0000000..f4a8d86 --- /dev/null +++ b/SOURCES/0020-luks-Check-return-values-from-malloc-more-carefully.patch @@ -0,0 +1,104 @@ +From 4e8599886ba4802fef1683811a725e7c4bc4fe72 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 12 Jul 2022 18:00:38 +0100 +Subject: [PATCH] luks: Check return values from malloc more carefully + +Found by Coverity: + + Error: GCC_ANALYZER_WARNING (CWE-688): [#def53] + nbdkit-1.30.7/filters/luks/luks-encryption.c: scope_hint: In function 'calculate_iv' + nbdkit-1.30.7/filters/luks/luks-encryption.c:175:5: warning[-Wanalyzer-possible-null-argument]: use of possibly-NULL 'iv' where non-null expected + nbdkit-1.30.7/filters/luks/luks-encryption.c:39: included_from: Included from here. + /usr/include/string.h:43:14: note: argument 1 of 'memcpy' must be non-null + # 173| sector32 = (uint32_t) sector; /* truncate to only lower bits */ + # 174| sector32 = htole32 (sector32); + # 175|-> memcpy (iv, §or32, prefixlen); + # 176| memset (iv + prefixlen, 0, ivlen - prefixlen); + # 177| break; + + Error: GCC_ANALYZER_WARNING (CWE-688): [#def54] + nbdkit-1.30.7/filters/luks/luks-encryption.c:184:5: warning[-Wanalyzer-possible-null-argument]: use of possibly-NULL 'iv' where non-null expected + nbdkit-1.30.7/filters/luks/luks-encryption.c:39: included_from: Included from here. + /usr/include/string.h:43:14: note: argument 1 of 'memcpy' must be non-null + # 182| prefixlen = ivlen; + # 183| sector = htole64 (sector); + # 184|-> memcpy (iv, §or, prefixlen); + # 185| memset (iv + prefixlen, 0, ivlen - prefixlen); + # 186| break; + + Error: NULL_RETURNS (CWE-476): [#def55] + nbdkit-1.30.7/filters/luks/luks-encryption.c:498: returned_null: "malloc" returns "NULL" (checked 86 out of 94 times). + nbdkit-1.30.7/filters/luks/luks-encryption.c:498: var_assigned: Assigning: "temp" = "NULL" return value from "malloc". + nbdkit-1.30.7/filters/luks/luks-encryption.c:523: dereference: Dereferencing a pointer that might be "NULL" "temp" when calling "memcpy". [Note: The source code implementation of the function has been overridden by a builtin model.] + # 521| gnutls_hash_deinit (hash, temp); + # 522| + # 523|-> memcpy (&block[i*digest_bytes], temp, blen); + # 524| } + # 525| + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit 00c8bbd9e321681843140f697985505de7177f34) +--- + filters/luks/luks-encryption.c | 28 +++++++++++++++++++++++----- + 1 file changed, 23 insertions(+), 5 deletions(-) + +diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c +index 8ee0eb35..19aaf06a 100644 +--- a/filters/luks/luks-encryption.c ++++ b/filters/luks/luks-encryption.c +@@ -495,9 +495,15 @@ af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) + size_t digest_bytes = gnutls_hash_get_len (hash_alg); + size_t nr_blocks, last_block_len; + size_t i; +- CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); + int r; + gnutls_hash_hd_t hash; ++ CLEANUP_FREE uint8_t *temp; ++ ++ temp = malloc (digest_bytes); ++ if (!temp) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } + + nr_blocks = len / digest_bytes; + last_block_len = len % digest_bytes; +@@ -874,9 +880,15 @@ int + do_decrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, + uint64_t sector, uint8_t *buf, size_t nr_sectors) + { +- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); +- CLEANUP_FREE uint8_t *iv = malloc (ivlen); + int r; ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ CLEANUP_FREE uint8_t *iv; ++ ++ iv = malloc (ivlen); ++ if (!iv) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } + + while (nr_sectors) { + calculate_iv (h->ivgen_alg, iv, ivlen, sector); +@@ -902,9 +914,15 @@ int + do_encrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, + uint64_t sector, uint8_t *buf, size_t nr_sectors) + { +- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); +- CLEANUP_FREE uint8_t *iv = malloc (ivlen); + int r; ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ CLEANUP_FREE uint8_t *iv; ++ ++ iv = malloc (ivlen); ++ if (!iv) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } + + while (nr_sectors) { + calculate_iv (h->ivgen_alg, iv, ivlen, sector); +-- +2.31.1 + diff --git a/SOURCES/0021-common-utils-vector-Rename-alloc-to-cap.patch b/SOURCES/0021-common-utils-vector-Rename-alloc-to-cap.patch deleted file mode 100644 index 569e302..0000000 --- a/SOURCES/0021-common-utils-vector-Rename-alloc-to-cap.patch +++ /dev/null @@ -1,188 +0,0 @@ -From 24e2694b302f6602e0fc7808a53a766cb983dfb4 Mon Sep 17 00:00:00 2001 -From: Nir Soffer -Date: Fri, 5 Nov 2021 22:59:38 +0200 -Subject: [PATCH] common/utils/vector: Rename `alloc` to `cap` - -The `alloc` field is the maximum number of items you can append to a -vector before it need to be resized. This may confuse users with the -size of the `ptr` array which is `alloc * itemsize`. Rename to "cap", -common term for this property in many languages (e.g C++, Rust, Go). - -Tested with "make check". Tests requiring root or external libraries -(vddk) not tested. - -Ported from libnbd commit e3c7f02a2a844295564c832108d36c939c4e4ecf. - -(cherry picked from commit 75a44237c4463524dbf7951bb62b59c373c85865) ---- - common/allocators/malloc.c | 24 ++++++++++++------------ - common/utils/vector.c | 16 ++++++++-------- - common/utils/vector.h | 12 ++++++------ - plugins/vddk/reexec.c | 2 +- - 4 files changed, 27 insertions(+), 27 deletions(-) - -diff --git a/common/allocators/malloc.c b/common/allocators/malloc.c -index 59409c24..f7474465 100644 ---- a/common/allocators/malloc.c -+++ b/common/allocators/malloc.c -@@ -88,16 +88,16 @@ extend (struct m_alloc *ma, uint64_t new_size) - ACQUIRE_WRLOCK_FOR_CURRENT_SCOPE (&ma->lock); - size_t old_size, n; - -- if (ma->ba.alloc < new_size) { -- old_size = ma->ba.alloc; -- n = new_size - ma->ba.alloc; -+ if (ma->ba.cap < new_size) { -+ old_size = ma->ba.cap; -+ n = new_size - ma->ba.cap; - - #ifdef HAVE_MUNLOCK - /* Since the memory might be moved by realloc, we must unlock the - * original array. - */ - if (ma->use_mlock) -- munlock (ma->ba.ptr, ma->ba.alloc); -+ munlock (ma->ba.ptr, ma->ba.cap); - #endif - - if (bytearray_reserve (&ma->ba, n) == -1) { -@@ -110,7 +110,7 @@ extend (struct m_alloc *ma, uint64_t new_size) - - #ifdef HAVE_MLOCK - if (ma->use_mlock) { -- if (mlock (ma->ba.ptr, ma->ba.alloc) == -1) { -+ if (mlock (ma->ba.ptr, ma->ba.cap) == -1) { - nbdkit_error ("allocator=malloc: mlock: %m"); - return -1; - } -@@ -138,11 +138,11 @@ m_alloc_read (struct allocator *a, void *buf, - /* Avoid reading beyond the end of the allocated array. Return - * zeroes for that part. - */ -- if (offset >= ma->ba.alloc) -+ if (offset >= ma->ba.cap) - memset (buf, 0, count); -- else if (offset + count > ma->ba.alloc) { -- memcpy (buf, ma->ba.ptr + offset, ma->ba.alloc - offset); -- memset (buf + ma->ba.alloc - offset, 0, offset + count - ma->ba.alloc); -+ else if (offset + count > ma->ba.cap) { -+ memcpy (buf, ma->ba.ptr + offset, ma->ba.cap - offset); -+ memset (buf + ma->ba.cap - offset, 0, offset + count - ma->ba.cap); - } - else - memcpy (buf, ma->ba.ptr + offset, count); -@@ -191,9 +191,9 @@ m_alloc_zero (struct allocator *a, uint64_t count, uint64_t offset) - /* Try to avoid extending the array, since the unallocated part - * always reads as zero. - */ -- if (offset < ma->ba.alloc) { -- if (offset + count > ma->ba.alloc) -- memset (ma->ba.ptr + offset, 0, ma->ba.alloc - offset); -+ if (offset < ma->ba.cap) { -+ if (offset + count > ma->ba.cap) -+ memset (ma->ba.ptr + offset, 0, ma->ba.cap - offset); - else - memset (ma->ba.ptr + offset, 0, count); - } -diff --git a/common/utils/vector.c b/common/utils/vector.c -index 7df17e1b..a4b43ce7 100644 ---- a/common/utils/vector.c -+++ b/common/utils/vector.c -@@ -41,21 +41,21 @@ int - generic_vector_reserve (struct generic_vector *v, size_t n, size_t itemsize) - { - void *newptr; -- size_t reqalloc, newalloc; -+ size_t reqcap, newcap; - -- reqalloc = v->alloc + n; -- if (reqalloc < v->alloc) -+ reqcap = v->cap + n; -+ if (reqcap < v->cap) - return -1; /* overflow */ - -- newalloc = (v->alloc * 3 + 1) / 2; -+ newcap = (v->cap * 3 + 1) / 2; - -- if (newalloc < reqalloc) -- newalloc = reqalloc; -+ if (newcap < reqcap) -+ newcap = reqcap; - -- newptr = realloc (v->ptr, newalloc * itemsize); -+ newptr = realloc (v->ptr, newcap * itemsize); - if (newptr == NULL) - return -1; - v->ptr = newptr; -- v->alloc = newalloc; -+ v->cap = newcap; - return 0; - } -diff --git a/common/utils/vector.h b/common/utils/vector.h -index f6a0af78..782dcba6 100644 ---- a/common/utils/vector.h -+++ b/common/utils/vector.h -@@ -86,7 +86,7 @@ - struct name { \ - type *ptr; /* Pointer to array of items. */ \ - size_t size; /* Number of valid items in the array. */ \ -- size_t alloc; /* Number of items allocated. */ \ -+ size_t cap; /* Maximum number of items. */ \ - }; \ - typedef struct name name; \ - \ -@@ -106,7 +106,7 @@ - name##_insert (name *v, type elem, size_t i) \ - { \ - assert (i <= v->size); \ -- if (v->size >= v->alloc) { \ -+ if (v->size >= v->cap) { \ - if (name##_reserve (v, 1) == -1) return -1; \ - } \ - memmove (&v->ptr[i+1], &v->ptr[i], (v->size-i) * sizeof (elem)); \ -@@ -137,7 +137,7 @@ - { \ - free (v->ptr); \ - v->ptr = NULL; \ -- v->size = v->alloc = 0; \ -+ v->size = v->cap = 0; \ - } \ - \ - /* Iterate over the vector, calling f() on each element. */ \ -@@ -181,17 +181,17 @@ - if (newptr == NULL) return -1; \ - memcpy (newptr, vptr, len); \ - copy->ptr = newptr; \ -- copy->size = copy->alloc = v->size; \ -+ copy->size = copy->cap = v->size; \ - return 0; \ - } \ - \ - --#define empty_vector { .ptr = NULL, .size = 0, .alloc = 0 } -+#define empty_vector { .ptr = NULL, .size = 0, .cap = 0 } - - struct generic_vector { - void *ptr; - size_t size; -- size_t alloc; -+ size_t cap; - }; - - extern int generic_vector_reserve (struct generic_vector *v, -diff --git a/plugins/vddk/reexec.c b/plugins/vddk/reexec.c -index 46acdb62..9e87025e 100644 ---- a/plugins/vddk/reexec.c -+++ b/plugins/vddk/reexec.c -@@ -116,7 +116,7 @@ perform_reexec (const char *env, const char *prepend) - nbdkit_error ("realloc: %m"); - exit (EXIT_FAILURE); - } -- r = read (fd, buf.ptr + buf.size, buf.alloc - buf.size); -+ r = read (fd, buf.ptr + buf.size, buf.cap - buf.size); - if (r == -1) { - nbdkit_error ("read: %s: %m", cmdline_file); - exit (EXIT_FAILURE); --- -2.31.1 - diff --git a/SOURCES/0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch b/SOURCES/0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch new file mode 100644 index 0000000..9c95dce --- /dev/null +++ b/SOURCES/0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch @@ -0,0 +1,57 @@ +From 1d593a76796574845d7e32aaadd9f7d1ed4e7987 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 12 Jul 2022 18:07:25 +0100 +Subject: [PATCH] luks: Avoid potential overflow when computing key material + offset and length + +Found by Coverity: + + Error: OVERFLOW_BEFORE_WIDEN (CWE-190): [#def58] + nbdkit-1.30.7/filters/luks/luks-encryption.c:558: overflow_before_widen: Potentially overflowing expression "h->phdr.master_key_len * h->phdr.keyslot[i].stripes" with type "unsigned int" (32 bits, unsigned) is evaluated using 32-bit arithmetic, and then used in a context that expects an expression of type "uint64_t" (64 bits, unsigned). + nbdkit-1.30.7/filters/luks/luks-encryption.c:558: remediation: To avoid overflow, cast either "h->phdr.master_key_len" or "h->phdr.keyslot[i].stripes" to type "uint64_t". + # 556| uint64_t len, r; + # 557| + # 558|-> len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; + # 559| r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); + # 560| r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); + + Error: OVERFLOW_BEFORE_WIDEN (CWE-190): [#def62] + nbdkit-1.30.7/filters/luks/luks-encryption.c:616: overflow_before_widen: Potentially overflowing expression "ks->key_material_offset * 512U" with type "unsigned int" (32 bits, unsigned) is evaluated using 32-bit arithmetic, and then used in a context that expects an expression of type "uint64_t" (64 bits, unsigned). + nbdkit-1.30.7/filters/luks/luks-encryption.c:616: remediation: To avoid overflow, cast either "ks->key_material_offset" or "512U" to type "uint64_t". + # 614| + # 615| /* Read master key material from plugin. */ + # 616|-> start = ks->key_material_offset * LUKS_SECTOR_SIZE; + # 617| if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { + # 618| errno = err; + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit 808d88fbc7b58b7c95e05f41fec729cba92ef518) +--- + filters/luks/luks-encryption.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c +index 19aaf06a..06435b27 100644 +--- a/filters/luks/luks-encryption.c ++++ b/filters/luks/luks-encryption.c +@@ -561,7 +561,7 @@ key_material_length_in_sectors (struct luks_data *h, size_t i) + { + uint64_t len, r; + +- len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; ++ len = (uint64_t) h->phdr.master_key_len * h->phdr.keyslot[i].stripes; + r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); + r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); + return r; +@@ -619,7 +619,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct luks_data *h, + } + + /* Read master key material from plugin. */ +- start = ks->key_material_offset * LUKS_SECTOR_SIZE; ++ start = (uint64_t) ks->key_material_offset * LUKS_SECTOR_SIZE; + if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { + errno = err; + return -1; +-- +2.31.1 + diff --git a/SOURCES/0022-common-utils-vector-Rename-size-to-len.patch b/SOURCES/0022-common-utils-vector-Rename-size-to-len.patch deleted file mode 100644 index da3f1cd..0000000 --- a/SOURCES/0022-common-utils-vector-Rename-size-to-len.patch +++ /dev/null @@ -1,1747 +0,0 @@ -From 1674e453d5652018c989059ea19eb79bed7e12c8 Mon Sep 17 00:00:00 2001 -From: Nir Soffer -Date: Sat, 6 Nov 2021 00:03:11 +0200 -Subject: [PATCH] common/utils/vector: Rename `size` to `len` - -The field `size` may be confusing with the size of the underlying array. -Rename to `len`, a common term for this concept. - -Tested with "make check" as regular user, tests that need root or -external libraries (vddk) are not tested. - -Ported from libnbd commit cc0567e9aed7e6b40a44bf8eac0a262ac7314fec. - -(cherry picked from commit 0b0eece73f04963a66b9adc507e5cbaba608660b) ---- - common/allocators/allocator.c | 2 +- - common/allocators/malloc.c | 2 +- - common/allocators/sparse.c | 6 +- - common/allocators/zstd.c | 6 +- - common/regions/regions.h | 6 +- - common/utils/environ.c | 2 +- - common/utils/test-vector.c | 2 +- - common/utils/vector.h | 52 ++++++------ - filters/ddrescue/ddrescue.c | 2 +- - filters/exitwhen/exitwhen.c | 2 +- - filters/extentlist/extentlist.c | 14 ++-- - filters/multi-conn/multi-conn.c | 16 ++-- - plugins/cc/cc.c | 4 +- - plugins/data/data.c | 4 +- - plugins/data/format.c | 114 +++++++++++++-------------- - plugins/eval/eval.c | 2 +- - plugins/floppy/directory-lfn.c | 24 +++--- - plugins/floppy/floppy.c | 2 +- - plugins/floppy/virtual-floppy.c | 32 ++++---- - plugins/iso/iso.c | 4 +- - plugins/nbd/nbd.c | 6 +- - plugins/partitioning/partition-mbr.c | 8 +- - plugins/partitioning/partitioning.c | 18 ++--- - plugins/partitioning/virtual-disk.c | 12 +-- - plugins/partitioning/virtual-disk.h | 2 +- - plugins/split/split.c | 14 ++-- - plugins/ssh/ssh.c | 2 +- - plugins/vddk/reexec.c | 8 +- - plugins/vddk/stats.c | 4 +- - plugins/vddk/worker.c | 4 +- - server/exports.c | 6 +- - server/extents.c | 22 +++--- - server/main.c | 2 +- - server/sockets.c | 8 +- - wrapper.c | 4 +- - 35 files changed, 209 insertions(+), 209 deletions(-) - -diff --git a/common/allocators/allocator.c b/common/allocators/allocator.c -index d306a842..019c68cd 100644 ---- a/common/allocators/allocator.c -+++ b/common/allocators/allocator.c -@@ -137,7 +137,7 @@ create_allocator (const char *type, bool debug) - return NULL; - - /* See if we can find the allocator. */ -- for (i = 0; i < allocators.size; ++i) { -+ for (i = 0; i < allocators.len; ++i) { - if (strncmp (type, allocators.ptr[i]->type, type_len) == 0) { - ret = allocators.ptr[i]->create (¶ms); - break; -diff --git a/common/allocators/malloc.c b/common/allocators/malloc.c -index f7474465..eea44432 100644 ---- a/common/allocators/malloc.c -+++ b/common/allocators/malloc.c -@@ -241,7 +241,7 @@ m_alloc_create (const void *paramsv) - size_t i; - - /* Parse the optional mlock=true|false parameter. */ -- for (i = 0; i < params->size; ++i) { -+ for (i = 0; i < params->len; ++i) { - if (strcmp (params->ptr[i].key, "mlock") == 0) { - int r = nbdkit_parse_bool (params->ptr[i].value); - if (r == -1) return NULL; -diff --git a/common/allocators/sparse.c b/common/allocators/sparse.c -index ca508c35..7c6d1636 100644 ---- a/common/allocators/sparse.c -+++ b/common/allocators/sparse.c -@@ -150,7 +150,7 @@ sparse_array_free (struct allocator *a) - size_t i; - - if (sa) { -- for (i = 0; i < sa->l1_dir.size; ++i) -+ for (i = 0; i < sa->l1_dir.len; ++i) - free_l2_dir (sa->l1_dir.ptr[i].l2_dir); - free (sa->l1_dir.ptr); - pthread_mutex_destroy (&sa->lock); -@@ -184,7 +184,7 @@ insert_l1_entry (struct sparse_array *sa, const struct l1_entry *entry) - { - size_t i; - -- for (i = 0; i < sa->l1_dir.size; ++i) { -+ for (i = 0; i < sa->l1_dir.len; ++i) { - if (entry->offset < sa->l1_dir.ptr[i].offset) { - /* Insert new entry before i'th directory entry. */ - if (l1_dir_insert (&sa->l1_dir, *entry, i) == -1) { -@@ -508,7 +508,7 @@ sparse_array_create (const void *paramsv) - const allocator_parameters *params = paramsv; - struct sparse_array *sa; - -- if (params->size > 0) { -+ if (params->len > 0) { - nbdkit_error ("allocator=sparse does not take extra parameters"); - return NULL; - } -diff --git a/common/allocators/zstd.c b/common/allocators/zstd.c -index 81fe4ed0..1675d21c 100644 ---- a/common/allocators/zstd.c -+++ b/common/allocators/zstd.c -@@ -136,7 +136,7 @@ zstd_array_free (struct allocator *a) - - ZSTD_freeCCtx (za->zcctx); - ZSTD_freeDStream (za->zdstrm); -- for (i = 0; i < za->l1_dir.size; ++i) -+ for (i = 0; i < za->l1_dir.len; ++i) - free_l2_dir (za->l1_dir.ptr[i].l2_dir); - free (za->l1_dir.ptr); - pthread_mutex_destroy (&za->lock); -@@ -170,7 +170,7 @@ insert_l1_entry (struct zstd_array *za, const struct l1_entry *entry) - { - size_t i; - -- for (i = 0; i < za->l1_dir.size; ++i) { -+ for (i = 0; i < za->l1_dir.len; ++i) { - if (entry->offset < za->l1_dir.ptr[i].offset) { - /* Insert new entry before i'th directory entry. */ - if (l1_dir_insert (&za->l1_dir, *entry, i) == -1) { -@@ -600,7 +600,7 @@ zstd_array_create (const void *paramsv) - const allocator_parameters *params = paramsv; - struct zstd_array *za; - -- if (params->size > 0) { -+ if (params->len > 0) { - nbdkit_error ("allocator=zstd does not take extra parameters"); - return NULL; - } -diff --git a/common/regions/regions.h b/common/regions/regions.h -index 6dfd5d88..3d562316 100644 ---- a/common/regions/regions.h -+++ b/common/regions/regions.h -@@ -84,17 +84,17 @@ extern void free_regions (regions *regions) - static inline size_t __attribute__((__nonnull__ (1))) - nr_regions (regions *rs) - { -- return rs->size; -+ return rs->len; - } - - /* Return the virtual size of the disk. */ - static inline int64_t __attribute__((__nonnull__ (1))) - virtual_size (regions *rs) - { -- if (rs->size == 0) -+ if (rs->len == 0) - return 0; - else -- return rs->ptr[rs->size-1].end + 1; -+ return rs->ptr[rs->len-1].end + 1; - } - - /* Look up the region corresponding to the given offset. If the -diff --git a/common/utils/environ.c b/common/utils/environ.c -index e70976cb..2ad996eb 100644 ---- a/common/utils/environ.c -+++ b/common/utils/environ.c -@@ -82,7 +82,7 @@ copy_environ (char **env, ...) - - /* Search for key in the existing environment. It's O(n^2) ... */ - len = strlen (key); -- for (i = 0; i < ret.size; ++i) { -+ for (i = 0; i < ret.len; ++i) { - if (strncmp (key, ret.ptr[i], len) == 0 && ret.ptr[i][len] == '=') { - /* Replace the existing key. */ - free (ret.ptr[i]); -diff --git a/common/utils/test-vector.c b/common/utils/test-vector.c -index 28af59b8..6d89a281 100644 ---- a/common/utils/test-vector.c -+++ b/common/utils/test-vector.c -@@ -73,7 +73,7 @@ test_int64_vector (void) - assert (v.ptr[i] == i); - - int64_vector_remove (&v, 1); -- assert (v.size == 9); -+ assert (v.len == 9); - assert (v.ptr[1] == 2); - - tmp = 10; -diff --git a/common/utils/vector.h b/common/utils/vector.h -index 782dcba6..1d04f812 100644 ---- a/common/utils/vector.h -+++ b/common/utils/vector.h -@@ -59,14 +59,14 @@ - * - * string_vector names = empty_vector; - * -- * where ‘names.ptr[]’ will be an array of strings and ‘names.size’ -+ * where ‘names.ptr[]’ will be an array of strings and ‘names.len’ - * will be the number of strings. There are no get/set accessors. To - * iterate over the strings you can use the ‘.ptr’ field directly: - * -- * for (size_t i = 0; i < names.size; ++i) -+ * for (size_t i = 0; i < names.len; ++i) - * printf ("%s\n", names.ptr[i]); - * -- * Initializing with ‘empty_vector’ sets ‘.ptr = NULL’ and ‘.size = 0’. -+ * Initializing with ‘empty_vector’ sets ‘.ptr = NULL’ and ‘.len = 0’. - * - * DEFINE_VECTOR_TYPE also defines utility functions. For the full - * list see the definition below, but useful functions include: -@@ -84,15 +84,15 @@ - */ - #define DEFINE_VECTOR_TYPE(name, type) \ - struct name { \ -- type *ptr; /* Pointer to array of items. */ \ -- size_t size; /* Number of valid items in the array. */ \ -- size_t cap; /* Maximum number of items. */ \ -+ type *ptr; /* Pointer to array of items. */ \ -+ size_t len; /* Number of valid items in the array. */ \ -+ size_t cap; /* Maximum number of items. */ \ - }; \ - typedef struct name name; \ - \ - /* Reserve n elements at the end of the vector. Note space is \ -- * allocated but the vector size is not increased and the new \ -- * elements are not initialized. \ -+ * allocated and capacity is increased, but the vector length \ -+ * is not increased and the new elements are not initialized. \ - */ \ - static inline int \ - name##_reserve (name *v, size_t n) \ -@@ -101,17 +101,17 @@ - sizeof (type)); \ - } \ - \ -- /* Insert at i'th element. i=0 => beginning i=size => append */ \ -+ /* Insert at i'th element. i=0 => beginning i=len => append */ \ - static inline int \ - name##_insert (name *v, type elem, size_t i) \ - { \ -- assert (i <= v->size); \ -- if (v->size >= v->cap) { \ -+ assert (i <= v->len); \ -+ if (v->len >= v->cap) { \ - if (name##_reserve (v, 1) == -1) return -1; \ - } \ -- memmove (&v->ptr[i+1], &v->ptr[i], (v->size-i) * sizeof (elem)); \ -+ memmove (&v->ptr[i+1], &v->ptr[i], (v->len-i) * sizeof (elem)); \ - v->ptr[i] = elem; \ -- v->size++; \ -+ v->len++; \ - return 0; \ - } \ - \ -@@ -119,16 +119,16 @@ - static inline int \ - name##_append (name *v, type elem) \ - { \ -- return name##_insert (v, elem, v->size); \ -+ return name##_insert (v, elem, v->len); \ - } \ - \ -- /* Remove i'th element. i=0 => beginning i=size-1 => end */ \ -+ /* Remove i'th element. i=0 => beginning i=len-1 => end */ \ - static inline void \ - name##_remove (name *v, size_t i) \ - { \ -- assert (i < v->size); \ -- memmove (&v->ptr[i], &v->ptr[i+1], (v->size-i-1) * sizeof (type)); \ -- v->size--; \ -+ assert (i < v->len); \ -+ memmove (&v->ptr[i], &v->ptr[i+1], (v->len-i-1) * sizeof (type)); \ -+ v->len--; \ - } \ - \ - /* Remove all elements and deallocate the vector. */ \ -@@ -137,7 +137,7 @@ - { \ - free (v->ptr); \ - v->ptr = NULL; \ -- v->size = v->cap = 0; \ -+ v->len = v->cap = 0; \ - } \ - \ - /* Iterate over the vector, calling f() on each element. */ \ -@@ -145,7 +145,7 @@ - name##_iter (name *v, void (*f) (type elem)) \ - { \ - size_t i; \ -- for (i = 0; i < v->size; ++i) \ -+ for (i = 0; i < v->len; ++i) \ - f (v->ptr[i]); \ - } \ - \ -@@ -154,7 +154,7 @@ - name##_sort (name *v, \ - int (*compare) (const type *p1, const type *p2)) \ - { \ -- qsort (v->ptr, v->size, sizeof (type), (void *) compare); \ -+ qsort (v->ptr, v->len, sizeof (type), (void *) compare); \ - } \ - \ - /* Search for an exactly matching element in the vector using a \ -@@ -164,7 +164,7 @@ - name##_search (const name *v, const void *key, \ - int (*compare) (const void *key, const type *v)) \ - { \ -- return bsearch (key, v->ptr, v->size, sizeof (type), \ -+ return bsearch (key, v->ptr, v->len, sizeof (type), \ - (void *) compare); \ - } \ - \ -@@ -175,22 +175,22 @@ - /* Note it's allowed for v and copy to be the same pointer. */ \ - type *vptr = v->ptr; \ - type *newptr; \ -- size_t len = v->size * sizeof (type); \ -+ size_t len = v->len * sizeof (type); \ - \ - newptr = malloc (len); \ - if (newptr == NULL) return -1; \ - memcpy (newptr, vptr, len); \ - copy->ptr = newptr; \ -- copy->size = copy->cap = v->size; \ -+ copy->len = copy->cap = v->len; \ - return 0; \ - } \ - \ - --#define empty_vector { .ptr = NULL, .size = 0, .cap = 0 } -+#define empty_vector { .ptr = NULL, .len = 0, .cap = 0 } - - struct generic_vector { - void *ptr; -- size_t size; -+ size_t len; - size_t cap; - }; - -diff --git a/filters/ddrescue/ddrescue.c b/filters/ddrescue/ddrescue.c -index 7b1c9c1e..218c8ee5 100644 ---- a/filters/ddrescue/ddrescue.c -+++ b/filters/ddrescue/ddrescue.c -@@ -180,7 +180,7 @@ ddrescue_pread (nbdkit_next *next, - { - size_t i; - -- for (i = 0; i < map.ranges.size; i++) { -+ for (i = 0; i < map.ranges.len; i++) { - if (map.ranges.ptr[i].status != '+') - continue; - if (offset >= map.ranges.ptr[i].start && offset <= map.ranges.ptr[i].end) { -diff --git a/filters/exitwhen/exitwhen.c b/filters/exitwhen/exitwhen.c -index 543af058..83e99953 100644 ---- a/filters/exitwhen/exitwhen.c -+++ b/filters/exitwhen/exitwhen.c -@@ -143,7 +143,7 @@ check_for_event (void) - size_t i; - - if (!exiting) { -- for (i = 0; i < events.size; ++i) { -+ for (i = 0; i < events.len; ++i) { - const struct event *event = &events.ptr[i]; - - switch (event->type) { -diff --git a/filters/extentlist/extentlist.c b/filters/extentlist/extentlist.c -index 7e6f1b78..c91fbfea 100644 ---- a/filters/extentlist/extentlist.c -+++ b/filters/extentlist/extentlist.c -@@ -134,7 +134,7 @@ parse_extentlist (void) - - assert (extentlist != NULL); - assert (extents.ptr == NULL); -- assert (extents.size == 0); -+ assert (extents.len == 0); - - fp = fopen (extentlist, "r"); - if (!fp) { -@@ -200,7 +200,7 @@ parse_extentlist (void) - - /* There must not be overlaps at this point. */ - end = 0; -- for (i = 0; i < extents.size; ++i) { -+ for (i = 0; i < extents.len; ++i) { - if (extents.ptr[i].offset < end || - extents.ptr[i].offset + extents.ptr[i].length < extents.ptr[i].offset) { - nbdkit_error ("extents in the extent list are overlapping"); -@@ -210,8 +210,8 @@ parse_extentlist (void) - } - - /* If there's a gap at the beginning, insert a hole|zero extent. */ -- if (extents.size == 0 || extents.ptr[0].offset > 0) { -- end = extents.size == 0 ? UINT64_MAX : extents.ptr[0].offset; -+ if (extents.len == 0 || extents.ptr[0].offset > 0) { -+ end = extents.len == 0 ? UINT64_MAX : extents.ptr[0].offset; - if (extent_list_insert (&extents, - (struct extent){.offset = 0, .length = end, - .type = HOLE}, -@@ -224,7 +224,7 @@ parse_extentlist (void) - /* Now insert hole|zero extents after every extent where there - * is a gap between that extent and the next one. - */ -- for (i = 0; i < extents.size-1; ++i) { -+ for (i = 0; i < extents.len-1; ++i) { - end = extents.ptr[i].offset + extents.ptr[i].length; - if (end < extents.ptr[i+1].offset) - if (extent_list_insert (&extents, -@@ -238,7 +238,7 @@ parse_extentlist (void) - } - - /* If there's a gap at the end, insert a hole|zero extent. */ -- end = extents.ptr[extents.size-1].offset + extents.ptr[extents.size-1].length; -+ end = extents.ptr[extents.len-1].offset + extents.ptr[extents.len-1].length; - if (end < UINT64_MAX) { - if (extent_list_append (&extents, - (struct extent){.offset = end, -@@ -250,7 +250,7 @@ parse_extentlist (void) - } - - /* Debug the final list. */ -- for (i = 0; i < extents.size; ++i) { -+ for (i = 0; i < extents.len; ++i) { - nbdkit_debug ("extentlist: " - "extent[%zu] = %" PRIu64 "-%" PRIu64 " (length %" PRIu64 ")" - " type %" PRIu32, -diff --git a/filters/multi-conn/multi-conn.c b/filters/multi-conn/multi-conn.c -index a6a25ef9..c7421a39 100644 ---- a/filters/multi-conn/multi-conn.c -+++ b/filters/multi-conn/multi-conn.c -@@ -207,14 +207,14 @@ multi_conn_prepare (nbdkit_next *next, void *handle, int readonly) - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); - if (byname) { - g = NULL; -- for (i = 0; i < groups.size; i++) -+ for (i = 0; i < groups.len; i++) - if (strcmp (groups.ptr[i]->name, h->name) == 0) { - g = groups.ptr[i]; - break; - } - } - else -- g = groups.size ? groups.ptr[0] : NULL; -+ g = groups.len ? groups.ptr[0] : NULL; - - if (!g) { - g = calloc (1, sizeof *g); -@@ -230,7 +230,7 @@ multi_conn_prepare (nbdkit_next *next, void *handle, int readonly) - } - if (conns_vector_append (&g->conns, h) == -1) { - if (new_group) { -- group_vector_remove (&groups, groups.size - 1); -+ group_vector_remove (&groups, groups.len - 1); - free (g->name); - free (g); - } -@@ -251,14 +251,14 @@ multi_conn_finalize (nbdkit_next *next, void *handle) - assert (h->group); - - /* XXX should we add a config param to flush if the client forgot? */ -- for (i = 0; i < h->group->conns.size; i++) { -+ for (i = 0; i < h->group->conns.len; i++) { - if (h->group->conns.ptr[i] == h) { - conns_vector_remove (&h->group->conns, i); - break; - } - } -- if (h->group->conns.size == 0) { -- for (i = 0; i < groups.size; i++) -+ if (h->group->conns.len == 0) { -+ for (i = 0; i < groups.len; i++) - if (groups.ptr[i] == h->group) { - group_vector_remove (&groups, i); - free (h->group->name); -@@ -451,7 +451,7 @@ multi_conn_flush (nbdkit_next *next, - assert (h->group); - if (h->mode == EMULATE) { - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); -- for (i = 0; i < h->group->conns.size; i++) { -+ for (i = 0; i < h->group->conns.len; i++) { - h2 = h->group->conns.ptr[i]; - if (track == OFF || (h->group->dirty && - (track == FAST || h2->dirty & READ)) || -@@ -474,7 +474,7 @@ multi_conn_flush (nbdkit_next *next, - case CONN: - if (next->can_multi_conn (next) == 1) { - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); -- for (i = 0; i < h->group->conns.size; i++) -+ for (i = 0; i < h->group->conns.len; i++) - h->group->conns.ptr[i]->dirty = 0; - h->group->dirty = 0; - } -diff --git a/plugins/cc/cc.c b/plugins/cc/cc.c -index 3251f312..c8c3c86b 100644 ---- a/plugins/cc/cc.c -+++ b/plugins/cc/cc.c -@@ -292,12 +292,12 @@ cc_config_complete (void) - if (subplugin.load) - subplugin.load (); - if (subplugin.config) { -- for (i = 0; i < params.size; ++i) { -+ for (i = 0; i < params.len; ++i) { - if (subplugin.config (params.ptr[i].key, params.ptr[i].value) == -1) - return -1; - } - } -- else if (params.size > 0) { -+ else if (params.len > 0) { - /* Just print the first one in the error message. */ - nbdkit_error ("unknown parameter: %s", params.ptr[0].key); - return -1; -diff --git a/plugins/data/data.c b/plugins/data/data.c -index 03bcc8a5..960cf97d 100644 ---- a/plugins/data/data.c -+++ b/plugins/data/data.c -@@ -158,7 +158,7 @@ get_extra_param (const char *name) - { - size_t i; - -- for (i = 0; i < params.size; ++i) { -+ for (i = 0; i < params.len; ++i) { - if (strcmp (params.ptr[i].key, name) == 0) - return params.ptr[i].value; - } -@@ -176,7 +176,7 @@ data_config_complete (void) - return -1; - } - -- if (data_seen != DATA && params.size != 0) { -+ if (data_seen != DATA && params.len != 0) { - nbdkit_error ("extra parameters passed and not using data='...'"); - return -1; - } -diff --git a/plugins/data/format.c b/plugins/data/format.c -index d351f79a..986a0f6e 100644 ---- a/plugins/data/format.c -+++ b/plugins/data/format.c -@@ -83,7 +83,7 @@ substring (string s, size_t offset, size_t len) - string r = empty_vector; - - for (i = 0; i < len; ++i) { -- assert (offset+i < s.size); -+ assert (offset+i < s.len); - if (string_append (&r, s.ptr[offset+i]) == -1) { - nbdkit_error ("realloc: %m"); - exit (EXIT_FAILURE); -@@ -167,7 +167,7 @@ static expr_list expr_table; - static node_id - new_node (const expr_t e) - { -- if (expr_table.size == 0) { -+ if (expr_table.len == 0) { - static const expr_t enull = { .t = EXPR_NULL }; - if (expr_list_append (&expr_table, enull) == -1) - goto out_of_memory; -@@ -179,14 +179,14 @@ new_node (const expr_t e) - nbdkit_error ("realloc"); - exit (EXIT_FAILURE); - } -- return expr_table.size-1; -+ return expr_table.len-1; - } - - /* Get an expression by node_id. */ - static expr_t - get_node (node_id id) - { -- assert (id < expr_table.size); -+ assert (id < expr_table.len); - return expr_table.ptr[id]; - } - -@@ -196,7 +196,7 @@ free_expr_table (void) - size_t i; - expr_t e; - -- for (i = 0; i < expr_table.size; ++i) { -+ for (i = 0; i < expr_table.len; ++i) { - e = get_node (i); - switch (e.t) { - case EXPR_LIST: free (e.list.ptr); break; -@@ -344,7 +344,7 @@ debug_expr (node_id id, int level) - break; - case EXPR_LIST: - nbdkit_debug ("%s(", debug_indent (level)); -- for (i = 0; i < e.list.size; ++i) -+ for (i = 0; i < e.list.len; ++i) - debug_expr (e.list.ptr[i], level+1); - nbdkit_debug ("%s)", debug_indent (level)); - break; -@@ -370,7 +370,7 @@ debug_expr (node_id id, int level) - CLEANUP_FREE_STRING string s = empty_vector; - static const char hex[] = "0123456789abcdef"; - -- for (i = 0; i < e.string.size; ++i) { -+ for (i = 0; i < e.string.len; ++i) { - char c = e.string.ptr[i]; - if (ascii_isprint ((char) c)) - string_append (&s, e.string.ptr[i]); -@@ -445,7 +445,7 @@ read_data_format (const char *value, struct allocator *a, uint64_t *size_rtn) - uint64_t offset = 0; - int r = -1; - -- assert (expr_table.size == 0); -+ assert (expr_table.len == 0); - - /* Run the parser across the entire string, returning the top level - * expression. -@@ -600,11 +600,11 @@ parser (int level, const char *value, size_t *start, size_t len, - - case '*': /* expr*N */ - i++; -- if (list.size == 0) { -+ if (list.len == 0) { - nbdkit_error ("*N must follow an expression"); - return -1; - } -- if (! is_data_expr (get_node (list.ptr[list.size-1]))) { -+ if (! is_data_expr (get_node (list.ptr[list.len-1]))) { - nbdkit_error ("*N cannot be applied to this type of expression"); - return -1; - } -@@ -619,18 +619,18 @@ parser (int level, const char *value, size_t *start, size_t len, - nbdkit_error ("*N not numeric"); - return -1; - } -- id = list.ptr[list.size-1]; -- list.size--; -+ id = list.ptr[list.len-1]; -+ list.len--; - APPEND_EXPR (new_node (expr (EXPR_REPEAT, id, (uint64_t) i64))); - break; - - case '[': /* expr[k:m] */ - i++; -- if (list.size == 0) { -+ if (list.len == 0) { - nbdkit_error ("[N:M] must follow an expression"); - return -1; - } -- if (! is_data_expr (get_node (list.ptr[list.size-1]))) { -+ if (! is_data_expr (get_node (list.ptr[list.len-1]))) { - nbdkit_error ("[N:M] cannot be applied to this type of expression"); - return -1; - } -@@ -647,8 +647,8 @@ parser (int level, const char *value, size_t *start, size_t len, - nbdkit_error ("enclosed pattern (...)[N:M] not numeric"); - return -1; - } -- id = list.ptr[list.size-1]; -- list.size--; -+ id = list.ptr[list.len-1]; -+ list.len--; - APPEND_EXPR (new_node (expr (EXPR_SLICE, id, i64, m))); - break; - -@@ -720,11 +720,11 @@ parser (int level, const char *value, size_t *start, size_t len, - i++; - if (value[i] != '>') goto parse_error; - i++; -- if (list.size == 0) { -+ if (list.len == 0) { - nbdkit_error ("-> must follow an expression"); - return -1; - } -- if (! is_data_expr (get_node (list.ptr[list.size-1]))) { -+ if (! is_data_expr (get_node (list.ptr[list.len-1]))) { - nbdkit_error ("-> cannot be applied to this type of expression"); - return -1; - } -@@ -735,9 +735,9 @@ parser (int level, const char *value, size_t *start, size_t len, - nbdkit_error ("strndup: %m"); - return -1; - } -- id = list.ptr[list.size-1]; -+ id = list.ptr[list.len-1]; - i += flen; -- list.size--; -+ list.len--; - APPEND_EXPR (new_node (expr (EXPR_ASSIGN, name, id))); - break; - } -@@ -1023,7 +1023,7 @@ parse_word (const char *value, size_t *start, size_t len, string *rtn) - } - memcpy (copy.ptr, &value[*start], n); - copy.ptr[n] = '\0'; -- copy.size = n + 1; -+ copy.len = n + 1; - *start = i; - - /* Reserve enough space in the return buffer for the longest -@@ -1036,22 +1036,22 @@ parse_word (const char *value, size_t *start, size_t len, string *rtn) - - /* Parse the rest of {le|be}{16|32|64}: */ - if (strncmp (copy.ptr, "le16:", 5) == 0) { -- endian = little; rtn->size = 2; -+ endian = little; rtn->len = 2; - } - else if (strncmp (copy.ptr, "le32:", 5) == 0) { -- endian = little; rtn->size = 4; -+ endian = little; rtn->len = 4; - } - else if (strncmp (copy.ptr, "le64:", 5) == 0) { -- endian = little; rtn->size = 8; -+ endian = little; rtn->len = 8; - } - else if (strncmp (copy.ptr, "be16:", 5) == 0) { -- endian = big; rtn->size = 2; -+ endian = big; rtn->len = 2; - } - else if (strncmp (copy.ptr, "be32:", 5) == 0) { -- endian = big; rtn->size = 4; -+ endian = big; rtn->len = 4; - } - else if (strncmp (copy.ptr, "be64:", 5) == 0) { -- endian = big; rtn->size = 8; -+ endian = big; rtn->len = 8; - } - else { - nbdkit_error ("data parameter: expected \"le16/32/64:\" " -@@ -1060,7 +1060,7 @@ parse_word (const char *value, size_t *start, size_t len, string *rtn) - } - - /* Parse the word field into a host-order unsigned int. */ -- switch (rtn->size) { -+ switch (rtn->len) { - case 2: - if (nbdkit_parse_uint16_t ("data", ©.ptr[5], &u16) == -1) - return -1; -@@ -1081,7 +1081,7 @@ parse_word (const char *value, size_t *start, size_t len, string *rtn) - */ - switch (endian) { - case little: -- switch (rtn->size) { -+ switch (rtn->len) { - case 2: /* le16: */ - *((uint16_t *) rtn->ptr) = htole16 (u16); - break; -@@ -1096,7 +1096,7 @@ parse_word (const char *value, size_t *start, size_t len, string *rtn) - break; - - case big: -- switch (rtn->size) { -+ switch (rtn->len) { - case 2: /* be16: */ - *((uint16_t *) rtn->ptr) = htobe16 (u16); - break; -@@ -1134,7 +1134,7 @@ optimize_ast (node_id root, node_id *root_rtn) - /* For convenience this makes a new list node. */ - - /* Optimize each element of the list. */ -- for (i = 0; i < get_node (root).list.size; ++i) { -+ for (i = 0; i < get_node (root).list.len; ++i) { - id = get_node (root).list.ptr[i]; - if (optimize_ast (id, &id) == -1) - return -1; -@@ -1148,7 +1148,7 @@ optimize_ast (node_id root, node_id *root_rtn) - * because flattening the list changes the scope. - */ - if (list_safe_to_inline (get_node (id).list)) { -- for (j = 0; j < get_node (id).list.size; ++j) { -+ for (j = 0; j < get_node (id).list.len; ++j) { - if (node_ids_append (&list, get_node (id).list.ptr[j]) == -1) - goto list_append_error; - } -@@ -1165,7 +1165,7 @@ optimize_ast (node_id root, node_id *root_rtn) - } - - /* Combine adjacent pairs of elements if possible. */ -- for (i = 1; i < list.size; ++i) { -+ for (i = 1; i < list.len; ++i) { - node_id id0, id1; - - id0 = list.ptr[i-1]; -@@ -1178,7 +1178,7 @@ optimize_ast (node_id root, node_id *root_rtn) - } - - /* List of length 0 is replaced with null. */ -- if (list.size == 0) { -+ if (list.len == 0) { - free (list.ptr); - *root_rtn = new_node (expr (EXPR_NULL)); - return 0; -@@ -1187,7 +1187,7 @@ optimize_ast (node_id root, node_id *root_rtn) - /* List of length 1 is replaced with the first element, but as - * above avoid inlining if it is not a safe expression. - */ -- if (list.size == 1 && expr_safe_to_inline (get_node (list.ptr[0]))) { -+ if (list.len == 1 && expr_safe_to_inline (get_node (list.ptr[0]))) { - id = list.ptr[0]; - free (list.ptr); - *root_rtn = id; -@@ -1242,13 +1242,13 @@ optimize_ast (node_id root, node_id *root_rtn) - */ - if (get_node (id).t == EXPR_STRING && - get_node (root).r.n <= 4 && -- get_node (id).string.size <= 512) { -+ get_node (id).string.len <= 512) { - string s = empty_vector; - size_t n = get_node (root).r.n; - const string sub = get_node (id).string; - - for (i = 0; i < n; ++i) { -- for (j = 0; j < sub.size; ++j) { -+ for (j = 0; j < sub.len; ++j) { - if (string_append (&s, sub.ptr[j]) == -1) { - nbdkit_error ("realloc: %m"); - return -1; -@@ -1307,7 +1307,7 @@ optimize_ast (node_id root, node_id *root_rtn) - } - break; - case EXPR_STRING: /* substring */ -- len = get_node (id).string.size; -+ len = get_node (id).string.len; - if (m >= 0 && n <= m && m <= len) { - if (m-n == 1) - *root_rtn = new_node (expr (EXPR_BYTE, get_node (id).string.ptr[n])); -@@ -1355,23 +1355,23 @@ optimize_ast (node_id root, node_id *root_rtn) - - case EXPR_STRING: - /* A zero length string can be replaced with null. */ -- if (get_node (root).string.size == 0) { -+ if (get_node (root).string.len == 0) { - *root_rtn = new_node (expr (EXPR_NULL)); - return 0; - } - /* Strings containing the same character can be replaced by a - * fill. These can be produced by other optimizations. - */ -- if (get_node (root).string.size > 1) { -+ if (get_node (root).string.len > 1) { - const string s = get_node (root).string; - uint8_t b = s.ptr[0]; - -- for (i = 1; i < s.size; ++i) -+ for (i = 1; i < s.len; ++i) - if (s.ptr[i] != b) - break; - -- if (i == s.size) { -- *root_rtn = new_node (expr (EXPR_FILL, b, (uint64_t) s.size)); -+ if (i == s.len) { -+ *root_rtn = new_node (expr (EXPR_FILL, b, (uint64_t) s.len)); - return 0; - } - } -@@ -1442,7 +1442,7 @@ list_safe_to_inline (const node_ids list) - { - size_t i; - -- for (i = 0; i < list.size; ++i) { -+ for (i = 0; i < list.len; ++i) { - if (!expr_safe_to_inline (get_node (list.ptr[i]))) - return false; - } -@@ -1461,11 +1461,11 @@ expr_is_single_byte (const expr_t e, uint8_t *b) - if (b) *b = e.b; - return true; - case EXPR_LIST: /* A single element list if it is single byte */ -- if (e.list.size != 1) -+ if (e.list.len != 1) - return false; - return expr_is_single_byte (get_node (e.list.ptr[0]), b); - case EXPR_STRING: /* A length-1 string. */ -- if (e.string.size != 1) -+ if (e.string.len != 1) - return false; - if (b) *b = e.string.ptr[0]; - return true; -@@ -1511,10 +1511,10 @@ exprs_can_combine (expr_t e0, expr_t e1, node_id *id_rtn) - } - return true; - case EXPR_STRING: /* byte string => string */ -- len = e1.string.size; -+ len = e1.string.len; - if (string_reserve (&s, len+1) == -1) - goto out_of_memory; -- s.size = len+1; -+ s.len = len+1; - s.ptr[0] = e0.b; - memcpy (&s.ptr[1], e1.string.ptr, len); - *id_rtn = new_node (expr (EXPR_STRING, s)); -@@ -1533,20 +1533,20 @@ exprs_can_combine (expr_t e0, expr_t e1, node_id *id_rtn) - case EXPR_STRING: - switch (e1.t) { - case EXPR_BYTE: /* string byte => string */ -- len = e0.string.size; -+ len = e0.string.len; - if (string_reserve (&s, len+1) == -1) - goto out_of_memory; -- s.size = len+1; -+ s.len = len+1; - memcpy (s.ptr, e0.string.ptr, len); - s.ptr[len] = e1.b; - *id_rtn = new_node (expr (EXPR_STRING, s)); - return true; - case EXPR_STRING: /* string string => string */ -- len = e0.string.size; -- len1 = e1.string.size; -+ len = e0.string.len; -+ len1 = e1.string.len; - if (string_reserve (&s, len+len1) == -1) - goto out_of_memory; -- s.size = len+len1; -+ s.len = len+len1; - memcpy (s.ptr, e0.string.ptr, len); - memcpy (&s.ptr[len], e1.string.ptr, len1); - *id_rtn = new_node (expr (EXPR_STRING, s)); -@@ -1618,11 +1618,11 @@ evaluate (const dict_t *dict, node_id root, - list = get_node (root).list; - } - else { -- list.size = 1; -+ list.len = 1; - list.ptr = &root; - } - -- for (i = 0; i < list.size; ++i) { -+ for (i = 0; i < list.len; ++i) { - const expr_t e = get_node (list.ptr[i]); - - switch (e.t) { -@@ -1667,9 +1667,9 @@ evaluate (const dict_t *dict, node_id root, - - case EXPR_STRING: - /* Copy the string into the allocator. */ -- if (a->f->write (a, e.string.ptr, e.string.size, *offset) == -1) -+ if (a->f->write (a, e.string.ptr, e.string.len, *offset) == -1) - return -1; -- *offset += e.string.size; -+ *offset += e.string.len; - break; - - case EXPR_FILL: -diff --git a/plugins/eval/eval.c b/plugins/eval/eval.c -index fa1c23ff..b312a59c 100644 ---- a/plugins/eval/eval.c -+++ b/plugins/eval/eval.c -@@ -114,7 +114,7 @@ insert_method_script (const char *method, char *script) - size_t i; - struct method_script new_entry = { .method = method, .script = script }; - -- for (i = 0; i < method_scripts.size; ++i) { -+ for (i = 0; i < method_scripts.len; ++i) { - r = compare_script (method, &method_scripts.ptr[i]); - /* This shouldn't happen. insert_method_script() must not be - * called if the method has already been added. Call get_script() -diff --git a/plugins/floppy/directory-lfn.c b/plugins/floppy/directory-lfn.c -index a87d376a..fe47e0b6 100644 ---- a/plugins/floppy/directory-lfn.c -+++ b/plugins/floppy/directory-lfn.c -@@ -79,8 +79,8 @@ create_directory (size_t di, const char *label, - struct virtual_floppy *floppy) - { - size_t i; -- const size_t nr_subdirs = floppy->dirs.ptr[di].subdirs.size; -- const size_t nr_files = floppy->dirs.ptr[di].fileidxs.size; -+ const size_t nr_subdirs = floppy->dirs.ptr[di].subdirs.len; -+ const size_t nr_files = floppy->dirs.ptr[di].fileidxs.len; - struct lfn *lfns, *lfn; - const char *name; - uint8_t attributes; -@@ -109,14 +109,14 @@ create_directory (size_t di, const char *label, - } - for (i = 0; i < nr_subdirs; ++i) { - const size_t sdi = floppy->dirs.ptr[di].subdirs.ptr[i]; -- assert (sdi < floppy->dirs.size); -+ assert (sdi < floppy->dirs.len); - - name = floppy->dirs.ptr[sdi].name; - lfns[i].name = name; - } - for (i = 0; i < nr_files; ++i) { - const size_t fi = floppy->dirs.ptr[di].fileidxs.ptr[i]; -- assert (fi < floppy->files.size); -+ assert (fi < floppy->files.len); - - name = floppy->files.ptr[fi].name; - lfns[nr_subdirs+i].name = name; -@@ -132,7 +132,7 @@ create_directory (size_t di, const char *label, - file_size = 0; - for (i = 0; i < nr_subdirs; ++i) { - const size_t sdi = floppy->dirs.ptr[di].subdirs.ptr[i]; -- assert (sdi < floppy->dirs.size); -+ assert (sdi < floppy->dirs.len); - - lfn = &lfns[i]; - statbuf = &floppy->dirs.ptr[sdi].statbuf; -@@ -148,7 +148,7 @@ create_directory (size_t di, const char *label, - attributes = DIR_ENTRY_ARCHIVE; /* Same as set by Linux kernel. */ - for (i = 0; i < nr_files; ++i) { - const size_t fi = floppy->dirs.ptr[di].fileidxs.ptr[i]; -- assert (fi < floppy->files.size); -+ assert (fi < floppy->files.len); - - lfn = &lfns[nr_subdirs+i]; - statbuf = &floppy->files.ptr[fi].statbuf; -@@ -532,7 +532,7 @@ append_dir_table (size_t di, const struct dir_entry *entry, - { - size_t i; - -- i = floppy->dirs.ptr[di].table.size; -+ i = floppy->dirs.ptr[di].table.len; - if (dir_entries_append (&floppy->dirs.ptr[di].table, *entry) == -1) { - nbdkit_error ("realloc: %m"); - return -1; -@@ -550,8 +550,8 @@ int - update_directory_first_cluster (size_t di, struct virtual_floppy *floppy) - { - size_t i, j, pdi; -- const size_t nr_subdirs = floppy->dirs.ptr[di].subdirs.size; -- const size_t nr_files = floppy->dirs.ptr[di].fileidxs.size; -+ const size_t nr_subdirs = floppy->dirs.ptr[di].subdirs.len; -+ const size_t nr_files = floppy->dirs.ptr[di].fileidxs.len; - uint32_t first_cluster; - struct dir_entry *entry; - -@@ -561,7 +561,7 @@ update_directory_first_cluster (size_t di, struct virtual_floppy *floppy) - * table entries. - */ - i = 0; -- for (j = 0; j < floppy->dirs.ptr[di].table.size; ++j) { -+ for (j = 0; j < floppy->dirs.ptr[di].table.len; ++j) { - entry = &floppy->dirs.ptr[di].table.ptr[j]; - - /* Skip LFN entries. */ -@@ -596,12 +596,12 @@ update_directory_first_cluster (size_t di, struct virtual_floppy *floppy) - */ - if (i < nr_subdirs) { - const size_t sdi = floppy->dirs.ptr[di].subdirs.ptr[i]; -- assert (sdi < floppy->dirs.size); -+ assert (sdi < floppy->dirs.len); - first_cluster = floppy->dirs.ptr[sdi].first_cluster; - } - else if (i < nr_subdirs + nr_files) { - const size_t fi = floppy->dirs.ptr[di].fileidxs.ptr[i-nr_subdirs]; -- assert (fi < floppy->files.size); -+ assert (fi < floppy->files.len); - first_cluster = floppy->files.ptr[fi].first_cluster; - } - else -diff --git a/plugins/floppy/floppy.c b/plugins/floppy/floppy.c -index 80f350af..938f5bec 100644 ---- a/plugins/floppy/floppy.c -+++ b/plugins/floppy/floppy.c -@@ -172,7 +172,7 @@ floppy_pread (void *handle, void *buf, uint32_t count, uint64_t offset) - switch (region->type) { - case region_file: - i = region->u.i; -- assert (i < floppy.files.size); -+ assert (i < floppy.files.len); - host_path = floppy.files.ptr[i].host_path; - fd = open (host_path, O_RDONLY|O_CLOEXEC); - if (fd == -1) { -diff --git a/plugins/floppy/virtual-floppy.c b/plugins/floppy/virtual-floppy.c -index 6eae5600..b1546bd5 100644 ---- a/plugins/floppy/virtual-floppy.c -+++ b/plugins/floppy/virtual-floppy.c -@@ -97,10 +97,10 @@ create_virtual_floppy (const char *dir, const char *label, uint64_t size, - return -1; - - nbdkit_debug ("floppy: %zu directories and %zu files", -- floppy->dirs.size, floppy->files.size); -+ floppy->dirs.len, floppy->files.len); - - /* Create the on disk directory tables. */ -- for (i = 0; i < floppy->dirs.size; ++i) { -+ for (i = 0; i < floppy->dirs.len; ++i) { - if (create_directory (i, label, floppy) == -1) - return -1; - } -@@ -115,10 +115,10 @@ create_virtual_floppy (const char *dir, const char *label, uint64_t size, - */ - data_used_size = 0; - cluster = 2; -- for (i = 0; i < floppy->dirs.size; ++i) { -+ for (i = 0; i < floppy->dirs.len; ++i) { - floppy->dirs.ptr[i].first_cluster = cluster; - nr_bytes = -- ROUND_UP (floppy->dirs.ptr[i].table.size * sizeof (struct dir_entry), -+ ROUND_UP (floppy->dirs.ptr[i].table.len * sizeof (struct dir_entry), - CLUSTER_SIZE); - data_used_size += nr_bytes; - nr_clusters = nr_bytes / CLUSTER_SIZE; -@@ -127,7 +127,7 @@ create_virtual_floppy (const char *dir, const char *label, uint64_t size, - floppy->dirs.ptr[i].nr_clusters = nr_clusters; - cluster += nr_clusters; - } -- for (i = 0; i < floppy->files.size; ++i) { -+ for (i = 0; i < floppy->files.len; ++i) { - floppy->files.ptr[i].first_cluster = cluster; - nr_bytes = ROUND_UP (floppy->files.ptr[i].statbuf.st_size, CLUSTER_SIZE); - data_used_size += nr_bytes; -@@ -187,7 +187,7 @@ create_virtual_floppy (const char *dir, const char *label, uint64_t size, - * directory entries (which we didn't have available during - * create_directory above). - */ -- for (i = 0; i < floppy->dirs.size; ++i) { -+ for (i = 0; i < floppy->dirs.len; ++i) { - if (update_directory_first_cluster (i, floppy) == -1) - return -1; - } -@@ -230,13 +230,13 @@ free_virtual_floppy (struct virtual_floppy *floppy) - - free (floppy->fat); - -- for (i = 0; i < floppy->files.size; ++i) { -+ for (i = 0; i < floppy->files.len; ++i) { - free (floppy->files.ptr[i].name); - free (floppy->files.ptr[i].host_path); - } - free (floppy->files.ptr); - -- for (i = 0; i < floppy->dirs.size; ++i) { -+ for (i = 0; i < floppy->dirs.len; ++i) { - free (floppy->dirs.ptr[i].name); - free (floppy->dirs.ptr[i].subdirs.ptr); - free (floppy->dirs.ptr[i].fileidxs.ptr); -@@ -266,7 +266,7 @@ visit (const char *dir, struct virtual_floppy *floppy) - * directory will always be at dirs[0]. - */ - memset (&null_dir, 0, sizeof null_dir); -- di = floppy->dirs.size; -+ di = floppy->dirs.len; - if (dirs_append (&floppy->dirs, null_dir) == -1) { - nbdkit_error ("realloc: %m"); - goto error0; -@@ -423,7 +423,7 @@ visit_file (const char *dir, const char *name, - } - new_file.host_path = host_path; - new_file.statbuf = *statbuf; -- fi = floppy->files.size; -+ fi = floppy->files.len; - if (files_append (&floppy->files, new_file) == -1) { - nbdkit_error ("realloc: %m"); - free (host_path); -@@ -574,11 +574,11 @@ create_fat (struct virtual_floppy *floppy) - floppy->fat[0] = htole32 (0x0ffffff8); - floppy->fat[1] = htole32 (0x0fffffff); - -- for (i = 0; i < floppy->dirs.size; ++i) { -+ for (i = 0; i < floppy->dirs.len; ++i) { - write_fat_file (floppy->dirs.ptr[i].first_cluster, - floppy->dirs.ptr[i].nr_clusters, floppy); - } -- for (i = 0; i < floppy->files.size; ++i) { -+ for (i = 0; i < floppy->files.len; ++i) { - write_fat_file (floppy->files.ptr[i].first_cluster, - floppy->files.ptr[i].nr_clusters, floppy); - } -@@ -676,15 +676,15 @@ create_regions (struct virtual_floppy *floppy) - /* Now we're into the data region. We add all directory tables - * first. - */ -- for (i = 0; i < floppy->dirs.size; ++i) { -+ for (i = 0; i < floppy->dirs.len; ++i) { - /* Directories can never be completely empty because of the volume - * label (root) or "." and ".." entries (non-root). - */ -- assert (floppy->dirs.ptr[i].table.size > 0); -+ assert (floppy->dirs.ptr[i].table.len > 0); - - if (append_region_len (&floppy->regions, - i == 0 ? "root directory" : floppy->dirs.ptr[i].name, -- floppy->dirs.ptr[i].table.size * -+ floppy->dirs.ptr[i].table.len * - sizeof (struct dir_entry), - 0, CLUSTER_SIZE, - region_data, -@@ -693,7 +693,7 @@ create_regions (struct virtual_floppy *floppy) - } - - /* Add all files. */ -- for (i = 0; i < floppy->files.size; ++i) { -+ for (i = 0; i < floppy->files.len; ++i) { - /* It's possible for a file to have zero size, in which case it - * doesn't occupy a region or cluster. - */ -diff --git a/plugins/iso/iso.c b/plugins/iso/iso.c -index cb621f41..e232175f 100644 ---- a/plugins/iso/iso.c -+++ b/plugins/iso/iso.c -@@ -105,7 +105,7 @@ make_iso (void) - fprintf (fp, " -quiet"); - if (params) - fprintf (fp, " %s", params); -- for (i = 0; i < dirs.size; ++i) { -+ for (i = 0; i < dirs.len; ++i) { - fputc (' ', fp); - shell_quote (dirs.ptr[i], fp); - } -@@ -169,7 +169,7 @@ iso_config (const char *key, const char *value) - static int - iso_config_complete (void) - { -- if (dirs.size == 0) { -+ if (dirs.len == 0) { - nbdkit_error ("you must supply the dir= parameter " - "after the plugin name on the command line"); - return -1; -diff --git a/plugins/nbd/nbd.c b/plugins/nbd/nbd.c -index 488eadb2..ae595ea7 100644 ---- a/plugins/nbd/nbd.c -+++ b/plugins/nbd/nbd.c -@@ -244,7 +244,7 @@ static int - nbdplug_config_complete (void) - { - int c = !!sockname + !!hostname + !!uri + -- (command.size > 0) + (socket_fd >= 0) + !!raw_cid; -+ (command.len > 0) + (socket_fd >= 0) + !!raw_cid; - - /* Check the user passed exactly one connection parameter. */ - if (c > 1) { -@@ -303,7 +303,7 @@ nbdplug_config_complete (void) - return -1; - #endif - } -- else if (command.size > 0) { -+ else if (command.len > 0) { - /* Add NULL sentinel to the command. */ - if (string_vector_append (&command, NULL) == -1) { - nbdkit_error ("realloc: %m"); -@@ -574,7 +574,7 @@ nbdplug_connect (struct nbd_handle *nbd) - #else - return nbd_connect_vsock (nbd, cid, vport); - #endif -- else if (command.size > 0) -+ else if (command.len > 0) - return nbd_connect_systemd_socket_activation (nbd, (char **) command.ptr); - else if (socket_fd >= 0) - return nbd_connect_socket (nbd, socket_fd); -diff --git a/plugins/partitioning/partition-mbr.c b/plugins/partitioning/partition-mbr.c -index 9a1a043c..1f178dcb 100644 ---- a/plugins/partitioning/partition-mbr.c -+++ b/plugins/partitioning/partition-mbr.c -@@ -62,9 +62,9 @@ create_mbr_layout (void) - primary[0x1fe] = 0x55; - primary[0x1ff] = 0xaa; - -- if (the_files.size <= 4) { -+ if (the_files.len <= 4) { - /* Basic MBR with no extended partition. */ -- for (i = 0; i < the_files.size; ++i) { -+ for (i = 0; i < the_files.len; ++i) { - const struct region *region = find_file_region (i, &j); - - create_mbr_partition_table_entry (region, i == 0, the_files.ptr[i].mbr_id, -@@ -97,7 +97,7 @@ create_mbr_layout (void) - /* The remaining files are mapped to logical partitions living in - * the fourth extended partition. - */ -- for (i = 3; i < the_files.size; ++i) { -+ for (i = 3; i < the_files.len; ++i) { - if (i == 3) - eptr = eptr0; - else -@@ -117,7 +117,7 @@ create_mbr_layout (void) - create_mbr_partition_table_entry (®ion, false, the_files.ptr[i].mbr_id, - &ebr[i-3][0x1be]); - -- if (i < the_files.size-1) { -+ if (i < the_files.len-1) { - size_t j2 = j; - const struct region *enext = find_ebr_region (i+1, &j2); - const struct region *rnext = find_file_region (i+1, &j2); -diff --git a/plugins/partitioning/partitioning.c b/plugins/partitioning/partitioning.c -index 231b2d77..2301ba4e 100644 ---- a/plugins/partitioning/partitioning.c -+++ b/plugins/partitioning/partitioning.c -@@ -104,7 +104,7 @@ partitioning_unload (void) - { - size_t i; - -- for (i = 0; i < the_files.size; ++i) -+ for (i = 0; i < the_files.len; ++i) - close (the_files.ptr[i].fd); - free (the_files.ptr); - -@@ -116,7 +116,7 @@ partitioning_unload (void) - free (primary); - free (secondary); - if (ebr) { -- for (i = 0; i < the_files.size-3; ++i) -+ for (i = 0; i < the_files.len-3; ++i) - free (ebr[i]); - free (ebr); - } -@@ -235,19 +235,19 @@ partitioning_config_complete (void) - bool needs_gpt; - - /* Not enough / too many files? */ -- if (the_files.size == 0) { -+ if (the_files.len == 0) { - nbdkit_error ("at least one file= parameter must be supplied"); - return -1; - } - - total_size = 0; -- for (i = 0; i < the_files.size; ++i) -+ for (i = 0; i < the_files.len; ++i) - total_size += the_files.ptr[i].statbuf.st_size; - needs_gpt = total_size > MAX_MBR_DISK_SIZE; - - /* Choose default parttype if not set. */ - if (parttype == PARTTYPE_UNSET) { -- if (needs_gpt || the_files.size > 4) { -+ if (needs_gpt || the_files.len > 4) { - parttype = PARTTYPE_GPT; - nbdkit_debug ("picking partition type GPT"); - } -@@ -262,7 +262,7 @@ partitioning_config_complete (void) - "but you requested %zu partition(s) " - "and a total size of %" PRIu64 " bytes (> %" PRIu64 "). " - "Try using: partition-type=gpt", -- the_files.size, total_size, (uint64_t) MAX_MBR_DISK_SIZE); -+ the_files.len, total_size, (uint64_t) MAX_MBR_DISK_SIZE); - return -1; - } - -@@ -327,7 +327,7 @@ partitioning_pread (void *handle, void *buf, uint32_t count, uint64_t offset) - switch (region->type) { - case region_file: - i = region->u.i; -- assert (i < the_files.size); -+ assert (i < the_files.len); - r = pread (the_files.ptr[i].fd, buf, len, offset - region->start); - if (r == -1) { - nbdkit_error ("pread: %s: %m", the_files.ptr[i].filename); -@@ -376,7 +376,7 @@ partitioning_pwrite (void *handle, - switch (region->type) { - case region_file: - i = region->u.i; -- assert (i < the_files.size); -+ assert (i < the_files.len); - r = pwrite (the_files.ptr[i].fd, buf, len, offset - region->start); - if (r == -1) { - nbdkit_error ("pwrite: %s: %m", the_files.ptr[i].filename); -@@ -418,7 +418,7 @@ partitioning_flush (void *handle) - { - size_t i; - -- for (i = 0; i < the_files.size; ++i) { -+ for (i = 0; i < the_files.len; ++i) { - if (fdatasync (the_files.ptr[i].fd) == -1) { - nbdkit_error ("fdatasync: %m"); - return -1; -diff --git a/plugins/partitioning/virtual-disk.c b/plugins/partitioning/virtual-disk.c -index 389a17b6..d46ca46a 100644 ---- a/plugins/partitioning/virtual-disk.c -+++ b/plugins/partitioning/virtual-disk.c -@@ -56,7 +56,7 @@ create_virtual_disk_layout (void) - size_t i; - - assert (nr_regions (&the_regions) == 0); -- assert (the_files.size > 0); -+ assert (the_files.len > 0); - assert (primary == NULL); - assert (secondary == NULL); - -@@ -68,17 +68,17 @@ create_virtual_disk_layout (void) - return -1; - } - -- if (the_files.size > 4) { -+ if (the_files.len > 4) { - /* The first 3 primary partitions will be real partitions, the - * 4th will be an extended partition, and so we need to store - * EBRs for the_files.size-3 logical partitions. - */ -- ebr = malloc (sizeof (unsigned char *) * (the_files.size-3)); -+ ebr = malloc (sizeof (unsigned char *) * (the_files.len-3)); - if (ebr == NULL) { - nbdkit_error ("malloc: %m"); - return -1; - } -- for (i = 0; i < the_files.size-3; ++i) { -+ for (i = 0; i < the_files.len-3; ++i) { - ebr[i] = calloc (1, SECTOR_SIZE); - if (ebr[i] == NULL) { - nbdkit_error ("malloc: %m"); -@@ -117,7 +117,7 @@ create_virtual_disk_layout (void) - } - - /* The partitions. */ -- for (i = 0; i < the_files.size; ++i) { -+ for (i = 0; i < the_files.len; ++i) { - uint64_t offset; - - offset = virtual_size (&the_regions); -@@ -127,7 +127,7 @@ create_virtual_disk_layout (void) - assert (IS_ALIGNED (offset, SECTOR_SIZE)); - - /* Logical partitions are preceeded by an EBR. */ -- if (parttype == PARTTYPE_MBR && the_files.size > 4 && i >= 3) { -+ if (parttype == PARTTYPE_MBR && the_files.len > 4 && i >= 3) { - if (append_region_len (&the_regions, "EBR", - SECTOR_SIZE, 0, 0, - region_data, ebr[i-3]) == -1) -diff --git a/plugins/partitioning/virtual-disk.h b/plugins/partitioning/virtual-disk.h -index 7032dfc8..d56c2b86 100644 ---- a/plugins/partitioning/virtual-disk.h -+++ b/plugins/partitioning/virtual-disk.h -@@ -55,7 +55,7 @@ - * 32 if the number of files is <= GPT_MIN_PARTITIONS, which is the - * normal case. - */ --#define GPT_PTA_SIZE ROUND_UP (the_files.size, GPT_MIN_PARTITIONS) -+#define GPT_PTA_SIZE ROUND_UP (the_files.len, GPT_MIN_PARTITIONS) - #define GPT_PTA_LBAs (GPT_PTA_SIZE * GPT_PT_ENTRY_SIZE / SECTOR_SIZE) - - /* Maximum possible and default alignment between partitions. */ -diff --git a/plugins/split/split.c b/plugins/split/split.c -index c559a0cd..4c9790a6 100644 ---- a/plugins/split/split.c -+++ b/plugins/split/split.c -@@ -121,13 +121,13 @@ split_open (int readonly) - return NULL; - } - -- h->files = malloc (filenames.size * sizeof (struct file)); -+ h->files = malloc (filenames.len * sizeof (struct file)); - if (h->files == NULL) { - nbdkit_error ("malloc: %m"); - free (h); - return NULL; - } -- for (i = 0; i < filenames.size; ++i) -+ for (i = 0; i < filenames.len; ++i) - h->files[i].fd = -1; - - /* Open the files. */ -@@ -137,7 +137,7 @@ split_open (int readonly) - else - flags |= O_RDWR; - -- for (i = 0; i < filenames.size; ++i) { -+ for (i = 0; i < filenames.len; ++i) { - h->files[i].fd = open (filenames.ptr[i], flags); - if (h->files[i].fd == -1) { - nbdkit_error ("open: %s: %m", filenames.ptr[i]); -@@ -146,7 +146,7 @@ split_open (int readonly) - } - - offset = 0; -- for (i = 0; i < filenames.size; ++i) { -+ for (i = 0; i < filenames.len; ++i) { - h->files[i].offset = offset; - - if (fstat (h->files[i].fd, &statbuf) == -1) { -@@ -179,7 +179,7 @@ split_open (int readonly) - return h; - - err: -- for (i = 0; i < filenames.size; ++i) { -+ for (i = 0; i < filenames.len; ++i) { - if (h->files[i].fd >= 0) - close (h->files[i].fd); - } -@@ -195,7 +195,7 @@ split_close (void *handle) - struct handle *h = handle; - size_t i; - -- for (i = 0; i < filenames.size; ++i) -+ for (i = 0; i < filenames.len; ++i) - close (h->files[i].fd); - free (h->files); - free (h); -@@ -242,7 +242,7 @@ static struct file * - get_file (struct handle *h, uint64_t offset) - { - return bsearch (&offset, h->files, -- filenames.size, sizeof (struct file), -+ filenames.len, sizeof (struct file), - compare_offset); - } - -diff --git a/plugins/ssh/ssh.c b/plugins/ssh/ssh.c -index 535caf1a..80623525 100644 ---- a/plugins/ssh/ssh.c -+++ b/plugins/ssh/ssh.c -@@ -397,7 +397,7 @@ ssh_open (int readonly) - * as this file is rarely present. - */ - } -- for (i = 0; i < identities.size; ++i) { -+ for (i = 0; i < identities.len; ++i) { - r = ssh_options_set (h->session, - SSH_OPTIONS_ADD_IDENTITY, identities.ptr[i]); - if (r != SSH_OK) { -diff --git a/plugins/vddk/reexec.c b/plugins/vddk/reexec.c -index 9e87025e..4eae2221 100644 ---- a/plugins/vddk/reexec.c -+++ b/plugins/vddk/reexec.c -@@ -116,20 +116,20 @@ perform_reexec (const char *env, const char *prepend) - nbdkit_error ("realloc: %m"); - exit (EXIT_FAILURE); - } -- r = read (fd, buf.ptr + buf.size, buf.cap - buf.size); -+ r = read (fd, buf.ptr + buf.len, buf.cap - buf.len); - if (r == -1) { - nbdkit_error ("read: %s: %m", cmdline_file); - exit (EXIT_FAILURE); - } - if (r == 0) - break; -- buf.size += r; -+ buf.len += r; - } - close (fd); -- nbdkit_debug ("original command line occupies %zu bytes", buf.size); -+ nbdkit_debug ("original command line occupies %zu bytes", buf.len); - - /* Split cmdline into argv, then append one more arg. */ -- for (len = 0; len < buf.size; len += strlen (buf.ptr + len) + 1) { -+ for (len = 0; len < buf.len; len += strlen (buf.ptr + len) + 1) { - char *arg = buf.ptr + len; /* Next \0-terminated argument. */ - - /* See below for why we eat password parameter(s). */ -diff --git a/plugins/vddk/stats.c b/plugins/vddk/stats.c -index 76e0c244..bb5401b1 100644 ---- a/plugins/vddk/stats.c -+++ b/plugins/vddk/stats.c -@@ -94,12 +94,12 @@ display_stats (void) - #undef STUB - #undef OPTIONAL_STUB - -- qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); -+ qsort (stats.ptr, stats.len, sizeof stats.ptr[0], stat_compare); - - nbdkit_debug ("VDDK function stats (-D vddk.stats=1):"); - nbdkit_debug ("%-24s %15s %5s %15s", - "VixDiskLib_...", "µs", "calls", "bytes"); -- for (i = 0; i < stats.size; ++i) { -+ for (i = 0; i < stats.len; ++i) { - if (stats.ptr[i].usecs) { - if (stats.ptr[i].bytes > 0) - nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64, -diff --git a/plugins/vddk/worker.c b/plugins/vddk/worker.c -index 2a1d4f26..c6e2fd22 100644 ---- a/plugins/vddk/worker.c -+++ b/plugins/vddk/worker.c -@@ -82,7 +82,7 @@ send_command_and_wait (struct vddk_handle *h, struct command *cmd) - return -1; - - /* Signal the caller if it could be sleeping on an empty queue. */ -- if (h->commands.size == 1) -+ if (h->commands.len == 1) - pthread_cond_signal (&h->commands_cond); - - /* This will be used to signal command completion back to us. */ -@@ -497,7 +497,7 @@ vddk_worker_thread (void *handle) - /* Wait until we are sent at least one command. */ - { - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&h->commands_lock); -- while (h->commands.size == 0) -+ while (h->commands.len == 0) - pthread_cond_wait (&h->commands_cond, &h->commands_lock); - cmd = h->commands.ptr[0]; - command_queue_remove (&h->commands, 0); -diff --git a/server/exports.c b/server/exports.c -index 7ce1eda9..12c8a879 100644 ---- a/server/exports.c -+++ b/server/exports.c -@@ -90,13 +90,13 @@ nbdkit_exports_free (struct nbdkit_exports *exps) - NBDKIT_DLL_PUBLIC size_t - nbdkit_exports_count (const struct nbdkit_exports *exps) - { -- return exps->exports.size; -+ return exps->exports.len; - } - - NBDKIT_DLL_PUBLIC const struct nbdkit_export - nbdkit_get_export (const struct nbdkit_exports *exps, size_t i) - { -- assert (i < exps->exports.size); -+ assert (i < exps->exports.len); - return exps->exports.ptr[i]; - } - -@@ -106,7 +106,7 @@ nbdkit_add_export (struct nbdkit_exports *exps, - { - struct nbdkit_export e = { NULL, NULL }; - -- if (exps->exports.size == MAX_EXPORTS) { -+ if (exps->exports.len == MAX_EXPORTS) { - nbdkit_error ("nbdkit_add_export: too many exports"); - errno = EINVAL; - return -1; -diff --git a/server/extents.c b/server/extents.c -index 8da82cf1..e180e313 100644 ---- a/server/extents.c -+++ b/server/extents.c -@@ -117,13 +117,13 @@ nbdkit_extents_free (struct nbdkit_extents *exts) - NBDKIT_DLL_PUBLIC size_t - nbdkit_extents_count (const struct nbdkit_extents *exts) - { -- return exts->extents.size; -+ return exts->extents.len; - } - - NBDKIT_DLL_PUBLIC struct nbdkit_extent - nbdkit_get_extent (const struct nbdkit_extents *exts, size_t i) - { -- assert (i < exts->extents.size); -+ assert (i < exts->extents.len); - return exts->extents.ptr[i]; - } - -@@ -160,7 +160,7 @@ nbdkit_add_extent (struct nbdkit_extents *exts, - return 0; - - /* Ignore extents beyond the end of the range, or if list is full. */ -- if (offset >= exts->end || exts->extents.size >= MAX_EXTENTS) -+ if (offset >= exts->end || exts->extents.len >= MAX_EXTENTS) - return 0; - - /* Shorten extents that overlap the end of the range. */ -@@ -169,7 +169,7 @@ nbdkit_add_extent (struct nbdkit_extents *exts, - length -= overlap; - } - -- if (exts->extents.size == 0) { -+ if (exts->extents.len == 0) { - /* If there are no existing extents, and the new extent is - * entirely before start, ignore it. - */ -@@ -196,10 +196,10 @@ nbdkit_add_extent (struct nbdkit_extents *exts, - } - - /* If we get here we are going to either add or extend. */ -- if (exts->extents.size > 0 && -- exts->extents.ptr[exts->extents.size-1].type == type) { -+ if (exts->extents.len > 0 && -+ exts->extents.ptr[exts->extents.len-1].type == type) { - /* Coalesce with the last extent. */ -- exts->extents.ptr[exts->extents.size-1].length += length; -+ exts->extents.ptr[exts->extents.len-1].length += length; - return 0; - } - else { -@@ -226,13 +226,13 @@ nbdkit_extents_aligned (struct context *next_c, - /* Perform an initial query, then scan for the first unaligned extent. */ - if (next->extents (next_c, count, offset, flags, exts, err) == -1) - return -1; -- for (i = 0; i < exts->extents.size; ++i) { -+ for (i = 0; i < exts->extents.len; ++i) { - e = &exts->extents.ptr[i]; - if (!IS_ALIGNED(e->length, align)) { - /* If the unalignment is past align, just truncate and return early */ - if (e->offset + e->length > offset + align) { - e->length = ROUND_DOWN (e->length, align); -- exts->extents.size = i + !!e->length; -+ exts->extents.len = i + !!e->length; - exts->next = e->offset + e->length; - break; - } -@@ -249,7 +249,7 @@ nbdkit_extents_aligned (struct context *next_c, - */ - assert (i == 0); - while (e->length < align) { -- if (exts->extents.size > 1) { -+ if (exts->extents.len > 1) { - e->length += exts->extents.ptr[1].length; - e->type &= exts->extents.ptr[1].type; - extents_remove (&exts->extents, 1); -@@ -284,7 +284,7 @@ nbdkit_extents_aligned (struct context *next_c, - } - } - e->length = align; -- exts->extents.size = 1; -+ exts->extents.len = 1; - exts->next = e->offset + e->length; - break; - } -diff --git a/server/main.c b/server/main.c -index 5fd8308f..225258de 100644 ---- a/server/main.c -+++ b/server/main.c -@@ -940,7 +940,7 @@ start_serving (void) - r = sockets_append (&socks, s); - assert (r == 0); - } -- debug ("using socket activation, nr_socks = %zu", socks.size); -+ debug ("using socket activation, nr_socks = %zu", socks.len); - change_user (); - write_pidfile (); - top->after_fork (top); -diff --git a/server/sockets.c b/server/sockets.c -index 18b68f0a..15a26f69 100644 ---- a/server/sockets.c -+++ b/server/sockets.c -@@ -246,14 +246,14 @@ bind_tcpip_socket (sockets *socks) - - freeaddrinfo (ai); - -- if (socks->size == 0 && addr_in_use) { -+ if (socks->len == 0 && addr_in_use) { - fprintf (stderr, "%s: unable to bind to any sockets: %s\n", - program_name, strerror (EADDRINUSE)); - exit (EXIT_FAILURE); - } - - debug ("bound to IP address %s:%s (%zu socket(s))", -- ipaddr ? ipaddr : "", port, socks->size); -+ ipaddr ? ipaddr : "", port, socks->len); - } - - void -@@ -443,7 +443,7 @@ accept_connection (int listen_sock) - static void - check_sockets_and_quit_fd (const sockets *socks) - { -- const size_t nr_socks = socks->size; -+ const size_t nr_socks = socks->len; - size_t i; - int r; - -@@ -552,7 +552,7 @@ accept_incoming_connections (const sockets *socks) - } - pthread_mutex_unlock (&count_mutex); - -- for (i = 0; i < socks->size; ++i) -+ for (i = 0; i < socks->len; ++i) - closesocket (socks->ptr[i]); - free (socks->ptr); - } -diff --git a/wrapper.c b/wrapper.c -index 3bab2074..87e5a033 100644 ---- a/wrapper.c -+++ b/wrapper.c -@@ -130,9 +130,9 @@ print_command (void) - { - size_t i; - -- if (cmd.size > 0) -+ if (cmd.len > 0) - shell_quote (cmd.ptr[0], stderr); -- for (i = 1; i < cmd.size && cmd.ptr[i] != NULL; ++i) { -+ for (i = 1; i < cmd.len && cmd.ptr[i] != NULL; ++i) { - fputc (' ', stderr); - shell_quote (cmd.ptr[i], stderr); - } --- -2.31.1 - diff --git a/SOURCES/0022-luks-Avoid-memory-leak-on-error-path.patch b/SOURCES/0022-luks-Avoid-memory-leak-on-error-path.patch new file mode 100644 index 0000000..22a2708 --- /dev/null +++ b/SOURCES/0022-luks-Avoid-memory-leak-on-error-path.patch @@ -0,0 +1,36 @@ +From ee25c1be953bf385caf23f96384a9834c1f1c250 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 12 Jul 2022 18:10:30 +0100 +Subject: [PATCH] luks: Avoid memory leak on error path + +Found by Coverity: + + Error: CPPCHECK_WARNING (CWE-401): [#def65] [important] + nbdkit-1.30.7/filters/luks/luks-encryption.c:707: error[memleak]: Memory leak: h + # 705| if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { + # 706| nbdkit_error ("this disk does not contain a LUKS header"); + # 707|-> return NULL; + # 708| } + # 709| h->phdr.version = be16toh (h->phdr.version); + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit a345cff137763f105f07bb8942c1bbefd0959cff) +--- + filters/luks/luks-encryption.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c +index 06435b27..207a4e46 100644 +--- a/filters/luks/luks-encryption.c ++++ b/filters/luks/luks-encryption.c +@@ -710,6 +710,7 @@ load_header (nbdkit_next *next, const char *passphrase) + + if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { + nbdkit_error ("this disk does not contain a LUKS header"); ++ free (h); + return NULL; + } + h->phdr.version = be16toh (h->phdr.version); +-- +2.31.1 + diff --git a/SOURCES/0023-podwrapper.pl.in-Use-short-commit-date.patch b/SOURCES/0023-podwrapper.pl.in-Use-short-commit-date.patch deleted file mode 100644 index dd289ac..0000000 --- a/SOURCES/0023-podwrapper.pl.in-Use-short-commit-date.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 2df98ef35c3b023a44983583f65379793599e57f Mon Sep 17 00:00:00 2001 -From: Nir Soffer -Date: Mon, 8 Nov 2021 19:47:57 +0200 -Subject: [PATCH] podwrapper.pl.in: Use short commit date - -We can use git short commit date format $cs. Maybe it was not available -when podwrapper.pl was created. - -Signed-off-by: Nir Soffer -(cherry picked from libnbd commit 0306fdcb08e8dc5957a9e344b54200711fca1220) -(cherry picked from commit 7a1e79c6b5ca4adcef47fc0929d25d54610fc417) ---- - podwrapper.pl.in | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - -diff --git a/podwrapper.pl.in b/podwrapper.pl.in -index abad578d..63c1025a 100755 ---- a/podwrapper.pl.in -+++ b/podwrapper.pl.in -@@ -233,8 +233,7 @@ my $date; - my $filename = "$abs_top_srcdir/.git"; - if (!$date && -d $filename) { - local $ENV{GIT_DIR} = $filename; -- $_ = `git show -O/dev/null -s --format=%ci`; -- $date = $1 if /^(\d+-\d+-\d+)\s/; -+ $date = `git show -O/dev/null -s --format=%cs`; - } - if (!$date) { - my ($day, $month, $year) = (gmtime($ENV{SOURCE_DATE_EPOCH} || time))[3,4,5]; --- -2.31.1 - diff --git a/SOURCES/0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch b/SOURCES/0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch new file mode 100644 index 0000000..0bfe9b4 --- /dev/null +++ b/SOURCES/0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch @@ -0,0 +1,48 @@ +From 5ccf1068703d300c8b5579b3a6ef0e409b5a713e Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 19 Jul 2022 11:56:47 +0100 +Subject: [PATCH] tests: Hoist some EXTRA_DIST out of automake conditionals + +We can fail to add some test files (test.tcl, test.lua) to the tarball +if compiling with those languages disabled, which would cause knock-on +failures when the tarball was used with the languages enabled. We +already fixed this for Ruby etc, this commit fixes it for Tcl and Lua. + +(cherry picked from commit 3b6763c82909c95431ff57c2fe9be1b98316b057) +--- + tests/Makefile.am | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 0f4b0746..2667be32 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1203,10 +1203,11 @@ EXTRA_DIST += \ + $(NULL) + + # Tcl plugin test. ++EXTRA_DIST += test.tcl ++ + if HAVE_TCL + + LIBGUESTFS_TESTS += test-tcl +-EXTRA_DIST += test.tcl + + test_tcl_SOURCES = test-lang-plugins.c test.h + test_tcl_CFLAGS = \ +@@ -1219,10 +1220,11 @@ test_tcl_LDADD = libtest.la $(LIBGUESTFS_LIBS) + endif HAVE_TCL + + # Lua plugin test. ++EXTRA_DIST += test.lua ++ + if HAVE_LUA + + LIBGUESTFS_TESTS += test-lua +-EXTRA_DIST += test.lua + + test_lua_SOURCES = test-lang-plugins.c test.h + test_lua_CFLAGS = \ +-- +2.31.1 + diff --git a/SOURCES/0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch b/SOURCES/0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch deleted file mode 100644 index 4c16d66..0000000 --- a/SOURCES/0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch +++ /dev/null @@ -1,89 +0,0 @@ -From e9f77e9da946c963e4ec5d82dfd144305f79ebb5 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 9 Nov 2021 09:07:42 +0000 -Subject: [PATCH] ocaml: Replace "noalloc" with [@@noalloc] annotation - -This requires OCaml >= 4.03 (released April 2016). The previous -minimum version was 4.02.2. - -(cherry picked from commit d15dd73845065cc9ca04aa785e2be994f76bf832) ---- - README | 2 +- - plugins/ocaml/NBDKit.ml | 16 ++++++++-------- - plugins/ocaml/nbdkit-ocaml-plugin.pod | 5 +---- - 3 files changed, 10 insertions(+), 13 deletions(-) - -diff --git a/README b/README -index b001620c..160856b6 100644 ---- a/README -+++ b/README -@@ -155,7 +155,7 @@ For the Python plugin: - - For the OCaml plugin: - -- - OCaml >= 4.02.2 -+ - OCaml >= 4.03 - - For the Tcl plugin: - -diff --git a/plugins/ocaml/NBDKit.ml b/plugins/ocaml/NBDKit.ml -index 75823ba5..4d45cc0c 100644 ---- a/plugins/ocaml/NBDKit.ml -+++ b/plugins/ocaml/NBDKit.ml -@@ -152,11 +152,11 @@ let default_callbacks = { - export_description = None; - } - --external set_name : string -> unit = "ocaml_nbdkit_set_name" "noalloc" --external set_longname : string -> unit = "ocaml_nbdkit_set_longname" "noalloc" --external set_version : string -> unit = "ocaml_nbdkit_set_version" "noalloc" --external set_description : string -> unit = "ocaml_nbdkit_set_description" "noalloc" --external set_config_help : string -> unit = "ocaml_nbdkit_set_config_help" "noalloc" -+external set_name : string -> unit = "ocaml_nbdkit_set_name" [@@noalloc] -+external set_longname : string -> unit = "ocaml_nbdkit_set_longname" [@@noalloc] -+external set_version : string -> unit = "ocaml_nbdkit_set_version" [@@noalloc] -+external set_description : string -> unit = "ocaml_nbdkit_set_description" [@@noalloc] -+external set_config_help : string -> unit = "ocaml_nbdkit_set_config_help" [@@noalloc] - - external set_field : string -> 'a -> unit = "ocaml_nbdkit_set_field" - -@@ -220,7 +220,7 @@ let register_plugin plugin = - - (* Bindings to nbdkit server functions. *) - --external _set_error : int -> unit = "ocaml_nbdkit_set_error" "noalloc" -+external _set_error : int -> unit = "ocaml_nbdkit_set_error" [@@noalloc] - - let set_error unix_error = - (* There's an awkward triple translation going on here, because -@@ -250,9 +250,9 @@ external read_password : string -> string = "ocaml_nbdkit_read_password" - external realpath : string -> string = "ocaml_nbdkit_realpath" - external nanosleep : int -> int -> unit = "ocaml_nbdkit_nanosleep" - external export_name : unit -> string = "ocaml_nbdkit_export_name" --external shutdown : unit -> unit = "ocaml_nbdkit_shutdown" "noalloc" -+external shutdown : unit -> unit = "ocaml_nbdkit_shutdown" [@@noalloc] - --external _debug : string -> unit = "ocaml_nbdkit_debug" "noalloc" -+external _debug : string -> unit = "ocaml_nbdkit_debug" [@@noalloc] - - let debug fs = - ksprintf _debug fs -diff --git a/plugins/ocaml/nbdkit-ocaml-plugin.pod b/plugins/ocaml/nbdkit-ocaml-plugin.pod -index 2bd0af25..293f8143 100644 ---- a/plugins/ocaml/nbdkit-ocaml-plugin.pod -+++ b/plugins/ocaml/nbdkit-ocaml-plugin.pod -@@ -11,10 +11,7 @@ nbdkit-ocaml-plugin - writing nbdkit plugins in OCaml - =head1 DESCRIPTION - - This manual page describes how to write nbdkit plugins in natively --compiled OCaml code. -- --Note this requires OCaml E 4.02.2, which has support for shared --libraries. See L -+compiled OCaml code. This requires OCaml E 4.03. - - =head1 WRITING AN OCAML NBDKIT PLUGIN - --- -2.31.1 - diff --git a/SOURCES/0025-vddk-Drop-obsolete-documentation-related-to-thread-m.patch b/SOURCES/0025-vddk-Drop-obsolete-documentation-related-to-thread-m.patch deleted file mode 100644 index 1abb6bb..0000000 --- a/SOURCES/0025-vddk-Drop-obsolete-documentation-related-to-thread-m.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 5da14da22c1e26aff24baf41fb2ae0f2832acae1 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 27 Nov 2021 16:44:41 +0000 -Subject: [PATCH] vddk: Drop obsolete documentation related to thread model - -Since commit 1eecf15fc3 ("vddk: Implement parallel thread model") we -have implemented a parallel thread model in this plugin, and thread -handling is believed to be safe and in conformity with the VDDK -documentation. Remove obsolete documentation contradicting this. - -Reported-by: Ming Xie -Fixes: commit 1eecf15fc3d8ea253ccec4f5883fdbb9aa6f8c2b -(cherry picked from commit 370ecb711c23f9143c933e13468e11d688d0d651) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 8 -------- - 1 file changed, 8 deletions(-) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index ce82a734..acec0bd2 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -452,14 +452,6 @@ sector boundaries. This is because the VDDK Read and Write APIs only - take sector numbers. If your client needs finer granularity, you can - use L with the setting C. - --=head2 Threads -- --Handling threads in the VDDK API is complex and does not map well to --any of the thread models offered by nbdkit (see --L). The plugin uses the nbdkit --C model, but technically even this is not --completely safe. This is a subject of future work. -- - =head2 Out of memory errors - - In the verbose log you may see errors like: --- -2.31.1 - diff --git a/SOURCES/0026-Revert-podwrapper.pl.in-Use-short-commit-date.patch b/SOURCES/0026-Revert-podwrapper.pl.in-Use-short-commit-date.patch deleted file mode 100644 index 16add6c..0000000 --- a/SOURCES/0026-Revert-podwrapper.pl.in-Use-short-commit-date.patch +++ /dev/null @@ -1,32 +0,0 @@ -From b986f25be4f013eb02cd327826fa225c8202571e Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 20 Nov 2021 17:50:25 +0000 -Subject: [PATCH] Revert "podwrapper.pl.in: Use short commit date" - -This commit breaks man page output because there is an extra newline -after the date which wasn't being removed. - -This reverts commit 7a1e79c6b5ca4adcef47fc0929d25d54610fc417. - -(cherry picked from commit 750ad5972bb082d188f17f8f71ef1ec0c616c676) ---- - podwrapper.pl.in | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/podwrapper.pl.in b/podwrapper.pl.in -index 63c1025a..abad578d 100755 ---- a/podwrapper.pl.in -+++ b/podwrapper.pl.in -@@ -233,7 +233,8 @@ my $date; - my $filename = "$abs_top_srcdir/.git"; - if (!$date && -d $filename) { - local $ENV{GIT_DIR} = $filename; -- $date = `git show -O/dev/null -s --format=%cs`; -+ $_ = `git show -O/dev/null -s --format=%ci`; -+ $date = $1 if /^(\d+-\d+-\d+)\s/; - } - if (!$date) { - my ($day, $month, $year) = (gmtime($ENV{SOURCE_DATE_EPOCH} || time))[3,4,5]; --- -2.31.1 - diff --git a/SOURCES/0027-Fix-podwrapper.pl.in-Use-short-commit-date.patch b/SOURCES/0027-Fix-podwrapper.pl.in-Use-short-commit-date.patch deleted file mode 100644 index 1077ed4..0000000 --- a/SOURCES/0027-Fix-podwrapper.pl.in-Use-short-commit-date.patch +++ /dev/null @@ -1,31 +0,0 @@ -From 0c430f02eec2671155c001c8a1d2f964b42022e5 Mon Sep 17 00:00:00 2001 -From: Eric Blake -Date: Tue, 30 Nov 2021 12:42:01 -0600 -Subject: [PATCH] Fix "podwrapper.pl.in: Use short commit date" - -This reverts commit 750ad5972bb082d188f17f8f71ef1ec0c616c676, then -fixes the broken newline as suggested in the thread at -https://listman.redhat.com/archives/libguestfs/2021-November/msg00275.html. - -(cherry picked from commit 80036dbb0b8f9e0aab5994d80de6321c2a55c669) ---- - podwrapper.pl.in | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - -diff --git a/podwrapper.pl.in b/podwrapper.pl.in -index abad578d..6f256ba8 100755 ---- a/podwrapper.pl.in -+++ b/podwrapper.pl.in -@@ -233,8 +233,7 @@ my $date; - my $filename = "$abs_top_srcdir/.git"; - if (!$date && -d $filename) { - local $ENV{GIT_DIR} = $filename; -- $_ = `git show -O/dev/null -s --format=%ci`; -- $date = $1 if /^(\d+-\d+-\d+)\s/; -+ $date = `git show -O/dev/null -s --format=format:%cs`; - } - if (!$date) { - my ($day, $month, $year) = (gmtime($ENV{SOURCE_DATE_EPOCH} || time))[3,4,5]; --- -2.31.1 - diff --git a/SOURCES/0028-scripts-Add-simple-script-for-automating-VDDK-disk-c.patch b/SOURCES/0028-scripts-Add-simple-script-for-automating-VDDK-disk-c.patch deleted file mode 100644 index c0e1b09..0000000 --- a/SOURCES/0028-scripts-Add-simple-script-for-automating-VDDK-disk-c.patch +++ /dev/null @@ -1,154 +0,0 @@ -From e00a8f2709fdf238daa195da03d8ea2aec9b05e1 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 30 Nov 2021 17:56:02 +0000 -Subject: [PATCH] scripts: Add simple script for automating VDDK disk - connections - -It's tedious to work out how to do this by hand every time. Include a -developer script to make connecting to a guest disk easy. - -(cherry picked from commit 44ee90ee01677032a14d5b71118b7af0651db3d5) ---- - .gitignore | 1 + - Makefile.am | 2 +- - configure.ac | 2 + - scripts/vddk-open.sh.in | 89 +++++++++++++++++++++++++++++++++++++++++ - 4 files changed, 93 insertions(+), 1 deletion(-) - create mode 100755 scripts/vddk-open.sh.in - -diff --git a/.gitignore b/.gitignore -index 847b72dd..6565600f 100644 ---- a/.gitignore -+++ b/.gitignore -@@ -89,6 +89,7 @@ plugins/*/*.3 - /plugins/S3/nbdkit-S3-plugin - /plugins/tmpdisk/default-command.c - /podwrapper.pl -+/scripts/vddk-open.sh - /server/libnbdkit.a - /server/local/nbdkit.pc - /server/nbdkit -diff --git a/Makefile.am b/Makefile.am -index 49f5d91c..6df5eba0 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -46,7 +46,7 @@ EXTRA_DIST = \ - SECURITY \ - $(NULL) - --CLEANFILES += html/*.html -+CLEANFILES += html/*.html scripts/*~ - - if !ENABLE_LIBFUZZER - # NB: This is not the real nbdkit binary. It's a wrapper that allows -diff --git a/configure.ac b/configure.ac -index 1b737fc1..08c307e9 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -1249,6 +1249,8 @@ dnl Produce output files. - AC_CONFIG_HEADERS([config.h]) - AC_CONFIG_FILES([podwrapper.pl], - [chmod +x,-w podwrapper.pl]) -+AC_CONFIG_FILES([scripts/vddk-open.sh], -+ [chmod +x,-w scripts/vddk-open.sh]) - AC_CONFIG_FILES([common/protocol/generate-protostrings.sh], - [chmod +x,-w common/protocol/generate-protostrings.sh]) - AC_CONFIG_FILES([Makefile -diff --git a/scripts/vddk-open.sh.in b/scripts/vddk-open.sh.in -new file mode 100755 -index 00000000..218bc93c ---- /dev/null -+++ b/scripts/vddk-open.sh.in -@@ -0,0 +1,89 @@ -+#!/bin/bash - -+# @configure_input@ -+# Copyright (C) 2013-2021 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+# Open an nbdkit-vddk-plugin connection to the first disk of a guest -+# on a VMware ESXi server. This script automates the tedious bits of -+# getting the disk name, moref, etc. However please read the -+# nbdkit-vddk-plugin documentation as well. -+# -+# Usage: -+# scripts/vddk-open.sh SERVER GUEST -r -f -v libdir=/path/to/vmware-vix-disklib-distrib [...] -+# -+# where SERVER is the hostname or IP address of the ESXi server and -+# GUEST is the name of the guest. -+# -+# These two required parameters are followed by any extra nbdkit -+# parameters you want to use, such as VDDK libdir, flags, filters etc. -+# -+# Note that the script runs ./nbdkit (ie. the wrapper in the top build -+# directory). -+ -+nbdkit="@abs_top_builddir@/nbdkit" -+ -+server="$1" -+guest="$2" -+shift 2 -+ -+# Get the libvirt XML, filename and moref. -+echo -n "root password? " -+xml="$( virsh -c "esx://root@$server/?no_verify=1" dumpxml "$guest" )" -+echo -+ -+file="$( echo "$xml" | grep '\(.*\)<.*,\1,' )" -+ -+#echo file="$file" -+#echo moref="$moref" -+ -+# Get the thumbprint. -+thumbprint="$( openssl s_client -connect "$server:443" /dev/null | -+ openssl x509 -in /dev/stdin -fingerprint -sha1 -noout 2>/dev/null | -+ grep '^sha1 Fingerprint=' | -+ sed 's/.*Fingerprint=\([A-F0-9:]\+\)/\1/' )" -+ -+#echo thumbprint="$thumbprint" -+ -+# Construct the nbdkit command line. -+declare -a args -+ -+args[${#args[@]}]="$nbdkit" -+args[${#args[@]}]="vddk" -+args[${#args[@]}]="file=$file" -+args[${#args[@]}]="vm=moref=$moref" -+args[${#args[@]}]="server=$server" -+args[${#args[@]}]="thumbprint=$thumbprint" -+args[${#args[@]}]="user=root" -+ -+echo "${args[@]}" "$@" -+"${args[@]}" "$@" --- -2.31.1 - diff --git a/SOURCES/0029-file-Fix-implementation-of-cache-none-for-writes.patch b/SOURCES/0029-file-Fix-implementation-of-cache-none-for-writes.patch deleted file mode 100644 index be65a3f..0000000 --- a/SOURCES/0029-file-Fix-implementation-of-cache-none-for-writes.patch +++ /dev/null @@ -1,181 +0,0 @@ -From 5cb4adb94a6ff4325205fea3512c037c91579263 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 7 Dec 2021 21:08:26 +0000 -Subject: [PATCH] file: Fix implementation of cache=none for writes - -When testing virt-v2v we found that cache=none had very pessimal -performance in its current implementation when writing. See: - - https://github.com/libguestfs/virt-v2v/commit/ac59d3b2310511b1537d408b675b19ec9a5d384e - -However we know of a much better implementation - the one in nbdcopy. -This commit copies that implementation (for writes only). - -A simple test is to do: - - $ ./nbdkit file out.img cache=none --run 'nbdcopy fedora-33.img $uri' - -and then check the cache usage of the output file, which should be -around 0% (using https://github.com/Feh/nocache): - - $ cachestats out.img - pages in cache: 409/1572864 (0.0%) [filesize=6291456.0K, pagesize=4K] - -For modular virt-v2v doing a local disk to local disk conversion: - - - before this change, without cache=none - virt-v2v took 93.7 seconds, 19.1% pages cached in output file - - - before this change, enabling cache=none - virt-v2v took 125.4 seconds, 0.0% pages cached in output file - ^^^ this is the bad case which caused the investigation - - - after this change, without cache=none - virt-v2v took 93.2 seconds, 19.1% pages cached in output file - - - after this change, enabling cache=none - virt-v2v took 97.9 seconds, 0.1% pages cached in output file - -I tried to adjust NR_WINDOWS to find an optimum. Increasing it made -no difference in performance but predictably caused a slight increase -in cached pages. Reducing it slowed performance slightly. So I -conclude that 8 is about right, but it probably depends on the -hardware. - -(cherry picked from commit a956e2e75d6c88eeefecd967505667c9f176e3af) ---- - plugins/file/file.c | 79 +++++++++++++++++++++++++---- - plugins/file/nbdkit-file-plugin.pod | 3 ++ - 2 files changed, 72 insertions(+), 10 deletions(-) - -diff --git a/plugins/file/file.c b/plugins/file/file.c -index 35270a24..caf24b2c 100644 ---- a/plugins/file/file.c -+++ b/plugins/file/file.c -@@ -85,6 +85,69 @@ static int fadvise_mode = - /* cache mode */ - static enum { cache_default, cache_none } cache_mode = cache_default; - -+/* Define EVICT_WRITES if we are going to evict the page cache -+ * (cache=none) after writing. This is only known to work on Linux. -+ */ -+#ifdef __linux__ -+#define EVICT_WRITES 1 -+#endif -+ -+#ifdef EVICT_WRITES -+/* Queue writes so they will be evicted from the cache. See -+ * libnbd.git copy/file-ops.c for the rationale behind this. -+ */ -+#define NR_WINDOWS 8 -+ -+struct write_window { -+ int fd; -+ uint64_t offset; -+ size_t len; -+}; -+ -+static pthread_mutex_t window_lock = PTHREAD_MUTEX_INITIALIZER; -+static struct write_window window[NR_WINDOWS]; -+ -+static void -+evict_writes (int fd, uint64_t offset, size_t len) -+{ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&window_lock); -+ -+ /* Evict the oldest window from the page cache. */ -+ if (window[0].len > 0) { -+ sync_file_range (window[0].fd, window[0].offset, window[0].len, -+ SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE| -+ SYNC_FILE_RANGE_WAIT_AFTER); -+ posix_fadvise (window[0].fd, window[0].offset, window[0].len, -+ POSIX_FADV_DONTNEED); -+ } -+ -+ /* Move the Nth window to N-1. */ -+ memmove (&window[0], &window[1], sizeof window[0] * (NR_WINDOWS-1)); -+ -+ /* Set up the current window and tell Linux to start writing it out -+ * to disk (asynchronously). -+ */ -+ sync_file_range (fd, offset, len, SYNC_FILE_RANGE_WRITE); -+ window[NR_WINDOWS-1].fd = fd; -+ window[NR_WINDOWS-1].offset = offset; -+ window[NR_WINDOWS-1].len = len; -+} -+ -+/* When we close the handle we must remove any windows which are still -+ * associated. They missed the boat, oh well :-( -+ */ -+static void -+remove_fd_from_window (int fd) -+{ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&window_lock); -+ size_t i; -+ -+ for (i = 0; i < NR_WINDOWS; ++i) -+ if (window[i].len > 0 && window[i].fd == fd) -+ window[i].len = 0; -+} -+#endif /* EVICT_WRITES */ -+ - /* Any callbacks using lseek must be protected by this lock. */ - static pthread_mutex_t lseek_lock = PTHREAD_MUTEX_INITIALIZER; - -@@ -431,6 +494,9 @@ file_close (void *handle) - { - struct handle *h = handle; - -+#ifdef EVICT_WRITES -+ remove_fd_from_window (h->fd); -+#endif - close (h->fd); - free (h); - } -@@ -583,15 +649,9 @@ file_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, - { - struct handle *h = handle; - --#if defined (HAVE_POSIX_FADVISE) && defined (POSIX_FADV_DONTNEED) -+#if EVICT_WRITES - uint32_t orig_count = count; - uint64_t orig_offset = offset; -- -- /* If cache=none we want to force pages we have just written to the -- * file to be flushed to disk so we can immediately evict them from -- * the page cache. -- */ -- if (cache_mode == cache_none) flags |= NBDKIT_FLAG_FUA; - #endif - - while (count > 0) { -@@ -608,10 +668,9 @@ file_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, - if ((flags & NBDKIT_FLAG_FUA) && file_flush (handle, 0) == -1) - return -1; - --#if defined (HAVE_POSIX_FADVISE) && defined (POSIX_FADV_DONTNEED) -- /* On Linux this will evict the pages we just wrote from the page cache. */ -+#if EVICT_WRITES - if (cache_mode == cache_none) -- posix_fadvise (h->fd, orig_offset, orig_count, POSIX_FADV_DONTNEED); -+ evict_writes (h->fd, orig_offset, orig_count); - #endif - - return 0; -diff --git a/plugins/file/nbdkit-file-plugin.pod b/plugins/file/nbdkit-file-plugin.pod -index 0ac0ee53..f8f0e198 100644 ---- a/plugins/file/nbdkit-file-plugin.pod -+++ b/plugins/file/nbdkit-file-plugin.pod -@@ -117,6 +117,9 @@ cache: - - nbdkit file disk.img fadvise=sequential cache=none - -+Only use fadvise=sequential if reading, and the reads are mainly -+sequential. -+ - =head2 Files on tmpfs - - If you want to expose a file that resides on a file system known to --- -2.31.1 - diff --git a/SOURCES/0030-tests-Add-configure-disable-libguestfs-tests-flag.patch b/SOURCES/0030-tests-Add-configure-disable-libguestfs-tests-flag.patch deleted file mode 100644 index 207f46f..0000000 --- a/SOURCES/0030-tests-Add-configure-disable-libguestfs-tests-flag.patch +++ /dev/null @@ -1,95 +0,0 @@ -From 92773e6852719354a136d31519948436f9adf7e9 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 18 Dec 2021 20:31:10 +0000 -Subject: [PATCH] tests: Add configure --disable-libguestfs-tests flag - -This can be used to disable tests which need libguestfs*. We were -already doing that in a hackish way in the Fedora build on some -architectures. This makes it more supportable. - -Note that you can use - - ./configure --enable-libguestfs --disable-libguestfs-tests - -to enable the bindings but disable the tests. - -The difference between without and with the new flag on an otherwise -fully configured Fedora machine: - - # TOTAL: 286 - # PASS: 273 - # SKIP: 13 - - # TOTAL: 263 - # PASS: 251 - # SKIP: 12 - -* except for those which directly test for requirements using -expressions like: - - requires guestfish --version - -(cherry picked from commit c09ae98ff3b4b786565de4aa173274531a753d30) ---- - configure.ac | 17 ++++++++++++++++- - tests/Makefile.am | 2 ++ - 2 files changed, 18 insertions(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 08c307e9..96d738d9 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -1146,7 +1146,8 @@ AS_IF([test "$with_libzstd" != "no"],[ - ]) - AM_CONDITIONAL([HAVE_LIBZSTD],[test "x$LIBZSTD_LIBS" != "x"]) - --dnl Check for libguestfs (only for the guestfs plugin and the test suite). -+dnl Check for libguestfs (only for the guestfs plugin and parts of -+dnl the test suite). - AC_ARG_WITH([libguestfs], - [AS_HELP_STRING([--without-libguestfs], - [disable guestfs plugin and tests @<:@default=check@:>@])], -@@ -1173,6 +1174,17 @@ AS_IF([test "$with_libguestfs" != "no"],[ - ]) - AM_CONDITIONAL([HAVE_LIBGUESTFS],[test "x$LIBGUESTFS_LIBS" != "x"]) - -+dnl Disable tests which need libguestfs. -+AC_ARG_ENABLE([libguestfs-tests], -+ [AS_HELP_STRING([--disable-libguestfs-tests], -+ [disable tests which need libguestfs])], -+ [], -+ [enable_libguestfs_tests=check] -+) -+AM_CONDITIONAL([USE_LIBGUESTFS_FOR_TESTS], -+ [test "x$LIBGUESTFS_LIBS" != "x" && \ -+ test "x$enable_libguestfs_tests" != "xno"]) -+ - dnl Check for ext2fs and com_err, for the ext2 filter. - AC_ARG_WITH([ext2], - [AS_HELP_STRING([--without-ext2], -@@ -1447,6 +1459,9 @@ echo "Other optional features:" - echo - feature "allocator=zstd ......................... " \ - test "x$HAVE_LIBZSTD_TRUE" = "x" -+feature "tests using libguestfs ................. " \ -+ test "x$HAVE_LIBGUESTFS_TRUE" = "x" -a \ -+ "x$USE_LIBGUESTFS_FOR_TESTS_TRUE" = "x" - - echo - echo "If any optional component is configured ‘no’ when you expected ‘yes’" -diff --git a/tests/Makefile.am b/tests/Makefile.am -index 2b7ae9f3..43b60943 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1888,6 +1888,8 @@ TESTS += $(LIBNBD_TESTS) - endif HAVE_LIBNBD - - if HAVE_LIBGUESTFS -+if USE_LIBGUESTFS_FOR_TESTS - check_PROGRAMS += $(LIBGUESTFS_TESTS) - TESTS += $(LIBGUESTFS_TESTS) -+endif USE_LIBGUESTFS_FOR_TESTS - endif HAVE_LIBGUESTFS --- -2.31.1 - diff --git a/SOURCES/0031-vddk-Implement-VMDK-creation.patch b/SOURCES/0031-vddk-Implement-VMDK-creation.patch deleted file mode 100644 index ffbaa30..0000000 --- a/SOURCES/0031-vddk-Implement-VMDK-creation.patch +++ /dev/null @@ -1,538 +0,0 @@ -From cf58241f19ed179e48c53f4d6c71df47dd2f5931 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 18 Jan 2022 08:58:15 +0000 -Subject: [PATCH] vddk: Implement VMDK creation - -Add the create=(true|false) parameter. Setting this to true causes -the VMDK local file to be created. Currently this is done on first -connection, but we might change that in future. Various other -parameters can be used to control aspects of the VMDK file. - -(cherry picked from commit a39d5773afc3ebab7e5768118a2bccb89a654585) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 102 ++++++++++++++++++++++- - plugins/vddk/vddk-structs.h | 32 +++++++ - plugins/vddk/vddk-stubs.h | 7 ++ - plugins/vddk/vddk.c | 125 ++++++++++++++++++++++++++++ - plugins/vddk/vddk.h | 5 ++ - tests/Makefile.am | 6 +- - tests/dummy-vddk.c | 10 +++ - tests/test-vddk-real-create.sh | 70 ++++++++++++++++ - 8 files changed, 354 insertions(+), 3 deletions(-) - create mode 100755 tests/test-vddk-real-create.sh - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index acec0bd2..b96192d0 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -6,7 +6,11 @@ nbdkit-vddk-plugin - nbdkit VMware VDDK plugin - - nbdkit vddk [file=]FILENAME - [compression=none|zlib|fastlz|skipz] -- [config=FILENAME] [cookie=COOKIE] [libdir=LIBRARY] -+ [config=FILENAME] [cookie=COOKIE] -+ [create=true] [create-adapter-type=ide|scsi-buslogic|...] -+ [create-hwversion=workstation4|workstation5|...] -+ [create-size=...] [create-type=monolithic-sparse|...] -+ [libdir=LIBRARY] - [nfchostport=PORT] [single-link=true] - [password=PASSWORD | password=- | password=+FILENAME | - password=-FD] -@@ -26,7 +30,7 @@ yourself (see L below). - - =head1 EXAMPLES - --=head2 Open a local VMDK file -+=head2 Open an existing local VMDK file - - nbdkit vddk /absolute/path/to/file.vmdk - -@@ -38,6 +42,18 @@ I<-r> option): - - nbdkit -r vddk /absolute/path/to/file.vmdk - -+=head2 Create a new local VMDK file -+ -+You can use VDDK to create a VMDK file and fill it with the contents -+of a disk image. Note the C parameter is the virtual -+size of the final VMDK disk image and must be at least as large as the -+input disk: -+ -+ nbdkit -U - vddk \ -+ /absolute/path/to/output.vmdk \ -+ create=1 create-size=100M \ -+ --run 'qemu-img convert input.qcow2 $uri' -+ - =head2 Open a file on a remote VMware ESXi hypervisor - - Connect directly to a VMware ESXi hypervisor and export a particular -@@ -136,6 +152,88 @@ C which can improve performance. The - cookie can be found by connecting to a VCenter Server over HTTPS and - retrieving the C cookie. - -+=item B -+ -+(nbdkit E 1.30) -+ -+Create a new, local VMDK file. Instead of opening an existing VMDK -+file, a new VMDK file is created and opened. The filename is given by -+the C parameter (see below). The file must not exist already. -+It is not possible to create a remote file using nbdkit. -+ -+If this is used, the C parameter is required to specify -+the virtual size of the disk. Other C parameters (see -+below) can be used to control the VMDK sub-format. -+ -+=item B -+ -+=item B -+ -+=item B -+ -+(nbdkit E 1.30) -+ -+Specify the VMDK disk adapter type. The default is C. -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+(nbdkit E 1.30) -+ -+Specify the VMDK virtual hardware version. The default is -+C. -+ -+=item BSIZE -+ -+(nbdkit E 1.30) -+ -+Specify the virtual size of the created disk. The C can use -+modifiers like C<100M> etc. It must be a multiple of 512 bytes -+because VMware only supports sector sizes. -+ -+If you use C then this parameter is required. -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+=item B -+ -+(nbdkit E 1.30) -+ -+Specify the VMDK sub-format. The default is C. -+ -+Some VMDK sub-formats use multiple files, where the C parameter -+specifies the "Disk Descriptor File" and the disk contents are stored -+in adjacent files. -+ - =item [B]FILENAME - - =item [B]B<[>datastoreB<] >vmname/vmnameB<.vmdk> -diff --git a/plugins/vddk/vddk-structs.h b/plugins/vddk/vddk-structs.h -index e97f017c..799c4aec 100644 ---- a/plugins/vddk/vddk-structs.h -+++ b/plugins/vddk/vddk-structs.h -@@ -43,6 +43,7 @@ - - typedef uint64_t VixError; - #define VIX_OK 0 -+#define VIX_E_NOT_SUPPORTED 6 - #define VIX_ASYNC 25000 - - #define VIXDISKLIB_FLAG_OPEN_UNBUFFERED 1 -@@ -54,6 +55,28 @@ typedef uint64_t VixError; - - #define VIXDISKLIB_SECTOR_SIZE 512 - -+enum VixDiskLibDiskType { -+ VIXDISKLIB_DISK_MONOLITHIC_SPARSE = 1, -+ VIXDISKLIB_DISK_MONOLITHIC_FLAT = 2, -+ VIXDISKLIB_DISK_SPLIT_SPARSE = 3, -+ VIXDISKLIB_DISK_SPLIT_FLAT = 4, -+ VIXDISKLIB_DISK_VMFS_FLAT = 5, -+ VIXDISKLIB_DISK_STREAM_OPTIMIZED = 6, -+ VIXDISKLIB_DISK_VMFS_THIN = 7, -+ VIXDISKLIB_DISK_VMFS_SPARSE = 8 -+}; -+ -+#define VIXDISKLIB_HWVERSION_WORKSTATION_4 3 -+#define VIXDISKLIB_HWVERSION_WORKSTATION_5 4 -+#define VIXDISKLIB_HWVERSION_WORKSTATION_6 6 -+#define VIXDISKLIB_HWVERSION_ESX30 4 -+#define VIXDISKLIB_HWVERSION_ESX4X 7 -+#define VIXDISKLIB_HWVERSION_ESX50 8 -+#define VIXDISKLIB_HWVERSION_ESX51 9 -+#define VIXDISKLIB_HWVERSION_ESX55 10 -+#define VIXDISKLIB_HWVERSION_ESX60 11 -+#define VIXDISKLIB_HWVERSION_ESX65 13 -+ - #define VIXDISKLIB_MIN_CHUNK_SIZE 128 - #define VIXDISKLIB_MAX_CHUNK_NUMBER (512*1024) - -@@ -148,4 +171,13 @@ typedef struct { - VixDiskLibBlock blocks[1]; - } VixDiskLibBlockList; - -+typedef struct { -+ enum VixDiskLibDiskType diskType; -+ enum VixDiskLibAdapterType adapterType; -+ uint16_t hwVersion; -+ uint64_t capacity; -+ uint32_t logicalSectorSize; -+ uint32_t physicalSectorSize; -+} VixDiskLibCreateParams; -+ - #endif /* NBDKIT_VDDK_STRUCTS_H */ -diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h -index 7d8644c3..d5affa10 100644 ---- a/plugins/vddk/vddk-stubs.h -+++ b/plugins/vddk/vddk-stubs.h -@@ -99,6 +99,13 @@ STUB (VixDiskLib_Write, - (VixDiskLibHandle handle, - uint64_t start_sector, uint64_t nr_sectors, - const unsigned char *buf)); -+STUB (VixDiskLib_Create, -+ VixError, -+ (const VixDiskLibConnection connection, -+ const char *path, -+ const VixDiskLibCreateParams *create_params, -+ void *progress_function_unused, -+ void *progress_data_unused)); - - /* Added in VDDK 6.0. */ - STUB (VixDiskLib_Flush, -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 31e5e23b..5ebf9a2c 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -81,6 +81,14 @@ bool is_remote; /* true if remote connection */ - enum compression_type compression; /* compression */ - char *config; /* config */ - const char *cookie; /* cookie */ -+bool create; /* create */ -+enum VixDiskLibAdapterType create_adapter_type = -+ VIXDISKLIB_ADAPTER_SCSI_BUSLOGIC; /* create-adapter-type */ -+uint16_t create_hwversion = -+ VIXDISKLIB_HWVERSION_WORKSTATION_5; /* create-hwversion */ -+uint64_t create_size; /* create-size */ -+enum VixDiskLibDiskType create_type = -+ VIXDISKLIB_DISK_MONOLITHIC_SPARSE; /* create-type */ - const char *filename; /* file */ - char *libdir; /* libdir */ - uint16_t nfc_host_port; /* nfchostport */ -@@ -119,6 +127,7 @@ static int - vddk_config (const char *key, const char *value) - { - int r; -+ int64_t r64; - - if (strcmp (key, "compression") == 0) { - if (strcmp (value, "zlib") == 0) -@@ -144,6 +153,82 @@ vddk_config (const char *key, const char *value) - else if (strcmp (key, "cookie") == 0) { - cookie = value; - } -+ else if (strcmp (key, "create") == 0) { -+ r = nbdkit_parse_bool (value); -+ if (r == -1) -+ return -1; -+ create = r; -+ } -+ else if (strcmp (key, "create-adapter-type") == 0) { -+ if (strcmp (value, "ide") == 0) -+ create_adapter_type = VIXDISKLIB_ADAPTER_IDE; -+ else if (strcmp (value, "scsi-buslogic") == 0) -+ create_adapter_type = VIXDISKLIB_ADAPTER_SCSI_BUSLOGIC; -+ else if (strcmp (value, "scsi-lsilogic") == 0) -+ create_adapter_type = VIXDISKLIB_ADAPTER_SCSI_LSILOGIC; -+ else { -+ nbdkit_error ("unknown create-adapter-type: %s", value); -+ return -1; -+ } -+ } -+ else if (strcmp (key, "create-hwversion") == 0) { -+ if (strcmp (value, "workstation4") == 0) -+ create_hwversion = VIXDISKLIB_HWVERSION_WORKSTATION_4; -+ else if (strcmp (value, "workstation5") == 0) -+ create_hwversion = VIXDISKLIB_HWVERSION_WORKSTATION_5; -+ else if (strcmp (value, "workstation6") == 0) -+ create_hwversion = VIXDISKLIB_HWVERSION_WORKSTATION_6; -+ else if (strcmp (value, "esx30") == 0) -+ create_hwversion = VIXDISKLIB_HWVERSION_ESX30; -+ else if (strcmp (value, "esx4x") == 0) -+ create_hwversion = VIXDISKLIB_HWVERSION_ESX4X; -+ else if (strcmp (value, "esx50") == 0) -+ create_hwversion = VIXDISKLIB_HWVERSION_ESX50; -+ else if (strcmp (value, "esx51") == 0) -+ create_hwversion = VIXDISKLIB_HWVERSION_ESX51; -+ else if (strcmp (value, "esx55") == 0) -+ create_hwversion = VIXDISKLIB_HWVERSION_ESX55; -+ else if (strcmp (value, "esx60") == 0) -+ create_hwversion = VIXDISKLIB_HWVERSION_ESX60; -+ else if (strcmp (value, "esx65") == 0) -+ create_hwversion = VIXDISKLIB_HWVERSION_ESX65; -+ else { -+ nbdkit_error ("unknown create-hwversion: %s", value); -+ return -1; -+ } -+ } -+ else if (strcmp (key, "create-size") == 0) { -+ r64 = nbdkit_parse_size (value); -+ if (r64 == -1) -+ return -1; -+ if (r64 <= 0 || (r64 & 511) != 0) { -+ nbdkit_error ("create-size must be greater than zero and a multiple of 512"); -+ return -1; -+ } -+ create_size = r64; -+ } -+ else if (strcmp (key, "create-type") == 0) { -+ if (strcmp (value, "monolithic-sparse") == 0) -+ create_type = VIXDISKLIB_DISK_MONOLITHIC_SPARSE; -+ else if (strcmp (value, "monolithic-flat") == 0) -+ create_type = VIXDISKLIB_DISK_MONOLITHIC_FLAT; -+ else if (strcmp (value, "split-sparse") == 0) -+ create_type = VIXDISKLIB_DISK_SPLIT_SPARSE; -+ else if (strcmp (value, "split-flat") == 0) -+ create_type = VIXDISKLIB_DISK_SPLIT_FLAT; -+ else if (strcmp (value, "vmfs-flat") == 0) -+ create_type = VIXDISKLIB_DISK_VMFS_FLAT; -+ else if (strcmp (value, "stream-optimized") == 0) -+ create_type = VIXDISKLIB_DISK_STREAM_OPTIMIZED; -+ else if (strcmp (value, "vmfs-thin") == 0) -+ create_type = VIXDISKLIB_DISK_VMFS_THIN; -+ else if (strcmp (value, "vmfs-sparse") == 0) -+ create_type = VIXDISKLIB_DISK_VMFS_SPARSE; -+ else { -+ nbdkit_error ("unknown create-type: %s", value); -+ return -1; -+ } -+ } - else if (strcmp (key, "file") == 0) { - /* NB: Don't convert this to an absolute path, because in the - * remote case this can be a path located on the VMware server. -@@ -266,6 +351,18 @@ vddk_config_complete (void) - #undef missing - } - -+ if (create) { -+ if (is_remote) { -+ nbdkit_error ("create=true can only be used to create local VMDK files"); -+ return -1; -+ } -+ -+ if (create_size == 0) { -+ nbdkit_error ("if using create=true you must specify the size using the create-size parameter"); -+ return -1; -+ } -+ } -+ - /* Restore original LD_LIBRARY_PATH after reexec. */ - if (restore_ld_library_path () == -1) - return -1; -@@ -618,6 +715,34 @@ vddk_open (int readonly) - goto err1; - } - -+ /* Creating a disk? The first time the connection is opened we will -+ * create it here (we need h->connection). Then set create=false so -+ * we don't create it again. This is all serialized through -+ * open_close_lock so it is safe. -+ */ -+ if (create) { -+ VixDiskLibCreateParams cparams = { -+ .diskType = create_type, -+ .adapterType = create_adapter_type, -+ .hwVersion = create_hwversion, -+ .capacity = create_size / VIXDISKLIB_SECTOR_SIZE, -+ .logicalSectorSize = 0, -+ .physicalSectorSize = 0 -+ }; -+ -+ VDDK_CALL_START (VixDiskLib_Create, -+ "h->connection, %s, &cparams, NULL, NULL", -+ filename) -+ err = VixDiskLib_Create (h->connection, filename, &cparams, NULL, NULL); -+ VDDK_CALL_END (VixDiskLib_Create, 0); -+ if (err != VIX_OK) { -+ VDDK_ERROR (err, "VixDiskLib_Create: %s", filename); -+ goto err2; -+ } -+ -+ create = false; /* Don't create it again. */ -+ } -+ - flags = 0; - if (readonly) - flags |= VIXDISKLIB_FLAG_OPEN_READ_ONLY; -diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h -index d99b6f4b..3a808013 100644 ---- a/plugins/vddk/vddk.h -+++ b/plugins/vddk/vddk.h -@@ -56,6 +56,11 @@ extern bool is_remote; - extern enum compression_type compression; - extern char *config; - extern const char *cookie; -+extern bool create; -+extern enum VixDiskLibAdapterType create_adapter_type; -+extern uint16_t create_hwversion; -+extern uint64_t create_size; -+extern enum VixDiskLibDiskType create_type; - extern const char *filename; - extern char *libdir; - extern uint16_t nfc_host_port; -diff --git a/tests/Makefile.am b/tests/Makefile.am -index 43b60943..ad2d43b9 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -171,7 +171,9 @@ if HAVE_VDDK - # - # make check-vddk vddkdir=vmware-vix-disklib-distrib - check-vddk: -- $(MAKE) check TESTS="test-vddk-real.sh test-vddk-real-dump-plugin.sh" -+ $(MAKE) check TESTS="test-vddk-real.sh -+ test-vddk-real-dump-plugin.sh -+ test-vddk-real-create.sh" - endif HAVE_VDDK - - #---------------------------------------------------------------------- -@@ -1033,6 +1035,7 @@ TESTS += \ - test-vddk-dump-plugin.sh \ - test-vddk-password-fd.sh \ - test-vddk-password-interactive.sh \ -+ test-vddk-real-create.sh \ - test-vddk-real-dump-plugin.sh \ - test-vddk-real.sh \ - test-vddk-reexec.sh \ -@@ -1063,6 +1066,7 @@ EXTRA_DIST += \ - test-vddk-dump-plugin.sh \ - test-vddk-password-fd.sh \ - test-vddk-password-interactive.sh \ -+ test-vddk-real-create.sh \ - test-vddk-real-dump-plugin.sh \ - test-vddk-real.sh \ - test-vddk-reexec.sh \ -diff --git a/tests/dummy-vddk.c b/tests/dummy-vddk.c -index b6f12042..0c5e505f 100644 ---- a/tests/dummy-vddk.c -+++ b/tests/dummy-vddk.c -@@ -236,3 +236,13 @@ VixDiskLib_Wait (VixDiskLibHandle handle) - { - return VIX_OK; - } -+ -+NBDKIT_DLL_PUBLIC VixError -+VixDiskLib_Create (const VixDiskLibConnection connection, -+ const char *path, -+ const VixDiskLibCreateParams *create_params, -+ void *progress_function_unused, -+ void *progress_data_unused) -+{ -+ return VIX_E_NOT_SUPPORTED; -+} -diff --git a/tests/test-vddk-real-create.sh b/tests/test-vddk-real-create.sh -new file mode 100755 -index 00000000..8f39a4c9 ---- /dev/null -+++ b/tests/test-vddk-real-create.sh -@@ -0,0 +1,70 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2022 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires test "x$vddkdir" != "x" -+requires test -d "$vddkdir" -+requires test -f "$vddkdir/lib64/libvixDiskLib.so" -+requires test -f disk -+requires nbdcopy --version -+requires stat --version -+ -+# Testing $LD_LIBRARY_PATH stuff breaks valgrind, so skip the rest of -+# this test if valgrinding. -+if [ "x$NBDKIT_VALGRIND" = "x1" ]; then -+ echo "$0: skipped LD_LIBRARY_PATH test when doing valgrind" -+ exit 77 -+fi -+ -+# VDDK > 5.1.1 only supports x86_64. -+if [ `uname -m` != "x86_64" ]; then -+ echo "$0: unsupported architecture" -+ exit 77 -+fi -+ -+vmdk=$PWD/test-vddk-real-create.vmdk ;# note must be an absolute path -+files="$vmdk" -+rm -f $files -+cleanup_fn rm -f $files -+ -+size="$(stat -c %s disk)" -+ -+nbdkit -fv -U - vddk libdir="$vddkdir" $vmdk \ -+ create=true create-size=$size \ -+ --run 'nbdcopy disk $uri' -+ -+# Check the VMDK file was created and looks reasonable. -+test -f $vmdk -+file $vmdk | grep 'VMware' --- -2.31.1 - diff --git a/SOURCES/0032-vddk-Fix-documentation-of-new-create-flag.patch b/SOURCES/0032-vddk-Fix-documentation-of-new-create-flag.patch deleted file mode 100644 index 2af8153..0000000 --- a/SOURCES/0032-vddk-Fix-documentation-of-new-create-flag.patch +++ /dev/null @@ -1,29 +0,0 @@ -From eb5d5a628968c7fd5401cf7e73a6cff6c43994aa Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 18 Jan 2022 13:14:01 +0000 -Subject: [PATCH] vddk: Fix documentation of new create flag - -create=1 works, but for consistency use create=true - -Fixes: commit a39d5773afc3ebab7e5768118a2bccb89a654585 -(cherry picked from commit 0b21897b64a6a1d97a8a7361e8f781ae743dedca) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index b96192d0..6c7ae989 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -51,7 +51,7 @@ input disk: - - nbdkit -U - vddk \ - /absolute/path/to/output.vmdk \ -- create=1 create-size=100M \ -+ create=true create-size=100M \ - --run 'qemu-img convert input.qcow2 $uri' - - =head2 Open a file on a remote VMware ESXi hypervisor --- -2.31.1 - diff --git a/SOURCES/0033-vddk-Allow-create-hwversion-to-be-specified-as-a-num.patch b/SOURCES/0033-vddk-Allow-create-hwversion-to-be-specified-as-a-num.patch deleted file mode 100644 index 1a7416b..0000000 --- a/SOURCES/0033-vddk-Allow-create-hwversion-to-be-specified-as-a-num.patch +++ /dev/null @@ -1,55 +0,0 @@ -From c8cdce47bc38d2f59ecc4b75d6db7f032b63d527 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 18 Jan 2022 20:49:21 +0000 -Subject: [PATCH] vddk: Allow create-hwversion to be specified as a number - -This gives us a bit of future-proofing so we don't always need to add -new hardware versions immediately. Another reason for this is that -VDDK allows you to specify seemingly any number here without -complaint. - -Updates: commit a39d5773afc3ebab7e5768118a2bccb89a654585 -(cherry picked from commit 071e32927237c2c00d78684c8a0f2e5fbca9963e) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 8 ++++++-- - plugins/vddk/vddk.c | 3 ++- - 2 files changed, 8 insertions(+), 3 deletions(-) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index 6c7ae989..e6972900 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -195,10 +195,14 @@ Specify the VMDK disk adapter type. The default is C. - - =item B - -+=item BN -+ - (nbdkit E 1.30) - --Specify the VMDK virtual hardware version. The default is --C. -+Specify the VMDK virtual hardware version. You can give either the -+named version or the equivalent 16 bit number. -+ -+The default is C (N = 4). - - =item BSIZE - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 5ebf9a2c..bab8de6f 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -192,7 +192,8 @@ vddk_config (const char *key, const char *value) - create_hwversion = VIXDISKLIB_HWVERSION_ESX60; - else if (strcmp (value, "esx65") == 0) - create_hwversion = VIXDISKLIB_HWVERSION_ESX65; -- else { -+ else if (nbdkit_parse_uint16_t ("create-hwversion", value, -+ &create_hwversion) == -1) { - nbdkit_error ("unknown create-hwversion: %s", value); - return -1; - } --- -2.31.1 - diff --git a/SOURCES/0034-tests-Fix-VDDK-tests.patch b/SOURCES/0034-tests-Fix-VDDK-tests.patch deleted file mode 100644 index 231d59d..0000000 --- a/SOURCES/0034-tests-Fix-VDDK-tests.patch +++ /dev/null @@ -1,31 +0,0 @@ -From 84c5bc4664607fdf1f051e9e52ac6d0e4f0be049 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 18 Jan 2022 21:02:32 +0000 -Subject: [PATCH] tests: Fix VDDK tests - -We need to use quoting for the subcommand split across lines. - -Fixes: commit a39d5773afc3ebab7e5768118a2bccb89a654585 -(cherry picked from commit 4df525566b38202ed8a7485ac8e7f06edd5ee49a) ---- - tests/Makefile.am | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/tests/Makefile.am b/tests/Makefile.am -index ad2d43b9..62a6f05b 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -171,8 +171,8 @@ if HAVE_VDDK - # - # make check-vddk vddkdir=vmware-vix-disklib-distrib - check-vddk: -- $(MAKE) check TESTS="test-vddk-real.sh -- test-vddk-real-dump-plugin.sh -+ $(MAKE) check TESTS="test-vddk-real.sh \ -+ test-vddk-real-dump-plugin.sh \ - test-vddk-real-create.sh" - endif HAVE_VDDK - --- -2.31.1 - diff --git a/SOURCES/0035-server-sockets-get-rid-of-AI_ADDRCONFIG.patch b/SOURCES/0035-server-sockets-get-rid-of-AI_ADDRCONFIG.patch deleted file mode 100644 index ad374bf..0000000 --- a/SOURCES/0035-server-sockets-get-rid-of-AI_ADDRCONFIG.patch +++ /dev/null @@ -1,204 +0,0 @@ -From 222bce6b83421db1afdad24cf4e8ab7b1aa7b273 Mon Sep 17 00:00:00 2001 -From: Laszlo Ersek -Date: Tue, 18 Jan 2022 14:48:33 +0100 -Subject: [PATCH] server/sockets: get rid of AI_ADDRCONFIG - -The AI_ADDRCONFIG hint of getaddrinfo() is supposed to restrict the name -resolution to such address families (IPv4 vs. IPv6) for which the -resolving host has publicly routable addresses assigned. - -The main problem with AI_ADDRCONFIG can be shown with the following -command line: - -$ nbdkit -f -p 32776 -P pidfile -i ::1 --exit-with-parent null - -On a host where ::1 is the only IPv6 address assigned (namely to the -loopback interface), the command fails with - -> nbdkit: getaddrinfo: ::1: 32776: Address family for hostname not -> supported - -due to the "publicly routable" requirement. - -Remove AI_ADDRCONFIG from the getaddrinfo() hints, and as a replacement, -introduce the "-4" and "-6" options, similarly to netcat and ssh. - -(1) This makes options of the form: - - -i 127.0.0.1 - -i ::1 - - work regardless of "public" IPv6 / IPv4 connectivity; - -(2) options of the form - - -i localhost - -i FQDN - - will bind both IPv4 and IPv6 addresses of the desired interface(s); - -(3) omitting the option "-i" will bind both IPv4 and IPv6 wildcard - addresses (0.0.0.0 and ::); - -(4) the configurations in (2) and (3) can be restricted to IPv4 or IPv6 - addresses by adding the "-4" or "-6" option, respectively. - -Importantly, this change allows the "connect-tcp6" test case of libnbd to -pass on such hosts that have no IPv6 connectivity (i.e., where the only -assigned IPv6 address is ::1, namely on the loopback interface). - -Ref: https://listman.redhat.com/archives/libguestfs/2022-January/msg00110.html -Signed-off-by: Laszlo Ersek -Message-Id: <20220118134833.13246-3-lersek@redhat.com> -[lersek@redhat.com: fix typo in "--exit-with-parent" (Eric)] -Acked-by: Eric Blake -(cherry picked from commit 9eec2335d630ae8ef947a927c1922d725d482f4a) ---- - common/utils/windows-compat.h | 7 ------- - docs/nbdkit.pod | 20 +++++++++++++++++++- - docs/synopsis.txt | 3 ++- - server/internal.h | 1 + - server/main.c | 9 +++++++++ - server/options.h | 4 +++- - server/sockets.c | 3 ++- - 7 files changed, 36 insertions(+), 11 deletions(-) - -diff --git a/common/utils/windows-compat.h b/common/utils/windows-compat.h -index 7695bf7e..658c1d8b 100644 ---- a/common/utils/windows-compat.h -+++ b/common/utils/windows-compat.h -@@ -75,13 +75,6 @@ struct sockaddr_un - #define O_NOCTTY 0 - #endif - --/* AI_ADDRCONFIG is not available on Windows. It enables a rather -- * obscure feature of getaddrinfo to do with IPv6. -- */ --#ifndef AI_ADDRCONFIG --#define AI_ADDRCONFIG 0 --#endif -- - /* Windows lacks certain errnos, so replace them here as - * best we can. - */ -diff --git a/docs/nbdkit.pod b/docs/nbdkit.pod -index 99cfb362..042607fb 100644 ---- a/docs/nbdkit.pod -+++ b/docs/nbdkit.pod -@@ -173,6 +173,24 @@ Display information about nbdkit or a specific plugin: - - Display brief command line usage information and exit. - -+=item B<-4> -+ -+=item B<--ipv4-only> -+ -+=item B<-6> -+ -+=item B<--ipv6-only> -+ -+When a non-numeric argument is passed to the I<-i> option (such as a -+Fully Qualified Domain Name, or a host name from C), -+restrict the name resolution to IPv4 or IPv6 addresses. -+ -+When the I<-i> option is omitted, listen on only the IPv4 or IPv6 -+address of all interfaces (C<0.0.0.0> or C<::>, respectively). -+ -+When both I<-4> and I<-6> options are present on the command line, the -+last one takes effect. -+ - =item B<-D> PLUGIN.FLAG=N - - =item B<-D> FILTER.FLAG=N -@@ -265,7 +283,7 @@ See also I<-u>. - =item B<--ipaddr> IPADDR - - Listen on the specified interface. The default is to listen on all --interfaces. See also I<-p>. -+interfaces. See also I<-4>, I<-6>, and I<-p>. - - =item B<--log=stderr> - -diff --git a/docs/synopsis.txt b/docs/synopsis.txt -index 07b9dcff..6154bb2e 100644 ---- a/docs/synopsis.txt -+++ b/docs/synopsis.txt -@@ -1,4 +1,5 @@ --nbdkit [-D|--debug PLUGIN|FILTER|nbdkit.FLAG=N] -+nbdkit [-4|--ipv4-only] [-6|--ipv6-only] -+ [-D|--debug PLUGIN|FILTER|nbdkit.FLAG=N] - [-e|--exportname EXPORTNAME] [--exit-with-parent] - [--filter FILTER ...] [-f|--foreground] - [-g|--group GROUP] [-i|--ipaddr IPADDR] -diff --git a/server/internal.h b/server/internal.h -index bc81b786..46fcdd46 100644 ---- a/server/internal.h -+++ b/server/internal.h -@@ -113,6 +113,7 @@ enum log_to { - LOG_TO_NULL, /* --log=null forced on the command line */ - }; - -+extern int tcpip_sock_af; - extern struct debug_flag *debug_flags; - extern const char *export_name; - extern bool foreground; -diff --git a/server/main.c b/server/main.c -index 225258de..8e7ac149 100644 ---- a/server/main.c -+++ b/server/main.c -@@ -86,6 +86,7 @@ static void error_if_stdio_closed (void); - static void switch_stdio (void); - static void winsock_init (void); - -+int tcpip_sock_af = AF_UNSPEC; /* -4, -6 */ - struct debug_flag *debug_flags; /* -D */ - bool exit_with_parent; /* --exit-with-parent */ - const char *export_name; /* -e */ -@@ -367,6 +368,14 @@ main (int argc, char *argv[]) - exit (EXIT_FAILURE); - #endif - -+ case '4': -+ tcpip_sock_af = AF_INET; -+ break; -+ -+ case '6': -+ tcpip_sock_af = AF_INET6; -+ break; -+ - case 'D': - add_debug_flag (optarg); - break; -diff --git a/server/options.h b/server/options.h -index e59ef17f..39299b9d 100644 ---- a/server/options.h -+++ b/server/options.h -@@ -59,8 +59,10 @@ enum { - VSOCK_OPTION, - }; - --static const char *short_options = "D:e:fg:i:nop:P:rst:u:U:vV"; -+static const char *short_options = "46D:e:fg:i:nop:P:rst:u:U:vV"; - static const struct option long_options[] = { -+ { "ipv4-only", no_argument, NULL, '4' }, -+ { "ipv6-only", no_argument, NULL, '6' }, - { "debug", required_argument, NULL, 'D' }, - { "dump-config", no_argument, NULL, DUMP_CONFIG_OPTION }, - { "dump-plugin", no_argument, NULL, DUMP_PLUGIN_OPTION }, -diff --git a/server/sockets.c b/server/sockets.c -index 15a26f69..4e4ccbc4 100644 ---- a/server/sockets.c -+++ b/server/sockets.c -@@ -179,7 +179,8 @@ bind_tcpip_socket (sockets *socks) - port = "10809"; - - memset (&hints, 0, sizeof hints); -- hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG; -+ hints.ai_flags = AI_PASSIVE; -+ hints.ai_family = tcpip_sock_af; - hints.ai_socktype = SOCK_STREAM; - - err = getaddrinfo (ipaddr, port, &hints, &ai); --- -2.31.1 - diff --git a/SOURCES/copy-patches.sh b/SOURCES/copy-patches.sh index 2aaf42c..10fca0f 100755 --- a/SOURCES/copy-patches.sh +++ b/SOURCES/copy-patches.sh @@ -6,7 +6,7 @@ set -e # directory. Use it like this: # ./copy-patches.sh -rhel_version=9.0 +rhel_version=9.1 # Check we're in the right directory. if [ ! -f nbdkit.spec ]; then diff --git a/SOURCES/nbdkit-1.28.5.tar.gz.sig b/SOURCES/nbdkit-1.28.5.tar.gz.sig deleted file mode 100644 index a1e3d70..0000000 --- a/SOURCES/nbdkit-1.28.5.tar.gz.sig +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN PGP SIGNATURE----- - -iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmHumIgRHHJpY2hAYW5u -ZXhpYS5vcmcACgkQkXOPc+G3aKAt/RAAlUnU/dJ6d1bec5ggmfTgWYwHcXqc6bGi -Tp0vxG+2LiE/D52xhGvaRWQwd2mR4vvHiDLlBitlBZNHItyuB9wM9r5SAIRw5PcR -1kCILAKGQN2dLSS+HUyib6cnlughFwIeSt4XuhwgedLlIBZ+2d38IRC3ia4ShbIX -btZMwRmpD/06Py8A+tM856zb1YgjpA3nQIC56r/ne25vLyAY4LE8T2BtjlkmBxBy -Lswg0KM3+SBsMWwbo0aCfyTOW9lpVa2WnLgu/9nsfIjA+m9kcjfpjfmhduV6lfrx -KRFGtKQnl+RWfhhfmxeCWo5/mXqrqOga7VIWltxRkjQ916TrzTWwMnWBtOvVfRSL -QS1tlJYbClRGHHkM9YHjvV0v+xHcUTt9VAd+RruVjnz2H4hZrEi8klAHWjOUe/1m -37PEoLAh9+ox+zcSODc+MWWOA98oJoXyS0PZvPOzzlSokxaLEY/TRUMrILGBJSyz -hIwdMi63gM1KzIw0ysNJ639Nvu0n/PiIgpPheXK81fNyNpzsThS1uEqonAMC9+Hr -QIMBgfRMdvbG791lVo7WRHSdGpSO+hun4scla+3VZpszqSwFX2+O/ji0+5cos4RR -dxfzV4gt/1FH72OhWGHMSmmTpd+G2ZjXFsjNTmzMYbS0kMeXdIXalVXJrfDroSoU -ITXkfk5uFtU= -=Wlxf ------END PGP SIGNATURE----- diff --git a/SOURCES/nbdkit-1.30.8.tar.gz.sig b/SOURCES/nbdkit-1.30.8.tar.gz.sig new file mode 100644 index 0000000..225116c --- /dev/null +++ b/SOURCES/nbdkit-1.30.8.tar.gz.sig @@ -0,0 +1,17 @@ +-----BEGIN PGP SIGNATURE----- + +iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmLWguQRHHJpY2hAYW5u +ZXhpYS5vcmcACgkQkXOPc+G3aKBpTQ//SBnebP9miDMyAnMS0zgK1LevsGMWwnLm +cZcXgvx+hvdomTlFElcxBjSPN51rhywI0vyN8fM1h/eckBkGH/fC8Ta+pQHBxFec +KZZfLZ+EhsBv1JAid3t284xxFkahtZ0UWpPmVlZG2EelGiYHpL+nckf0hTS6R2a2 +cWDgpSwxIlPNEvDmDqZyUPCTY4vdxtdlI4JwRcqYn7nH6TbBLWK0/pjQH6NtZ/bu +cBQJQpi+ne3mIU6P8GocFJhNEMO1//aZRJsAX5uURuIaAToH+QDtcXCVPTZDogqh +5BPL/nGQu+LbXFlRTGFIwNRyfSLDGFPWnZ/356NssaPpX4cYB9zsEmdnf9enz/Ez +cYrwoxQbilzkvRH0oovb+2UL4V/dEoUgdND6amHWF+3uuT+c/j9CsIk5IrL5y9Ly +VhkttvYL96rOGQoIeZfg0+Icjx6aSau6L7qRvjZBP6egZN3VUVhMFn7knynENXlv +DxDg4XwqWP6OKsBiBteED48Hn2nF0jniq/6Mx/fVrblGlYGxOSLIupdTXl5n10OK +9vwsp7qSYJP1160Mt5r3MFNH0DygmAHbH5jqzDFn5H6TqSV31PuF4XZg/DnBedaJ ++PI9qOnXjkM9gYeVO3FOosf70kh7zfjP6/kOOJ4BGiaZ5T9Y12oJAwxqkjZhUHHF +QDvUE7KDOM8= +=I81A +-----END PGP SIGNATURE----- diff --git a/SOURCES/nbdkit-find-provides b/SOURCES/nbdkit-find-provides new file mode 100755 index 0000000..7013ccd --- /dev/null +++ b/SOURCES/nbdkit-find-provides @@ -0,0 +1,23 @@ +#!/bin/bash - + +# Generate RPM provides automatically for nbdkit packages and filters. +# Copyright (C) 2009-2022 Red Hat Inc. + +# To test: +# find /usr/lib64/nbdkit/plugins | ./nbdkit-find-provides VER REL +# find /usr/lib64/nbdkit/filters | ./nbdkit-find-provides VER REL + +ver="$1" +rel="$2" + +function process_file +{ + if [[ $1 =~ /plugins/nbdkit-.*-plugin ]] || + [[ $1 =~ /filters/nbdkit-.*-filter ]]; then + echo "Provides:" "$(basename $1 .so)" "=" "$ver-$rel" + fi +} + +while read line; do + process_file "$line" +done diff --git a/SOURCES/nbdkit.attr b/SOURCES/nbdkit.attr new file mode 100644 index 0000000..2757679 --- /dev/null +++ b/SOURCES/nbdkit.attr @@ -0,0 +1,3 @@ +%__nbdkit_provides %{_rpmconfigdir}/nbdkit-find-provides %{version} %{release} +%__nbdkit_path %{_libdir}/nbdkit/(plugins|filters)/nbdkit-.*-(plugin|filter)(\.so)?$ +%__nbdkit_flags exeonly diff --git a/SPECS/nbdkit.spec b/SPECS/nbdkit.spec index 8089073..30019fe 100644 --- a/SPECS/nbdkit.spec +++ b/SPECS/nbdkit.spec @@ -1,6 +1,9 @@ +%undefine _package_note_flags %global _hardened_build 1 %ifarch %{kernel_arches} +# ppc64le broken in rawhide: +# https://bugzilla.redhat.com/show_bug.cgi?id=2006709 # riscv64 tests fail with # qemu-system-riscv64: invalid accelerator kvm # qemu-system-riscv64: falling back to tcg @@ -8,7 +11,7 @@ # This seems to require changes in libguestfs and/or qemu to support # -cpu max or -cpu virt. # s390x builders can't run libguestfs -%ifnarch riscv64 s390 s390x +%ifnarch %{power64} riscv64 s390 s390x %global have_libguestfs 1 %endif %endif @@ -29,8 +32,6 @@ # If the test suite is broken on a particular architecture, document # it as a bug and add it to this list. -# -# armv7, aarch64: https://bugzilla.redhat.com/show_bug.cgi?id=1893892 %global broken_test_arches NONE %if 0%{?rhel} == 7 @@ -48,10 +49,10 @@ ExclusiveArch: x86_64 %global patches_touch_autotools 1 # The source directory. -%global source_directory 1.28-stable +%global source_directory 1.30-stable Name: nbdkit -Version: 1.28.5 +Version: 1.30.8 Release: 1%{?dist} Summary: NBD server @@ -74,44 +75,37 @@ Source2: libguestfs.keyring Source3: copy-patches.sh # Patches come from the upstream repository: -# https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.0/ +# https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.1/ # Patches. -Patch0001: 0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch -Patch0002: 0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch -Patch0003: 0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch -Patch0004: 0004-vddk-Document-troubleshooting-performance-problems.patch -Patch0005: 0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch -Patch0006: 0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch -Patch0007: 0007-vddk-Fix-typo-in-debug-message.patch -Patch0008: 0008-vddk-Only-print-vddk_library_version-when-we-managed.patch -Patch0009: 0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch -Patch0010: 0010-vddk-Move-minimum-version-to-VDDK-6.5.patch -Patch0011: 0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch -Patch0012: 0012-vddk-Start-to-split-VDDK-over-several-files.patch -Patch0013: 0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch -Patch0014: 0014-vddk-Implement-parallel-thread-model.patch -Patch0015: 0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch -Patch0016: 0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch -Patch0017: 0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch -Patch0018: 0018-vddk-Move-config-debug-error-and-utility-functions-a.patch -Patch0019: 0019-common-utils-test-vector.c-Add-vector-benchmarks.patch -Patch0020: 0020-common-urils-vector.c-Optimize-vector-append.patch -Patch0021: 0021-common-utils-vector-Rename-alloc-to-cap.patch -Patch0022: 0022-common-utils-vector-Rename-size-to-len.patch -Patch0023: 0023-podwrapper.pl.in-Use-short-commit-date.patch -Patch0024: 0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch -Patch0025: 0025-vddk-Drop-obsolete-documentation-related-to-thread-m.patch -Patch0026: 0026-Revert-podwrapper.pl.in-Use-short-commit-date.patch -Patch0027: 0027-Fix-podwrapper.pl.in-Use-short-commit-date.patch -Patch0028: 0028-scripts-Add-simple-script-for-automating-VDDK-disk-c.patch -Patch0029: 0029-file-Fix-implementation-of-cache-none-for-writes.patch -Patch0030: 0030-tests-Add-configure-disable-libguestfs-tests-flag.patch -Patch0031: 0031-vddk-Implement-VMDK-creation.patch -Patch0032: 0032-vddk-Fix-documentation-of-new-create-flag.patch -Patch0033: 0033-vddk-Allow-create-hwversion-to-be-specified-as-a-num.patch -Patch0034: 0034-tests-Fix-VDDK-tests.patch -Patch0035: 0035-server-sockets-get-rid-of-AI_ADDRCONFIG.patch +Patch0001: 0001-ssh-Allow-the-remote-file-to-be-created.patch +Patch0002: 0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch +Patch0003: 0003-readahead-Fix-test.patch +Patch0004: 0004-New-filter-luks.patch +Patch0005: 0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch +Patch0006: 0006-luks-Various-fixes-for-Clang.patch +Patch0007: 0007-luks-Link-with-libcompat-on-Windows.patch +Patch0008: 0008-luks-Refactor-the-filter.patch +Patch0009: 0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch +Patch0010: 0010-Add-nbdkit.parse_size-Python-function.patch +Patch0011: 0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch +Patch0012: 0012-curl-Don-t-document-curl-plugin-readahead-filter.patch +Patch0013: 0013-New-filter-scan.patch +Patch0014: 0014-scan-Remove-condition-variable.patch +Patch0015: 0015-scan-Small-typographical-fix-in-manual.patch +Patch0016: 0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch +Patch0017: 0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch +Patch0018: 0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch +Patch0019: 0019-rate-Allow-burstiness-to-be-controlled.patch +Patch0020: 0020-luks-Check-return-values-from-malloc-more-carefully.patch +Patch0021: 0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch +Patch0022: 0022-luks-Avoid-memory-leak-on-error-path.patch +Patch0023: 0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch + +# For automatic RPM Provides generation. +# See: https://rpm-software-management.github.io/rpm/manual/dependency_generators.html +Source4: nbdkit.attr +Source5: nbdkit-find-provides BuildRequires: make %if 0%{patches_touch_autotools} @@ -150,9 +144,7 @@ BuildRequires: python3-boto3 %endif %if !0%{?rhel} %if 0%{?have_ocaml} -# Requires OCaml 4.02.2 which contains fix for -# http://caml.inria.fr/mantis/view.php?id=6693 -BuildRequires: ocaml >= 4.02.2 +BuildRequires: ocaml >= 4.03 BuildRequires: ocaml-ocamldoc %endif BuildRequires: ruby-devel @@ -185,6 +177,11 @@ BuildRequires: %{_sbindir}/ss BuildRequires: %{_bindir}/stat BuildRequires: %{_bindir}/ssh-keygen +# This package has RPM rules that create the automatic Provides: for +# nbdkit plugins and filters. This means nbdkit build depends on +# itself, but it's a simple noarch package so easy to install. +BuildRequires: nbdkit-srpm-macros >= 1.30.0 + # nbdkit is a metapackage pulling the server and a useful subset # of the plugins and filters. Requires: nbdkit-server%{?_isa} = %{version}-%{release} @@ -226,7 +223,6 @@ reading the nbdkit(1) and nbdkit-plugin(3) manual pages. %package server Summary: The %{name} server License: BSD -Provides: %{name}-null-plugin = %{version}-%{release} %description server This package contains the %{name} server with only the null plugin @@ -239,21 +235,6 @@ the metapackage "nbdkit". Summary: Basic plugins for %{name} License: BSD Requires: %{name}-server%{?_isa} = %{version}-%{release} -Provides: %{name}-data-plugin = %{version}-%{release} -Provides: %{name}-eval-plugin = %{version}-%{release} -Provides: %{name}-file-plugin = %{version}-%{release} -Provides: %{name}-floppy-plugin = %{version}-%{release} -Provides: %{name}-full-plugin = %{version}-%{release} -Provides: %{name}-info-plugin = %{version}-%{release} -Provides: %{name}-memory-plugin = %{version}-%{release} -Provides: %{name}-ondemand-plugin = %{version}-%{release} -Provides: %{name}-pattern-plugin = %{version}-%{release} -Provides: %{name}-partitioning-plugin = %{version}-%{release} -Provides: %{name}-random-plugin = %{version}-%{release} -Provides: %{name}-sh-plugin = %{version}-%{release} -Provides: %{name}-sparse-random-plugin = %{version}-%{release} -Provides: %{name}-split-plugin = %{version}-%{release} -Provides: %{name}-zero-plugin = %{version}-%{release} %description basic-plugins @@ -549,38 +530,6 @@ VMware VDDK for accessing VMware disks and servers. Summary: Basic filters for %{name} License: BSD Requires: %{name}-server%{?_isa} = %{version}-%{release} -Provides: %{name}-blocksize-filter = %{version}-%{release} -Provides: %{name}-cache-filter = %{version}-%{release} -Provides: %{name}-cacheextents-filter = %{version}-%{release} -Provides: %{name}-checkwrite-filter = %{version}-%{release} -Provides: %{name}-cow-filter = %{version}-%{release} -Provides: %{name}-ddrescue-filter = %{version}-%{release} -Provides: %{name}-delay-filter = %{version}-%{release} -Provides: %{name}-error-filter = %{version}-%{release} -Provides: %{name}-exitlast-filter = %{version}-%{release} -Provides: %{name}-exitwhen-filter = %{version}-%{release} -Provides: %{name}-exportname-filter = %{version}-%{release} -Provides: %{name}-extentlist-filter = %{version}-%{release} -Provides: %{name}-fua-filter = %{version}-%{release} -Provides: %{name}-ip-filter = %{version}-%{release} -Provides: %{name}-limit-filter = %{version}-%{release} -Provides: %{name}-log-filter = %{version}-%{release} -Provides: %{name}-multi-conn-filter = %{version}-%{release} -Provides: %{name}-nocache-filter = %{version}-%{release} -Provides: %{name}-noextents-filter = %{version}-%{release} -Provides: %{name}-nofilter-filter = %{version}-%{release} -Provides: %{name}-noparallel-filter = %{version}-%{release} -Provides: %{name}-nozero-filter = %{version}-%{release} -Provides: %{name}-offset-filter = %{version}-%{release} -Provides: %{name}-partition-filter = %{version}-%{release} -Provides: %{name}-pause-filter = %{version}-%{release} -Provides: %{name}-rate-filter = %{version}-%{release} -Provides: %{name}-readahead-filter = %{version}-%{release} -Provides: %{name}-retry-filter = %{version}-%{release} -Provides: %{name}-stats-filter = %{version}-%{release} -Provides: %{name}-swab-filter = %{version}-%{release} -Provides: %{name}-tls-fallback-filter = %{version}-%{release} -Provides: %{name}-truncate-filter = %{version}-%{release} %description basic-filters This package contains filters for %{name} which only depend on simple @@ -589,6 +538,8 @@ complex dependencies are packaged separately. nbdkit-blocksize-filter Adjust block size of requests sent to plugins. +nbdkit-blocksize-policy-filter Set block size constraints and policy. + nbdkit-cache-filter Server-side cache. nbdkit-cacheextents-filter Cache extents. @@ -619,6 +570,8 @@ nbdkit-limit-filter Limit nr clients that can connect concurrently. nbdkit-log-filter Log all transactions to a file. +nbdkit-luks-filter Read and write LUKS-encrypted disks. + nbdkit-multi-conn-filter Enable, emulate or disable multi-conn. nbdkit-nocache-filter Disable cache requests in the underlying plugin. @@ -637,12 +590,18 @@ nbdkit-partition-filter Serve a single partition. nbdkit-pause-filter Pause NBD requests. +nbdkit-protect-filter Write-protect parts of a plugin. + nbdkit-rate-filter Limit bandwidth by connection or server. nbdkit-readahead-filter Prefetch data when reading sequentially. nbdkit-retry-filter Reopen connection on error. +nbdkit-retry-request-filter Retry single requests on error. + +nbdkit-scan-filter Prefetch data ahead of sequential reads. + nbdkit-stats-filter Display statistics about operations. nbdkit-swab-filter Filter for swapping byte order. @@ -705,8 +664,19 @@ for %{name}. Install this package if you want to develop plugins for %{name}. +%package srpm-macros +Summary: RPM Provides rules for %{name} plugins and filters +License: BSD +BuildArch: noarch + +%description srpm-macros +This package contains RPM rules that create the automatic Provides: +for %{name} plugins and filters found in the plugins directory. + + %package bash-completion Summary: Bash tab-completion for %{name} +License: BSD BuildArch: noarch Requires: bash-completion >= 2.0 Requires: %{name}-server = %{version}-%{release} @@ -779,10 +749,6 @@ find $RPM_BUILD_ROOT -name '*.la' -delete # rust plugin is built. Delete it if this happens. rm -f $RPM_BUILD_ROOT%{_mandir}/man3/nbdkit-rust-plugin.3* -# Remove the streaming plugin (deprecated upstream). -rm $RPM_BUILD_ROOT%{_libdir}/%{name}/plugins/nbdkit-streaming-plugin.so -rm $RPM_BUILD_ROOT%{_mandir}/man1/nbdkit-streaming-plugin.1* - %if 0%{?rhel} # In RHEL, remove some plugins we cannot --disable. for f in cc cdi torrent; do @@ -793,6 +759,11 @@ rm -f $RPM_BUILD_ROOT%{_libdir}/%{name}/plugins/nbdkit-S3-plugin rm -f $RPM_BUILD_ROOT%{_mandir}/man1/nbdkit-S3-plugin.1* %endif +# Install RPM dependency generator. +mkdir -p $RPM_BUILD_ROOT%{_rpmconfigdir}/fileattrs/ +install -m 0644 %{SOURCE4} $RPM_BUILD_ROOT%{_rpmconfigdir}/fileattrs/ +install -m 0755 %{SOURCE5} $RPM_BUILD_ROOT%{_rpmconfigdir}/ + %check %ifnarch %{broken_test_arches} @@ -1085,6 +1056,7 @@ export LIBGUESTFS_TRACE=1 %doc README %license LICENSE %{_libdir}/%{name}/filters/nbdkit-blocksize-filter.so +%{_libdir}/%{name}/filters/nbdkit-blocksize-policy-filter.so %{_libdir}/%{name}/filters/nbdkit-cache-filter.so %{_libdir}/%{name}/filters/nbdkit-cacheextents-filter.so %{_libdir}/%{name}/filters/nbdkit-checkwrite-filter.so @@ -1100,6 +1072,7 @@ export LIBGUESTFS_TRACE=1 %{_libdir}/%{name}/filters/nbdkit-ip-filter.so %{_libdir}/%{name}/filters/nbdkit-limit-filter.so %{_libdir}/%{name}/filters/nbdkit-log-filter.so +%{_libdir}/%{name}/filters/nbdkit-luks-filter.so %{_libdir}/%{name}/filters/nbdkit-multi-conn-filter.so %{_libdir}/%{name}/filters/nbdkit-nocache-filter.so %{_libdir}/%{name}/filters/nbdkit-noextents-filter.so @@ -1109,14 +1082,18 @@ export LIBGUESTFS_TRACE=1 %{_libdir}/%{name}/filters/nbdkit-offset-filter.so %{_libdir}/%{name}/filters/nbdkit-partition-filter.so %{_libdir}/%{name}/filters/nbdkit-pause-filter.so +%{_libdir}/%{name}/filters/nbdkit-protect-filter.so %{_libdir}/%{name}/filters/nbdkit-rate-filter.so %{_libdir}/%{name}/filters/nbdkit-readahead-filter.so %{_libdir}/%{name}/filters/nbdkit-retry-filter.so +%{_libdir}/%{name}/filters/nbdkit-retry-request-filter.so +%{_libdir}/%{name}/filters/nbdkit-scan-filter.so %{_libdir}/%{name}/filters/nbdkit-stats-filter.so %{_libdir}/%{name}/filters/nbdkit-swab-filter.so %{_libdir}/%{name}/filters/nbdkit-tls-fallback-filter.so %{_libdir}/%{name}/filters/nbdkit-truncate-filter.so %{_mandir}/man1/nbdkit-blocksize-filter.1* +%{_mandir}/man1/nbdkit-blocksize-policy-filter.1* %{_mandir}/man1/nbdkit-cache-filter.1* %{_mandir}/man1/nbdkit-cacheextents-filter.1* %{_mandir}/man1/nbdkit-checkwrite-filter.1* @@ -1132,6 +1109,7 @@ export LIBGUESTFS_TRACE=1 %{_mandir}/man1/nbdkit-ip-filter.1* %{_mandir}/man1/nbdkit-limit-filter.1* %{_mandir}/man1/nbdkit-log-filter.1* +%{_mandir}/man1/nbdkit-luks-filter.1* %{_mandir}/man1/nbdkit-multi-conn-filter.1* %{_mandir}/man1/nbdkit-nocache-filter.1* %{_mandir}/man1/nbdkit-noextents-filter.1* @@ -1141,9 +1119,12 @@ export LIBGUESTFS_TRACE=1 %{_mandir}/man1/nbdkit-offset-filter.1* %{_mandir}/man1/nbdkit-partition-filter.1* %{_mandir}/man1/nbdkit-pause-filter.1* +%{_mandir}/man1/nbdkit-protect-filter.1* %{_mandir}/man1/nbdkit-rate-filter.1* %{_mandir}/man1/nbdkit-readahead-filter.1* %{_mandir}/man1/nbdkit-retry-filter.1* +%{_mandir}/man1/nbdkit-retry-request-filter.1* +%{_mandir}/man1/nbdkit-scan-filter.1* %{_mandir}/man1/nbdkit-stats-filter.1* %{_mandir}/man1/nbdkit-swab-filter.1* %{_mandir}/man1/nbdkit-tls-fallback-filter.1* @@ -1214,6 +1195,12 @@ export LIBGUESTFS_TRACE=1 %{_libdir}/pkgconfig/nbdkit.pc +%files srpm-macros +%license LICENSE +%{_rpmconfigdir}/fileattrs/nbdkit.attr +%{_rpmconfigdir}/nbdkit-find-provides + + %files bash-completion %license LICENSE %dir %{_datadir}/bash-completion/completions @@ -1221,6 +1208,37 @@ export LIBGUESTFS_TRACE=1 %changelog +* Tue Jul 19 2022 Richard W.M. Jones - 1.30.8-1 +- Rebase to new stable branch version 1.30.8 + resolves: rhbz#2059289 +- Add automatic provides generator and subpackage nbdkit-srpm-macros + resolves: rhbz#2059291 +- New filters: blocksize-policy, protect, retry-request +- Fix license of bash-completion subpackage +- vddk: Fix use of uninitialized memory when computing block size + resolves: rhbz#2066655 +- Skip vsock tests unless the vsock_loopback module is loaded + resolves: rhbz#2069558 +- Add support for ssh create remote file. +- Suppress excess messages from nbdkit-nbd-plugin + resolves: rhbz#2083498 +- Suppress incorrect VDDK error when converting guests from vCenter + resolves: rhbz#2083617 +- Backport new readahead filter from 1.32. +- Backport new LUKS filter from 1.32. +- Backport new scan filter from 1.32. +- Add new Python binding for nbdkit_parse_size from 1.32 +- Add new rate filter burstiness setting from 1.32 +- vddk: Suppress new VDDK "phone home" messages + resolves: rhbz#2104720 +- vddk: Clearer error message when thumbprint is wrong + resolves: rhbz#1905772 +- Fix memory allocator=malloc,mlock=true + resolves: rhbz#2044432 +- Fix multiple Coverity problems +- Fix bounds error in nbdkit-checkwrite-filter + resolves: rhbz#2108545 + * Mon Jan 24 2022 Richard W.M. Jones - 1.28.5-1 - Rebase to new stable branch version 1.28.5 resolves: rhbz#2011709