|
 |
f3fc18 |
From 3f74004478d3590840d7eba97a590b7ec954957f Mon Sep 17 00:00:00 2001
|
|
|
f3fc18 |
From: "Richard W.M. Jones" <rjones@redhat.com>
|
|
|
f3fc18 |
Date: Thu, 2 Feb 2023 13:59:32 +0000
|
|
|
f3fc18 |
Subject: [PATCH] curl: Enable multi-conn for read-only connections
|
|
|
f3fc18 |
MIME-Version: 1.0
|
|
|
f3fc18 |
Content-Type: text/plain; charset=UTF-8
|
|
|
f3fc18 |
Content-Transfer-Encoding: 8bit
|
|
|
f3fc18 |
|
|
|
f3fc18 |
Comparing before and after this commit shows approximately double the
|
|
|
f3fc18 |
performance. In other tests this allowed us to download files from
|
|
|
f3fc18 |
web servers at line speed.
|
|
|
f3fc18 |
|
|
|
f3fc18 |
Benchmark 1: nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:"
|
|
|
f3fc18 |
Time (mean ± σ): 943.8 ms ± 18.8 ms [User: 316.2 ms, System: 1029.7 ms]
|
|
|
f3fc18 |
Range (min … max): 923.7 ms … 989.2 ms 10 runs
|
|
|
f3fc18 |
|
|
|
f3fc18 |
Benchmark 2: ~/d/nbdkit/nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:"
|
|
|
f3fc18 |
Time (mean ± σ): 455.0 ms ± 6.2 ms [User: 542.2 ms, System: 1824.7 ms]
|
|
|
f3fc18 |
Range (min … max): 449.1 ms … 471.6 ms 10 runs
|
|
|
f3fc18 |
|
|
|
f3fc18 |
Summary
|
|
|
f3fc18 |
' ~/d/nbdkit/nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:" ' ran
|
|
|
f3fc18 |
2.07 ± 0.05 times faster than ' nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:" '
|
|
|
f3fc18 |
|
|
|
f3fc18 |
Multi-conn is enabled only when we know the connection is read-only:
|
|
|
f3fc18 |
|
|
|
f3fc18 |
$ ./nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run ' nbdinfo $uri ' | grep can_multi_conn
|
|
|
f3fc18 |
can_multi_conn: true
|
|
|
f3fc18 |
$ ./nbdkit curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run ' nbdinfo $uri ' | grep can_multi_conn
|
|
|
f3fc18 |
can_multi_conn: false
|
|
|
f3fc18 |
|
|
|
f3fc18 |
See also:
|
|
|
f3fc18 |
https://listman.redhat.com/archives/libguestfs/2023-February/030581.html
|
|
|
f3fc18 |
|
|
|
f3fc18 |
Reviewed-by: Eric Blake <eblake@redhat.com>
|
|
|
f3fc18 |
(cherry picked from commit bb0f93ad7b9de451874d0c54188bf69cd37c5409)
|
|
|
f3fc18 |
---
|
|
|
f3fc18 |
plugins/curl/curl.c | 14 ++++++++++++++
|
|
|
f3fc18 |
plugins/curl/curldefs.h | 1 +
|
|
|
f3fc18 |
2 files changed, 15 insertions(+)
|
|
|
f3fc18 |
|
|
|
f3fc18 |
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
|
|
|
f3fc18 |
index e89bea99..eeba5aa4 100644
|
|
|
f3fc18 |
--- a/plugins/curl/curl.c
|
|
|
f3fc18 |
+++ b/plugins/curl/curl.c
|
|
|
f3fc18 |
@@ -455,6 +455,7 @@ curl_open (int readonly)
|
|
|
f3fc18 |
nbdkit_error ("calloc: %m");
|
|
|
f3fc18 |
return NULL;
|
|
|
f3fc18 |
}
|
|
|
f3fc18 |
+ h->readonly = readonly;
|
|
|
f3fc18 |
|
|
|
f3fc18 |
h->c = curl_easy_init ();
|
|
|
f3fc18 |
if (h->c == NULL) {
|
|
|
f3fc18 |
@@ -764,6 +765,18 @@ curl_get_size (void *handle)
|
|
|
f3fc18 |
return h->exportsize;
|
|
|
f3fc18 |
}
|
|
|
f3fc18 |
|
|
|
f3fc18 |
+/* Multi-conn is safe for read-only connections, but HTTP does not
|
|
|
f3fc18 |
+ * have any concept of flushing so we cannot use it for read-write
|
|
|
f3fc18 |
+ * connections.
|
|
|
f3fc18 |
+ */
|
|
|
f3fc18 |
+static int
|
|
|
f3fc18 |
+curl_can_multi_conn (void *handle)
|
|
|
f3fc18 |
+{
|
|
|
f3fc18 |
+ struct curl_handle *h = handle;
|
|
|
f3fc18 |
+
|
|
|
f3fc18 |
+ return !! h->readonly;
|
|
|
f3fc18 |
+}
|
|
|
f3fc18 |
+
|
|
|
f3fc18 |
/* NB: The terminology used by libcurl is confusing!
|
|
|
f3fc18 |
*
|
|
|
f3fc18 |
* WRITEFUNCTION / write_cb is used when reading from the remote server
|
|
|
f3fc18 |
@@ -907,6 +920,7 @@ static struct nbdkit_plugin plugin = {
|
|
|
f3fc18 |
.open = curl_open,
|
|
|
f3fc18 |
.close = curl_close,
|
|
|
f3fc18 |
.get_size = curl_get_size,
|
|
|
f3fc18 |
+ .can_multi_conn = curl_can_multi_conn,
|
|
|
f3fc18 |
.pread = curl_pread,
|
|
|
f3fc18 |
.pwrite = curl_pwrite,
|
|
|
f3fc18 |
};
|
|
|
f3fc18 |
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
|
|
|
f3fc18 |
index f3095f92..9d4949f3 100644
|
|
|
f3fc18 |
--- a/plugins/curl/curldefs.h
|
|
|
f3fc18 |
+++ b/plugins/curl/curldefs.h
|
|
|
f3fc18 |
@@ -64,6 +64,7 @@ extern const char *user_agent;
|
|
|
f3fc18 |
/* The per-connection handle. */
|
|
|
f3fc18 |
struct curl_handle {
|
|
|
f3fc18 |
CURL *c;
|
|
|
f3fc18 |
+ int readonly;
|
|
|
f3fc18 |
bool accept_range;
|
|
|
f3fc18 |
int64_t exportsize;
|
|
|
f3fc18 |
char errbuf[CURL_ERROR_SIZE];
|
|
|
f3fc18 |
--
|
|
|
f3fc18 |
2.31.1
|
|
|
f3fc18 |
|