From 9f163418fabbe6219ab04cfe9bf81d2f33bd54d7 Mon Sep 17 00:00:00 2001
From: Richy Kim <richy@fb.com>
Date: Tue, 20 Dec 2016 05:48:15 -0500
Subject: [PATCH 01/18] CURLOPT_BUFFERSIZE: support enlarging receive buffer
Replace use of fixed macro BUFSIZE to define the size of the receive
buffer. Reappropriate CURLOPT_BUFFERSIZE to include enlarging receive
buffer size. Upon setting, resize buffer if larger than the current
default size up to a MAX_BUFSIZE (512KB). This can benefit protocols
like SFTP.
Closes #1222
Upstream-commit: 6b7616690e5370c21e3a760321af6bf4edbabfb6
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
docs/libcurl/curl_easy_setopt.3 | 12 ++++++------
docs/libcurl/symbols-in-versions | 1 +
include/curl/curl.h | 5 +++++
lib/easy.c | 6 ++++++
lib/file.c | 2 +-
lib/ftp.c | 4 ++--
lib/http.c | 3 ++-
lib/telnet.c | 5 +++--
lib/url.c | 28 +++++++++++++++++++++++-----
lib/urldata.h | 5 ++++-
10 files changed, 53 insertions(+), 18 deletions(-)
diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
index cbebfba..17b632f 100644
--- a/docs/libcurl/curl_easy_setopt.3
+++ b/docs/libcurl/curl_easy_setopt.3
@@ -938,12 +938,12 @@ to using the share interface instead! See \fICURLOPT_SHARE\fP and
.IP CURLOPT_BUFFERSIZE
Pass a long specifying your preferred size (in bytes) for the receive buffer
in libcurl. The main point of this would be that the write callback gets
-called more often and with smaller chunks. This is just treated as a request,
-not an order. You cannot be guaranteed to actually get the given size. (Added
-in 7.10)
-
-This size is by default set as big as possible (CURL_MAX_WRITE_SIZE), so it
-only makes sense to use this option if you want it smaller.
+called more often and with smaller chunks. Secondly, for some protocols,
+there's a benefit of having a larger buffer for performance. This is just
+treated as a request, not an order. You cannot be guaranteed to actually get
+the given size. This buffer size is by default \fICURL_MAX_WRITE_SIZE\fP
+(16kB). The maximum buffer size allowed to set is \fICURL_MAX_READ_SIZE\fP
+(512kB). (Added in 7.10)
.IP CURLOPT_PORT
Pass a long specifying what remote port number to connect to, instead of the
one specified in the URL or the default port for the used protocol.
diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
index b0b6232..e2cce4c 100644
--- a/docs/libcurl/symbols-in-versions
+++ b/docs/libcurl/symbols-in-versions
@@ -639,6 +639,7 @@ CURL_LOCK_TYPE_DNS 7.10 - 7.10.2
CURL_LOCK_TYPE_NONE 7.10 - 7.10.2
CURL_LOCK_TYPE_SSL_SESSION 7.10 - 7.10.2
CURL_MAX_HTTP_HEADER 7.19.7
+CURL_MAX_READ_SIZE 7.53.0
CURL_MAX_WRITE_SIZE 7.9.7
CURL_NETRC_IGNORED 7.9.8
CURL_NETRC_OPTIONAL 7.9.8
diff --git a/include/curl/curl.h b/include/curl/curl.h
index 0375a64..8b639fa 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -170,6 +170,11 @@ typedef int (*curl_progress_callback)(void *clientp,
double ultotal,
double ulnow);
+#ifndef CURL_MAX_READ_SIZE
+ /* The maximum receive buffer size configurable via CURLOPT_BUFFERSIZE. */
+#define CURL_MAX_READ_SIZE 524288
+#endif
+
#ifndef CURL_MAX_WRITE_SIZE
/* Tests have proven that 20K is a very bad buffer size for uploads on
Windows, while 16K for some odd reason performed a lot better.
diff --git a/lib/easy.c b/lib/easy.c
index 0e9ba18..5d4d5ae 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -563,6 +563,11 @@ CURL *curl_easy_duphandle(CURL *incurl)
* get setup on-demand in the code, as that would probably decrease
* the likeliness of us forgetting to init a buffer here in the future.
*/
+ outcurl->set.buffer_size = data->set.buffer_size;
+ outcurl->state.buffer = malloc(CURL_BUFSIZE(outcurl->set.buffer_size) + 1);
+ if(!outcurl->state.buffer)
+ goto fail;
+
outcurl->state.headerbuff = malloc(HEADERSIZE);
if(!outcurl->state.headerbuff)
goto fail;
@@ -633,6 +638,7 @@ CURL *curl_easy_duphandle(CURL *incurl)
if(outcurl) {
curl_slist_free_all(outcurl->change.cookielist);
outcurl->change.cookielist = NULL;
+ Curl_safefree(outcurl->state.buffer);
Curl_safefree(outcurl->state.headerbuff);
Curl_safefree(outcurl->change.url);
Curl_safefree(outcurl->change.referer);
diff --git a/lib/file.c b/lib/file.c
index 038bf42..1ad4758 100644
--- a/lib/file.c
+++ b/lib/file.c
@@ -473,7 +473,7 @@ static CURLcode file_do(struct connectdata *conn, bool *done)
date. */
if(data->set.opt_no_body && data->set.include_header && fstated) {
CURLcode result;
- snprintf(buf, sizeof(data->state.buffer),
+ snprintf(buf, CURL_BUFSIZE(data->set.buffer_size),
"Content-Length: %" FORMAT_OFF_T "\r\n", expected_size);
result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
if(result)
diff --git a/lib/ftp.c b/lib/ftp.c
index a9826ce..730b695 100644
--- a/lib/ftp.c
+++ b/lib/ftp.c
@@ -2136,7 +2136,7 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn,
/* we have a time, reformat it */
time_t secs=time(NULL);
/* using the good old yacc/bison yuck */
- snprintf(buf, sizeof(conn->data->state.buffer),
+ snprintf(buf, CURL_BUFSIZE(conn->data->set.buffer_size),
"%04d%02d%02d %02d:%02d:%02d GMT",
year, month, day, hour, minute, second);
/* now, convert this into a time() value: */
@@ -2347,7 +2347,7 @@ static CURLcode ftp_state_size_resp(struct connectdata *conn,
if(instate == FTP_SIZE) {
#ifdef CURL_FTP_HTTPSTYLE_HEAD
if(-1 != filesize) {
- snprintf(buf, sizeof(data->state.buffer),
+ snprintf(buf, CURL_BUFSIZE(data->set.buffer_size),
"Content-Length: %" FORMAT_OFF_T "\r\n", filesize);
result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
if(result)
diff --git a/lib/http.c b/lib/http.c
index 1487fb2..f4368c4 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -247,7 +247,8 @@ static CURLcode http_output_basic(struct connectdata *conn, bool proxy)
pwd = conn->passwd;
}
- snprintf(data->state.buffer, sizeof(data->state.buffer), "%s:%s", user, pwd);
+ snprintf(data->state.buffer, CURL_BUFSIZE(data->set.buffer_size),
+ "%s:%s", user, pwd);
error = Curl_base64_encode(data,
data->state.buffer, strlen(data->state.buffer),
diff --git a/lib/telnet.c b/lib/telnet.c
index 77d8b7b..89452dd 100644
--- a/lib/telnet.c
+++ b/lib/telnet.c
@@ -1421,6 +1421,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
/* Keep on listening and act on events */
while(keepon) {
+ const size_t buf_size = CURL_BUFSIZE(data->set.buffer_size);
waitret = WaitForMultipleObjects(obj_count, objs, FALSE, wait_timeout);
switch(waitret) {
case WAIT_TIMEOUT:
@@ -1455,7 +1456,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
if(!readfile_read)
break;
- if(!ReadFile(stdin_handle, buf, sizeof(data->state.buffer),
+ if(!ReadFile(stdin_handle, buf, buf_size,
&readfile_read, NULL)) {
keepon = FALSE;
code = CURLE_READ_ERROR;
@@ -1474,7 +1475,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
case WAIT_OBJECT_0 + 1:
{
- if(!ReadFile(stdin_handle, buf, sizeof(data->state.buffer),
+ if(!ReadFile(stdin_handle, buf, buf_size,
&readfile_read, NULL)) {
keepon = FALSE;
code = CURLE_READ_ERROR;
diff --git a/lib/url.c b/lib/url.c
index 89958a7..32e7e2e 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -441,6 +441,7 @@ CURLcode Curl_close(struct SessionHandle *data)
}
data->change.url = NULL;
+ Curl_safefree(data->state.buffer);
Curl_safefree(data->state.headerbuff);
Curl_flush_cookies(data, 1);
@@ -612,6 +613,12 @@ CURLcode Curl_open(struct SessionHandle **curl)
/* We do some initial setup here, all those fields that can't be just 0 */
+ data->state.buffer = malloc(BUFSIZE + 1);
+ if(!data->state.buffer) {
+ DEBUGF(fprintf(stderr, "Error: malloc of buffer failed\n"));
+ res = CURLE_OUT_OF_MEMORY;
+ }
+
data->state.headerbuff = malloc(HEADERSIZE);
if(!data->state.headerbuff) {
DEBUGF(fprintf(stderr, "Error: malloc of headerbuff failed\n"));
@@ -642,8 +649,8 @@ CURLcode Curl_open(struct SessionHandle **curl)
if(res) {
Curl_resolver_cleanup(data->state.resolver);
- if(data->state.headerbuff)
- free(data->state.headerbuff);
+ free(data->state.buffer);
+ free(data->state.headerbuff);
Curl_freeset(data);
free(data);
data = NULL;
@@ -1960,9 +1967,20 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
*/
data->set.buffer_size = va_arg(param, long);
- if((data->set.buffer_size> (BUFSIZE -1 )) ||
- (data->set.buffer_size < 1))
- data->set.buffer_size = 0; /* huge internal default */
+ if(data->set.buffer_size > MAX_BUFSIZE)
+ data->set.buffer_size = MAX_BUFSIZE; /* huge internal default */
+ else if(data->set.buffer_size < 1)
+ data->set.buffer_size = BUFSIZE;
+
+ /* Resize only if larger than default buffer size. */
+ if(data->set.buffer_size > BUFSIZE) {
+ data->state.buffer = realloc(data->state.buffer,
+ data->set.buffer_size + 1);
+ if(!data->state.buffer) {
+ DEBUGF(fprintf(stderr, "Error: realloc of buffer failed\n"));
+ result = CURLE_OUT_OF_MEMORY;
+ }
+ }
break;
diff --git a/lib/urldata.h b/lib/urldata.h
index 7431825..a7807cf 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -196,6 +196,9 @@
/* Download buffer size, keep it fairly big for speed reasons */
#undef BUFSIZE
#define BUFSIZE CURL_MAX_WRITE_SIZE
+#undef MAX_BUFSIZE
+#define MAX_BUFSIZE CURL_MAX_READ_SIZE
+#define CURL_BUFSIZE(x) ((x)?(x):(BUFSIZE))
/* Initial size of the buffer to store headers in, it'll be enlarged in case
of need. */
@@ -1174,7 +1177,7 @@ struct UrlState {
char *headerbuff; /* allocated buffer to store headers in */
size_t headersize; /* size of the allocation */
- char buffer[BUFSIZE+1]; /* download buffer */
+ char *buffer; /* download buffer */
char uploadbuffer[BUFSIZE+1]; /* upload buffer */
curl_off_t current_speed; /* the ProgressShow() funcion sets this,
bytes / second */
--
2.14.3
From f175a713c964d351012baaf8c78c1b468cc6aba0 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Mon, 24 Apr 2017 15:33:57 +0200
Subject: [PATCH 02/18] http: use private user:password output buffer
Don't clobber the receive buffer.
Upstream-commit: 94460878cc634b590a7282e3fe60ceafb62d141a
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/http.c | 32 +++++++++++++++++++-------------
1 file changed, 19 insertions(+), 13 deletions(-)
diff --git a/lib/http.c b/lib/http.c
index f4368c4..12e7dc3 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -234,7 +234,8 @@ static CURLcode http_output_basic(struct connectdata *conn, bool proxy)
char **userp;
const char *user;
const char *pwd;
- CURLcode error;
+ CURLcode result;
+ char *out;
if(proxy) {
userp = &conn->allocptr.proxyuserpwd;
@@ -247,27 +248,32 @@ static CURLcode http_output_basic(struct connectdata *conn, bool proxy)
pwd = conn->passwd;
}
- snprintf(data->state.buffer, CURL_BUFSIZE(data->set.buffer_size),
- "%s:%s", user, pwd);
+ out = aprintf("%s:%s", user, pwd);
+ if(!out)
+ return CURLE_OUT_OF_MEMORY;
- error = Curl_base64_encode(data,
- data->state.buffer, strlen(data->state.buffer),
- &authorization, &size);
- if(error)
- return error;
+ result = Curl_base64_encode(data, out, strlen(out), &authorization, &size);
+ if(result)
+ goto fail;
- if(!authorization)
- return CURLE_REMOTE_ACCESS_DENIED;
+ if(!authorization) {
+ result = CURLE_REMOTE_ACCESS_DENIED;
+ goto fail;
+ }
Curl_safefree(*userp);
*userp = aprintf("%sAuthorization: Basic %s\r\n",
proxy?"Proxy-":"",
authorization);
free(authorization);
- if(!*userp)
- return CURLE_OUT_OF_MEMORY;
+ if(!*userp) {
+ result = CURLE_OUT_OF_MEMORY;
+ goto fail;
+ }
- return CURLE_OK;
+ fail:
+ free(out);
+ return result;
}
/* pickoneauth() selects the most favourable authentication method from the
--
2.14.3
From 6ff175806c338223a2a9a69f6ae8ae2b91dc2b56 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Mon, 24 Apr 2017 16:05:46 +0200
Subject: [PATCH 03/18] ftp: use private buffer for temp storage, not receive
buffer
Upstream-commit: 349789e645a306a6ee467ef90a57f6cc306ca92e
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/ftp.c | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/lib/ftp.c b/lib/ftp.c
index 730b695..10a21ce 100644
--- a/lib/ftp.c
+++ b/lib/ftp.c
@@ -2130,17 +2130,17 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn,
/* we got a time. Format should be: "YYYYMMDDHHMMSS[.sss]" where the
last .sss part is optional and means fractions of a second */
int year, month, day, hour, minute, second;
- char *buf = data->state.buffer;
- if(6 == sscanf(buf+4, "%04d%02d%02d%02d%02d%02d",
+ if(6 == sscanf(&data->state.buffer[4], "%04d%02d%02d%02d%02d%02d",
&year, &month, &day, &hour, &minute, &second)) {
/* we have a time, reformat it */
+ char timebuf[24];
time_t secs=time(NULL);
- /* using the good old yacc/bison yuck */
- snprintf(buf, CURL_BUFSIZE(conn->data->set.buffer_size),
+
+ snprintf(timebuf, sizeof(timebuf),
"%04d%02d%02d %02d:%02d:%02d GMT",
year, month, day, hour, minute, second);
/* now, convert this into a time() value: */
- data->info.filetime = (long)curl_getdate(buf, &secs);
+ data->info.filetime = (long)curl_getdate(timebuf, &secs);
}
#ifdef CURL_FTP_HTTPSTYLE_HEAD
@@ -2151,6 +2151,7 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn,
ftpc->file &&
data->set.get_filetime &&
(data->info.filetime>=0) ) {
+ char headerbuf[128];
time_t filetime = (time_t)data->info.filetime;
struct tm buffer;
const struct tm *tm = &buffer;
@@ -2160,7 +2161,7 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn,
return result;
/* format: "Tue, 15 Nov 1994 12:45:26" */
- snprintf(buf, BUFSIZE-1,
+ snprintf(headerbuf, sizeof(headerbuf),
"Last-Modified: %s, %02d %s %4d %02d:%02d:%02d GMT\r\n",
Curl_wkday[tm->tm_wday?tm->tm_wday-1:6],
tm->tm_mday,
@@ -2169,7 +2170,7 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn,
tm->tm_hour,
tm->tm_min,
tm->tm_sec);
- result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
+ result = Curl_client_write(conn, CLIENTWRITE_BOTH, headerbuf, 0);
if(result)
return result;
} /* end of a ridiculous amount of conditionals */
@@ -2347,9 +2348,10 @@ static CURLcode ftp_state_size_resp(struct connectdata *conn,
if(instate == FTP_SIZE) {
#ifdef CURL_FTP_HTTPSTYLE_HEAD
if(-1 != filesize) {
- snprintf(buf, CURL_BUFSIZE(data->set.buffer_size),
+ char clbuf[128];
+ snprintf(clbuf, sizeof(clbuf),
"Content-Length: %" FORMAT_OFF_T "\r\n", filesize);
- result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
+ result = Curl_client_write(conn, CLIENTWRITE_BOTH, clbuf, 0);
if(result)
return result;
}
@@ -2450,7 +2452,6 @@ static CURLcode ftp_state_get_resp(struct connectdata *conn,
CURLcode result = CURLE_OK;
struct SessionHandle *data = conn->data;
struct FTP *ftp = data->state.proto.ftp;
- char *buf = data->state.buffer;
if((ftpcode == 150) || (ftpcode == 125)) {
@@ -2494,6 +2495,7 @@ static CURLcode ftp_state_get_resp(struct connectdata *conn,
*
* Example D above makes this parsing a little tricky */
char *bytes;
+ char *buf = data->state.buffer;
bytes=strstr(buf, " bytes");
if(bytes--) {
long in=(long)(bytes-buf);
--
2.14.3
From b67324919089fc4f9bb7a38a6a31174883a4bc24 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 00:09:22 +0200
Subject: [PATCH 04/18] CURLOPT_BUFFERSIZE: 1024 bytes is now the minimum size
The buffer is needed to receive FTP, HTTP CONNECT responses etc so
already at this size things risk breaking and smaller is certainly not
wise.
Upstream-commit: c2ddc12d6086b522703c8b80a72ab791680f1a28
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/url.c | 15 +++++++++------
lib/urldata.h | 1 +
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/lib/url.c b/lib/url.c
index 32e7e2e..f87dca4 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -1965,15 +1965,17 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
* The application kindly asks for a differently sized receive buffer.
* If it seems reasonable, we'll use it.
*/
- data->set.buffer_size = va_arg(param, long);
+ arg = va_arg(param, long);
- if(data->set.buffer_size > MAX_BUFSIZE)
- data->set.buffer_size = MAX_BUFSIZE; /* huge internal default */
- else if(data->set.buffer_size < 1)
- data->set.buffer_size = BUFSIZE;
+ if(arg > MAX_BUFSIZE)
+ arg = MAX_BUFSIZE; /* huge internal default */
+ else if(arg < 1)
+ arg = BUFSIZE;
+ else if(arg < MIN_BUFSIZE)
+ arg = BUFSIZE;
/* Resize only if larger than default buffer size. */
- if(data->set.buffer_size > BUFSIZE) {
+ if(arg > BUFSIZE) {
data->state.buffer = realloc(data->state.buffer,
data->set.buffer_size + 1);
if(!data->state.buffer) {
@@ -1981,6 +1983,7 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
result = CURLE_OUT_OF_MEMORY;
}
}
+ data->set.buffer_size = arg;
break;
diff --git a/lib/urldata.h b/lib/urldata.h
index a7807cf..cd96e8f 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -198,6 +198,7 @@
#define BUFSIZE CURL_MAX_WRITE_SIZE
#undef MAX_BUFSIZE
#define MAX_BUFSIZE CURL_MAX_READ_SIZE
+#define MIN_BUFSIZE 1024
#define CURL_BUFSIZE(x) ((x)?(x):(BUFSIZE))
/* Initial size of the buffer to store headers in, it'll be enlarged in case
--
2.14.3
From 9798012315c087168c5a4a1dc56eacfe82c69626 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 00:15:28 +0200
Subject: [PATCH 05/18] file: use private buffer for C-L output
... instead of clobbering the download buffer.
Upstream-commit: 7c312f84ea930d89c0f0f774b50032c4f9ae30e4
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/file.c | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/file.c b/lib/file.c
index 1ad4758..b6bf18e 100644
--- a/lib/file.c
+++ b/lib/file.c
@@ -473,9 +473,10 @@ static CURLcode file_do(struct connectdata *conn, bool *done)
date. */
if(data->set.opt_no_body && data->set.include_header && fstated) {
CURLcode result;
- snprintf(buf, CURL_BUFSIZE(data->set.buffer_size),
+ char header[80];
+ snprintf(header, sizeof(header),
"Content-Length: %" FORMAT_OFF_T "\r\n", expected_size);
- result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
+ result = Curl_client_write(conn, CLIENTWRITE_BOTH, header, 0);
if(result)
return result;
@@ -493,7 +494,7 @@ static CURLcode file_do(struct connectdata *conn, bool *done)
return result;
/* format: "Tue, 15 Nov 1994 12:45:26 GMT" */
- snprintf(buf, BUFSIZE-1,
+ snprintf(header, sizeof(header),
"Last-Modified: %s, %02d %s %4d %02d:%02d:%02d GMT\r\n",
Curl_wkday[tm->tm_wday?tm->tm_wday-1:6],
tm->tm_mday,
--
2.14.3
From f4868e737e9f8d719cb9897506da2c7f92dfd87d Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 00:16:10 +0200
Subject: [PATCH 06/18] buffer_size: make sure it always has the correct size
Removes the need for CURL_BUFSIZE
Upstream-commit: f535f4f5fc6cbdce1aec5a3481cec37369dca468
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/easy.c | 2 +-
lib/telnet.c | 2 +-
lib/url.c | 2 ++
lib/urldata.h | 1 -
4 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/easy.c b/lib/easy.c
index 5d4d5ae..9cad5f1 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -564,7 +564,7 @@ CURL *curl_easy_duphandle(CURL *incurl)
* the likeliness of us forgetting to init a buffer here in the future.
*/
outcurl->set.buffer_size = data->set.buffer_size;
- outcurl->state.buffer = malloc(CURL_BUFSIZE(outcurl->set.buffer_size) + 1);
+ outcurl->state.buffer = malloc(outcurl->set.buffer_size + 1);
if(!outcurl->state.buffer)
goto fail;
diff --git a/lib/telnet.c b/lib/telnet.c
index 89452dd..e43b423 100644
--- a/lib/telnet.c
+++ b/lib/telnet.c
@@ -1421,7 +1421,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
/* Keep on listening and act on events */
while(keepon) {
- const size_t buf_size = CURL_BUFSIZE(data->set.buffer_size);
+ const size_t buf_size = (DWORD)data->set.buffer_size;
waitret = WaitForMultipleObjects(obj_count, objs, FALSE, wait_timeout);
switch(waitret) {
case WAIT_TIMEOUT:
diff --git a/lib/url.c b/lib/url.c
index f87dca4..81de7c2 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -577,6 +577,8 @@ CURLcode Curl_init_userdefined(struct UserDefined *set)
set->tcp_keepintvl = 60;
set->tcp_keepidle = 60;
+ set->buffer_size = BUFSIZE;
+
return res;
}
diff --git a/lib/urldata.h b/lib/urldata.h
index cd96e8f..fbe69c2 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -199,7 +199,6 @@
#undef MAX_BUFSIZE
#define MAX_BUFSIZE CURL_MAX_READ_SIZE
#define MIN_BUFSIZE 1024
-#define CURL_BUFSIZE(x) ((x)?(x):(BUFSIZE))
/* Initial size of the buffer to store headers in, it'll be enlarged in case
of need. */
--
2.14.3
From 43d9e7acc44c9849f85882d2ec18c2a609f90809 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 00:48:56 +0200
Subject: [PATCH 07/18] http: don't clobber the receive buffer for timecond
Upstream-commit: 87eb8d5b30ce2adfe2673cc0b1abf6ca68891cc4
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/http.c | 37 +++++++++++++++++++------------------
1 file changed, 19 insertions(+), 18 deletions(-)
diff --git a/lib/http.c b/lib/http.c
index 12e7dc3..6ecdda2 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -1607,9 +1607,10 @@ CURLcode Curl_add_timecondition(struct SessionHandle *data,
Curl_send_buffer *req_buffer)
{
const struct tm *tm;
- char *buf = data->state.buffer;
CURLcode result = CURLE_OK;
struct tm keeptime;
+ char datestr[80];
+ const char *condp;
result = Curl_gmtime(data->set.timevalue, &keeptime);
if(result) {
@@ -1618,6 +1619,19 @@ CURLcode Curl_add_timecondition(struct SessionHandle *data,
}
tm = &keeptime;
+ switch(data->set.timecondition) {
+ default:
+ case CURL_TIMECOND_IFMODSINCE:
+ condp = "If-Modified-Since";
+ break;
+ case CURL_TIMECOND_IFUNMODSINCE:
+ condp = "If-Unmodified-Since";
+ break;
+ case CURL_TIMECOND_LASTMOD:
+ condp = "Last-Modified";
+ break;
+ }
+
/* The If-Modified-Since header family should have their times set in
* GMT as RFC2616 defines: "All HTTP date/time stamps MUST be
* represented in Greenwich Mean Time (GMT), without exception. For the
@@ -1626,8 +1640,9 @@ CURLcode Curl_add_timecondition(struct SessionHandle *data,
*/
/* format: "Tue, 15 Nov 1994 12:45:26 GMT" */
- snprintf(buf, BUFSIZE-1,
- "%s, %02d %s %4d %02d:%02d:%02d GMT",
+ snprintf(datestr, sizeof(datestr),
+ "%s: %s, %02d %s %4d %02d:%02d:%02d GMT\r\n",
+ condp,
Curl_wkday[tm->tm_wday?tm->tm_wday-1:6],
tm->tm_mday,
Curl_month[tm->tm_mon],
@@ -1636,21 +1651,7 @@ CURLcode Curl_add_timecondition(struct SessionHandle *data,
tm->tm_min,
tm->tm_sec);
- switch(data->set.timecondition) {
- case CURL_TIMECOND_IFMODSINCE:
- default:
- result = Curl_add_bufferf(req_buffer,
- "If-Modified-Since: %s\r\n", buf);
- break;
- case CURL_TIMECOND_IFUNMODSINCE:
- result = Curl_add_bufferf(req_buffer,
- "If-Unmodified-Since: %s\r\n", buf);
- break;
- case CURL_TIMECOND_LASTMOD:
- result = Curl_add_bufferf(req_buffer,
- "Last-Modified: %s\r\n", buf);
- break;
- }
+ result = Curl_add_buffer(req_buffer, datestr, strlen(datestr));
return result;
}
--
2.17.2
From 91ca8ea3e4e984070ef07dbc1a7258ced3b6982a Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 00:50:04 +0200
Subject: [PATCH 08/18] pingpong: use the set buffer size
Upstream-commit: b8191e975faa7810ed3d858205b0b3f0d297f0b2
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/pingpong.c | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/lib/pingpong.c b/lib/pingpong.c
index 16b4ad3..c4666fe 100644
--- a/lib/pingpong.c
+++ b/lib/pingpong.c
@@ -309,7 +309,8 @@ CURLcode Curl_pp_readresp(curl_socket_t sockfd,
keepon=TRUE;
- while((pp->nread_resp<BUFSIZE) && (keepon && !result)) {
+ while((pp->nread_resp < (size_t)data->set.buffer_size) &&
+ (keepon && !result)) {
if(pp->cache) {
/* we had data in the "cache", copy that instead of doing an actual
@@ -319,7 +320,7 @@ CURLcode Curl_pp_readresp(curl_socket_t sockfd,
* it would have been populated with something of size int to begin
* with, even though its datatype may be larger than an int.
*/
- DEBUGASSERT((ptr+pp->cache_size) <= (buf+BUFSIZE+1));
+ DEBUGASSERT((ptr+pp->cache_size) <= (buf+data->set.buffer_size+1));
memcpy(ptr, pp->cache, pp->cache_size);
gotbytes = (ssize_t)pp->cache_size;
free(pp->cache); /* free the cache */
@@ -332,8 +333,10 @@ CURLcode Curl_pp_readresp(curl_socket_t sockfd,
enum protection_level prot = conn->data_prot;
conn->data_prot = PROT_CLEAR;
#endif
- DEBUGASSERT((ptr+BUFSIZE-pp->nread_resp) <= (buf+BUFSIZE+1));
- res = Curl_read(conn, sockfd, ptr, BUFSIZE-pp->nread_resp,
+ DEBUGASSERT((ptr + data->set.buffer_size - pp->nread_resp) <=
+ (buf + data->set.buffer_size + 1));
+ res = Curl_read(conn, sockfd, ptr,
+ data->set.buffer_size - pp->nread_resp,
&gotbytes);
#if defined(HAVE_KRB4) || defined(HAVE_GSSAPI)
DEBUGASSERT(prot > PROT_NONE && prot < PROT_LAST);
@@ -430,7 +433,7 @@ CURLcode Curl_pp_readresp(curl_socket_t sockfd,
}
else if(keepon) {
- if((perline == gotbytes) && (gotbytes > BUFSIZE/2)) {
+ if((perline == gotbytes) && (gotbytes > data->set.buffer_size/2)) {
/* We got an excessive line without newlines and we need to deal
with it. We keep the first bytes of the line then we throw
away the rest. */
@@ -442,7 +445,7 @@ CURLcode Curl_pp_readresp(curl_socket_t sockfd,
interested in the first piece */
clipamount = 40;
}
- else if(pp->nread_resp > BUFSIZE/2) {
+ else if(pp->nread_resp > (size_t)data->set.buffer_size/2) {
/* We got a large chunk of data and there's potentially still
trailing data to take care of, so we put any such part in the
"cache", clear the buffer to make space and restart. */
--
2.17.2
From 3bc2bc2f1c5e3afd7a7cbd208d97fbb37f2dfc5f Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 00:50:21 +0200
Subject: [PATCH 09/18] failf: use private buffer, don't clobber receive buffer
Upstream-commit: f2fadf490f66ad364f5a6f0356d626dda5f9a82f
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/sendf.c | 15 +++++++--------
1 file changed, 7 insertions(+), 8 deletions(-)
diff --git a/lib/sendf.c b/lib/sendf.c
index c64d686..403025b 100644
--- a/lib/sendf.c
+++ b/lib/sendf.c
@@ -155,21 +155,20 @@ void Curl_failf(struct SessionHandle *data, const char *fmt, ...)
{
va_list ap;
size_t len;
+ char error[CURL_ERROR_SIZE + 2];
va_start(ap, fmt);
- vsnprintf(data->state.buffer, BUFSIZE, fmt, ap);
+ vsnprintf(error, CURL_ERROR_SIZE, fmt, ap);
+ len = strlen(error);
if(data->set.errorbuffer && !data->state.errorbuf) {
- snprintf(data->set.errorbuffer, CURL_ERROR_SIZE, "%s", data->state.buffer);
+ strcpy(data->set.errorbuffer, error);
data->state.errorbuf = TRUE; /* wrote error string */
}
if(data->set.verbose) {
- len = strlen(data->state.buffer);
- if(len < BUFSIZE - 1) {
- data->state.buffer[len] = '\n';
- data->state.buffer[++len] = '\0';
- }
- Curl_debug(data, CURLINFO_TEXT, data->state.buffer, len, NULL);
+ error[len] = '\n';
+ error[++len] = '\0';
+ Curl_debug(data, CURLINFO_TEXT, error, len, NULL);
}
va_end(ap);
--
2.17.2
From 0a1107c2ad898e837f0aa4742c1b17ef186a3243 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 00:50:50 +0200
Subject: [PATCH 10/18] transfer: fix minor buffer_size mistake
Upstream-commit: 40a074f255de90003ab753f5d0ad61b74c62ca9b
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/transfer.c | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/lib/transfer.c b/lib/transfer.c
index dff6838..9b932de 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -385,8 +385,7 @@ static CURLcode readwrite_data(struct SessionHandle *data,
/* This is where we loop until we have read everything there is to
read or we get a CURLE_AGAIN */
do {
- size_t buffersize = data->set.buffer_size?
- data->set.buffer_size : BUFSIZE;
+ size_t buffersize = data->set.buffer_size;
size_t bytestoread = buffersize;
if(k->size != -1 && !k->header) {
--
2.17.2
From 15408839da1b768974650bdf187c749432fdaefa Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 01:03:17 +0200
Subject: [PATCH 11/18] http-proxy: use a dedicated CONNECT response buffer
To make it suitably independent of the receive buffer and its flexible
size.
Upstream-commit: 0cab3a394a190c1cdf900cf2887ccdd6a7f213ef
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/http_proxy.c | 69 +++++++++++++++++++++++++++++++++++-------------
lib/url.c | 1 +
lib/urldata.h | 3 ++-
3 files changed, 53 insertions(+), 20 deletions(-)
diff --git a/lib/http_proxy.c b/lib/http_proxy.c
index 4ab280f..13fb228 100644
--- a/lib/http_proxy.c
+++ b/lib/http_proxy.c
@@ -83,20 +83,13 @@ CURLcode Curl_proxy_connect(struct connectdata *conn)
return CURLE_OK;
}
-/*
- * Curl_proxyCONNECT() requires that we're connected to a HTTP proxy. This
- * function will issue the necessary commands to get a seamless tunnel through
- * this proxy. After that, the socket can be used just as a normal socket.
- *
- * 'blocking' set to TRUE means that this function will do the entire CONNECT
- * + response in a blocking fashion. Should be avoided!
- */
+#define CONNECT_BUFFER_SIZE 16384
-CURLcode Curl_proxyCONNECT(struct connectdata *conn,
- int sockindex,
- const char *hostname,
- unsigned short remote_port,
- bool blocking)
+CURLcode CONNECT(struct connectdata *conn,
+ int sockindex,
+ const char *hostname,
+ unsigned short remote_port,
+ bool blocking)
{
int subversion=0;
struct SessionHandle *data=conn->data;
@@ -253,14 +246,19 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
char *ptr;
char *line_start;
- ptr=data->state.buffer;
+ ptr = conn->connect_buffer;
line_start = ptr;
nread=0;
perline=0;
keepon=TRUE;
- while((nread<BUFSIZE) && (keepon && !error)) {
+ while((nread < (size_t)CONNECT_BUFFER_SIZE) && (keepon && !error)) {
+
+ if(ptr >= &conn->connect_buffer[CONNECT_BUFFER_SIZE]) {
+ failf(data, "CONNECT response too large!");
+ return CURLE_RECV_ERROR;
+ }
check = Curl_timeleft(data, NULL, TRUE);
if(check <= 0) {
@@ -279,8 +277,8 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
case 0: /* timeout */
break;
default:
- DEBUGASSERT(ptr+BUFSIZE-nread <= data->state.buffer+BUFSIZE+1);
- result = Curl_read(conn, tunnelsocket, ptr, BUFSIZE-nread,
+ result = Curl_read(conn, tunnelsocket, ptr,
+ CONNECT_BUFFER_SIZE - nread,
&gotbytes);
if(result==CURLE_AGAIN)
continue; /* go loop yourself */
@@ -313,7 +311,7 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
/* This means we are currently ignoring a response-body */
nread = 0; /* make next read start over in the read buffer */
- ptr=data->state.buffer;
+ ptr = conn->connect_buffer;
if(cl) {
/* A Content-Length based body: simply count down the counter
and make sure to break out of the loop when we're done! */
@@ -386,7 +384,7 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
/* end of response-headers from the proxy */
nread = 0; /* make next read start over in the read
buffer */
- ptr=data->state.buffer;
+ ptr = conn->connect_buffer;
if((407 == k->httpcode) && !data->state.authproblem) {
/* If we get a 407 response code with content length
when we have no auth problem, we must ignore the
@@ -585,4 +583,37 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
data->req.ignorebody = FALSE; /* put it (back) to non-ignore state */
return CURLE_OK;
}
+
+/*
+ * Curl_proxyCONNECT() requires that we're connected to a HTTP proxy. This
+ * function will issue the necessary commands to get a seamless tunnel through
+ * this proxy. After that, the socket can be used just as a normal socket.
+ *
+ * 'blocking' set to TRUE means that this function will do the entire CONNECT
+ * + response in a blocking fashion. Should be avoided!
+ */
+
+CURLcode Curl_proxyCONNECT(struct connectdata *conn,
+ int sockindex,
+ const char *hostname,
+ unsigned short remote_port,
+ bool blocking)
+{
+ CURLcode result;
+ if(TUNNEL_INIT == conn->tunnel_state[sockindex]) {
+ if(!conn->connect_buffer) {
+ conn->connect_buffer = malloc(CONNECT_BUFFER_SIZE);
+ if(!conn->connect_buffer)
+ return CURLE_OUT_OF_MEMORY;
+ }
+ }
+ result = CONNECT(conn, sockindex, hostname, remote_port, blocking);
+
+ if(result || (TUNNEL_COMPLETE == conn->tunnel_state[sockindex]))
+ Curl_safefree(conn->connect_buffer);
+
+ return result;
+}
+
+
#endif /* CURL_DISABLE_PROXY */
diff --git a/lib/url.c b/lib/url.c
index 00fd15d..b86243b 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -2523,6 +2523,7 @@ static void conn_free(struct connectdata *conn)
Curl_safefree(conn->host.rawalloc); /* host name buffer */
Curl_safefree(conn->proxy.rawalloc); /* proxy name buffer */
Curl_safefree(conn->master_buffer);
+ Curl_safefree(conn->connect_buffer);
Curl_llist_destroy(conn->send_pipe, NULL);
Curl_llist_destroy(conn->recv_pipe, NULL);
diff --git a/lib/urldata.h b/lib/urldata.h
index fbe69c2..c362c58 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1018,7 +1018,8 @@ struct connectdata {
TUNNEL_COMPLETE /* CONNECT response received completely */
} tunnel_state[2]; /* two separate ones to allow FTP */
- struct connectbundle *bundle; /* The bundle we are member of */
+ struct connectbundle *bundle; /* The bundle we are member of */
+ char *connect_buffer; /* for CONNECT business */
};
/* The end of connectdata. */
--
2.17.2
From 1122ecc4b522f24d466bcc2e658dd098a32d982f Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 14:37:06 +0200
Subject: [PATCH 12/18] upload: UPLOAD_BUFSIZE is now for the upload buffer
Upstream-commit: 89cf6f38d2f525cbc8537a60061f5f37bb2f35f7
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/file.c | 2 +-
lib/transfer.c | 2 +-
lib/urldata.h | 6 +++++-
3 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/lib/file.c b/lib/file.c
index 98536cd..2f2e466 100644
--- a/lib/file.c
+++ b/lib/file.c
@@ -364,7 +364,7 @@ static CURLcode file_upload(struct connectdata *conn)
while(res == CURLE_OK) {
int readcount;
- res = Curl_fillreadbuffer(conn, BUFSIZE, &readcount);
+ res = Curl_fillreadbuffer(conn, (int)data->set.buffer_size, &readcount);
if(res)
break;
diff --git a/lib/transfer.c b/lib/transfer.c
index 9b932de..b707f8a 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -835,7 +835,7 @@ static CURLcode readwrite_upload(struct SessionHandle *data,
sending_http_headers = FALSE;
}
- result = Curl_fillreadbuffer(conn, BUFSIZE, &fillcount);
+ result = Curl_fillreadbuffer(conn, UPLOAD_BUFSIZE, &fillcount);
if(result)
return result;
diff --git a/lib/urldata.h b/lib/urldata.h
index c362c58..afe37c0 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -200,6 +200,10 @@
#define MAX_BUFSIZE CURL_MAX_READ_SIZE
#define MIN_BUFSIZE 1024
+/* The upload buffer size, should not be smaller than CURL_MAX_WRITE_SIZE, as
+ it needs to hold a full buffer as could be sent in a write callback */
+#define UPLOAD_BUFSIZE CURL_MAX_WRITE_SIZE
+
/* Initial size of the buffer to store headers in, it'll be enlarged in case
of need. */
#define HEADERSIZE 256
@@ -1179,7 +1183,7 @@ struct UrlState {
size_t headersize; /* size of the allocation */
char *buffer; /* download buffer */
- char uploadbuffer[BUFSIZE+1]; /* upload buffer */
+ char uploadbuffer[UPLOAD_BUFSIZE+1]; /* upload buffer */
curl_off_t current_speed; /* the ProgressShow() funcion sets this,
bytes / second */
bool this_is_a_follow; /* this is a followed Location: request */
--
2.17.2
From 4666a96fce7a2aa8c294cef7aec23e39a3d49673 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 14:37:45 +0200
Subject: [PATCH 13/18] krb5: use private buffer for temp string, not receive
buffer
Upstream-commit: c79f4908d461cecaa9976099dcbb8a63b351f19e
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/krb5.c | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/lib/krb5.c b/lib/krb5.c
index 1e99c70..e4eb0b0 100644
--- a/lib/krb5.c
+++ b/lib/krb5.c
@@ -174,6 +174,7 @@ krb5_auth(void *app_data, struct connectdata *conn)
gss_ctx_id_t *context = app_data;
struct gss_channel_bindings_struct chan;
size_t base64_sz = 0;
+ char *stringp;
if(getsockname(conn->sock[FIRSTSOCKET],
(struct sockaddr *)LOCAL_ADDR, &l) < 0)
@@ -205,16 +206,19 @@ krb5_auth(void *app_data, struct connectdata *conn)
return -1;
}
- input_buffer.value = data->state.buffer;
- input_buffer.length = snprintf(input_buffer.value, BUFSIZE, "%s@%s",
- service, host);
+ stringp = aprintf("%s@%s", service, host);
+ if(!stringp)
+ return -2;
+
+ input_buffer.value = stringp;
+ input_buffer.length = strlen(stringp);
maj = gss_import_name(&min, &input_buffer, GSS_C_NT_HOSTBASED_SERVICE,
&gssname);
+ free(stringp);
if(maj != GSS_S_COMPLETE) {
gss_release_name(&min, &gssname);
if(service == srv_host) {
- Curl_failf(data, "Error importing service name %s",
- input_buffer.value);
+ Curl_failf(data, "Error importing service name %s@%s", service, host);
return AUTH_ERROR;
}
service = srv_host;
--
2.17.2
From 4725e3cbe9af074304a4735b6ac667a154081b99 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 14:38:34 +0200
Subject: [PATCH 14/18] buffer: use data->set.buffer_size instead of BUFSIZE
... to properly use the dynamically set buffer size!
Upstream-commit: e40e9d7f0decc799e3ccfe2c418632f8bb52031a
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/file.c | 4 ++--
lib/ftp.c | 5 +++--
lib/http.c | 7 ++++---
lib/sendf.c | 9 ++++-----
lib/smtp.c | 2 +-
lib/ssh.c | 5 +++--
lib/telnet.c | 10 +++++-----
lib/transfer.c | 4 +---
8 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/lib/file.c b/lib/file.c
index 2f2e466..ddac28f 100644
--- a/lib/file.c
+++ b/lib/file.c
@@ -558,8 +558,8 @@ static CURLcode file_do(struct connectdata *conn, bool *done)
while(res == CURLE_OK) {
/* Don't fill a whole buffer if we want less than all data */
size_t bytestoread =
- (expected_size < CURL_OFF_T_C(BUFSIZE) - CURL_OFF_T_C(1)) ?
- curlx_sotouz(expected_size) : BUFSIZE - 1;
+ (expected_size < data->set.buffer_size) ?
+ curlx_sotouz(expected_size) : (size_t)data->set.buffer_size;
nread = read(fd, buf, bytestoread);
diff --git a/lib/ftp.c b/lib/ftp.c
index c40d7e4..3a8b4b2 100644
--- a/lib/ftp.c
+++ b/lib/ftp.c
@@ -1646,8 +1646,9 @@ static CURLcode ftp_state_ul_setup(struct connectdata *conn,
curl_off_t passed=0;
do {
size_t readthisamountnow =
- (data->state.resume_from - passed > CURL_OFF_T_C(BUFSIZE)) ?
- BUFSIZE : curlx_sotouz(data->state.resume_from - passed);
+ (data->state.resume_from - passed > data->set.buffer_size) ?
+ (size_t)data->set.buffer_size :
+ curlx_sotouz(data->state.resume_from - passed);
size_t actuallyread =
conn->fread_func(data->state.buffer, 1, readthisamountnow,
diff --git a/lib/http.c b/lib/http.c
index 6ecdda2..665447b 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -1061,7 +1061,7 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
buffer is using this size.
*/
- sendsize= (size > CURL_MAX_WRITE_SIZE)?CURL_MAX_WRITE_SIZE:size;
+ sendsize = CURLMIN(size, CURL_MAX_WRITE_SIZE);
/* OpenSSL is very picky and we must send the SAME buffer pointer to the
library when we attempt to re-send this buffer. Sending the same data
@@ -2036,8 +2036,9 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
curl_off_t passed=0;
do {
size_t readthisamountnow =
- (data->state.resume_from - passed > CURL_OFF_T_C(BUFSIZE)) ?
- BUFSIZE : curlx_sotouz(data->state.resume_from - passed);
+ (data->state.resume_from - passed > data->set.buffer_size) ?
+ (size_t)data->set.buffer_size :
+ curlx_sotouz(data->state.resume_from - passed);
size_t actuallyread =
data->set.fread_func(data->state.buffer, 1, readthisamountnow,
diff --git a/lib/sendf.c b/lib/sendf.c
index 403025b..fb0e6e8 100644
--- a/lib/sendf.c
+++ b/lib/sendf.c
@@ -528,7 +528,8 @@ CURLcode Curl_read(struct connectdata *conn, /* connection data */
ssize_t nread = 0;
size_t bytesfromsocket = 0;
char *buffertofill = NULL;
- bool pipelining = (conn->data->multi &&
+ struct SessionHandle *data = conn->data;
+ bool pipelining = (data->multi &&
Curl_multi_canPipeline(conn->data->multi)) ? TRUE : FALSE;
/* Set 'num' to 0 or 1, depending on which socket that has been sent here.
@@ -554,13 +555,11 @@ CURLcode Curl_read(struct connectdata *conn, /* connection data */
}
/* If we come here, it means that there is no data to read from the buffer,
* so we read from the socket */
- bytesfromsocket = CURLMIN(sizerequested, BUFSIZE * sizeof (char));
+ bytesfromsocket = CURLMIN(sizerequested, (size_t)data->set.buffer_size);
buffertofill = conn->master_buffer;
}
else {
- bytesfromsocket = CURLMIN((long)sizerequested,
- conn->data->set.buffer_size ?
- conn->data->set.buffer_size : BUFSIZE);
+ bytesfromsocket = CURLMIN(sizerequested, (size_t)data->set.buffer_size);
buffertofill = buf;
}
diff --git a/lib/smtp.c b/lib/smtp.c
index d2d4aeb..ccc2dc7 100644
--- a/lib/smtp.c
+++ b/lib/smtp.c
@@ -1665,7 +1665,7 @@ CURLcode Curl_smtp_escape_eob(struct connectdata *conn, ssize_t nread)
/* Do we need to allocate the scatch buffer? */
if(!data->state.scratch) {
- data->state.scratch = malloc(2 * BUFSIZE);
+ data->state.scratch = malloc(2 * data->set.buffer_size);
if(!data->state.scratch) {
failf (data, "Failed to alloc scratch buffer!");
diff --git a/lib/ssh.c b/lib/ssh.c
index 589d4a3..169ef1b 100644
--- a/lib/ssh.c
+++ b/lib/ssh.c
@@ -1711,8 +1711,9 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block)
curl_off_t passed=0;
do {
size_t readthisamountnow =
- (data->state.resume_from - passed > CURL_OFF_T_C(BUFSIZE)) ?
- BUFSIZE : curlx_sotouz(data->state.resume_from - passed);
+ (data->state.resume_from - passed > data->set.buffer_size) ?
+ data->set.buffer_size :
+ curlx_sotouz(data->state.resume_from - passed);
size_t actuallyread =
conn->fread_func(data->state.buffer, 1, readthisamountnow,
diff --git a/lib/telnet.c b/lib/telnet.c
index e43b423..4492514 100644
--- a/lib/telnet.c
+++ b/lib/telnet.c
@@ -1429,7 +1429,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
for(;;) {
if(obj_count == 1) {
/* read from user-supplied method */
- code = (int)conn->fread_func(buf, 1, BUFSIZE - 1, conn->fread_in);
+ code = (int)conn->fread_func(buf, 1, buf_size, conn->fread_in);
if(code == CURL_READFUNC_ABORT) {
keepon = FALSE;
code = CURLE_READ_ERROR;
@@ -1502,7 +1502,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
}
if(events.lNetworkEvents & FD_READ) {
/* read data from network */
- code = Curl_read(conn, sockfd, buf, BUFSIZE - 1, &nread);
+ code = Curl_read(conn, sockfd, buf, data->set.buffer_size, &nread);
/* read would've blocked. Loop again */
if(code == CURLE_AGAIN)
break;
@@ -1591,7 +1591,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
default: /* read! */
if(pfd[0].revents & POLLIN) {
/* read data from network */
- code = Curl_read(conn, sockfd, buf, BUFSIZE - 1, &nread);
+ code = Curl_read(conn, sockfd, buf, data->set.buffer_size, &nread);
/* read would've blocked. Loop again */
if(code == CURLE_AGAIN)
break;
@@ -1627,12 +1627,12 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
nread = 0;
if(poll_cnt == 2) {
if(pfd[1].revents & POLLIN) { /* read from in file */
- nread = read(pfd[1].fd, buf, BUFSIZE - 1);
+ nread = read(pfd[1].fd, buf, data->set.buffer_size);
}
}
else {
/* read from user-supplied method */
- nread = (int)conn->fread_func(buf, 1, BUFSIZE - 1, conn->fread_in);
+ nread = (int)conn->fread_func(buf, 1, data->set.buffer_size, conn->fread_in);
if(nread == CURL_READFUNC_ABORT) {
keepon = FALSE;
break;
diff --git a/lib/transfer.c b/lib/transfer.c
index b707f8a..77f0937 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -626,8 +626,6 @@ static CURLcode readwrite_data(struct SessionHandle *data,
excess = (size_t)(k->bytecount + nread - k->maxdownload);
if(excess > 0 && !k->ignorebody) {
if(conn->data->multi && Curl_multi_canPipeline(conn->data->multi)) {
- /* The 'excess' amount below can't be more than BUFSIZE which
- always will fit in a size_t */
infof(data,
"Rewinding stream by : %zu"
" bytes on url %s (size = %" FORMAT_OFF_T
@@ -880,7 +878,7 @@ static CURLcode readwrite_upload(struct SessionHandle *data,
#endif
(data->set.crlf))) {
if(data->state.scratch == NULL)
- data->state.scratch = malloc(2*BUFSIZE);
+ data->state.scratch = malloc(2 * data->set.buffer_size);
if(data->state.scratch == NULL) {
failf (data, "Failed to alloc scratch buffer!");
return CURLE_OUT_OF_MEMORY;
--
2.17.2
From a8725233e1af133dd1d89c89b30ef4396c3f2e5b Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 15:19:19 +0200
Subject: [PATCH 15/18] sendf: remove use of BUFSIZE from debug data
conversions
The buffer can have other sizes.
Upstream-commit: 7ee52c25f35816968b86d32eaef70ab743a457d4
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/sendf.c | 51 ++++++++++++++++++++++++++-------------------------
1 file changed, 26 insertions(+), 25 deletions(-)
diff --git a/lib/sendf.c b/lib/sendf.c
index fb0e6e8..bad4683 100644
--- a/lib/sendf.c
+++ b/lib/sendf.c
@@ -584,21 +584,19 @@ static int showit(struct SessionHandle *data, curl_infotype type,
{
static const char s_infotype[CURLINFO_END][3] = {
"* ", "< ", "> ", "{ ", "} ", "{ ", "} " };
+ int rc = 0;
#ifdef CURL_DOES_CONVERSIONS
- char buf[BUFSIZE+1];
+ char *buf = NULL;
size_t conv_size = 0;
switch(type) {
case CURLINFO_HEADER_OUT:
- /* assume output headers are ASCII */
- /* copy the data into my buffer so the original is unchanged */
- if(size > BUFSIZE) {
- size = BUFSIZE; /* truncate if necessary */
- buf[BUFSIZE] = '\0';
- }
+ buf = Curl_memdup(ptr, size);
+ if(!buf)
+ return 1;
conv_size = size;
- memcpy(buf, ptr, size);
+
/* Special processing is needed for this block if it
* contains both headers and data (separated by CRLFCRLF).
* We want to convert just the headers, leaving the data as-is.
@@ -626,26 +624,29 @@ static int showit(struct SessionHandle *data, curl_infotype type,
#endif /* CURL_DOES_CONVERSIONS */
if(data->set.fdebug)
- return (*data->set.fdebug)(data, type, ptr, size,
- data->set.debugdata);
-
- switch(type) {
- case CURLINFO_TEXT:
- case CURLINFO_HEADER_OUT:
- case CURLINFO_HEADER_IN:
- fwrite(s_infotype[type], 2, 1, data->set.err);
- fwrite(ptr, size, 1, data->set.err);
+ rc = (*data->set.fdebug)(data, type, ptr, size, data->set.debugdata);
+ else {
+ switch(type) {
+ case CURLINFO_TEXT:
+ case CURLINFO_HEADER_OUT:
+ case CURLINFO_HEADER_IN:
+ fwrite(s_infotype[type], 2, 1, data->set.err);
+ fwrite(ptr, size, 1, data->set.err);
#ifdef CURL_DOES_CONVERSIONS
- if(size != conv_size) {
- /* we had untranslated data so we need an explicit newline */
- fwrite("\n", 1, 1, data->set.err);
- }
+ if(size != conv_size) {
+ /* we had untranslated data so we need an explicit newline */
+ fwrite("\n", 1, 1, data->set.err);
+ }
#endif
- break;
- default: /* nada */
- break;
+ break;
+ default: /* nada */
+ break;
+ }
}
- return 0;
+#ifdef CURL_DOES_CONVERSIONS
+ free(buf);
+#endif
+ return rc;
}
int Curl_debug(struct SessionHandle *data, curl_infotype type,
--
2.17.2
From a600c050d0fcaa5bff272f47b9be0a653a4071c6 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 25 Apr 2017 15:31:14 +0200
Subject: [PATCH 16/18] BUFSIZE: rename to READBUFFER_*, make separate
MASTERBUF_SIZE
Upstream-commit: e3ed5cb380e615e91d99b09da9f0ead0eaf3e0b5
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/multi.c | 4 ++--
lib/sendf.c | 2 +-
lib/url.c | 25 ++++++++++++++-----------
lib/urldata.h | 10 +++-------
4 files changed, 20 insertions(+), 21 deletions(-)
diff --git a/lib/multi.c b/lib/multi.c
index 3029fa6..39a0938 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -1452,7 +1452,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
/* calculate upload rate-limitation timeout. */
buffersize = (int)(data->set.buffer_size ?
- data->set.buffer_size : BUFSIZE);
+ data->set.buffer_size : MASTERBUF_SIZE);
timeout_ms = Curl_sleep_time(data->set.max_send_speed,
data->progress.ulspeed, buffersize);
Curl_expire(data, timeout_ms);
@@ -1468,7 +1468,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
/* Calculate download rate-limitation timeout. */
buffersize = (int)(data->set.buffer_size ?
- data->set.buffer_size : BUFSIZE);
+ data->set.buffer_size : MASTERBUF_SIZE);
timeout_ms = Curl_sleep_time(data->set.max_recv_speed,
data->progress.dlspeed, buffersize);
Curl_expire(data, timeout_ms);
diff --git a/lib/sendf.c b/lib/sendf.c
index bad4683..95d1a51 100644
--- a/lib/sendf.c
+++ b/lib/sendf.c
@@ -555,7 +555,7 @@ CURLcode Curl_read(struct connectdata *conn, /* connection data */
}
/* If we come here, it means that there is no data to read from the buffer,
* so we read from the socket */
- bytesfromsocket = CURLMIN(sizerequested, (size_t)data->set.buffer_size);
+ bytesfromsocket = CURLMIN(sizerequested, MASTERBUF_SIZE);
buffertofill = conn->master_buffer;
}
else {
diff --git a/lib/url.c b/lib/url.c
index b86243b..cb3f3c3 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -242,6 +242,10 @@ static const struct Curl_handler * const protocols[] = {
(struct Curl_handler *) NULL
};
+#define READBUFFER_SIZE CURL_MAX_WRITE_SIZE
+#define READBUFFER_MAX CURL_MAX_READ_SIZE
+#define READBUFFER_MIN 1024
+
/*
* Dummy handler for undefined protocol schemes.
*/
@@ -577,7 +581,7 @@ CURLcode Curl_init_userdefined(struct UserDefined *set)
set->tcp_keepintvl = 60;
set->tcp_keepidle = 60;
- set->buffer_size = BUFSIZE;
+ set->buffer_size = READBUFFER_SIZE;
return res;
}
@@ -615,7 +619,7 @@ CURLcode Curl_open(struct SessionHandle **curl)
/* We do some initial setup here, all those fields that can't be just 0 */
- data->state.buffer = malloc(BUFSIZE + 1);
+ data->state.buffer = malloc(READBUFFER_SIZE + 1);
if(!data->state.buffer) {
DEBUGF(fprintf(stderr, "Error: malloc of buffer failed\n"));
res = CURLE_OUT_OF_MEMORY;
@@ -1969,17 +1973,16 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
*/
arg = va_arg(param, long);
- if(arg > MAX_BUFSIZE)
- arg = MAX_BUFSIZE; /* huge internal default */
+ if(arg > READBUFFER_MAX)
+ arg = READBUFFER_MAX; /* huge internal default */
else if(arg < 1)
- arg = BUFSIZE;
- else if(arg < MIN_BUFSIZE)
- arg = BUFSIZE;
+ arg = READBUFFER_SIZE;
+ else if(arg < READBUFFER_MIN)
+ arg = READBUFFER_MIN;
/* Resize only if larger than default buffer size. */
- if(arg > BUFSIZE) {
- data->state.buffer = realloc(data->state.buffer,
- data->set.buffer_size + 1);
+ if(arg > READBUFFER_SIZE) {
+ data->state.buffer = realloc(data->state.buffer, arg + 1);
if(!data->state.buffer) {
DEBUGF(fprintf(stderr, "Error: realloc of buffer failed\n"));
result = CURLE_OUT_OF_MEMORY;
@@ -3537,7 +3540,7 @@ static struct connectdata *allocate_conn(struct SessionHandle *data)
if(data->multi && Curl_multi_canPipeline(data->multi) &&
!conn->master_buffer) {
/* Allocate master_buffer to be used for pipelining */
- conn->master_buffer = calloc(BUFSIZE, sizeof (char));
+ conn->master_buffer = calloc(MASTERBUF_SIZE, sizeof (char));
if(!conn->master_buffer)
goto error;
}
diff --git a/lib/urldata.h b/lib/urldata.h
index afe37c0..d10c784 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -193,17 +193,13 @@
#include <libssh2_sftp.h>
#endif /* HAVE_LIBSSH2_H */
-/* Download buffer size, keep it fairly big for speed reasons */
-#undef BUFSIZE
-#define BUFSIZE CURL_MAX_WRITE_SIZE
-#undef MAX_BUFSIZE
-#define MAX_BUFSIZE CURL_MAX_READ_SIZE
-#define MIN_BUFSIZE 1024
-
/* The upload buffer size, should not be smaller than CURL_MAX_WRITE_SIZE, as
it needs to hold a full buffer as could be sent in a write callback */
#define UPLOAD_BUFSIZE CURL_MAX_WRITE_SIZE
+/* The "master buffer" is for HTTP pipelining */
+#define MASTERBUF_SIZE 16384
+
/* Initial size of the buffer to store headers in, it'll be enlarged in case
of need. */
#define HEADERSIZE 256
--
2.17.2
From 40c7b1ccca9fcf3b0ce261f098faebd58ed439ac Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 2 May 2017 08:32:04 +0200
Subject: [PATCH 17/18] ssh: fix compiler warning from e40e9d7f0de
Upstream-commit: eab6732fde094073480af5039b6fb495cbc3fb8a
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/ssh.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/ssh.c b/lib/ssh.c
index 169ef1b..b1f136e 100644
--- a/lib/ssh.c
+++ b/lib/ssh.c
@@ -1712,7 +1712,7 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block)
do {
size_t readthisamountnow =
(data->state.resume_from - passed > data->set.buffer_size) ?
- data->set.buffer_size :
+ (size_t)data->set.buffer_size :
curlx_sotouz(data->state.resume_from - passed);
size_t actuallyread =
--
2.17.2
From 9f3810bae5fad685e848a39750863557e17a0163 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Thu, 8 Mar 2018 10:33:16 +0100
Subject: [PATCH 18/18] readwrite: make sure excess reads don't go beyond
buffer end
CVE-2018-1000122
Bug: https://curl.haxx.se/docs/adv_2018-b047.html
Detected by OSS-fuzz
Upstream-commit: d52dc4760f6d9ca1937eefa2093058a952465128
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/transfer.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/lib/transfer.c b/lib/transfer.c
index dff6838..7ad6e3c 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -735,10 +735,15 @@ static CURLcode readwrite_data(struct SessionHandle *data,
} /* if(! header and data to read ) */
- if(conn->handler->readwrite &&
- (excess > 0 && !conn->bits.stream_was_rewound)) {
+ if(conn->handler->readwrite && excess && !conn->bits.stream_was_rewound) {
/* Parse the excess data */
k->str += nread;
+
+ if(&k->str[excess] > &k->buf[data->set.buffer_size]) {
+ /* the excess amount was too excessive(!), make sure
+ it doesn't read out of buffer */
+ excess = &k->buf[data->set.buffer_size] - k->str;
+ }
nread = (ssize_t)excess;
result = conn->handler->readwrite(data, conn, &nread, &readmore);
--
2.14.3