From 95f873ff983a1ae57415b3c16a881e74432cf8b8 Mon Sep 17 00:00:00 2001
From: Fabian Keil <fk@fabiankeil.de>
Date: Tue, 9 Feb 2021 14:04:32 +0100
Subject: [PATCH 1/4] runtests.pl: support the nonewline attribute for the data
part
Added to FILEFORMAT
Closes #8239
Upstream-commit: 736847611a40c01e7c290407e22e2f0f5f8efd6a
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
tests/FILEFORMAT.md | 5 ++++-
tests/runtests.pl | 7 +++++++
tests/server/getpart.c | 11 ++++++++++-
3 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/tests/FILEFORMAT.md b/tests/FILEFORMAT.md
index b75a02a..0b98787 100644
--- a/tests/FILEFORMAT.md
+++ b/tests/FILEFORMAT.md
@@ -185,7 +185,7 @@ which are treated together as a single identifier.
## `<reply>`
-### `<data [nocheck="yes"] [sendzero="yes"] [base64="yes"] [hex="yes"]>`
+### `<data [nocheck="yes"] [sendzero="yes"] [base64="yes"] [hex="yes"] [nonewline="yes"]>`
data to be sent to the client on its request and later verified that it
arrived safely. Set `nocheck="yes"` to prevent the test script from verifying
@@ -211,6 +211,9 @@ much sense for other sections than "data").
`hex=yes` means that the data is a sequence of hex pairs. It will get decoded
and used as "raw" data.
+`nonewline=yes` means that the last byte (the trailing newline character)
+should be cut off from the data before sending or comparing it.
+
For FTP file listings, the `<data>` section will be used *only* if you make
sure that there has been a CWD done first to a directory named `test-[num]`
where [num] is the test case number. Otherwise the ftp server can't know from
diff --git a/tests/runtests.pl b/tests/runtests.pl
index 40315aa..2e1500d 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -3837,6 +3837,13 @@ sub singletest {
else {
# check against the data section
@reply = getpart("reply", "data");
+ if(@reply) {
+ my %hash = getpartattr("reply", "data");
+ if($hash{'nonewline'}) {
+ # cut off the final newline from the final line of the data
+ chomp($reply[$#reply]);
+ }
+ }
# get the mode attribute
my $filemode=$replyattr{'mode'};
if($filemode && ($filemode eq "text") && $has_textaware) {
diff --git a/tests/server/getpart.c b/tests/server/getpart.c
index 32b55bc..f8fe3f6 100644
--- a/tests/server/getpart.c
+++ b/tests/server/getpart.c
@@ -5,7 +5,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
- * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
@@ -295,6 +295,7 @@ int getpart(char **outbuf, size_t *outlen,
size_t outalloc = 256;
int in_wanted_part = 0;
int base64 = 0;
+ int nonewline = 0;
int error;
enum {
@@ -360,6 +361,8 @@ int getpart(char **outbuf, size_t *outlen,
if(error)
return error;
}
+ if(nonewline)
+ (*outlen)--;
break;
}
}
@@ -377,6 +380,8 @@ int getpart(char **outbuf, size_t *outlen,
if(error)
return error;
}
+ if(nonewline)
+ (*outlen)--;
break;
}
}
@@ -451,6 +456,10 @@ int getpart(char **outbuf, size_t *outlen,
/* bit rough test, but "mostly" functional, */
/* treat wanted part data as base64 encoded */
base64 = 1;
+ if(strstr(patt, "nonewline=")) {
+ show(("* setting nonewline\n"));
+ nonewline = 1;
+ }
}
continue;
}
--
2.39.1
From 6e244e1bcb04012e11c537253e76e6f968d8bb72 Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Thu, 1 Dec 2022 09:21:04 +0100
Subject: [PATCH 2/4] runtests: do CRLF replacements per section only
The `crlf="yes"` attribute and "hyper mode" are now only applied on a
subset of dedicated sections: data, datacheck, stdout and protocol.
Updated test 2500 accordingly.
Also made test1 use crlf="yes" for <protocol>, mostly because it is
often used as a template test case. Going forward, using this attribute
we should be able to write test cases using linefeeds only and avoid
mixed line ending encodings.
Follow-up to ca15b7512e8d11
Fixes #10009
Closes #10010
Upstream-commit: 2f34a7347f315513bfda9ef14770d287fb246bcd
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
tests/FILEFORMAT.md | 22 +++++++++++++++------
tests/data/test1 | 14 ++++++-------
tests/runtests.pl | 48 ++++++++++++++++++++++++++++++++++++++++++---
3 files changed, 68 insertions(+), 16 deletions(-)
diff --git a/tests/FILEFORMAT.md b/tests/FILEFORMAT.md
index c1fbc57..dcb5695 100644
--- a/tests/FILEFORMAT.md
+++ b/tests/FILEFORMAT.md
@@ -185,7 +185,7 @@ which are treated together as a single identifier.
## `<reply>`
-### `<data [nocheck="yes"] [sendzero="yes"] [base64="yes"] [hex="yes"] [nonewline="yes"]>`
+### `<data [nocheck="yes"] [sendzero="yes"] [base64="yes"] [hex="yes"] [nonewline="yes"] [crlf="yes"]>`
data to be sent to the client on its request and later verified that it
arrived safely. Set `nocheck="yes"` to prevent the test script from verifying
@@ -214,12 +214,16 @@ and used as "raw" data.
`nonewline=yes` means that the last byte (the trailing newline character)
should be cut off from the data before sending or comparing it.
+`crlf=yes` forces *header* newlines to become CRLF even if not written so in
+the source file. Note that this makes runtests.pl parse and "guess" what is a
+header and what is not in order to apply the CRLF line endings appropriately.
+
For FTP file listings, the `<data>` section will be used *only* if you make
sure that there has been a CWD done first to a directory named `test-[num]`
where [num] is the test case number. Otherwise the ftp server can't know from
which test file to load the list content.
-### `<dataNUM>`
+### `<dataNUM [crlf="yes"]>`
Send back this contents instead of the <data> one. The num is set by:
@@ -243,7 +247,7 @@ The connect section is used instead of the 'data' for all CONNECT
requests. The remainder of the rules for the data section then apply but with
a connect prefix.
-### `<datacheck [mode="text"] [nonewline="yes"]>`
+### `<datacheck [mode="text"] [nonewline="yes"] [crlf="yes"]>`
if the data is sent but this is what should be checked afterwards. If
`nonewline=yes` is set, runtests will cut off the trailing newline from the
data before comparing with the one actually received by the client.
@@ -251,7 +255,7 @@ data before comparing with the one actually received by the client.
Use the `mode="text"` attribute if the output is in text mode on platforms
that have a text/binary difference.
-### `<datacheckNUM [nonewline="yes"] [mode="text"]>`
+### `<datacheckNUM [nonewline="yes"] [mode="text"] [crlf="yes"]>`
The contents of numbered datacheck sections are appended to the non-numbered
one.
@@ -528,13 +532,16 @@ changing protocol data such as port numbers or user-agent strings.
One perl op per line that operates on the protocol dump. This is pretty
advanced. Example: `s/^EPRT .*/EPRT stripped/`.
-### `<protocol [nonewline="yes"]>`
+### `<protocol [nonewline="yes"] crlf="yes">`
the protocol dump curl should transmit, if 'nonewline' is set, we will cut off
the trailing newline of this given data before comparing with the one actually
sent by the client The `<strip>` and `<strippart>` rules are applied before
comparisons are made.
+`crlf=yes` forces the newlines to become CRLF even if not written so in the
+test.
+
### `<proxy [nonewline="yes"]>`
The protocol dump curl should transmit to a HTTP proxy (when the http-proxy
@@ -551,7 +558,7 @@ have a text/binary difference.
If 'nonewline' is set, we will cut off the trailing newline of this given data
before comparing with the one actually received by the client
-### `<stdout [mode="text"] [nonewline="yes"]>`
+### `<stdout [mode="text"] [nonewline="yes"] [crlf="yes"]>`
This verifies that this data was passed to stdout.
Use the mode="text" attribute if the output is in text mode on platforms that
@@ -560,6 +567,9 @@ have a text/binary difference.
If 'nonewline' is set, we will cut off the trailing newline of this given data
before comparing with the one actually received by the client
+`crlf=yes` forces the newlines to become CRLF even if not written so in the
+test.
+
### `<file name="log/filename" [mode="text"]>`
The file's contents must be identical to this after the test is complete. Use
the mode="text" attribute if the output is in text mode on platforms that have
diff --git a/tests/data/test1 b/tests/data/test1
index f39a08b..700bed8 100644
--- a/tests/data/test1
+++ b/tests/data/test1
@@ -9,7 +9,7 @@ HTTP GET
#
# Server-side
<reply>
-<data>
+<data crlf="yes">
HTTP/1.1 200 OK
Date: Tue, 09 Nov 2010 14:49:00 GMT
Server: test-server/fake
@@ -42,12 +42,12 @@ http://%HOSTIP:%HTTPPORT/%TESTNUMBER
#
# Verify data after the test has been "shot"
<verify>
-<protocol>
-GET /%TESTNUMBER HTTP/1.1
-Host: %HOSTIP:%HTTPPORT
-User-Agent: curl/%VERSION
-Accept: */*
-
+<protocol crlf="yes">
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
</protocol>
</verify>
</testcase>
diff --git a/tests/runtests.pl b/tests/runtests.pl
index 72a9989..b12a42d 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -3410,7 +3410,13 @@ sub subBase64 {
my $prevupdate;
sub subNewlines {
- my ($thing) = @_;
+ my ($force, $thing) = @_;
+
+ if($force) {
+ # enforce CRLF newline
+ $$thing =~ s/\x0d*\x0a/\x0d\x0a/;
+ return;
+ }
# When curl is built with Hyper, it gets all response headers delivered as
# name/value pairs and curl "invents" the newlines when it saves the
@@ -3424,7 +3430,7 @@ sub subNewlines {
# skip curl error messages
($$thing !~ /^curl: \(\d+\) /))) {
# enforce CRLF newline
- $$thing =~ s/\x0a/\x0d\x0a/;
+ $$thing =~ s/\x0d*\x0a/\x0d\x0a/;
$prevupdate = 1;
}
else {
@@ -3496,6 +3502,7 @@ sub prepro {
my (@entiretest) = @_;
my $show = 1;
my @out;
+ my $data_crlf;
for my $s (@entiretest) {
my $f = $s;
if($s =~ /^ *%if (.*)/) {
@@ -3519,9 +3526,19 @@ sub prepro {
next;
}
if($show) {
+ # The processor does CRLF replacements in the <data*> sections if
+ # necessary since those parts might be read by separate servers.
+ if($s =~ /^ *<data(.*)\>/) {
+ if($1 =~ /crlf="yes"/ || $has_hyper) {
+ $data_crlf = 1;
+ }
+ }
+ elsif(($s =~ /^ *<\/data/) && $data_crlf) {
+ $data_crlf = 0;
+ }
subVariables(\$s, $testnum, "%");
subBase64(\$s);
- subNewlines(\$s) if($has_hyper);
+ subNewlines(0, \$s) if($data_crlf);
push @out, $s;
}
}
@@ -3830,6 +3847,11 @@ sub singletest {
# of the datacheck
chomp($replycheckpart[$#replycheckpart]);
}
+ if($replycheckpartattr{'crlf'} ||
+ ($has_hyper && ($keywords{"HTTP"}
+ || $keywords{"HTTPS"}))) {
+ map subNewlines(0, \$_), @replycheckpart;
+ }
push(@reply, @replycheckpart);
}
}
@@ -3851,6 +3873,11 @@ sub singletest {
map s/\r\n/\n/g, @reply;
map s/\n/\r\n/g, @reply;
}
+ if($replyattr{'crlf'} ||
+ ($has_hyper && ($keywords{"HTTP"}
+ || $keywords{"HTTPS"}))) {
+ map subNewlines(0, \$_), @reply;
+ }
}
# this is the valid protocol blurb curl should generate
@@ -4287,6 +4314,12 @@ sub singletest {
chomp($validstdout[$#validstdout]);
}
+ if($hash{'crlf'} ||
+ ($has_hyper && ($keywords{"HTTP"}
+ || $keywords{"HTTPS"}))) {
+ map subNewlines(0, \$_), @validstdout;
+ }
+
$res = compare($testnum, $testname, "stdout", \@actual, \@validstdout);
if($res) {
return $errorreturncode;
@@ -4381,6 +4414,10 @@ sub singletest {
}
}
+ if($hash{'crlf'}) {
+ map subNewlines(1, \$_), @protstrip;
+ }
+
if((!$out[0] || ($out[0] eq "")) && $protstrip[0]) {
logmsg "\n $testnum: protocol FAILED!\n".
" There was no content at all in the file $SERVERIN.\n".
@@ -4512,6 +4549,11 @@ sub singletest {
map s/\r\n/\n/g, @outfile;
map s/\n/\r\n/g, @outfile;
}
+ if($hash{'crlf'} ||
+ ($has_hyper && ($keywords{"HTTP"}
+ || $keywords{"HTTPS"}))) {
+ map subNewlines(0, \$_), @outfile;
+ }
my $strip;
for $strip (@stripfile) {
--
2.39.1
From 228ed11bf33c63d9208a3fb38fe5a0d19c0764bd Mon Sep 17 00:00:00 2001
From: Daniel Stenberg <daniel@haxx.se>
Date: Tue, 27 Dec 2022 11:50:23 +0100
Subject: [PATCH 3/4] runtests: support crlf="yes" for verify/proxy
Upstream-commit: dc0725244a3163f1e2d5f51165db3a1a430f3ba0
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
tests/FILEFORMAT.md | 4 ++--
tests/runtests.pl | 5 +++++
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/tests/FILEFORMAT.md b/tests/FILEFORMAT.md
index dcb5695..6646793 100644
--- a/tests/FILEFORMAT.md
+++ b/tests/FILEFORMAT.md
@@ -532,7 +532,7 @@ changing protocol data such as port numbers or user-agent strings.
One perl op per line that operates on the protocol dump. This is pretty
advanced. Example: `s/^EPRT .*/EPRT stripped/`.
-### `<protocol [nonewline="yes"] crlf="yes">`
+### `<protocol [nonewline="yes"][crlf="yes"]>`
the protocol dump curl should transmit, if 'nonewline' is set, we will cut off
the trailing newline of this given data before comparing with the one actually
@@ -542,7 +542,7 @@ comparisons are made.
`crlf=yes` forces the newlines to become CRLF even if not written so in the
test.
-### `<proxy [nonewline="yes"]>`
+### `<proxy [nonewline="yes"][crlf="yes"]>`
The protocol dump curl should transmit to a HTTP proxy (when the http-proxy
server is used), if 'nonewline' is set, we will cut off the trailing newline
diff --git a/tests/runtests.pl b/tests/runtests.pl
index b12a42d..5cdc83d 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -4510,6 +4510,11 @@ sub singletest {
}
}
+ if($hash{'crlf'} ||
+ ($has_hyper && ($keywords{"HTTP"} || $keywords{"HTTPS"}))) {
+ map subNewlines(0, \$_), @protstrip;
+ }
+
$res = compare($testnum, $testname, "proxy", \@out, \@protstrip);
if($res) {
return $errorreturncode;
--
2.39.1
From bc5fc958b017895728962c9d44c469418cbec1a0 Mon Sep 17 00:00:00 2001
From: Patrick Monnerat <patrick@monnerat.net>
Date: Mon, 13 Feb 2023 08:33:09 +0100
Subject: [PATCH 4/4] content_encoding: do not reset stage counter for each
header
Test 418 verifies
Closes #10492
Upstream-commit: 119fb187192a9ea13dc90d9d20c215fc82799ab9
Signed-off-by: Kamil Dudka <kdudka@redhat.com>
---
lib/content_encoding.c | 7 +-
lib/urldata.h | 1 +
tests/data/Makefile.inc | 1 +
tests/data/test387 | 2 +-
tests/data/test418 | 152 ++++++++++++++++++++++++++++++++++++++++
5 files changed, 158 insertions(+), 5 deletions(-)
create mode 100644 tests/data/test418
diff --git a/lib/content_encoding.c b/lib/content_encoding.c
index bfc13e2..94344d6 100644
--- a/lib/content_encoding.c
+++ b/lib/content_encoding.c
@@ -1033,7 +1033,6 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
const char *enclist, int maybechunked)
{
struct SingleRequest *k = &data->req;
- int counter = 0;
do {
const char *name;
@@ -1068,9 +1067,9 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
if(!encoding)
encoding = &error_encoding; /* Defer error at stack use. */
- if(++counter >= MAX_ENCODE_STACK) {
- failf(data, "Reject response due to %u content encodings",
- counter);
+ if(k->writer_stack_depth++ >= MAX_ENCODE_STACK) {
+ failf(data, "Reject response due to more than %u content encodings",
+ MAX_ENCODE_STACK);
return CURLE_BAD_CONTENT_ENCODING;
}
/* Stack the unencoding stage. */
diff --git a/lib/urldata.h b/lib/urldata.h
index 5b4b34f..8c8c20b 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -700,6 +700,7 @@ struct SingleRequest {
#ifndef CURL_DISABLE_DOH
struct dohdata *doh; /* DoH specific data for this request */
#endif
+ unsigned char writer_stack_depth; /* Unencoding stack depth. */
BIT(header); /* incoming data has HTTP header */
BIT(content_range); /* set TRUE if Content-Range: was found */
BIT(upload_done); /* set to TRUE when doing chunked transfer-encoding
diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index fb51cd6..86b6f85 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -67,6 +67,7 @@ test393 test394 test395 test396 test397 \
\
test400 test401 test402 test403 test404 test405 test406 test407 test408 \
test409 test410 \
+ test418 \
\
test430 test431 test432 test433 test434 \
\
diff --git a/tests/data/test387 b/tests/data/test387
index 015ec25..644fc7f 100644
--- a/tests/data/test387
+++ b/tests/data/test387
@@ -47,7 +47,7 @@ Accept: */*
61
</errorcode>
<stderr mode="text">
-curl: (61) Reject response due to 5 content encodings
+curl: (61) Reject response due to more than 5 content encodings
</stderr>
</verify>
</testcase>
diff --git a/tests/data/test418 b/tests/data/test418
new file mode 100644
index 0000000..50e974e
--- /dev/null
+++ b/tests/data/test418
@@ -0,0 +1,152 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+gzip
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+Transfer-Encoding: gzip
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+ <name>
+Response with multiple Transfer-Encoding headers
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -sS
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+
+# CURLE_BAD_CONTENT_ENCODING is 61
+<errorcode>
+61
+</errorcode>
+<stderr mode="text">
+curl: (61) Reject response due to more than 5 content encodings
+</stderr>
+</verify>
+</testcase>
--
2.39.1