Backported for 5.0.5
From 6facfb7a103b26b9a602253a738b2130afb7c5d3 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 3 Jun 2021 12:10:02 +0300
Subject: [PATCH] Fix ziplist and listpack overflows and truncations
(CVE-2021-32627, CVE-2021-32628)
- fix possible heap corruption in ziplist and listpack resulting by trying to
allocate more than the maximum size of 4GB.
- prevent ziplist (hash and zset) from reaching size of above 1GB, will be
converted to HT encoding, that's not a useful size.
- prevent listpack (stream) from reaching size of above 1GB.
- XADD will start a new listpack if the new record may cause the previous
listpack to grow over 1GB.
- XADD will respond with an error if a single stream record is over 1GB
- List type (ziplist in quicklist) was truncating strings that were over 4GB,
now it'll respond with an error.
(cherry picked from commit 68e221a3f98a427805d31c1760b4cdf37ba810ab)
---
src/geo.c | 5 +-
src/listpack.c | 2 +-
src/quicklist.c | 17 ++++-
src/rdb.c | 36 ++++++---
src/server.h | 2 +-
src/t_hash.c | 13 +++-
src/t_list.c | 25 ++++++
src/t_stream.c | 48 +++++++++---
src/t_zset.c | 43 +++++++----
src/ziplist.c | 17 ++++-
src/ziplist.h | 1 +
tests/support/util.tcl | 21 +++++
tests/unit/violations.tcl | 156 ++++++++++++++++++++++++++++++++++++++
13 files changed, 338 insertions(+), 48 deletions(-)
create mode 100644 tests/unit/violations.tcl
diff --git a/src/geo.c b/src/geo.c
index f1d3f18d46e7..b94fcc1b3d70 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -635,7 +635,7 @@ void georadiusGeneric(client *c, int fla
robj *zobj;
zset *zs;
int i;
- size_t maxelelen = 0;
+ size_t maxelelen = 0, totelelen = 0;
if (returned_items) {
zobj = createZsetObject();
@@ -650,13 +650,14 @@ void georadiusGeneric(client *c, int fla
size_t elelen = sdslen(gp->member);
if (maxelelen < elelen) maxelelen = elelen;
+ totelelen += elelen;
znode = zslInsert(zs->zsl,score,gp->member);
serverAssert(dictAdd(zs->dict,gp->member,&znode->score) == DICT_OK);
gp->member = NULL;
}
if (returned_items) {
- zsetConvertToZiplistIfNeeded(zobj,maxelelen);
+ zsetConvertToZiplistIfNeeded(zobj,maxelelen,totelelen);
setKey(c->db,storekey,zobj);
decrRefCount(zobj);
notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey,
diff --git a/src/listpack.c b/src/listpack.c
index e1f4d9a02ee8..cd5583ccb258 100644
--- a/src/listpack.c
+++ b/src/listpack.c
@@ -283,7 +283,7 @@ int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, ui
} else {
if (size < 64) *enclen = 1+size;
else if (size < 4096) *enclen = 2+size;
- else *enclen = 5+size;
+ else *enclen = 5+(uint64_t)size;
return LP_ENCODING_STRING;
}
}
diff --git a/src/quicklist.c b/src/quicklist.c
index 7b5484116785..d5cc758b2fa0 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -29,6 +29,7 @@
*/
#include <string.h> /* for memcpy */
+#include "redisassert.h"
#include "quicklist.h"
#include "zmalloc.h"
#include "ziplist.h"
@@ -43,11 +44,16 @@
#define REDIS_STATIC static
#endif
-/* Optimization levels for size-based filling */
+/* Optimization levels for size-based filling.
+ * Note that the largest possible limit is 16k, so even if each record takes
+ * just one byte, it still won't overflow the 16 bit count field. */
static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536};
/* Maximum size in bytes of any multi-element ziplist.
- * Larger values will live in their own isolated ziplists. */
+ * Larger values will live in their own isolated ziplists.
+ * This is used only if we're limited by record count. when we're limited by
+ * size, the maximum limit is bigger, but still safe.
+ * 8k is a recommended / default size limit */
#define SIZE_SAFETY_LIMIT 8192
/* Minimum ziplist size in bytes for attempting compression. */
@@ -441,6 +447,8 @@ REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node,
unsigned int new_sz = node->sz + sz + ziplist_overhead;
if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill)))
return 1;
+ /* when we return 1 above we know that the limit is a size limit (which is
+ * safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
else if (!sizeMeetsSafetyLimit(new_sz))
return 0;
else if ((int)node->count < fill)
@@ -460,6 +468,8 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,
unsigned int merge_sz = a->sz + b->sz - 11;
if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill)))
return 1;
+ /* when we return 1 above we know that the limit is a size limit (which is
+ * safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
else if (!sizeMeetsSafetyLimit(merge_sz))
return 0;
else if ((int)(a->count + b->count) <= fill)
@@ -479,6 +489,7 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,
* Returns 1 if new head created. */
int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) {
quicklistNode *orig_head = quicklist->head;
+ assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */
if (likely(
_quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz))) {
quicklist->head->zl =
@@ -502,6 +513,7 @@ int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) {
* Returns 1 if new tail created. */
int quicklistPushTail(quicklist *quicklist, void *value, size_t sz) {
quicklistNode *orig_tail = quicklist->tail;
+ assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */
if (likely(
_quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz))) {
quicklist->tail->zl =
@@ -835,6 +847,7 @@ REDIS_STATIC void _quicklistInsert(quicklist *quicklist, quicklistEntry *entry,
int fill = quicklist->fill;
quicklistNode *node = entry->node;
quicklistNode *new_node = NULL;
+ assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */
if (!node) {
/* we have no reference node, so let's create only node in the list */
diff --git a/src/rdb.c b/src/rdb.c
index 3c58a1eaf7fb..c7dc724f3df6 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1452,7 +1452,7 @@ robj *rdbLoadObject(int rdbtype, rio *rd
} else if (rdbtype == RDB_TYPE_ZSET_2 || rdbtype == RDB_TYPE_ZSET) {
/* Read list/set value. */
uint64_t zsetlen;
- size_t maxelelen = 0;
+ size_t maxelelen = 0, totelelen = 0;
zset *zs;
if ((zsetlen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
@@ -1479,6 +1479,7 @@ robj *rdbLoadObject(int rdbtype, rio *rd
/* Don't care about integer-encoded strings. */
if (sdslen(sdsele) > maxelelen) maxelelen = sdslen(sdsele);
+ totelelen += sdslen(sdsele);
znode = zslInsert(zs->zsl,score,sdsele);
dictAdd(zs->dict,sdsele,&znode->score);
@@ -1486,8 +1487,11 @@ robj *rdbLoadObject(int rdbtype, rio *rd
/* Convert *after* loading, since sorted sets are not stored ordered. */
if (zsetLength(o) <= server.zset_max_ziplist_entries &&
- maxelelen <= server.zset_max_ziplist_value)
- zsetConvert(o,OBJ_ENCODING_ZIPLIST);
+ maxelelen <= server.zset_max_ziplist_value &&
+ ziplistSafeToAdd(NULL, totelelen))
+ {
+ zsetConvert(o,OBJ_ENCODING_ZIPLIST);
+ }
} else if (rdbtype == RDB_TYPE_HASH) {
uint64_t len;
int ret;
@@ -1511,21 +1515,25 @@ robj *rdbLoadObject(int rdbtype, rio *rd
if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
== NULL) return NULL;
- /* Add pair to ziplist */
- o->ptr = ziplistPush(o->ptr, (unsigned char*)field,
- sdslen(field), ZIPLIST_TAIL);
- o->ptr = ziplistPush(o->ptr, (unsigned char*)value,
- sdslen(value), ZIPLIST_TAIL);
-
/* Convert to hash table if size threshold is exceeded */
if (sdslen(field) > server.hash_max_ziplist_value ||
- sdslen(value) > server.hash_max_ziplist_value)
+ sdslen(value) > server.hash_max_ziplist_value ||
+ !ziplistSafeToAdd(o->ptr, sdslen(field)+sdslen(value)))
{
- sdsfree(field);
- sdsfree(value);
hashTypeConvert(o, OBJ_ENCODING_HT);
+ ret = dictAdd((dict*)o->ptr, field, value);
+ if (ret == DICT_ERR) {
+ rdbExitReportCorruptRDB("Duplicate hash fields detected");
+ }
break;
}
+
+ /* Add pair to ziplist */
+ o->ptr = ziplistPush(o->ptr, (unsigned char*)field,
+ sdslen(field), ZIPLIST_TAIL);
+ o->ptr = ziplistPush(o->ptr, (unsigned char*)value,
+ sdslen(value), ZIPLIST_TAIL);
+
sdsfree(field);
sdsfree(value);
}
@@ -1594,6 +1602,10 @@ robj *rdbLoadObject(int rdbtype, rio *rd
while ((zi = zipmapNext(zi, &fstr, &flen, &vstr, &vlen)) != NULL) {
if (flen > maxlen) maxlen = flen;
if (vlen > maxlen) maxlen = vlen;
+ if (!ziplistSafeToAdd(zl, (size_t)flen + vlen)) {
+ rdbExitReportCorruptRDB("Hash zipmap too big (%u)", flen);
+ }
+
zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL);
zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL);
}
diff --git a/src/server.h b/src/server.h
index ca868939cf6d..164a82271f44 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1677,7 +1677,7 @@ unsigned char *zzlFirstInRange(unsigned char *zl, zrangespec *range);
unsigned char *zzlLastInRange(unsigned char *zl, zrangespec *range);
unsigned long zsetLength(const robj *zobj);
void zsetConvert(robj *zobj, int encoding);
-void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen);
+void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen);
int zsetScore(robj *zobj, sds member, double *score);
unsigned long zslGetRank(zskiplist *zsl, double score, sds o);
int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore);
diff --git a/src/t_hash.c b/src/t_hash.c
index 0ca152df78cc..109522c1322f 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -39,17 +39,22 @@
* as their string length can be queried in constant time. */
void hashTypeTryConversion(robj *o, robj **argv, int start, int end) {
int i;
+ size_t sum = 0;
if (o->encoding != OBJ_ENCODING_ZIPLIST) return;
for (i = start; i <= end; i++) {
- if (sdsEncodedObject(argv[i]) &&
- sdslen(argv[i]->ptr) > server.hash_max_ziplist_value)
- {
+ if (!sdsEncodedObject(argv[i]))
+ continue;
+ size_t len = sdslen(argv[i]->ptr);
+ if (len > server.hash_max_ziplist_value) {
hashTypeConvert(o, OBJ_ENCODING_HT);
- break;
+ return;
}
+ sum += len;
}
+ if (!ziplistSafeToAdd(o->ptr, sum))
+ hashTypeConvert(o, OBJ_ENCODING_HT);
}
/* Get the value from a ziplist encoded hash, identified by field.
diff --git a/src/t_list.c b/src/t_list.c
index de417f4705f4..67541554f616 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -29,6 +29,8 @@
#include "server.h"
+#define LIST_MAX_ITEM_SIZE ((1ull<<32)-1024)
+
/*-----------------------------------------------------------------------------
* List API
*----------------------------------------------------------------------------*/
@@ -196,6 +198,14 @@ void listTypeConvert(robj *subject, int enc) {
void pushGenericCommand(client *c, int where) {
int j, pushed = 0;
+
+ for (j = 2; j < c->argc; j++) {
+ if (sdslen(c->argv[j]->ptr) > LIST_MAX_ITEM_SIZE) {
+ addReplyError(c, "Element too large");
+ return;
+ }
+ }
+
robj *lobj = lookupKeyWrite(c->db,c->argv[1]);
if (lobj && lobj->type != OBJ_LIST) {
@@ -277,6 +287,11 @@ void linsertCommand(client *c) {
return;
}
+ if (sdslen(c->argv[4]->ptr) > LIST_MAX_ITEM_SIZE) {
+ addReplyError(c, "Element too large");
+ return;
+ }
+
if ((subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,subject,OBJ_LIST)) return;
@@ -344,6 +359,11 @@ void lsetCommand(client *c) {
long index;
robj *value = c->argv[3];
+ if (sdslen(value->ptr) > LIST_MAX_ITEM_SIZE) {
+ addReplyError(c, "Element too large");
+ return;
+ }
+
if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != C_OK))
return;
@@ -493,6 +513,11 @@ void lremCommand(client *c) {
long toremove;
long removed = 0;
+ if (sdslen(obj->ptr) > LIST_MAX_ITEM_SIZE) {
+ addReplyError(c, "Element too large");
+ return;
+ }
+
if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != C_OK))
return;
diff --git a/src/t_stream.c b/src/t_stream.c
index d7754985dd03..e7263d68a28f 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -40,6 +40,12 @@
#define STREAM_ITEM_FLAG_DELETED (1<<0) /* Entry is delted. Skip it. */
#define STREAM_ITEM_FLAG_SAMEFIELDS (1<<1) /* Same fields as master entry. */
+/* Don't let listpacks grow too big, even if the user config allows it.
+ * doing so can lead to an overflow (trying to store more than 32bit length
+ * into the listpack header), or actually an assertion since lpInsert
+ * will return NULL. */
+#define STREAM_LISTPACK_MAX_SIZE (1<<30)
+
void streamFreeCG(streamCG *cg);
void streamFreeNACK(streamNACK *na);
size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start, streamID *end, size_t count, streamConsumer *consumer);
@@ -170,12 +176,31 @@ int streamCompareID(streamID *a, streamI
*
* The function returns C_OK if the item was added, this is always true
* if the ID was generated by the function. However the function may return
- * C_ERR if an ID was given via 'use_id', but adding it failed since the
- * current top ID is greater or equal. */
+ * C_ERR in several cases:
+ * 1. If an ID was given via 'use_id', but adding it failed since the
+ * current top ID is greater or equal. errno will be set to EDOM.
+ * 2. If a size of a single element or the sum of the elements is too big to
+ * be stored into the stream. errno will be set to ERANGE. */
int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id) {
/* If an ID was given, check that it's greater than the last entry ID
* or return an error. */
- if (use_id && streamCompareID(use_id,&s->last_id) <= 0) return C_ERR;
+ if (use_id && streamCompareID(use_id,&s->last_id) <= 0) {
+ errno = EDOM;
+ return C_ERR;
+ }
+
+ /* Avoid overflow when trying to add an element to the stream (listpack
+ * can only host up to 32bit length sttrings, and also a total listpack size
+ * can't be bigger than 32bit length. */
+ size_t totelelen = 0;
+ for (int64_t i = 0; i < numfields*2; i++) {
+ sds ele = argv[i]->ptr;
+ totelelen += sdslen(ele);
+ }
+ if (totelelen > STREAM_LISTPACK_MAX_SIZE) {
+ errno = ERANGE;
+ return C_ERR;
+ }
/* Add the new entry. */
raxIterator ri;
@@ -241,9 +266,10 @@ int streamAppendItem(stream *s, robj **a
* if we need to switch to the next one. 'lp' will be set to NULL if
* the current node is full. */
if (lp != NULL) {
- if (server.stream_node_max_bytes &&
- lp_bytes > server.stream_node_max_bytes)
- {
+ size_t node_max_bytes = server.stream_node_max_bytes;
+ if (node_max_bytes == 0 || node_max_bytes > STREAM_LISTPACK_MAX_SIZE)
+ node_max_bytes = STREAM_LISTPACK_MAX_SIZE;
+ if (lp_bytes + totelelen >= node_max_bytes) {
lp = NULL;
} else if (server.stream_node_max_entries) {
int64_t count = lpGetInteger(lpFirst(lp));
@@ -1224,11 +1250,13 @@ void xaddCommand(client *c) {
/* Append using the low level function and return the ID. */
if (streamAppendItem(s,c->argv+field_pos,(c->argc-field_pos)/2,
- &id, id_given ? &id : NULL)
- == C_ERR)
+ &id, id_given ? &id : NULL) == C_ERR)
{
- addReplyError(c,"The ID specified in XADD is equal or smaller than the "
- "target stream top item");
+ if (errno == EDOM)
+ addReplyError(c,"The ID specified in XADD is equal or smaller than "
+ "the target stream top item");
+ else
+ addReplyError(c,"Elements are too large to be stored");
return;
}
addReplyStreamID(c,&id);
diff --git a/src/t_zset.c b/src/t_zset.c
index 56ea39607b52..989d5855e1ea 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1237,15 +1237,18 @@ void zsetConvert(robj *zobj, int encodin
}
/* Convert the sorted set object into a ziplist if it is not already a ziplist
- * and if the number of elements and the maximum element size is within the
- * expected ranges. */
-void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen) {
+ * and if the number of elements and the maximum element size and total elements size
+ * are within the expected ranges. */
+void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen) {
if (zobj->encoding == OBJ_ENCODING_ZIPLIST) return;
zset *zset = zobj->ptr;
if (zset->zsl->length <= server.zset_max_ziplist_entries &&
- maxelelen <= server.zset_max_ziplist_value)
- zsetConvert(zobj,OBJ_ENCODING_ZIPLIST);
+ maxelelen <= server.zset_max_ziplist_value &&
+ ziplistSafeToAdd(NULL, totelelen))
+ {
+ zsetConvert(zobj,OBJ_ENCODING_ZIPLIST);
+ }
}
/* Return (by reference) the score of the specified member of the sorted set
@@ -1354,21 +1357,28 @@ int zsetAdd(robj *zobj, double score, sd
}
return 1;
} else if (!xx) {
- /* Optimize: check if the element is too large or the list
+ /* check if the element is too large or the list
* becomes too long *before* executing zzlInsert. */
- zobj->ptr = zzlInsert(zobj->ptr,ele,score);
- if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries)
- zsetConvert(zobj,OBJ_ENCODING_SKIPLIST);
- if (sdslen(ele) > server.zset_max_ziplist_value)
+ if (zzlLength(zobj->ptr)+1 > server.zset_max_ziplist_entries ||
+ sdslen(ele) > server.zset_max_ziplist_value ||
+ !ziplistSafeToAdd(zobj->ptr, sdslen(ele)))
+ {
zsetConvert(zobj,OBJ_ENCODING_SKIPLIST);
- if (newscore) *newscore = score;
- *flags |= ZADD_ADDED;
- return 1;
+ } else {
+ zobj->ptr = zzlInsert(zobj->ptr,ele,score);
+ if (newscore) *newscore = score;
+ *flags |= ZADD_ADDED;
+ return 1;
+ }
} else {
*flags |= ZADD_NOP;
return 1;
}
- } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
+ }
+
+ /* Note that the above block handling ziplist would have either returned or
+ * converted the key to skiplist. */
+ if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
zset *zs = zobj->ptr;
zskiplistNode *znode;
dictEntry *de;
@@ -2180,7 +2190,7 @@ void zunionInterGenericCommand(client *c
zsetopsrc *src;
zsetopval zval;
sds tmp;
- size_t maxelelen = 0;
+ size_t maxelelen = 0, totelelen = 0;
robj *dstobj;
zset *dstzset;
zskiplistNode *znode;
@@ -2304,6 +2314,7 @@ void zunionInterGenericCommand(client *c
tmp = zuiNewSdsFromValue(&zval);
znode = zslInsert(dstzset->zsl,score,tmp);
dictAdd(dstzset->dict,tmp,&znode->score);
+ totelelen += sdslen(tmp);
if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp);
}
}
@@ -2340,6 +2351,7 @@ void zunionInterGenericCommand(client *c
/* Remember the longest single element encountered,
* to understand if it's possible to convert to ziplist
* at the end. */
+ totelelen += sdslen(tmp);
if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp);
/* Update the element with its initial score. */
dictSetKey(accumulator, de, tmp);
@@ -2380,7 +2392,7 @@ void zunionInterGenericCommand(client *c
if (dbDelete(c->db,dstkey))
touched = 1;
if (dstzset->zsl->length) {
- zsetConvertToZiplistIfNeeded(dstobj,maxelelen);
+ zsetConvertToZiplistIfNeeded(dstobj,maxelelen,totelelen);
dbAdd(c->db,dstkey,dstobj);
addReplyLongLong(c,zsetLength(dstobj));
signalModifiedKey(c->db,dstkey);
diff --git a/src/ziplist.c b/src/ziplist.c
index dbd804b11dfc..1a8566698972 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -265,6 +265,17 @@
ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \
}
+/* Don't let ziplists grow over 1GB in any case, don't wanna risk overflow in
+ * zlbytes*/
+#define ZIPLIST_MAX_SAFETY_SIZE (1<<30)
+int ziplistSafeToAdd(unsigned char* zl, size_t add) {
+ size_t len = zl? ziplistBlobLen(zl): 0;
+ if (len + add > ZIPLIST_MAX_SAFETY_SIZE)
+ return 0;
+ return 1;
+}
+
+
/* We use this function to receive information about a ziplist entry.
* Note that this is not how the data is actually encoded, is just what we
* get filled by a function in order to operate more easily. */
@@ -586,7 +597,8 @@ unsigned char *ziplistNew(void) {
}
/* Resize the ziplist. */
-unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
+unsigned char *ziplistResize(unsigned char *zl, size_t len) {
+ assert(len < UINT32_MAX);
zl = zrealloc(zl,len);
ZIPLIST_BYTES(zl) = intrev32ifbe(len);
zl[len-1] = ZIP_END;
@@ -898,6 +910,9 @@ unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) {
/* Combined zl length should be limited within UINT16_MAX */
zllength = zllength < UINT16_MAX ? zllength : UINT16_MAX;
+ /* larger values can't be stored into ZIPLIST_BYTES */
+ assert(zlbytes < UINT32_MAX);
+
/* Save offset positions before we start ripping memory apart. */
size_t first_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*first));
size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second));
diff --git a/src/ziplist.h b/src/ziplist.h
index 964a47f6dc29..f6ba6c8be47d 100644
--- a/src/ziplist.h
+++ b/src/ziplist.h
@@ -49,6 +49,7 @@ unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int v
unsigned int ziplistLen(unsigned char *zl);
size_t ziplistBlobLen(unsigned char *zl);
void ziplistRepr(unsigned char *zl);
+int ziplistSafeToAdd(unsigned char* zl, size_t add);
#ifdef REDIS_TEST
int ziplistTest(int argc, char *argv[]);
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 74f491e483a5..46b56cc2822a 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -99,6 +99,27 @@ proc wait_for_ofs_sync {r1 r2} {
}
}
+# count current log lines in server's stdout
+proc count_log_lines {srv_idx} {
+ set _ [string trim [exec wc -l < [srv $srv_idx stdout]]]
+}
+
+# returns the number of times a line with that pattern appears in a file
+proc count_message_lines {file pattern} {
+ set res 0
+ # exec fails when grep exists with status other than 0 (when the patter wasn't found)
+ catch {
+ set res [string trim [exec grep $pattern $file 2> /dev/null | wc -l]]
+ }
+ return $res
+}
+
+# returns the number of times a line with that pattern appears in the log
+proc count_log_message {srv_idx pattern} {
+ set stdout [srv $srv_idx stdout]
+ return [count_message_lines $stdout $pattern]
+}
+
# Random integer between 0 and max (excluded).
proc randomInt {max} {
expr {int(rand()*$max)}
diff --git a/tests/unit/violations.tcl b/tests/unit/violations.tcl
new file mode 100644
index 000000000000..d87b9236528e
--- /dev/null
+++ b/tests/unit/violations.tcl
@@ -0,0 +1,156 @@
+# These tests consume massive amounts of memory, and are not
+# suitable to be executed as part of the normal test suite
+set ::str500 [string repeat x 500000000] ;# 500mb
+
+# Utility function to write big argument into redis client connection
+proc write_big_bulk {size} {
+ r write "\$$size\r\n"
+ while {$size >= 500000000} {
+ r write $::str500
+ incr size -500000000
+ }
+ if {$size > 0} {
+ r write [string repeat x $size]
+ }
+ r write "\r\n"
+}
+
+# One XADD with one huge 5GB field
+# Expected to fail resulting in an empty stream
+start_server [list overrides [list save ""] ] {
+ test {XADD one huge field} {
+ r config set proto-max-bulk-len 10000000000 ;#10gb
+ r config set client-query-buffer-limit 10000000000 ;#10gb
+ r write "*5\r\n\$4\r\nXADD\r\n\$2\r\nS1\r\n\$1\r\n*\r\n"
+ r write "\$1\r\nA\r\n"
+ write_big_bulk 5000000000 ;#5gb
+ r flush
+ catch {r read} err
+ assert_match {*too large*} $err
+ r xlen S1
+ } {0}
+}
+
+# One XADD with one huge (exactly nearly) 4GB field
+# This uncovers the overflow in lpEncodeGetType
+# Expected to fail resulting in an empty stream
+start_server [list overrides [list save ""] ] {
+ test {XADD one huge field - 1} {
+ r config set proto-max-bulk-len 10000000000 ;#10gb
+ r config set client-query-buffer-limit 10000000000 ;#10gb
+ r write "*5\r\n\$4\r\nXADD\r\n\$2\r\nS1\r\n\$1\r\n*\r\n"
+ r write "\$1\r\nA\r\n"
+ write_big_bulk 4294967295 ;#4gb-1
+ r flush
+ catch {r read} err
+ assert_match {*too large*} $err
+ r xlen S1
+ } {0}
+}
+
+# Gradually add big stream fields using repeated XADD calls
+start_server [list overrides [list save ""] ] {
+ test {several XADD big fields} {
+ r config set stream-node-max-bytes 0
+ for {set j 0} {$j<10} {incr j} {
+ r xadd stream * 1 $::str500 2 $::str500
+ }
+ r ping
+ r xlen stream
+ } {10}
+}
+
+# Add over 4GB to a single stream listpack (one XADD command)
+# Expected to fail resulting in an empty stream
+start_server [list overrides [list save ""] ] {
+ test {single XADD big fields} {
+ r write "*23\r\n\$4\r\nXADD\r\n\$1\r\nS\r\n\$1\r\n*\r\n"
+ for {set j 0} {$j<10} {incr j} {
+ r write "\$1\r\n$j\r\n"
+ write_big_bulk 500000000 ;#500mb
+ }
+ r flush
+ catch {r read} err
+ assert_match {*too large*} $err
+ r xlen S
+ } {0}
+}
+
+# Gradually add big hash fields using repeated HSET calls
+# This reproduces the overflow in the call to ziplistResize
+# Object will be converted to hashtable encoding
+start_server [list overrides [list save ""] ] {
+ r config set hash-max-ziplist-value 1000000000 ;#1gb
+ test {hash with many big fields} {
+ for {set j 0} {$j<10} {incr j} {
+ r hset h $j $::str500
+ }
+ r object encoding h
+ } {hashtable}
+}
+
+# Add over 4GB to a single hash field (one HSET command)
+# Object will be converted to hashtable encoding
+start_server [list overrides [list save ""] ] {
+ test {hash with one huge field} {
+ catch {r config set hash-max-ziplist-value 10000000000} ;#10gb
+ r config set proto-max-bulk-len 10000000000 ;#10gb
+ r config set client-query-buffer-limit 10000000000 ;#10gb
+ r write "*4\r\n\$4\r\nHSET\r\n\$2\r\nH1\r\n"
+ r write "\$1\r\nA\r\n"
+ write_big_bulk 5000000000 ;#5gb
+ r flush
+ r read
+ r object encoding H1
+ } {hashtable}
+}
+
+# Add over 4GB to a single list member (one LPUSH command)
+# Currently unsupported, and expected to fail rather than being truncated
+# Expected to fail resulting in a non-existing list
+start_server [list overrides [list save ""] ] {
+ test {list with one huge field} {
+ r config set proto-max-bulk-len 10000000000 ;#10gb
+ r config set client-query-buffer-limit 10000000000 ;#10gb
+ r write "*3\r\n\$5\r\nLPUSH\r\n\$2\r\nL1\r\n"
+ write_big_bulk 5000000000 ;#5gb
+ r flush
+ catch {r read} err
+ assert_match {*too large*} $err
+ r exists L1
+ } {0}
+}
+
+# SORT which attempts to store an element larger than 4GB into a list.
+# Currently unsupported and results in an assertion instead of truncation
+start_server [list overrides [list save ""] ] {
+ test {SORT adds huge field to list} {
+ r config set proto-max-bulk-len 10000000000 ;#10gb
+ r config set client-query-buffer-limit 10000000000 ;#10gb
+ r write "*3\r\n\$3\r\nSET\r\n\$2\r\nS1\r\n"
+ write_big_bulk 5000000000 ;#5gb
+ r flush
+ r read
+ assert_equal [r strlen S1] 5000000000
+ r set S2 asdf
+ r sadd myset 1 2
+ r mset D1 1 D2 2
+ catch {r sort myset by D* get S* store mylist}
+ # assert_equal [count_log_message 0 "crashed by signal"] 0 - not suitable for 6.0
+ assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+ }
+}
+
+# SORT which stores an integer encoded element into a list.
+# Just for coverage, no news here.
+start_server [list overrides [list save ""] ] {
+ test {SORT adds integer field to list} {
+ r set S1 asdf
+ r set S2 123 ;# integer encoded
+ assert_encoding "int" S2
+ r sadd myset 1 2
+ r mset D1 1 D2 2
+ r sort myset by D* get S* store mylist
+ r llen mylist
+ } {2}
+}