Backported for 5.0.5

From 6facfb7a103b26b9a602253a738b2130afb7c5d3 Mon Sep 17 00:00:00 2001

From: Oran Agra <oran@redislabs.com>

Date: Thu, 3 Jun 2021 12:10:02 +0300

Subject: [PATCH] Fix ziplist and listpack overflows and truncations

 (CVE-2021-32627, CVE-2021-32628)

- fix possible heap corruption in ziplist and listpack resulting by trying to

  allocate more than the maximum size of 4GB.

- prevent ziplist (hash and zset) from reaching size of above 1GB, will be

  converted to HT encoding, that's not a useful size.

- prevent listpack (stream) from reaching size of above 1GB.

- XADD will start a new listpack if the new record may cause the previous

  listpack to grow over 1GB.

- XADD will respond with an error if a single stream record is over 1GB

- List type (ziplist in quicklist) was truncating strings that were over 4GB,

  now it'll respond with an error.

(cherry picked from commit 68e221a3f98a427805d31c1760b4cdf37ba810ab)

---

 src/geo.c                 |   5 +-

 src/listpack.c            |   2 +-

 src/quicklist.c           |  17 ++++-

 src/rdb.c                 |  36 ++++++---

 src/server.h              |   2 +-

 src/t_hash.c              |  13 +++-

 src/t_list.c              |  25 ++++++

 src/t_stream.c            |  48 +++++++++---

 src/t_zset.c              |  43 +++++++----

 src/ziplist.c             |  17 ++++-

 src/ziplist.h             |   1 +

 tests/support/util.tcl    |  21 +++++

 tests/unit/violations.tcl | 156 ++++++++++++++++++++++++++++++++++++++

 13 files changed, 338 insertions(+), 48 deletions(-)

 create mode 100644 tests/unit/violations.tcl

diff --git a/src/geo.c b/src/geo.c

index f1d3f18d46e7..b94fcc1b3d70 100644

--- a/src/geo.c

+++ b/src/geo.c

@@ -635,7 +635,7 @@ void georadiusGeneric(client *c, int fla

         robj *zobj;

         zset *zs;

         int i;

-        size_t maxelelen = 0;

+        size_t maxelelen = 0, totelelen = 0;

         if (returned_items) {

             zobj = createZsetObject();

@@ -650,13 +650,14 @@ void georadiusGeneric(client *c, int fla

             size_t elelen = sdslen(gp->member);

             if (maxelelen < elelen) maxelelen = elelen;

+            totelelen += elelen;

             znode = zslInsert(zs->zsl,score,gp->member);

             serverAssert(dictAdd(zs->dict,gp->member,&znode->score) == DICT_OK);

             gp->member = NULL;

         }

         if (returned_items) {

-            zsetConvertToZiplistIfNeeded(zobj,maxelelen);

+            zsetConvertToZiplistIfNeeded(zobj,maxelelen,totelelen);

             setKey(c->db,storekey,zobj);

             decrRefCount(zobj);

             notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey,

diff --git a/src/listpack.c b/src/listpack.c

index e1f4d9a02ee8..cd5583ccb258 100644

--- a/src/listpack.c

+++ b/src/listpack.c

@@ -283,7 +283,7 @@ int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, ui

     } else {

         if (size < 64) *enclen = 1+size;

         else if (size < 4096) *enclen = 2+size;

-        else *enclen = 5+size;

+        else *enclen = 5+(uint64_t)size;

         return LP_ENCODING_STRING;

     }

 }

diff --git a/src/quicklist.c b/src/quicklist.c

index 7b5484116785..d5cc758b2fa0 100644

--- a/src/quicklist.c

+++ b/src/quicklist.c

@@ -29,6 +29,7 @@

  */

 #include <string.h> /* for memcpy */

+#include "redisassert.h"

 #include "quicklist.h"

 #include "zmalloc.h"

 #include "ziplist.h"

@@ -43,11 +44,16 @@

 #define REDIS_STATIC static

 #endif

-/* Optimization levels for size-based filling */

+/* Optimization levels for size-based filling.

+ * Note that the largest possible limit is 16k, so even if each record takes

+ * just one byte, it still won't overflow the 16 bit count field. */

 static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536};

 /* Maximum size in bytes of any multi-element ziplist.

- * Larger values will live in their own isolated ziplists. */

+ * Larger values will live in their own isolated ziplists.

+ * This is used only if we're limited by record count. when we're limited by

+ * size, the maximum limit is bigger, but still safe.

+ * 8k is a recommended / default size limit */

 #define SIZE_SAFETY_LIMIT 8192

 /* Minimum ziplist size in bytes for attempting compression. */

@@ -441,6 +447,8 @@ REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node,

     unsigned int new_sz = node->sz + sz + ziplist_overhead;

     if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill)))

         return 1;

+    /* when we return 1 above we know that the limit is a size limit (which is

+     * safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */

     else if (!sizeMeetsSafetyLimit(new_sz))

         return 0;

     else if ((int)node->count < fill)

@@ -460,6 +468,8 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,

     unsigned int merge_sz = a->sz + b->sz - 11;

     if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill)))

         return 1;

+    /* when we return 1 above we know that the limit is a size limit (which is

+     * safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */

     else if (!sizeMeetsSafetyLimit(merge_sz))

         return 0;

     else if ((int)(a->count + b->count) <= fill)

@@ -479,6 +489,7 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,

  * Returns 1 if new head created. */

 int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) {

     quicklistNode *orig_head = quicklist->head;

+    assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */

     if (likely(

             _quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz))) {

         quicklist->head->zl =

@@ -502,6 +513,7 @@ int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) {

  * Returns 1 if new tail created. */

 int quicklistPushTail(quicklist *quicklist, void *value, size_t sz) {

     quicklistNode *orig_tail = quicklist->tail;

+    assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */

     if (likely(

             _quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz))) {

         quicklist->tail->zl =

@@ -835,6 +847,7 @@ REDIS_STATIC void _quicklistInsert(quicklist *quicklist, quicklistEntry *entry,

     int fill = quicklist->fill;

     quicklistNode *node = entry->node;

     quicklistNode *new_node = NULL;

+    assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */

     if (!node) {

         /* we have no reference node, so let's create only node in the list */

diff --git a/src/rdb.c b/src/rdb.c

index 3c58a1eaf7fb..c7dc724f3df6 100644

--- a/src/rdb.c

+++ b/src/rdb.c

@@ -1452,7 +1452,7 @@ robj *rdbLoadObject(int rdbtype, rio *rd

     } else if (rdbtype == RDB_TYPE_ZSET_2 || rdbtype == RDB_TYPE_ZSET) {

         /* Read list/set value. */

         uint64_t zsetlen;

-        size_t maxelelen = 0;

+        size_t maxelelen = 0, totelelen = 0;

         zset *zs;

         if ((zsetlen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;

@@ -1479,6 +1479,7 @@ robj *rdbLoadObject(int rdbtype, rio *rd

             /* Don't care about integer-encoded strings. */

             if (sdslen(sdsele) > maxelelen) maxelelen = sdslen(sdsele);

+            totelelen += sdslen(sdsele);

             znode = zslInsert(zs->zsl,score,sdsele);

             dictAdd(zs->dict,sdsele,&znode->score);

@@ -1486,8 +1487,11 @@ robj *rdbLoadObject(int rdbtype, rio *rd

         /* Convert *after* loading, since sorted sets are not stored ordered. */

         if (zsetLength(o) <= server.zset_max_ziplist_entries &&

-            maxelelen <= server.zset_max_ziplist_value)

-                zsetConvert(o,OBJ_ENCODING_ZIPLIST);

+            maxelelen <= server.zset_max_ziplist_value &&

+            ziplistSafeToAdd(NULL, totelelen))

+        {

+            zsetConvert(o,OBJ_ENCODING_ZIPLIST);

+        }

     } else if (rdbtype == RDB_TYPE_HASH) {

         uint64_t len;

         int ret;

@@ -1511,21 +1515,25 @@ robj *rdbLoadObject(int rdbtype, rio *rd

             if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))

                 == NULL) return NULL;

-            /* Add pair to ziplist */

-            o->ptr = ziplistPush(o->ptr, (unsigned char*)field,

-                    sdslen(field), ZIPLIST_TAIL);

-            o->ptr = ziplistPush(o->ptr, (unsigned char*)value,

-                    sdslen(value), ZIPLIST_TAIL);

-

             /* Convert to hash table if size threshold is exceeded */

             if (sdslen(field) > server.hash_max_ziplist_value ||

-                sdslen(value) > server.hash_max_ziplist_value)

+                sdslen(value) > server.hash_max_ziplist_value ||

+                !ziplistSafeToAdd(o->ptr, sdslen(field)+sdslen(value)))

             {

-                sdsfree(field);

-                sdsfree(value);

                 hashTypeConvert(o, OBJ_ENCODING_HT);

+                ret = dictAdd((dict*)o->ptr, field, value);

+                if (ret == DICT_ERR) {

+                    rdbExitReportCorruptRDB("Duplicate hash fields detected");

+                }

                 break;

             }

+

+            /* Add pair to ziplist */

+            o->ptr = ziplistPush(o->ptr, (unsigned char*)field,

+                    sdslen(field), ZIPLIST_TAIL);

+            o->ptr = ziplistPush(o->ptr, (unsigned char*)value,

+                    sdslen(value), ZIPLIST_TAIL);

+

             sdsfree(field);

             sdsfree(value);

         }

@@ -1594,6 +1602,10 @@ robj *rdbLoadObject(int rdbtype, rio *rd

                     while ((zi = zipmapNext(zi, &fstr, &flen, &vstr, &vlen)) != NULL) {

                         if (flen > maxlen) maxlen = flen;

                         if (vlen > maxlen) maxlen = vlen;

+                        if (!ziplistSafeToAdd(zl, (size_t)flen + vlen)) {

+                            rdbExitReportCorruptRDB("Hash zipmap too big (%u)", flen);

+                        }

+

                         zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL);

                         zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL);

                     }

diff --git a/src/server.h b/src/server.h

index ca868939cf6d..164a82271f44 100644

--- a/src/server.h

+++ b/src/server.h

@@ -1677,7 +1677,7 @@ unsigned char *zzlFirstInRange(unsigned char *zl, zrangespec *range);

 unsigned char *zzlLastInRange(unsigned char *zl, zrangespec *range);

 unsigned long zsetLength(const robj *zobj);

 void zsetConvert(robj *zobj, int encoding);

-void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen);

+void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen);

 int zsetScore(robj *zobj, sds member, double *score);

 unsigned long zslGetRank(zskiplist *zsl, double score, sds o);

 int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore);

diff --git a/src/t_hash.c b/src/t_hash.c

index 0ca152df78cc..109522c1322f 100644

--- a/src/t_hash.c

+++ b/src/t_hash.c

@@ -39,17 +39,22 @@

  * as their string length can be queried in constant time. */

 void hashTypeTryConversion(robj *o, robj **argv, int start, int end) {

     int i;

+    size_t sum = 0;

     if (o->encoding != OBJ_ENCODING_ZIPLIST) return;

     for (i = start; i <= end; i++) {

-        if (sdsEncodedObject(argv[i]) &&

-            sdslen(argv[i]->ptr) > server.hash_max_ziplist_value)

-        {

+        if (!sdsEncodedObject(argv[i]))

+            continue;

+        size_t len = sdslen(argv[i]->ptr);

+        if (len > server.hash_max_ziplist_value) {

             hashTypeConvert(o, OBJ_ENCODING_HT);

-            break;

+            return;

         }

+        sum += len;

     }

+    if (!ziplistSafeToAdd(o->ptr, sum))

+        hashTypeConvert(o, OBJ_ENCODING_HT);

 }

 /* Get the value from a ziplist encoded hash, identified by field.

diff --git a/src/t_list.c b/src/t_list.c

index de417f4705f4..67541554f616 100644

--- a/src/t_list.c

+++ b/src/t_list.c

@@ -29,6 +29,8 @@

 #include "server.h"

+#define LIST_MAX_ITEM_SIZE ((1ull<<32)-1024)

+

 /*-----------------------------------------------------------------------------

  * List API

  *----------------------------------------------------------------------------*/

@@ -196,6 +198,14 @@ void listTypeConvert(robj *subject, int enc) {

 void pushGenericCommand(client *c, int where) {

     int j, pushed = 0;

+

+    for (j = 2; j < c->argc; j++) {

+        if (sdslen(c->argv[j]->ptr) > LIST_MAX_ITEM_SIZE) {

+            addReplyError(c, "Element too large");

+            return;

+        }

+    }

+

     robj *lobj = lookupKeyWrite(c->db,c->argv[1]);

     if (lobj && lobj->type != OBJ_LIST) {

@@ -277,6 +287,11 @@ void linsertCommand(client *c) {

         return;

     }

+    if (sdslen(c->argv[4]->ptr) > LIST_MAX_ITEM_SIZE) {

+        addReplyError(c, "Element too large");

+        return;

+    }

+

     if ((subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||

         checkType(c,subject,OBJ_LIST)) return;

@@ -344,6 +359,11 @@ void lsetCommand(client *c) {

     long index;

     robj *value = c->argv[3];

+    if (sdslen(value->ptr) > LIST_MAX_ITEM_SIZE) {

+        addReplyError(c, "Element too large");

+        return;

+    }

+

     if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != C_OK))

         return;

@@ -493,6 +513,11 @@ void lremCommand(client *c) {

     long toremove;

     long removed = 0;

+    if (sdslen(obj->ptr) > LIST_MAX_ITEM_SIZE) {

+        addReplyError(c, "Element too large");

+        return;

+    }

+

     if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != C_OK))

         return;

diff --git a/src/t_stream.c b/src/t_stream.c

index d7754985dd03..e7263d68a28f 100644

--- a/src/t_stream.c

+++ b/src/t_stream.c

@@ -40,6 +40,12 @@

 #define STREAM_ITEM_FLAG_DELETED (1<<0)     /* Entry is delted. Skip it. */

 #define STREAM_ITEM_FLAG_SAMEFIELDS (1<<1)  /* Same fields as master entry. */

+/* Don't let listpacks grow too big, even if the user config allows it.

+ * doing so can lead to an overflow (trying to store more than 32bit length

+ * into the listpack header), or actually an assertion since lpInsert

+ * will return NULL. */

+#define STREAM_LISTPACK_MAX_SIZE (1<<30)

+

 void streamFreeCG(streamCG *cg);

 void streamFreeNACK(streamNACK *na);

 size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start, streamID *end, size_t count, streamConsumer *consumer);

@@ -170,12 +176,31 @@ int streamCompareID(streamID *a, streamI

  *

  * The function returns C_OK if the item was added, this is always true

  * if the ID was generated by the function. However the function may return

- * C_ERR if an ID was given via 'use_id', but adding it failed since the

- * current top ID is greater or equal. */

+ * C_ERR in several cases:

+ * 1. If an ID was given via 'use_id', but adding it failed since the

+ *    current top ID is greater or equal. errno will be set to EDOM.

+ * 2. If a size of a single element or the sum of the elements is too big to

+ *    be stored into the stream. errno will be set to ERANGE. */

 int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id) {

     /* If an ID was given, check that it's greater than the last entry ID

      * or return an error. */

-    if (use_id && streamCompareID(use_id,&s->last_id) <= 0) return C_ERR;

+    if (use_id && streamCompareID(use_id,&s->last_id) <= 0) {

+        errno = EDOM;

+        return C_ERR;

+    }

+

+    /* Avoid overflow when trying to add an element to the stream (listpack

+     * can only host up to 32bit length sttrings, and also a total listpack size

+     * can't be bigger than 32bit length. */

+    size_t totelelen = 0;

+    for (int64_t i = 0; i < numfields*2; i++) {

+        sds ele = argv[i]->ptr;

+        totelelen += sdslen(ele);

+    }

+    if (totelelen > STREAM_LISTPACK_MAX_SIZE) {

+        errno = ERANGE;

+        return C_ERR;

+    }

     /* Add the new entry. */

     raxIterator ri;

@@ -241,9 +266,10 @@ int streamAppendItem(stream *s, robj **a

      * if we need to switch to the next one. 'lp' will be set to NULL if

      * the current node is full. */

     if (lp != NULL) {

-        if (server.stream_node_max_bytes &&

-            lp_bytes > server.stream_node_max_bytes)

-        {

+        size_t node_max_bytes = server.stream_node_max_bytes;

+        if (node_max_bytes == 0 || node_max_bytes > STREAM_LISTPACK_MAX_SIZE)

+            node_max_bytes = STREAM_LISTPACK_MAX_SIZE;

+        if (lp_bytes + totelelen >= node_max_bytes) {

             lp = NULL;

         } else if (server.stream_node_max_entries) {

             int64_t count = lpGetInteger(lpFirst(lp));

@@ -1224,11 +1250,13 @@ void xaddCommand(client *c) {

     /* Append using the low level function and return the ID. */

     if (streamAppendItem(s,c->argv+field_pos,(c->argc-field_pos)/2,

-        &id, id_given ? &id : NULL)

-        == C_ERR)

+        &id, id_given ? &id : NULL) == C_ERR)

     {

-        addReplyError(c,"The ID specified in XADD is equal or smaller than the "

-                        "target stream top item");

+        if (errno == EDOM)

+            addReplyError(c,"The ID specified in XADD is equal or smaller than "

+                            "the target stream top item");

+        else

+            addReplyError(c,"Elements are too large to be stored");

         return;

     }

     addReplyStreamID(c,&id;;

diff --git a/src/t_zset.c b/src/t_zset.c

index 56ea39607b52..989d5855e1ea 100644

--- a/src/t_zset.c

+++ b/src/t_zset.c

@@ -1237,15 +1237,18 @@ void zsetConvert(robj *zobj, int encodin

 }

 /* Convert the sorted set object into a ziplist if it is not already a ziplist

- * and if the number of elements and the maximum element size is within the

- * expected ranges. */

-void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen) {

+ * and if the number of elements and the maximum element size and total elements size

+ * are within the expected ranges. */

+void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen) {

     if (zobj->encoding == OBJ_ENCODING_ZIPLIST) return;

     zset *zset = zobj->ptr;

     if (zset->zsl->length <= server.zset_max_ziplist_entries &&

-        maxelelen <= server.zset_max_ziplist_value)

-            zsetConvert(zobj,OBJ_ENCODING_ZIPLIST);

+        maxelelen <= server.zset_max_ziplist_value &&

+        ziplistSafeToAdd(NULL, totelelen))

+    {

+        zsetConvert(zobj,OBJ_ENCODING_ZIPLIST);

+    }

 }

 /* Return (by reference) the score of the specified member of the sorted set

@@ -1354,21 +1357,28 @@ int zsetAdd(robj *zobj, double score, sd

             }

             return 1;

         } else if (!xx) {

-            /* Optimize: check if the element is too large or the list

+            /* check if the element is too large or the list

              * becomes too long *before* executing zzlInsert. */

-            zobj->ptr = zzlInsert(zobj->ptr,ele,score);

-            if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries)

-                zsetConvert(zobj,OBJ_ENCODING_SKIPLIST);

-            if (sdslen(ele) > server.zset_max_ziplist_value)

+            if (zzlLength(zobj->ptr)+1 > server.zset_max_ziplist_entries ||

+                sdslen(ele) > server.zset_max_ziplist_value ||

+                !ziplistSafeToAdd(zobj->ptr, sdslen(ele)))

+            {

                 zsetConvert(zobj,OBJ_ENCODING_SKIPLIST);

-            if (newscore) *newscore = score;

-            *flags |= ZADD_ADDED;

-            return 1;

+            } else {

+                zobj->ptr = zzlInsert(zobj->ptr,ele,score);

+                if (newscore) *newscore = score;

+                *flags |= ZADD_ADDED;

+                return 1;

+            }

         } else {

             *flags |= ZADD_NOP;

             return 1;

         }

-    } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {

+    }

+

+    /* Note that the above block handling ziplist would have either returned or

+     * converted the key to skiplist. */

+    if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {

         zset *zs = zobj->ptr;

         zskiplistNode *znode;

         dictEntry *de;

@@ -2180,7 +2190,7 @@ void zunionInterGenericCommand(client *c

     zsetopsrc *src;

     zsetopval zval;

     sds tmp;

-    size_t maxelelen = 0;

+    size_t maxelelen = 0, totelelen = 0;

     robj *dstobj;

     zset *dstzset;

     zskiplistNode *znode;

@@ -2304,6 +2314,7 @@ void zunionInterGenericCommand(client *c

                     tmp = zuiNewSdsFromValue(&zval);

                     znode = zslInsert(dstzset->zsl,score,tmp);

                     dictAdd(dstzset->dict,tmp,&znode->score);

+                    totelelen += sdslen(tmp);

                     if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp);

                 }

             }

@@ -2340,6 +2351,7 @@ void zunionInterGenericCommand(client *c

                     /* Remember the longest single element encountered,

                      * to understand if it's possible to convert to ziplist

                      * at the end. */

+                     totelelen += sdslen(tmp);

                      if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp);

                     /* Update the element with its initial score. */

                     dictSetKey(accumulator, de, tmp);

@@ -2380,7 +2392,7 @@ void zunionInterGenericCommand(client *c

     if (dbDelete(c->db,dstkey))

         touched = 1;

     if (dstzset->zsl->length) {

-        zsetConvertToZiplistIfNeeded(dstobj,maxelelen);

+        zsetConvertToZiplistIfNeeded(dstobj,maxelelen,totelelen);

         dbAdd(c->db,dstkey,dstobj);

         addReplyLongLong(c,zsetLength(dstobj));

         signalModifiedKey(c->db,dstkey);

diff --git a/src/ziplist.c b/src/ziplist.c

index dbd804b11dfc..1a8566698972 100644

--- a/src/ziplist.c

+++ b/src/ziplist.c

@@ -265,6 +265,17 @@

         ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \

 }

+/* Don't let ziplists grow over 1GB in any case, don't wanna risk overflow in

+ * zlbytes*/

+#define ZIPLIST_MAX_SAFETY_SIZE (1<<30)

+int ziplistSafeToAdd(unsigned char* zl, size_t add) {

+    size_t len = zl? ziplistBlobLen(zl): 0;

+    if (len + add > ZIPLIST_MAX_SAFETY_SIZE)

+        return 0;

+    return 1;

+}

+

+

 /* We use this function to receive information about a ziplist entry.

  * Note that this is not how the data is actually encoded, is just what we

  * get filled by a function in order to operate more easily. */

@@ -586,7 +597,8 @@ unsigned char *ziplistNew(void) {

 }

 /* Resize the ziplist. */

-unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {

+unsigned char *ziplistResize(unsigned char *zl, size_t len) {

+    assert(len < UINT32_MAX);

     zl = zrealloc(zl,len);

     ZIPLIST_BYTES(zl) = intrev32ifbe(len);

     zl[len-1] = ZIP_END;

@@ -898,6 +910,9 @@ unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) {

     /* Combined zl length should be limited within UINT16_MAX */

     zllength = zllength < UINT16_MAX ? zllength : UINT16_MAX;

+    /* larger values can't be stored into ZIPLIST_BYTES */

+    assert(zlbytes < UINT32_MAX);

+

     /* Save offset positions before we start ripping memory apart. */

     size_t first_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*first));

     size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second));

diff --git a/src/ziplist.h b/src/ziplist.h

index 964a47f6dc29..f6ba6c8be47d 100644

--- a/src/ziplist.h

+++ b/src/ziplist.h

@@ -49,6 +49,7 @@ unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int v

 unsigned int ziplistLen(unsigned char *zl);

 size_t ziplistBlobLen(unsigned char *zl);

 void ziplistRepr(unsigned char *zl);

+int ziplistSafeToAdd(unsigned char* zl, size_t add);

 #ifdef REDIS_TEST

 int ziplistTest(int argc, char *argv[]);

diff --git a/tests/support/util.tcl b/tests/support/util.tcl

index 74f491e483a5..46b56cc2822a 100644

--- a/tests/support/util.tcl

+++ b/tests/support/util.tcl

@@ -99,6 +99,27 @@ proc wait_for_ofs_sync {r1 r2} {

     }

 }

+# count current log lines in server's stdout

+proc count_log_lines {srv_idx} {

+    set _ [string trim [exec wc -l < [srv $srv_idx stdout]]]

+}

+

+# returns the number of times a line with that pattern appears in a file

+proc count_message_lines {file pattern} {

+    set res 0

+    # exec fails when grep exists with status other than 0 (when the patter wasn't found)

+    catch {

+        set res [string trim [exec grep $pattern $file 2> /dev/null | wc -l]]

+    }

+    return $res

+}

+

+# returns the number of times a line with that pattern appears in the log

+proc count_log_message {srv_idx pattern} {

+    set stdout [srv $srv_idx stdout]

+    return [count_message_lines $stdout $pattern]

+}

+

 # Random integer between 0 and max (excluded).

 proc randomInt {max} {

     expr {int(rand()*$max)}

diff --git a/tests/unit/violations.tcl b/tests/unit/violations.tcl

new file mode 100644

index 000000000000..d87b9236528e

--- /dev/null

+++ b/tests/unit/violations.tcl

@@ -0,0 +1,156 @@

+# These tests consume massive amounts of memory, and are not

+# suitable to be executed as part of the normal test suite

+set ::str500 [string repeat x 500000000] ;# 500mb

+

+# Utility function to write big argument into redis client connection

+proc write_big_bulk {size} {

+    r write "\$$size\r\n"

+    while {$size >= 500000000} {

+        r write $::str500

+        incr size -500000000

+    }

+    if {$size > 0} {

+        r write [string repeat x $size]

+    }

+    r write "\r\n"

+}

+

+# One XADD with one huge 5GB field

+# Expected to fail resulting in an empty stream

+start_server [list overrides [list save ""] ] {

+    test {XADD one huge field} {

+        r config set proto-max-bulk-len 10000000000 ;#10gb

+        r config set client-query-buffer-limit 10000000000 ;#10gb

+        r write "*5\r\n\$4\r\nXADD\r\n\$2\r\nS1\r\n\$1\r\n*\r\n"

+        r write "\$1\r\nA\r\n"

+        write_big_bulk 5000000000 ;#5gb

+        r flush

+        catch {r read} err

+        assert_match {*too large*} $err

+        r xlen S1

+    } {0}

+}

+

+# One XADD with one huge (exactly nearly) 4GB field

+# This uncovers the overflow in lpEncodeGetType

+# Expected to fail resulting in an empty stream

+start_server [list overrides [list save ""] ] {

+    test {XADD one huge field - 1} {

+        r config set proto-max-bulk-len 10000000000 ;#10gb

+        r config set client-query-buffer-limit 10000000000 ;#10gb

+        r write "*5\r\n\$4\r\nXADD\r\n\$2\r\nS1\r\n\$1\r\n*\r\n"

+        r write "\$1\r\nA\r\n"

+        write_big_bulk 4294967295 ;#4gb-1

+        r flush

+        catch {r read} err

+        assert_match {*too large*} $err

+        r xlen S1

+    } {0}

+}

+

+# Gradually add big stream fields using repeated XADD calls

+start_server [list overrides [list save ""] ] {

+    test {several XADD big fields} {

+        r config set stream-node-max-bytes 0

+        for {set j 0} {$j<10} {incr j} {

+            r xadd stream * 1 $::str500 2 $::str500

+        }

+        r ping

+        r xlen stream

+    } {10}

+}

+

+# Add over 4GB to a single stream listpack (one XADD command)

+# Expected to fail resulting in an empty stream

+start_server [list overrides [list save ""] ] {

+    test {single XADD big fields} {

+        r write "*23\r\n\$4\r\nXADD\r\n\$1\r\nS\r\n\$1\r\n*\r\n"

+        for {set j 0} {$j<10} {incr j} {

+            r write "\$1\r\n$j\r\n"

+            write_big_bulk 500000000 ;#500mb

+        }

+        r flush

+        catch {r read} err

+        assert_match {*too large*} $err

+        r xlen S

+    } {0}

+}

+

+# Gradually add big hash fields using repeated HSET calls

+# This reproduces the overflow in the call to ziplistResize

+# Object will be converted to hashtable encoding

+start_server [list overrides [list save ""] ] {

+    r config set hash-max-ziplist-value 1000000000 ;#1gb

+    test {hash with many big fields} {

+        for {set j 0} {$j<10} {incr j} {

+            r hset h $j $::str500

+        }

+        r object encoding h

+    } {hashtable}

+}

+

+# Add over 4GB to a single hash field (one HSET command)

+# Object will be converted to hashtable encoding

+start_server [list overrides [list save ""] ] {

+    test {hash with one huge field} {

+        catch {r config set hash-max-ziplist-value 10000000000} ;#10gb

+        r config set proto-max-bulk-len 10000000000 ;#10gb

+        r config set client-query-buffer-limit 10000000000 ;#10gb

+        r write "*4\r\n\$4\r\nHSET\r\n\$2\r\nH1\r\n"

+        r write "\$1\r\nA\r\n"

+        write_big_bulk 5000000000 ;#5gb