Backport of the relevant parts of the upstream commit:
From 4a59cfaf9ab3d48fca4a15c0d2674bf7138e3d1a Mon Sep 17 00:00:00 2001
From: Michael Adams <mdadams@ece.uvic.ca>
Date: Sat, 26 Nov 2016 20:23:23 -0800
Subject: [PATCH] Fixed a buffer overrun problem in the QMFB code in the JPC
codec that was caused by a buffer being allocated with a size that was too
small in some cases. Added a new regression test case.
diff -pruN jasper-1.900.1.orig/src/libjasper/jpc/jpc_qmfb.c jasper-1.900.1/src/libjasper/jpc/jpc_qmfb.c
--- jasper-1.900.1.orig/src/libjasper/jpc/jpc_qmfb.c 2017-03-29 14:47:26.000000000 +0200
+++ jasper-1.900.1/src/libjasper/jpc/jpc_qmfb.c 2017-03-29 16:24:55.425985016 +0200
@@ -439,7 +439,7 @@ void jpc_qmfb_split_colgrp(jpc_fix_t *a,
/* Get a buffer. */
if (bufsize > QMFB_SPLITBUFSIZE) {
- if (!(buf = jas_alloc2(bufsize, sizeof(jpc_fix_t)))) {
+ if (!(buf = jas_alloc2(bufsize, JPC_QMFB_COLGRPSIZE * sizeof(jpc_fix_t)))) {
/* We have no choice but to commit suicide in this case. */
abort();
}
@@ -520,7 +520,7 @@ void jpc_qmfb_split_colres(jpc_fix_t *a,
/* Get a buffer. */
if (bufsize > QMFB_SPLITBUFSIZE) {
- if (!(buf = jas_alloc2(bufsize, sizeof(jpc_fix_t)))) {
+ if (!(buf = jas_alloc3(bufsize, numcols, sizeof(jpc_fix_t)))) {
/* We have no choice but to commit suicide in this case. */
abort();
}