Backport of the relevant parts of the upstream commit:

From 4a59cfaf9ab3d48fca4a15c0d2674bf7138e3d1a Mon Sep 17 00:00:00 2001
From: Michael Adams <mdadams@ece.uvic.ca>
Date: Sat, 26 Nov 2016 20:23:23 -0800
Subject: [PATCH] Fixed a buffer overrun problem in the QMFB code in the JPC
 codec that was caused by a buffer being allocated with a size that was too
 small in some cases. Added a new regression test case.

diff -pruN jasper-1.900.1.orig/src/libjasper/jpc/jpc_qmfb.c jasper-1.900.1/src/libjasper/jpc/jpc_qmfb.c
--- jasper-1.900.1.orig/src/libjasper/jpc/jpc_qmfb.c	2017-03-29 14:47:26.000000000 +0200
+++ jasper-1.900.1/src/libjasper/jpc/jpc_qmfb.c	2017-03-29 16:24:55.425985016 +0200
@@ -439,7 +439,7 @@ void jpc_qmfb_split_colgrp(jpc_fix_t *a,
 
 	/* Get a buffer. */
 	if (bufsize > QMFB_SPLITBUFSIZE) {
-		if (!(buf = jas_alloc2(bufsize, sizeof(jpc_fix_t)))) {
+		if (!(buf = jas_alloc2(bufsize, JPC_QMFB_COLGRPSIZE * sizeof(jpc_fix_t)))) {
 			/* We have no choice but to commit suicide in this case. */
 			abort();
 		}
@@ -520,7 +520,7 @@ void jpc_qmfb_split_colres(jpc_fix_t *a,
 
 	/* Get a buffer. */
 	if (bufsize > QMFB_SPLITBUFSIZE) {
-		if (!(buf = jas_alloc2(bufsize, sizeof(jpc_fix_t)))) {
+		if (!(buf = jas_alloc3(bufsize, numcols, sizeof(jpc_fix_t)))) {
 			/* We have no choice but to commit suicide in this case. */
 			abort();
 		}