Blame SOURCES/jasper-CVE-2016-9583.patch

83be9e
Backport of upstream commits:
83be9e
83be9e
From aa0b0f79ade5eef8b0e7a214c03f5af54b36ba7d Mon Sep 17 00:00:00 2001
83be9e
From: Michael Adams <mdadams@ece.uvic.ca>
83be9e
Date: Sat, 26 Nov 2016 17:14:09 -0800
83be9e
Subject: [PATCH] Fixed numerous integer overflow problems in the code for
83be9e
 packet iterators in the JPC decoder.
83be9e
83be9e
From f25486c3d4aa472fec79150f2c41ed4333395d3d Mon Sep 17 00:00:00 2001
83be9e
From: Michael Adams <mdadams@ece.uvic.ca>
83be9e
Date: Sat, 26 Nov 2016 20:54:24 -0800
83be9e
Subject: [PATCH] Fixed a bug in the packet iterator code. Added a new
83be9e
 regression test case.
83be9e
83be9e
From 99a50593254d1b53002719bbecfc946c84b23d27 Mon Sep 17 00:00:00 2001
83be9e
From: Michael Adams <mdadams@ece.uvic.ca>
83be9e
Date: Fri, 9 Dec 2016 05:42:39 -0800
83be9e
Subject: [PATCH] Apply a patch for the following bug report:    
83be9e
 https://github.com/mdadams/jasper/issues/103     Heap-Buffer-Overflow or
83be9e
 Null-pointer-dereference vulnerability due     to a programming mistake
83be9e
 (off-by-1)
83be9e
83be9e
diff -pruN jasper-1.900.1.orig/src/libjasper/include/jasper/jas_types.h jasper-1.900.1/src/libjasper/include/jasper/jas_types.h
83be9e
--- jasper-1.900.1.orig/src/libjasper/include/jasper/jas_types.h	2007-01-19 22:43:04.000000000 +0100
83be9e
+++ jasper-1.900.1/src/libjasper/include/jasper/jas_types.h	2017-03-30 22:12:18.000000000 +0200
83be9e
@@ -217,6 +217,10 @@ typedef ulonglong uint_fast64_t;
83be9e
 #define	JAS_CAST(t, e) \
83be9e
 	((t) (e))
83be9e
 
83be9e
+/* The number of bits in the integeral type uint_fast32_t. */
83be9e
+/* NOTE: This could underestimate the size on some exotic architectures. */
83be9e
+#define JAS_UINTFAST32_NUMBITS (8 * sizeof(uint_fast32_t))
83be9e
+
83be9e
 #ifdef __cplusplus
83be9e
 extern "C" {
83be9e
 #endif
83be9e
diff -pruN jasper-1.900.1.orig/src/libjasper/jpc/jpc_t2cod.c jasper-1.900.1/src/libjasper/jpc/jpc_t2cod.c
83be9e
--- jasper-1.900.1.orig/src/libjasper/jpc/jpc_t2cod.c	2017-03-30 18:03:55.000000000 +0200
83be9e
+++ jasper-1.900.1/src/libjasper/jpc/jpc_t2cod.c	2017-03-30 22:14:39.000000000 +0200
83be9e
@@ -249,10 +249,17 @@ static int jpc_pi_nextrpcl(register jpc_
83be9e
 		  ++compno, ++picomp) {
83be9e
 			for (rlvlno = 0, pirlvl = picomp->pirlvls; rlvlno <
83be9e
 			  picomp->numrlvls; ++rlvlno, ++pirlvl) {
83be9e
-				xstep = picomp->hsamp * (1 << (pirlvl->prcwidthexpn +
83be9e
-				  picomp->numrlvls - rlvlno - 1));
83be9e
-				ystep = picomp->vsamp * (1 << (pirlvl->prcheightexpn +
83be9e
-				  picomp->numrlvls - rlvlno - 1));
83be9e
+				// Check for the potential for overflow problems.
83be9e
+				if (pirlvl->prcwidthexpn + picomp->numrlvls >
83be9e
+				  JAS_UINTFAST32_NUMBITS - 2 ||
83be9e
+				  pirlvl->prcheightexpn + picomp->numrlvls >
83be9e
+				  JAS_UINTFAST32_NUMBITS - 2) {
83be9e
+					return -1;
83be9e
+				}
83be9e
+				xstep = picomp->hsamp * (JAS_CAST(uint_fast32_t, 1) <<
83be9e
+				  (pirlvl->prcwidthexpn + picomp->numrlvls - rlvlno - 1));
83be9e
+				ystep = picomp->vsamp * (JAS_CAST(uint_fast32_t, 1) <<
83be9e
+				  (pirlvl->prcheightexpn + picomp->numrlvls - rlvlno - 1));
83be9e
 				pi->xstep = (!pi->xstep) ? xstep : JAS_MIN(pi->xstep, xstep);
83be9e
 				pi->ystep = (!pi->ystep) ? ystep : JAS_MIN(pi->ystep, ystep);
83be9e
 			}
83be9e
@@ -282,21 +289,24 @@ static int jpc_pi_nextrpcl(register jpc_
83be9e
 					rpy = r + pi->pirlvl->prcheightexpn;
83be9e
 					trx0 = JPC_CEILDIV(pi->xstart, pi->picomp->hsamp << r);
83be9e
 					try0 = JPC_CEILDIV(pi->ystart, pi->picomp->vsamp << r);
83be9e
-					if (((pi->x == pi->xstart && ((trx0 << r) % (1 << rpx)))
83be9e
-					  || !(pi->x % (1 << rpx))) &&
83be9e
-					  ((pi->y == pi->ystart && ((try0 << r) % (1 << rpy)))
83be9e
-					  || !(pi->y % (1 << rpy)))) {
83be9e
-						prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x, pi->picomp->hsamp
83be9e
-						  << r), pi->pirlvl->prcwidthexpn) - JPC_FLOORDIVPOW2(trx0,
83be9e
-						  pi->pirlvl->prcwidthexpn);
83be9e
-						prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y, pi->picomp->vsamp
83be9e
-						  << r), pi->pirlvl->prcheightexpn) - JPC_FLOORDIVPOW2(try0,
83be9e
-						  pi->pirlvl->prcheightexpn);
83be9e
+					if (((pi->x == pi->xstart &&
83be9e
+					  ((trx0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpx)))
83be9e
+					  || !(pi->x % (JAS_CAST(uint_fast32_t, 1) << rpx))) &&
83be9e
+					  ((pi->y == pi->ystart &&
83be9e
+					  ((try0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpy)))
83be9e
+					  || !(pi->y % (JAS_CAST(uint_fast32_t, 1) << rpy)))) {
83be9e
+						prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x,
83be9e
+						  pi->picomp->hsamp << r), pi->pirlvl->prcwidthexpn) -
83be9e
+						  JPC_FLOORDIVPOW2(trx0, pi->pirlvl->prcwidthexpn);
83be9e
+						prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y,
83be9e
+						  pi->picomp->vsamp << r), pi->pirlvl->prcheightexpn) -
83be9e
+						  JPC_FLOORDIVPOW2(try0, pi->pirlvl->prcheightexpn);
83be9e
 						pi->prcno = prcvind * pi->pirlvl->numhprcs + prchind;
83be9e
 
83be9e
 						assert(pi->prcno < pi->pirlvl->numprcs);
83be9e
 						for (pi->lyrno = 0; pi->lyrno <
83be9e
-						  pi->numlyrs && pi->lyrno < JAS_CAST(int, pchg->lyrnoend); ++pi->lyrno) {
83be9e
+						  pi->numlyrs && pi->lyrno < JAS_CAST(int,
83be9e
+						  pchg->lyrnoend); ++pi->lyrno) {
83be9e
 							prclyrno = &pi->pirlvl->prclyrnos[pi->prcno];
83be9e
 							if (pi->lyrno >= *prclyrno) {
83be9e
 								++(*prclyrno);
83be9e
@@ -341,16 +351,19 @@ static int jpc_pi_nextpcrl(register jpc_
83be9e
 		  ++compno, ++picomp) {
83be9e
 			for (rlvlno = 0, pirlvl = picomp->pirlvls; rlvlno <
83be9e
 			  picomp->numrlvls; ++rlvlno, ++pirlvl) {
83be9e
-				xstep = picomp->hsamp * (1 <<
83be9e
-				  (pirlvl->prcwidthexpn + picomp->numrlvls -
83be9e
-				  rlvlno - 1));
83be9e
-				ystep = picomp->vsamp * (1 <<
83be9e
-				  (pirlvl->prcheightexpn + picomp->numrlvls -
83be9e
-				  rlvlno - 1));
83be9e
-				pi->xstep = (!pi->xstep) ? xstep :
83be9e
-				  JAS_MIN(pi->xstep, xstep);
83be9e
-				pi->ystep = (!pi->ystep) ? ystep :
83be9e
-				  JAS_MIN(pi->ystep, ystep);
83be9e
+				// Check for the potential for overflow problems.
83be9e
+				if (pirlvl->prcwidthexpn + picomp->numrlvls >
83be9e
+				  JAS_UINTFAST32_NUMBITS - 2 ||
83be9e
+				  pirlvl->prcheightexpn + picomp->numrlvls >
83be9e
+				  JAS_UINTFAST32_NUMBITS - 2) {
83be9e
+					return -1;
83be9e
+				}
83be9e
+				xstep = picomp->hsamp * (JAS_CAST(uint_fast32_t, 1) <<
83be9e
+				  (pirlvl->prcwidthexpn + picomp->numrlvls - rlvlno - 1));
83be9e
+				ystep = picomp->vsamp * (JAS_CAST(uint_fast32_t, 1) <<
83be9e
+				  (pirlvl->prcheightexpn + picomp->numrlvls - rlvlno - 1));
83be9e
+				pi->xstep = (!pi->xstep) ? xstep : JAS_MIN(pi->xstep, xstep);
83be9e
+				pi->ystep = (!pi->ystep) ? ystep : JAS_MIN(pi->ystep, ystep);
83be9e
 			}
83be9e
 		}
83be9e
 		pi->prgvolfirst = 0;
83be9e
@@ -377,20 +390,23 @@ static int jpc_pi_nextpcrl(register jpc_
83be9e
 					try0 = JPC_CEILDIV(pi->ystart, pi->picomp->vsamp << r);
83be9e
 					rpx = r + pi->pirlvl->prcwidthexpn;
83be9e
 					rpy = r + pi->pirlvl->prcheightexpn;
83be9e
-					if (((pi->x == pi->xstart && ((trx0 << r) % (1 << rpx))) ||
83be9e
+					if (((pi->x == pi->xstart &&
83be9e
+					  ((trx0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpx))) ||
83be9e
 					  !(pi->x % (pi->picomp->hsamp << rpx))) &&
83be9e
-					  ((pi->y == pi->ystart && ((try0 << r) % (1 << rpy))) ||
83be9e
+					  ((pi->y == pi->ystart &&
83be9e
+					  ((try0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpy))) ||
83be9e
 					  !(pi->y % (pi->picomp->vsamp << rpy)))) {
83be9e
-						prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x, pi->picomp->hsamp
83be9e
-						  << r), pi->pirlvl->prcwidthexpn) - JPC_FLOORDIVPOW2(trx0,
83be9e
-						  pi->pirlvl->prcwidthexpn);
83be9e
-						prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y, pi->picomp->vsamp
83be9e
-						  << r), pi->pirlvl->prcheightexpn) - JPC_FLOORDIVPOW2(try0,
83be9e
-						  pi->pirlvl->prcheightexpn);
83be9e
+						prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x,
83be9e
+						  pi->picomp->hsamp << r), pi->pirlvl->prcwidthexpn) -
83be9e
+						  JPC_FLOORDIVPOW2(trx0, pi->pirlvl->prcwidthexpn);
83be9e
+						prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y,
83be9e
+						  pi->picomp->vsamp << r), pi->pirlvl->prcheightexpn) -
83be9e
+						  JPC_FLOORDIVPOW2(try0, pi->pirlvl->prcheightexpn);
83be9e
 						pi->prcno = prcvind * pi->pirlvl->numhprcs + prchind;
83be9e
 						assert(pi->prcno < pi->pirlvl->numprcs);
83be9e
 						for (pi->lyrno = 0; pi->lyrno < pi->numlyrs &&
83be9e
-						  pi->lyrno < JAS_CAST(int, pchg->lyrnoend); ++pi->lyrno) {
83be9e
+						  pi->lyrno < JAS_CAST(int, pchg->lyrnoend);
83be9e
+						  ++pi->lyrno) {
83be9e
 							prclyrno = &pi->pirlvl->prclyrnos[pi->prcno];
83be9e
 							if (pi->lyrno >= *prclyrno) {
83be9e
 								++(*prclyrno);
83be9e
@@ -428,10 +444,17 @@ static int jpc_pi_nextcprl(register jpc_
83be9e
 		pi->prgvolfirst = 0;
83be9e
 	}
83be9e
 
83be9e
-	for (pi->compno = pchg->compnostart, pi->picomp =
83be9e
-	  &pi->picomps[pi->compno]; pi->compno < JAS_CAST(int, pchg->compnoend) && pi->compno < pi->numcomps; ++pi->compno,
83be9e
-	  ++pi->picomp) {
83be9e
+	for (pi->compno = pchg->compnostart, pi->picomp = &pi->picomps[pi->compno];
83be9e
+	  pi->compno < JAS_CAST(int, pchg->compnoend) && pi->compno < pi->numcomps;
83be9e
+	  ++pi->compno, ++pi->picomp) {
83be9e
 		pirlvl = pi->picomp->pirlvls;
83be9e
+		// Check for the potential for overflow problems.
83be9e
+		if (pirlvl->prcwidthexpn + pi->picomp->numrlvls >
83be9e
+		  JAS_UINTFAST32_NUMBITS - 2 ||
83be9e
+		  pirlvl->prcheightexpn + pi->picomp->numrlvls >
83be9e
+		  JAS_UINTFAST32_NUMBITS - 2) {
83be9e
+			return -1;
83be9e
+		}
83be9e
 		pi->xstep = pi->picomp->hsamp * (JAS_CAST(uint_fast32_t, 1) <<
83be9e
 		  (pirlvl->prcwidthexpn + pi->picomp->numrlvls - 1));
83be9e
 		pi->ystep = pi->picomp->vsamp * (JAS_CAST(uint_fast32_t, 1) <<
83be9e
@@ -461,23 +484,23 @@ static int jpc_pi_nextcprl(register jpc_
83be9e
 					try0 = JPC_CEILDIV(pi->ystart, pi->picomp->vsamp << r);
83be9e
 					rpx = r + pi->pirlvl->prcwidthexpn;
83be9e
 					rpy = r + pi->pirlvl->prcheightexpn;
83be9e
-					if (((pi->x == pi->xstart && ((trx0 << r) % (1 << rpx))) ||
83be9e
+					if (((pi->x == pi->xstart &&
83be9e
+					  ((trx0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpx))) ||
83be9e
 					  !(pi->x % (pi->picomp->hsamp << rpx))) &&
83be9e
-					  ((pi->y == pi->ystart && ((try0 << r) % (1 << rpy))) ||
83be9e
+					  ((pi->y == pi->ystart &&
83be9e
+					  ((try0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpy))) ||
83be9e
 					  !(pi->y % (pi->picomp->vsamp << rpy)))) {
83be9e
-						prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x, pi->picomp->hsamp
83be9e
-						  << r), pi->pirlvl->prcwidthexpn) - JPC_FLOORDIVPOW2(trx0,
83be9e
-						  pi->pirlvl->prcwidthexpn);
83be9e
-						prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y, pi->picomp->vsamp
83be9e
-						  << r), pi->pirlvl->prcheightexpn) - JPC_FLOORDIVPOW2(try0,
83be9e
-						  pi->pirlvl->prcheightexpn);
83be9e
-						pi->prcno = prcvind *
83be9e
-						  pi->pirlvl->numhprcs +
83be9e
-						  prchind;
83be9e
-						assert(pi->prcno <
83be9e
-						  pi->pirlvl->numprcs);
83be9e
-						for (pi->lyrno = 0; pi->lyrno <
83be9e
-						  pi->numlyrs && pi->lyrno < JAS_CAST(int, pchg->lyrnoend); ++pi->lyrno) {
83be9e
+						prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x,
83be9e
+						  pi->picomp->hsamp << r), pi->pirlvl->prcwidthexpn) -
83be9e
+						  JPC_FLOORDIVPOW2(trx0, pi->pirlvl->prcwidthexpn);
83be9e
+						prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y,
83be9e
+						  pi->picomp->vsamp << r), pi->pirlvl->prcheightexpn) -
83be9e
+						  JPC_FLOORDIVPOW2(try0, pi->pirlvl->prcheightexpn);
83be9e
+						pi->prcno = prcvind * pi->pirlvl->numhprcs + prchind;
83be9e
+						assert(pi->prcno < pi->pirlvl->numprcs);
83be9e
+						for (pi->lyrno = 0; pi->lyrno < pi->numlyrs &&
83be9e
+						  pi->lyrno < JAS_CAST(int, pchg->lyrnoend);
83be9e
+						  ++pi->lyrno) {
83be9e
 							prclyrno = &pi->pirlvl->prclyrnos[pi->prcno];
83be9e
 							if (pi->lyrno >= *prclyrno) {
83be9e
 								++(*prclyrno);