Backport of the upstream commit:

From b35a05635e56f554870ce85f64293a3868793f69 Mon Sep 17 00:00:00 2001

From: Michael Adams <mdadams@ece.uvic.ca>

Date: Wed, 19 Oct 2016 08:42:25 -0700

Subject: [PATCH] Fixed potential integer overflow problem.

Further enhanced by a change from d42b2388f7f8e0332c846675133acea151fc557a to

use jas_safe_size_mul3() and an explicit check to ensure that size not only

fits into size_t, but that it also does not exceed INT_MAX.  This is similar

approach to what upstream used in a712a2041085e7cd5f2b153e1532ac2a2954ffaa.

This also adds all jas_safe_size_*() functions, including changes from the

following upstream commits:

f596a0766825b48cdc07b28d2051977a382cfb95

65536647d380571d1a9a6c91fa03775fb5bbd256

3afacc174867cc9d1f74ef2683bc780de4b0b2df

d42b2388f7f8e0332c846675133acea151fc557a

diff -pruN jasper-1.900.1.orig/src/libjasper/base/jas_image.c jasper-1.900.1/src/libjasper/base/jas_image.c

--- jasper-1.900.1.orig/src/libjasper/base/jas_image.c	2017-03-24 16:09:34.000000000 +0100

+++ jasper-1.900.1/src/libjasper/base/jas_image.c	2017-03-24 22:28:46.620880896 +0100

@@ -76,6 +76,7 @@

 #include <string.h>

 #include <assert.h>

 #include <ctype.h>

+#include <limits.h>

 #include "jasper/jas_math.h"

 #include "jasper/jas_image.h"

@@ -307,10 +308,10 @@ static jas_image_cmpt_t *jas_image_cmpt_

   height, uint_fast16_t depth, bool sgnd, uint_fast32_t inmem)

 {

 	jas_image_cmpt_t *cmpt;

-	long size;

+	size_t size;

 	if (!(cmpt = jas_malloc(sizeof(jas_image_cmpt_t)))) {

-		return 0;

+		goto error;

 	}

 	cmpt->type_ = JAS_IMAGE_CT_UNKNOWN;

@@ -325,11 +326,14 @@ static jas_image_cmpt_t *jas_image_cmpt_

 	cmpt->stream_ = 0;

 	cmpt->cps_ = (depth + 7) / 8;

-	size = cmpt->width_ * cmpt->height_ * cmpt->cps_;

+	//size = cmpt->width_ * cmpt->height_ * cmpt->cps_;

+	if (!jas_safe_size_mul3(cmpt->width_, cmpt->height_, cmpt->cps_, &size) ||

+		size > INT_MAX) {

+		goto error;

+	}

 	cmpt->stream_ = (inmem) ? jas_stream_memopen(0, size) : jas_stream_tmpfile();

 	if (!cmpt->stream_) {

-		jas_image_cmpt_destroy(cmpt);

-		return 0;

+		goto error;

 	}

 	/* Zero the component data.  This isn't necessary, but it is

@@ -337,11 +341,16 @@ static jas_image_cmpt_t *jas_image_cmpt_

 	if (jas_stream_seek(cmpt->stream_, size - 1, SEEK_SET) < 0 ||

 	  jas_stream_putc(cmpt->stream_, 0) == EOF ||

 	  jas_stream_seek(cmpt->stream_, 0, SEEK_SET) < 0) {

-		jas_image_cmpt_destroy(cmpt);

-		return 0;

+		goto error;

 	}

 	return cmpt;

+

+error:

+	if (cmpt) {

+		jas_image_cmpt_destroy(cmpt);

+	}

+	return 0;

 }

 static void jas_image_cmpt_destroy(jas_image_cmpt_t *cmpt)

diff -pruN jasper-1.900.1.orig/src/libjasper/include/jasper/jas_math.h jasper-1.900.1/src/libjasper/include/jasper/jas_math.h

--- jasper-1.900.1.orig/src/libjasper/include/jasper/jas_math.h	2007-01-19 22:43:04.000000000 +0100

+++ jasper-1.900.1/src/libjasper/include/jasper/jas_math.h	2017-03-24 22:29:36.085024105 +0100

@@ -76,9 +76,13 @@

 #include <jasper/jas_config.h>

-#include	<assert.h>

-#include	<stdio.h>

-#include	<string.h>

+#include <jasper/jas_types.h>

+

+#include <assert.h>

+#include <stdio.h>

+#include <string.h>

+#include <stdbool.h>

+#include <stdint.h>

 #ifdef __cplusplus

 extern "C" {

@@ -110,6 +114,62 @@ extern "C" {

 #define	JAS_ONES(n) \

   ((1 << (n)) - 1)

+/******************************************************************************\

+* Safe integer arithmetic (i.e., with overflow checking).

+\******************************************************************************/

+

+/* Compute the product of two size_t integers with overflow checking. */

+inline static bool jas_safe_size_mul(size_t x, size_t y, size_t *result)

+{

+	/* Check if overflow would occur */

+	if (x && y > SIZE_MAX / x) {

+		/* Overflow would occur. */

+		return false;

+	}

+	if (result) {

+		*result = x * y;

+	}

+	return true;

+}

+

+inline static bool jas_safe_size_mul3(size_t a, size_t b, size_t c,

+  size_t *result)

+{

+	size_t tmp;

+	if (!jas_safe_size_mul(a, b, &tmp) ||

+	  !jas_safe_size_mul(tmp, c, &tmp)) {

+		return false;

+	}

+	if (result) {

+		*result = tmp;

+	}

+	return true;

+}

+

+/* Compute the sum of two size_t integer with overflow checking. */

+inline static bool jas_safe_size_add(size_t x, size_t y, size_t *result)

+{

+	if (y > SIZE_MAX - x) {

+		return false;

+	}

+	if (result) {

+		*result = x + y;

+	}

+	return true;

+}

+

+/* Compute the difference of two size_t integer with overflow checking. */

+inline static bool jas_safe_size_sub(size_t x, size_t y, size_t *result)

+{

+	if (y > x) {

+		return false;

+	}

+	if (result) {

+		*result = x - y;

+	}

+	return true;

+}

+

 #ifdef __cplusplus

 }

 #endif