Backport of the upstream commit:

From b35a05635e56f554870ce85f64293a3868793f69 Mon Sep 17 00:00:00 2001
From: Michael Adams <mdadams@ece.uvic.ca>
Date: Wed, 19 Oct 2016 08:42:25 -0700
Subject: [PATCH] Fixed potential integer overflow problem.

Further enhanced by a change from d42b2388f7f8e0332c846675133acea151fc557a to
use jas_safe_size_mul3() and an explicit check to ensure that size not only
fits into size_t, but that it also does not exceed INT_MAX.  This is similar
approach to what upstream used in a712a2041085e7cd5f2b153e1532ac2a2954ffaa.

This also adds all jas_safe_size_*() functions, including changes from the
following upstream commits:

f596a0766825b48cdc07b28d2051977a382cfb95
65536647d380571d1a9a6c91fa03775fb5bbd256
3afacc174867cc9d1f74ef2683bc780de4b0b2df
d42b2388f7f8e0332c846675133acea151fc557a

diff -pruN jasper-1.900.1.orig/src/libjasper/base/jas_image.c jasper-1.900.1/src/libjasper/base/jas_image.c
--- jasper-1.900.1.orig/src/libjasper/base/jas_image.c	2017-03-24 16:09:34.000000000 +0100
+++ jasper-1.900.1/src/libjasper/base/jas_image.c	2017-03-24 22:28:46.620880896 +0100
@@ -76,6 +76,7 @@
 #include <string.h>
 #include <assert.h>
 #include <ctype.h>
+#include <limits.h>
 
 #include "jasper/jas_math.h"
 #include "jasper/jas_image.h"
@@ -307,10 +308,10 @@ static jas_image_cmpt_t *jas_image_cmpt_
   height, uint_fast16_t depth, bool sgnd, uint_fast32_t inmem)
 {
 	jas_image_cmpt_t *cmpt;
-	long size;
+	size_t size;
 
 	if (!(cmpt = jas_malloc(sizeof(jas_image_cmpt_t)))) {
-		return 0;
+		goto error;
 	}
 
 	cmpt->type_ = JAS_IMAGE_CT_UNKNOWN;
@@ -325,11 +326,14 @@ static jas_image_cmpt_t *jas_image_cmpt_
 	cmpt->stream_ = 0;
 	cmpt->cps_ = (depth + 7) / 8;
 
-	size = cmpt->width_ * cmpt->height_ * cmpt->cps_;
+	//size = cmpt->width_ * cmpt->height_ * cmpt->cps_;
+	if (!jas_safe_size_mul3(cmpt->width_, cmpt->height_, cmpt->cps_, &size) ||
+		size > INT_MAX) {
+		goto error;
+	}
 	cmpt->stream_ = (inmem) ? jas_stream_memopen(0, size) : jas_stream_tmpfile();
 	if (!cmpt->stream_) {
-		jas_image_cmpt_destroy(cmpt);
-		return 0;
+		goto error;
 	}
 
 	/* Zero the component data.  This isn't necessary, but it is
@@ -337,11 +341,16 @@ static jas_image_cmpt_t *jas_image_cmpt_
 	if (jas_stream_seek(cmpt->stream_, size - 1, SEEK_SET) < 0 ||
 	  jas_stream_putc(cmpt->stream_, 0) == EOF ||
 	  jas_stream_seek(cmpt->stream_, 0, SEEK_SET) < 0) {
-		jas_image_cmpt_destroy(cmpt);
-		return 0;
+		goto error;
 	}
 
 	return cmpt;
+
+error:
+	if (cmpt) {
+		jas_image_cmpt_destroy(cmpt);
+	}
+	return 0;
 }
 
 static void jas_image_cmpt_destroy(jas_image_cmpt_t *cmpt)
diff -pruN jasper-1.900.1.orig/src/libjasper/include/jasper/jas_math.h jasper-1.900.1/src/libjasper/include/jasper/jas_math.h
--- jasper-1.900.1.orig/src/libjasper/include/jasper/jas_math.h	2007-01-19 22:43:04.000000000 +0100
+++ jasper-1.900.1/src/libjasper/include/jasper/jas_math.h	2017-03-24 22:29:36.085024105 +0100
@@ -76,9 +76,13 @@
 
 #include <jasper/jas_config.h>
 
-#include	<assert.h>
-#include	<stdio.h>
-#include	<string.h>
+#include <jasper/jas_types.h>
+
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+#include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -110,6 +114,62 @@ extern "C" {
 #define	JAS_ONES(n) \
   ((1 << (n)) - 1)
 
+/******************************************************************************\
+* Safe integer arithmetic (i.e., with overflow checking).
+\******************************************************************************/
+
+/* Compute the product of two size_t integers with overflow checking. */
+inline static bool jas_safe_size_mul(size_t x, size_t y, size_t *result)
+{
+	/* Check if overflow would occur */
+	if (x && y > SIZE_MAX / x) {
+		/* Overflow would occur. */
+		return false;
+	}
+	if (result) {
+		*result = x * y;
+	}
+	return true;
+}
+
+inline static bool jas_safe_size_mul3(size_t a, size_t b, size_t c,
+  size_t *result)
+{
+	size_t tmp;
+	if (!jas_safe_size_mul(a, b, &tmp) ||
+	  !jas_safe_size_mul(tmp, c, &tmp)) {
+		return false;
+	}
+	if (result) {
+		*result = tmp;
+	}
+	return true;
+}
+
+/* Compute the sum of two size_t integer with overflow checking. */
+inline static bool jas_safe_size_add(size_t x, size_t y, size_t *result)
+{
+	if (y > SIZE_MAX - x) {
+		return false;
+	}
+	if (result) {
+		*result = x + y;
+	}
+	return true;
+}
+
+/* Compute the difference of two size_t integer with overflow checking. */
+inline static bool jas_safe_size_sub(size_t x, size_t y, size_t *result)
+{
+	if (y > x) {
+		return false;
+	}
+	if (result) {
+		*result = x - y;
+	}
+	return true;
+}
+
 #ifdef __cplusplus
 }
 #endif