Backport of upstream commit:

From 411a4068f8c464e883358bf403a3e25158863823 Mon Sep 17 00:00:00 2001

From: Michael Adams <mdadams@ece.uvic.ca>

Date: Mon, 24 Oct 2016 06:56:08 -0700

Subject: [PATCH] Fixed a few bugs in the RAS encoder and decoder where errors

 were tested with assertions instead of being gracefully handled.

diff -pruN jasper-1.900.1.orig/src/libjasper/ras/ras_dec.c jasper-1.900.1/src/libjasper/ras/ras_dec.c

--- jasper-1.900.1.orig/src/libjasper/ras/ras_dec.c	2007-01-19 22:43:04.000000000 +0100

+++ jasper-1.900.1/src/libjasper/ras/ras_dec.c	2017-03-31 22:38:04.000000000 +0200

@@ -257,9 +257,16 @@ static int ras_getdatastd(jas_stream_t *

 	/* Avoid compiler warnings about unused parameters. */

 	cmap = 0;

+	assert(jas_image_numcmpts(image) <= 3);

+

+	for (i = 0; i < 3; ++i) {

+		data[i] = 0;

+	}

+

 	for (i = 0; i < jas_image_numcmpts(image); ++i) {

-		data[i] = jas_matrix_create(1, jas_image_width(image));

-		assert(data[i]);

+		if (!(data[i] = jas_matrix_create(1, jas_image_width(image)))) {

+			goto error;

+		}

 	}

 	pad = RAS_ROWSIZE(hdr) - (hdr->width * hdr->depth + 7) / 8;

@@ -270,7 +277,7 @@ static int ras_getdatastd(jas_stream_t *

 		for (x = 0; x < hdr->width; x++) {

 			while (nz < hdr->depth) {

 				if ((c = jas_stream_getc(in)) == EOF) {

-					return -1;

+					goto error;

 				}

 				z = (z << 8) | c;

 				nz += 8;

@@ -290,22 +297,31 @@ static int ras_getdatastd(jas_stream_t *

 		}

 		if (pad) {

 			if ((c = jas_stream_getc(in)) == EOF) {

-				return -1;

+				goto error;

 			}

 		}

 		for (i = 0; i < jas_image_numcmpts(image); ++i) {

 			if (jas_image_writecmpt(image, i, 0, y, hdr->width, 1,

 			  data[i])) {

-				return -1;

+				goto error;

 			}

 		}

 	}

 	for (i = 0; i < jas_image_numcmpts(image); ++i) {

 		jas_matrix_destroy(data[i]);

+		data[i] = 0;

 	}

 	return 0;

+

+error:

+	for (i = 0; i < 3; ++i) {

+		if (data[i]) {

+			jas_matrix_destroy(data[i]);

+		}

+	}

+	return -1;

 }

 static int ras_getcmap(jas_stream_t *in, ras_hdr_t *hdr, ras_cmap_t *cmap)

@@ -324,7 +340,9 @@ static int ras_getcmap(jas_stream_t *in,

 		{

 		jas_eprintf("warning: palettized images not fully supported\n");

 		numcolors = 1 << hdr->depth;

-		assert(numcolors <= RAS_CMAP_MAXSIZ);

+		if (numcolors > RAS_CMAP_MAXSIZ) {

+			return -1;

+		}

 		actualnumcolors = hdr->maplength / 3;

 		for (i = 0; i < numcolors; i++) {

 			cmap->data[i] = 0;

diff -pruN jasper-1.900.1.orig/src/libjasper/ras/ras_enc.c jasper-1.900.1/src/libjasper/ras/ras_enc.c

--- jasper-1.900.1.orig/src/libjasper/ras/ras_enc.c	2017-03-31 22:20:38.000000000 +0200

+++ jasper-1.900.1/src/libjasper/ras/ras_enc.c	2017-03-31 22:38:04.000000000 +0200

@@ -230,9 +230,17 @@ static int ras_putdatastd(jas_stream_t *

 	jas_matrix_t *data[3];

 	int i;

+	assert(numcmpts <= 3);

+

+	for (i = 0; i < 3; ++i) {

+		data[i] = 0;

+	}

+

 	for (i = 0; i < numcmpts; ++i) {

-		data[i] = jas_matrix_create(jas_image_height(image), jas_image_width(image));

-		assert(data[i]);

+		if (!(data[i] = jas_matrix_create(jas_image_height(image),

+		  jas_image_width(image)))) {

+			goto error;

+		}

 	}

 	rowsize = RAS_ROWSIZE(hdr);

@@ -244,7 +252,7 @@ static int ras_putdatastd(jas_stream_t *

 		for (i = 0; i < numcmpts; ++i) {

 			if (jas_image_readcmpt(image, cmpts[i], 0, y,

 					jas_image_width(image), 1, data[i])) {

-				return -1;

+				goto error;

 			}

 		}

 		z = 0;

@@ -263,7 +271,7 @@ static int ras_putdatastd(jas_stream_t *

 			while (nz >= 8) {

 				c = (z >> (nz - 8)) & 0xff;

 				if (jas_stream_putc(out, c) == EOF) {

-					return -1;

+					goto error;

 				}

 				nz -= 8;

 				z &= RAS_ONES(nz);

@@ -272,21 +280,30 @@ static int ras_putdatastd(jas_stream_t *

 		if (nz > 0) {

 			c = (z >> (8 - nz)) & RAS_ONES(nz);

 			if (jas_stream_putc(out, c) == EOF) {

-				return -1;

+				goto error;

 			}

 		}

 		if (pad % 2) {

 			if (jas_stream_putc(out, 0) == EOF) {

-				return -1;

+				goto error;

 			}

 		}

 	}

 	for (i = 0; i < numcmpts; ++i) {

 		jas_matrix_destroy(data[i]);

+		data[i] = 0;

 	}

 	return 0;

+

+error:

+	for (i = 0; i < numcmpts; ++i) {

+		if (data[i]) {

+			jas_matrix_destroy(data[i]);

+		}

+	}

+	return -1;

 }

 static int ras_puthdr(jas_stream_t *out, ras_hdr_t *hdr)