Backport of the upstream commit:
From 988f8365f7d8ad8073b6786e433d34c553ecf568 Mon Sep 17 00:00:00 2001
From: Michael Adams <mdadams@ece.uvic.ca>
Date: Sat, 22 Oct 2016 14:36:49 -0700
Subject: [PATCH] Fixed an integer overflow problem.
Further enhanced by an explicit check to ensure that size not only fits into
size_t, but that it also does not exceed INT_FAST32_MAX (the type of
matrix->datasize_). This is similar approach to what upstream used in
a712a2041085e7cd5f2b153e1532ac2a2954ffaa.
diff -pruN jasper-1.900.1.orig/src/libjasper/base/jas_seq.c jasper-1.900.1/src/libjasper/base/jas_seq.c
--- jasper-1.900.1.orig/src/libjasper/base/jas_seq.c 2017-03-31 15:02:01.000000000 +0200
+++ jasper-1.900.1/src/libjasper/base/jas_seq.c 2017-03-31 15:41:17.527623038 +0200
@@ -101,13 +101,16 @@ jas_matrix_t *jas_matrix_create(int numr
{
jas_matrix_t *matrix;
int i;
+ size_t size;
+
+ matrix = 0;
if (numrows < 0 || numcols < 0) {
- return 0;
+ goto error;
}
if (!(matrix = jas_malloc(sizeof(jas_matrix_t)))) {
- return 0;
+ goto error;
}
matrix->flags_ = 0;
matrix->numrows_ = numrows;
@@ -115,21 +118,25 @@ jas_matrix_t *jas_matrix_create(int numr
matrix->rows_ = 0;
matrix->maxrows_ = numrows;
matrix->data_ = 0;
- matrix->datasize_ = numrows * numcols;
+ matrix->datasize_ = 0;
+
+ // matrix->datasize_ = numrows * numcols;
+ if (!jas_safe_size_mul(numrows, numcols, &size) || size > INT_FAST32_MAX) {
+ goto error;
+ }
+ matrix->datasize_ = size;
if (matrix->maxrows_ > 0) {
if (!(matrix->rows_ = jas_alloc2(matrix->maxrows_,
sizeof(jas_seqent_t *)))) {
- jas_matrix_destroy(matrix);
- return 0;
+ goto error;
}
}
if (matrix->datasize_ > 0) {
if (!(matrix->data_ = jas_alloc2(matrix->datasize_,
sizeof(jas_seqent_t)))) {
- jas_matrix_destroy(matrix);
- return 0;
+ goto error;
}
}
@@ -147,6 +154,12 @@ jas_matrix_t *jas_matrix_create(int numr
matrix->yend_ = matrix->numrows_;
return matrix;
+
+error:
+ if (matrix) {
+ jas_matrix_destroy(matrix);
+ }
+ return 0;
}
void jas_matrix_destroy(jas_matrix_t *matrix)