Defend against integer overflow in buffer size calculations within tiff2pdf.

CVE-2012-2113

diff -Naur tiff-3.9.4.orig/tools/tiff2pdf.c tiff-3.9.4/tools/tiff2pdf.c

--- tiff-3.9.4.orig/tools/tiff2pdf.c	2010-06-13 16:51:44.000000000 -0400

+++ tiff-3.9.4/tools/tiff2pdf.c	2012-06-27 13:13:44.581229393 -0400

@@ -44,6 +44,7 @@

 # include <io.h>

 #endif

+#include "tiffiop.h"

 #include "tiffio.h"

 #ifndef HAVE_GETOPT

@@ -414,6 +415,34 @@

 	(void) handle, (void) data, (void) offset;

 }

+static uint64

+checkAdd64(uint64 summand1, uint64 summand2, T2P* t2p)

+{

+	uint64 bytes = summand1 + summand2;

+

+	if (bytes - summand1 != summand2) {

+		TIFFError(TIFF2PDF_MODULE, "Integer overflow");

+		t2p->t2p_error = T2P_ERR_ERROR;

+		bytes = 0;

+	}

+

+	return bytes;

+}

+

+static uint64

+checkMultiply64(uint64 first, uint64 second, T2P* t2p)

+{

+	uint64 bytes = first * second;

+

+	if (second && bytes / second != first) {

+		TIFFError(TIFF2PDF_MODULE, "Integer overflow");

+		t2p->t2p_error = T2P_ERR_ERROR;

+		bytes = 0;

+	}

+

+	return bytes;

+}

+

 /*

   This is the main function.

@@ -1828,9 +1857,7 @@

 	tstrip_t i=0;

 	tstrip_t stripcount=0;

 #endif

-#ifdef OJPEG_SUPPORT

-        tsize_t k = 0;

-#endif

+        uint64 k = 0;

 	if(t2p->pdf_transcode == T2P_TRANSCODE_RAW){

 #ifdef CCITT_SUPPORT

@@ -1858,19 +1885,25 @@

 			}

 			stripcount=TIFFNumberOfStrips(input);

 			for(i=0;i

-				k += sbc[i];

+				k = checkAdd64(k, sbc[i], t2p);

 			}

 			if(TIFFGetField(input, TIFFTAG_JPEGIFOFFSET, &(t2p->tiff_dataoffset))){

 				if(t2p->tiff_dataoffset != 0){

 					if(TIFFGetField(input, TIFFTAG_JPEGIFBYTECOUNT, &(t2p->tiff_datasize))!=0){

 						if(t2p->tiff_datasize < k) {

-							t2p->pdf_ojpegiflength=t2p->tiff_datasize;

-							t2p->tiff_datasize+=k;

-							t2p->tiff_datasize+=6;

-							t2p->tiff_datasize+=2*stripcount;

 							TIFFWarning(TIFF2PDF_MODULE, 

 								"Input file %s has short JPEG interchange file byte count", 

 								TIFFFileName(input));

+							t2p->pdf_ojpegiflength=t2p->tiff_datasize;

+							k = checkAdd64(k, t2p->tiff_datasize, t2p);

+							k = checkAdd64(k, 6, t2p);

+							k = checkAdd64(k, stripcount, t2p);

+							k = checkAdd64(k, stripcount, t2p);

+							t2p->tiff_datasize = (tsize_t) k;

+							if ((uint64) t2p->tiff_datasize != k) {

+								TIFFError(TIFF2PDF_MODULE, "Integer overflow");

+								t2p->t2p_error = T2P_ERR_ERROR;

+							}

 							return;

 						}

 						return;

@@ -1883,9 +1916,14 @@

 					}

 				}

 			}

-			t2p->tiff_datasize+=k;

-			t2p->tiff_datasize+=2*stripcount;

-			t2p->tiff_datasize+=2048;

+			k = checkAdd64(k, stripcount, t2p);

+			k = checkAdd64(k, stripcount, t2p);

+			k = checkAdd64(k, 2048, t2p);

+			t2p->tiff_datasize = (tsize_t) k;

+			if ((uint64) t2p->tiff_datasize != k) {

+				TIFFError(TIFF2PDF_MODULE, "Integer overflow");

+				t2p->t2p_error = T2P_ERR_ERROR;

+			}

 			return;

 		}

 #endif

@@ -1894,11 +1932,11 @@

 			uint32 count = 0;

 			if(TIFFGetField(input, TIFFTAG_JPEGTABLES, &count, &jpt) != 0 ){

 				if(count > 4){

-					t2p->tiff_datasize += count;

-					t2p->tiff_datasize -= 2; /* don't use EOI of header */

+					k += count;

+					k -= 2; /* don't use EOI of header */

 				}

 			} else {

-				t2p->tiff_datasize = 2; /* SOI for first strip */

+				k = 2; /* SOI for first strip */

 			}

 			stripcount=TIFFNumberOfStrips(input);

 			if(!TIFFGetField(input, TIFFTAG_STRIPBYTECOUNTS, &sbc)){

@@ -1909,18 +1947,33 @@

 				return;

 			}

 			for(i=0;i

-				t2p->tiff_datasize += sbc[i];

-				t2p->tiff_datasize -=4; /* don't use SOI or EOI of strip */

+				k = checkAdd64(k, sbc[i], t2p);

+				k -=4; /* don't use SOI or EOI of strip */

+			}

+			k = checkAdd64(k, 2, t2p); /* use EOI of last strip */

+			t2p->tiff_datasize = (tsize_t) k;

+			if ((uint64) t2p->tiff_datasize != k) {

+				TIFFError(TIFF2PDF_MODULE, "Integer overflow");

+				t2p->t2p_error = T2P_ERR_ERROR;

 			}

-			t2p->tiff_datasize +=2; /* use EOI of last strip */

 			return;

 		}

 #endif

 		(void) 0;

 	}

-	t2p->tiff_datasize=TIFFScanlineSize(input) * t2p->tiff_length;

+	k = checkMultiply64(TIFFScanlineSize(input), t2p->tiff_length, t2p);

 	if(t2p->tiff_planar==PLANARCONFIG_SEPARATE){

-		t2p->tiff_datasize*= t2p->tiff_samplesperpixel;

+		k = checkMultiply64(k, t2p->tiff_samplesperpixel, t2p);

+	}

+	if (k == 0) {

+		/* Assume we had overflow inside TIFFScanlineSize */

+		t2p->t2p_error = T2P_ERR_ERROR;

+	}

+

+	t2p->tiff_datasize = (tsize_t) k;

+	if ((uint64) t2p->tiff_datasize != k) {

+		TIFFError(TIFF2PDF_MODULE, "Integer overflow");

+		t2p->t2p_error = T2P_ERR_ERROR;

 	}

 	return;

@@ -1938,6 +1991,7 @@

 #ifdef JPEG_SUPPORT

 	unsigned char* jpt;

 #endif

+        uint64 k;

 	edge |= t2p_tile_is_right_edge(t2p->tiff_tiles[t2p->pdf_page], tile);

 	edge |= t2p_tile_is_bottom_edge(t2p->tiff_tiles[t2p->pdf_page], tile);

@@ -1949,14 +2003,17 @@

 #endif

 		){

 			t2p->tiff_datasize=TIFFTileSize(input);

+			if (t2p->tiff_datasize == 0) {

+				/* Assume we had overflow inside TIFFTileSize */

+				t2p->t2p_error = T2P_ERR_ERROR;

+			}

 			return;

 		} else {

 			TIFFGetField(input, TIFFTAG_TILEBYTECOUNTS, &tbc;;

-			t2p->tiff_datasize=tbc[tile];

+			k=tbc[tile];

 #ifdef OJPEG_SUPPORT

 			if(t2p->tiff_compression==COMPRESSION_OJPEG){

-				t2p->tiff_datasize+=2048;

-				return;

+			  	k = checkAdd64(k, 2048, t2p);

 			}

 #endif

 #ifdef JPEG_SUPPORT

@@ -1964,18 +2021,33 @@

 				uint32 count = 0;

 				if(TIFFGetField(input, TIFFTAG_JPEGTABLES, &count, &jpt)!=0){

 					if(count > 4){

-						t2p->tiff_datasize += count;

-						t2p->tiff_datasize -= 4; /* don't use EOI of header or SOI of tile */

+						k = checkAdd64(k, count, t2p);

+						k -= 4; /* don't use EOI of header or SOI of tile */

 					}

 				}

 			}

 #endif

+			t2p->tiff_datasize = (tsize_t) k;

+			if ((uint64) t2p->tiff_datasize != k) {

+				TIFFError(TIFF2PDF_MODULE, "Integer overflow");

+				t2p->t2p_error = T2P_ERR_ERROR;

+			}

 			return;

 		}

 	}

-	t2p->tiff_datasize=TIFFTileSize(input);

+	k = TIFFTileSize(input);

 	if(t2p->tiff_planar==PLANARCONFIG_SEPARATE){

-		t2p->tiff_datasize*= t2p->tiff_samplesperpixel;

+		k = checkMultiply64(k, t2p->tiff_samplesperpixel, t2p);

+	}

+	if (k == 0) {

+		/* Assume we had overflow inside TIFFTileSize */

+		t2p->t2p_error = T2P_ERR_ERROR;

+	}

+

+	t2p->tiff_datasize = (tsize_t) k;

+	if ((uint64) t2p->tiff_datasize != k) {

+		TIFFError(TIFF2PDF_MODULE, "Integer overflow");

+		t2p->t2p_error = T2P_ERR_ERROR;

 	}

 	return;

@@ -2068,6 +2140,10 @@

 	uint32 max_striplength=0;

 #endif

+	/* Fail if prior error (in particular, can't trust tiff_datasize) */

+	if (t2p->t2p_error != T2P_ERR_OK)

+		return(0);

+

 	if(t2p->pdf_transcode == T2P_TRANSCODE_RAW){

 #ifdef CCITT_SUPPORT

 		if(t2p->pdf_compression == T2P_COMPRESS_G4){

@@ -2641,6 +2717,10 @@

 	uint32 xuint32=0;

 #endif

+	/* Fail if prior error (in particular, can't trust tiff_datasize) */

+	if (t2p->t2p_error != T2P_ERR_OK)

+		return(0);

+

 	edge |= t2p_tile_is_right_edge(t2p->tiff_tiles[t2p->pdf_page], tile);

 	edge |= t2p_tile_is_bottom_edge(t2p->tiff_tiles[t2p->pdf_page], tile);