Blame SOURCES/libtiff-CVE-2012-2088.patch

6456d8
Do strip and tile size calculations in unsigned arithmetic, and then
6456d8
complain if the result overflows signed int32, because callers of these
6456d8
functions expect signed results (tsize_t is signed).  CVE-2012-2088
6456d8
6456d8
NB: must be applied after libtiff-subsampling.patch to avoid fuzz issues.
6456d8
6456d8
6456d8
diff -Naur tiff-3.9.4.orig/libtiff/tif_strip.c tiff-3.9.4/libtiff/tif_strip.c
6456d8
--- tiff-3.9.4.orig/libtiff/tif_strip.c	2010-06-08 14:50:43.000000000 -0400
6456d8
+++ tiff-3.9.4/libtiff/tif_strip.c	2012-06-27 12:37:55.054788399 -0400
6456d8
@@ -107,6 +107,7 @@
6456d8
 TIFFVStripSize(TIFF* tif, uint32 nrows)
6456d8
 {
6456d8
 	TIFFDirectory *td = &tif->tif_dir;
6456d8
+	uint32 stripsize;
6456d8
 
6456d8
 	if (nrows == (uint32) -1)
6456d8
 		nrows = td->td_imagelength;
6456d8
@@ -122,7 +123,7 @@
6456d8
 		 * YCbCr data for the extended image.
6456d8
 		 */
6456d8
 		uint16 ycbcrsubsampling[2];
6456d8
-		tsize_t w, scanline, samplingarea;
6456d8
+		uint32 w, scanline, samplingarea;
6456d8
 
6456d8
 		TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING,
6456d8
 				      ycbcrsubsampling + 0,
6456d8
@@ -141,13 +142,27 @@
6456d8
 		nrows = TIFFroundup(nrows, ycbcrsubsampling[1]);
6456d8
 		/* NB: don't need TIFFhowmany here 'cuz everything is rounded */
6456d8
 		scanline = multiply(tif, nrows, scanline, "TIFFVStripSize");
6456d8
-		return ((tsize_t)
6456d8
-		    summarize(tif, scanline,
6456d8
-			      multiply(tif, 2, scanline / samplingarea,
6456d8
-				       "TIFFVStripSize"), "TIFFVStripSize"));
6456d8
+		/* a zero anywhere in here means overflow, must return zero */
6456d8
+		if (scanline > 0) {
6456d8
+			uint32 extra =
6456d8
+			    multiply(tif, 2, scanline / samplingarea,
6456d8
+				     "TIFFVStripSize");
6456d8
+			if (extra > 0)
6456d8
+				stripsize = summarize(tif, scanline, extra,
6456d8
+						      "TIFFVStripSize");
6456d8
+			else
6456d8
+				stripsize = 0;
6456d8
+		} else
6456d8
+			stripsize = 0;
6456d8
 	} else
6456d8
-		return ((tsize_t) multiply(tif, nrows, TIFFScanlineSize(tif),
6456d8
-					   "TIFFVStripSize"));
6456d8
+		stripsize = multiply(tif, nrows, TIFFScanlineSize(tif),
6456d8
+				     "TIFFVStripSize");
6456d8
+	/* Because tsize_t is signed, we might have conversion overflow */
6456d8
+	if (((tsize_t) stripsize) < 0) {
6456d8
+		TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Integer overflow in %s", "TIFFVStripSize");
6456d8
+		stripsize = 0;
6456d8
+	}
6456d8
+	return (tsize_t) stripsize;
6456d8
 }
6456d8
 
6456d8
 
6456d8
diff -Naur tiff-3.9.4.orig/libtiff/tif_tile.c tiff-3.9.4/libtiff/tif_tile.c
6456d8
--- tiff-3.9.4.orig/libtiff/tif_tile.c	2010-06-08 14:50:43.000000000 -0400
6456d8
+++ tiff-3.9.4/libtiff/tif_tile.c	2012-06-27 12:37:55.055788446 -0400
6456d8
@@ -174,7 +174,7 @@
6456d8
 TIFFTileRowSize(TIFF* tif)
6456d8
 {
6456d8
 	TIFFDirectory *td = &tif->tif_dir;
6456d8
-	tsize_t rowsize;
6456d8
+	uint32 rowsize;
6456d8
 	
6456d8
 	if (td->td_tilelength == 0 || td->td_tilewidth == 0)
6456d8
 		return ((tsize_t) 0);
6456d8
@@ -193,7 +193,7 @@
6456d8
 TIFFVTileSize(TIFF* tif, uint32 nrows)
6456d8
 {
6456d8
 	TIFFDirectory *td = &tif->tif_dir;
6456d8
-	tsize_t tilesize;
6456d8
+	uint32 tilesize;
6456d8
 
6456d8
 	if (td->td_tilelength == 0 || td->td_tilewidth == 0 ||
6456d8
 	    td->td_tiledepth == 0)
6456d8
@@ -209,12 +209,12 @@
6456d8
 		 * horizontal/vertical subsampling area include
6456d8
 		 * YCbCr data for the extended image.
6456d8
 		 */
6456d8
-		tsize_t w =
6456d8
+		uint32 w =
6456d8
 		    TIFFroundup(td->td_tilewidth, td->td_ycbcrsubsampling[0]);
6456d8
-		tsize_t rowsize =
6456d8
+		uint32 rowsize =
6456d8
 		    TIFFhowmany8(multiply(tif, w, td->td_bitspersample,
6456d8
 					  "TIFFVTileSize"));
6456d8
-		tsize_t samplingarea =
6456d8
+		uint32 samplingarea =
6456d8
 		    td->td_ycbcrsubsampling[0]*td->td_ycbcrsubsampling[1];
6456d8
 		if (samplingarea == 0) {
6456d8
 			TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Invalid YCbCr subsampling");
6456d8
@@ -223,15 +223,27 @@
6456d8
 		nrows = TIFFroundup(nrows, td->td_ycbcrsubsampling[1]);
6456d8
 		/* NB: don't need TIFFhowmany here 'cuz everything is rounded */
6456d8
 		tilesize = multiply(tif, nrows, rowsize, "TIFFVTileSize");
6456d8
-		tilesize = summarize(tif, tilesize,
6456d8
-				     multiply(tif, 2, tilesize / samplingarea,
6456d8
-					      "TIFFVTileSize"),
6456d8
+		/* a zero anywhere in here means overflow, must return zero */
6456d8
+		if (tilesize > 0) {
6456d8
+			uint32 extra =
6456d8
+			    multiply(tif, 2, tilesize / samplingarea,
6456d8
 				     "TIFFVTileSize");
6456d8
+			if (extra > 0)
6456d8
+				tilesize = summarize(tif, tilesize, extra,
6456d8
+						     "TIFFVTileSize");
6456d8
+			else
6456d8
+				tilesize = 0;
6456d8
+		}
6456d8
 	} else
6456d8
 		tilesize = multiply(tif, nrows, TIFFTileRowSize(tif),
6456d8
 				    "TIFFVTileSize");
6456d8
-	return ((tsize_t)
6456d8
-	    multiply(tif, tilesize, td->td_tiledepth, "TIFFVTileSize"));
6456d8
+	tilesize = multiply(tif, tilesize, td->td_tiledepth, "TIFFVTileSize");
6456d8
+	/* Because tsize_t is signed, we might have conversion overflow */
6456d8
+	if (((tsize_t) tilesize) < 0) {
6456d8
+		TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Integer overflow in %s", "TIFFVTileSize");
6456d8
+		tilesize = 0;
6456d8
+	}
6456d8
+	return (tsize_t) tilesize;
6456d8
 }
6456d8
 
6456d8
 /*