Blob Blame Raw
From 64aa150a92ccb082db6a3383fa734a6ac91cf1bf Mon Sep 17 00:00:00 2001
From: Marek Kasik <mkasik@redhat.com>
Date: Tue, 30 Apr 2019 18:47:44 +0200
Subject: [PATCH] PSOutputDev: Don't read outside of image buffer

Check whether input image is RGB or BGR to not treat
it as CMYK in those cases in PSOutputDev::checkPageSlice().

Fixes #751
---
 poppler/PSOutputDev.cc | 248 ++++++++++++++++++++++++++++++++---------
 1 file changed, 196 insertions(+), 52 deletions(-)

diff --git a/poppler/PSOutputDev.cc b/poppler/PSOutputDev.cc
index 0d201835..155a8cbe 100644
--- a/poppler/PSOutputDev.cc
+++ b/poppler/PSOutputDev.cc
@@ -3342,13 +3342,21 @@ GBool PSOutputDev::checkPageSlice(Page *
       }
       break;
     case psLevel1Sep:
+      GfxColor inputColor;
+      GfxCMYK cmyk;
+      unsigned char cmykColor[4];
+      GfxDeviceRGBColorSpace *rgbCS;
+      SplashColorMode colorMode;
+
+      colorMode = bitmap->getMode();
+
       useBinary = globalParams->getPSBinary();
       p = bitmap->getDataPtr();
       // Check for an all gray image
       isGray = gTrue;
       for (y = 0; y < h; ++y) {
 	for (x = 0; x < w; ++x) {
-	  if (p[4*x] != p[4*x + 1] || p[4*x] != p[4*x + 2]) {
+	  if (p[numComps*x] != p[numComps*x + 1] || p[numComps*x] != p[numComps*x + 2]) {
 	    isGray = gFalse;
 	    y = h;
 	    break;
@@ -3365,7 +3373,9 @@ GBool PSOutputDev::checkPageSlice(Page *
       col[0] = col[1] = col[2] = col[3] = 0;
       if (isGray) {
         int g;
-        if ((psProcessBlack & processColors) == 0) {
+        if ((psProcessBlack & processColors) == 0 &&
+            colorMode != splashModeRGB8 &&
+            colorMode != splashModeBGR8) {
 	  // Check if the image uses black
 	  for (y = 0; y < h; ++y) {
 	    for (x = 0; x < w; ++x) {
@@ -3379,59 +3389,23 @@ GBool PSOutputDev::checkPageSlice(Page *
 	  }
           p = bitmap->getDataPtr() + (h - 1) * bitmap->getRowSize();
         }
-        for (y = 0; y < h; ++y) {
-	  if (useBinary) {
-	    // Binary gray image
-	    for (x = 0; x < w; ++x) {
-	      g = p[4*x] + p[4*x + 3];
-	      g = 255 - g;
-	      if (g < 0) g = 0;
-	      hexBuf[i++] = (Guchar) g;
-	      if (i >= 64) {
-	        writePSBuf(hexBuf, i);
-	        i = 0;
-	      }
-	    }
-	  } else {
-	    // Hex gray image
-	    for (x = 0; x < w; ++x) {
-	      g = p[4*x] + p[4*x + 3];
-	      g = 255 - g;
-	      if (g < 0) g = 0;
-	      digit = g / 16;
-	      hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
-	      digit = g % 16;
-	      hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
-	      if (i >= 64) {
-	        hexBuf[i++] = '\n';
-	        writePSBuf(hexBuf, i);
-	        i = 0;
-	      }
-	    }
-          }
-          p -= bitmap->getRowSize();
-        }
-      } else if (((psProcessCyan | psProcessMagenta | psProcessYellow | psProcessBlack) & ~processColors) != 0) {
-	// Color image, need to check color flags for each dot
-        for (y = 0; y < h; ++y) {
-          for (comp = 0; comp < 4; ++comp) {
+        if (colorMode == splashModeRGB8 || colorMode != splashModeBGR8) {
+          for (y = 0; y < h; ++y) {
 	    if (useBinary) {
-	      // Binary color image
+	      // Binary gray image
 	      for (x = 0; x < w; ++x) {
-	        col[comp] |= p[4*x + comp];
-	        hexBuf[i++] = p[4*x + comp];
+	        hexBuf[i++] = (Guchar) p[3*x];
 	        if (i >= 64) {
 	          writePSBuf(hexBuf, i);
 	          i = 0;
 	        }
 	      }
 	    } else {
-	      // Gray color image
+	      // Hex gray image
 	      for (x = 0; x < w; ++x) {
-	        col[comp] |= p[4*x + comp];
-	        digit = p[4*x + comp] / 16;
+	        digit = p[3*x] / 16;
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
-	        digit = p[4*x + comp] % 16;
+	        digit = p[3*x] % 16;
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
 	        if (i >= 64) {
 	          hexBuf[i++] = '\n';
@@ -3439,29 +3413,31 @@ GBool PSOutputDev::checkPageSlice(Page *
 	          i = 0;
 	        }
 	      }
-	    }
+            }
           }
-          p -= bitmap->getRowSize();
-        }
-      } else {
-	// Color image, do not need to check color flags
-        for (y = 0; y < h; ++y) {
-          for (comp = 0; comp < 4; ++comp) {
+        } else {
+          for (y = 0; y < h; ++y) {
 	    if (useBinary) {
-	      // Binary color image
+	      // Binary gray image
 	      for (x = 0; x < w; ++x) {
-	        hexBuf[i++] = p[4*x + comp];
+	        g = p[4*x] + p[4*x + 3];
+	        g = 255 - g;
+	        if (g < 0) g = 0;
+	        hexBuf[i++] = (Guchar) g;
 	        if (i >= 64) {
 	          writePSBuf(hexBuf, i);
 	          i = 0;
 	        }
 	      }
 	    } else {
-	      // Hex color image
+	      // Hex gray image
 	      for (x = 0; x < w; ++x) {
-	        digit = p[4*x + comp] / 16;
+	        g = p[4*x] + p[4*x + 3];
+	        g = 255 - g;
+	        if (g < 0) g = 0;
+	        digit = g / 16;
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
-	        digit = p[4*x + comp] % 16;
+	        digit = g % 16;
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
 	        if (i >= 64) {
 	          hexBuf[i++] = '\n';
@@ -3469,9 +3445,207 @@ GBool PSOutputDev::checkPageSlice(Page *
 	          i = 0;
 	        }
 	      }
-	    }
+            }
           }
-          p -= bitmap->getRowSize();
+        }
+        p -= bitmap->getRowSize();
+      } else if (((psProcessCyan | psProcessMagenta | psProcessYellow | psProcessBlack) & ~processColors) != 0) {
+	// Color image, need to check color flags for each dot
+        switch (colorMode) {
+          case splashModeRGB8:
+          case splashModeBGR8:
+            rgbCS = new GfxDeviceRGBColorSpace();
+            for (y = 0; y < h; ++y) {
+              for (comp = 0; comp < 4; ++comp) {
+	        if (useBinary) {
+	          // Binary color image
+	          for (x = 0; x < w; ++x) {
+	            if (likely(colorMode == splashModeRGB8)) {
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
+                    } else {
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
+                    }
+                    rgbCS->getCMYK(&inputColor, &cmyk);
+                    cmykColor[0] = colToByte(cmyk.c);
+                    cmykColor[1] = colToByte(cmyk.m);
+                    cmykColor[2] = colToByte(cmyk.y);
+                    cmykColor[3] = colToByte(cmyk.k);
+
+	            col[comp] |= cmykColor[comp];
+	            hexBuf[i++] = cmykColor[comp];
+	            if (i >= 64) {
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        } else {
+	          // Gray color image
+	          for (x = 0; x < w; ++x) {
+	            if (likely(colorMode == splashModeRGB8)) {
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
+                    } else {
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
+                    }
+                    rgbCS->getCMYK(&inputColor, &cmyk);
+                    cmykColor[0] = colToByte(cmyk.c);
+                    cmykColor[1] = colToByte(cmyk.m);
+                    cmykColor[2] = colToByte(cmyk.y);
+                    cmykColor[3] = colToByte(cmyk.k);
+
+	            col[comp] |= cmykColor[comp];
+	            digit = cmykColor[comp] / 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            digit = cmykColor[comp] % 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            if (i >= 64) {
+	              hexBuf[i++] = '\n';
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        }
+              }
+              p -= bitmap->getRowSize();
+            }
+            delete rgbCS;
+            break;
+          default:
+            for (y = 0; y < h; ++y) {
+              for (comp = 0; comp < 4; ++comp) {
+	        if (useBinary) {
+	          // Binary color image
+	          for (x = 0; x < w; ++x) {
+	            col[comp] |= p[4*x + comp];
+	            hexBuf[i++] = p[4*x + comp];
+	            if (i >= 64) {
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        } else {
+	          // Gray color image
+	          for (x = 0; x < w; ++x) {
+	            col[comp] |= p[4*x + comp];
+	            digit = p[4*x + comp] / 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            digit = p[4*x + comp] % 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            if (i >= 64) {
+	              hexBuf[i++] = '\n';
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        }
+              }
+              p -= bitmap->getRowSize();
+            }
+          break;
+        }
+      } else {
+	// Color image, do not need to check color flags
+        switch (colorMode) {
+          case splashModeRGB8:
+          case splashModeBGR8:
+            rgbCS = new GfxDeviceRGBColorSpace();
+            for (y = 0; y < h; ++y) {
+              for (comp = 0; comp < 4; ++comp) {
+	        if (useBinary) {
+	          // Binary color image
+	          for (x = 0; x < w; ++x) {
+	            if (likely(colorMode == splashModeRGB8)) {
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
+                    } else {
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
+                    }
+                    rgbCS->getCMYK(&inputColor, &cmyk);
+                    cmykColor[0] = colToByte(cmyk.c);
+                    cmykColor[1] = colToByte(cmyk.m);
+                    cmykColor[2] = colToByte(cmyk.y);
+                    cmykColor[3] = colToByte(cmyk.k);
+
+	            hexBuf[i++] = cmykColor[comp];
+	            if (i >= 64) {
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        } else {
+	          // Hex color image
+	          for (x = 0; x < w; ++x) {
+	            if (likely(colorMode == splashModeRGB8)) {
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
+                    } else {
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
+                    }
+                    rgbCS->getCMYK(&inputColor, &cmyk);
+                    cmykColor[0] = colToByte(cmyk.c);
+                    cmykColor[1] = colToByte(cmyk.m);
+                    cmykColor[2] = colToByte(cmyk.y);
+                    cmykColor[3] = colToByte(cmyk.k);
+
+	            digit = cmykColor[comp] / 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            digit = cmykColor[comp] % 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            if (i >= 64) {
+	              hexBuf[i++] = '\n';
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        }
+              }
+              p -= bitmap->getRowSize();
+            }
+            delete rgbCS;
+            break;
+          default:
+            for (y = 0; y < h; ++y) {
+              for (comp = 0; comp < 4; ++comp) {
+	        if (useBinary) {
+	          // Binary color image
+	          for (x = 0; x < w; ++x) {
+	            hexBuf[i++] = p[4*x + comp];
+	            if (i >= 64) {
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        } else {
+	          // Hex color image
+	          for (x = 0; x < w; ++x) {
+	            digit = p[4*x + comp] / 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            digit = p[4*x + comp] % 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            if (i >= 64) {
+	              hexBuf[i++] = '\n';
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        }
+              }
+              p -= bitmap->getRowSize();
+            }
+          break;
         }
       }
       if (i != 0) {