3587f3
From 64aa150a92ccb082db6a3383fa734a6ac91cf1bf Mon Sep 17 00:00:00 2001
3587f3
From: Marek Kasik <mkasik@redhat.com>
3587f3
Date: Tue, 30 Apr 2019 18:47:44 +0200
3587f3
Subject: [PATCH] PSOutputDev: Don't read outside of image buffer
3587f3
3587f3
Check whether input image is RGB or BGR to not treat
3587f3
it as CMYK in those cases in PSOutputDev::checkPageSlice().
3587f3
3587f3
Fixes #751
3587f3
---
3587f3
 poppler/PSOutputDev.cc | 248 ++++++++++++++++++++++++++++++++---------
3587f3
 1 file changed, 196 insertions(+), 52 deletions(-)
3587f3
3587f3
diff --git a/poppler/PSOutputDev.cc b/poppler/PSOutputDev.cc
3587f3
index 0d201835..155a8cbe 100644
3587f3
--- a/poppler/PSOutputDev.cc
3587f3
+++ b/poppler/PSOutputDev.cc
3587f3
@@ -3385,13 +3385,21 @@ GBool PSOutputDev::checkPageSlice(Page *
3587f3
       }
3587f3
       break;
3587f3
     case psLevel1Sep:
3587f3
+      GfxColor inputColor;
3587f3
+      GfxCMYK cmyk;
3587f3
+      unsigned char cmykColor[4];
3587f3
+      GfxDeviceRGBColorSpace *rgbCS;
3587f3
+      SplashColorMode colorMode;
3587f3
+
3587f3
+      colorMode = bitmap->getMode();
3587f3
+
3587f3
       p = bitmap->getDataPtr();
3587f3
       // Check for an all gray image
3587f3
       if (getOptimizeColorSpace()) {
3587f3
         isGray = gTrue;
3587f3
         for (y = 0; y < h; ++y) {
3587f3
 	  for (x = 0; x < w; ++x) {
3587f3
-	    if (p[4*x] != p[4*x + 1] || p[4*x] != p[4*x + 2]) {
3587f3
+	    if (p[numComps*x] != p[numComps*x + 1] || p[numComps*x] != p[numComps*x + 2]) {
3587f3
 	      isGray = gFalse;
3587f3
 	      y = h;
3587f3
 	      break;
3587f3
@@ -3411,7 +3419,9 @@ GBool PSOutputDev::checkPageSlice(Page *
3587f3
       col[0] = col[1] = col[2] = col[3] = 0;
3587f3
       if (isGray) {
3587f3
         int g;
3587f3
-        if ((psProcessBlack & processColors) == 0) {
3587f3
+        if ((psProcessBlack & processColors) == 0 &&
3587f3
+            colorMode != splashModeRGB8 &&
3587f3
+            colorMode != splashModeBGR8) {
3587f3
 	  // Check if the image uses black
3587f3
 	  for (y = 0; y < h; ++y) {
3587f3
 	    for (x = 0; x < w; ++x) {
3587f3
@@ -3425,59 +3435,23 @@ GBool PSOutputDev::checkPageSlice(Page *
3587f3
 	  }
3587f3
           p = bitmap->getDataPtr() + (h - 1) * bitmap->getRowSize();
3587f3
         }
3587f3
-        for (y = 0; y < h; ++y) {
3587f3
-	  if (useBinary) {
3587f3
-	    // Binary gray image
3587f3
-	    for (x = 0; x < w; ++x) {
3587f3
-	      g = p[4*x] + p[4*x + 3];
3587f3
-	      g = 255 - g;
3587f3
-	      if (g < 0) g = 0;
3587f3
-	      hexBuf[i++] = (Guchar) g;
3587f3
-	      if (i >= 64) {
3587f3
-	        writePSBuf(hexBuf, i);
3587f3
-	        i = 0;
3587f3
-	      }
3587f3
-	    }
3587f3
-	  } else {
3587f3
-	    // Hex gray image
3587f3
-	    for (x = 0; x < w; ++x) {
3587f3
-	      g = p[4*x] + p[4*x + 3];
3587f3
-	      g = 255 - g;
3587f3
-	      if (g < 0) g = 0;
3587f3
-	      digit = g / 16;
3587f3
-	      hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
-	      digit = g % 16;
3587f3
-	      hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
-	      if (i >= 64) {
3587f3
-	        hexBuf[i++] = '\n';
3587f3
-	        writePSBuf(hexBuf, i);
3587f3
-	        i = 0;
3587f3
-	      }
3587f3
-	    }
3587f3
-          }
3587f3
-          p -= bitmap->getRowSize();
3587f3
-        }
3587f3
-      } else if (((psProcessCyan | psProcessMagenta | psProcessYellow | psProcessBlack) & ~processColors) != 0) {
3587f3
-	// Color image, need to check color flags for each dot
3587f3
-        for (y = 0; y < h; ++y) {
3587f3
-          for (comp = 0; comp < 4; ++comp) {
3587f3
+        if (colorMode == splashModeRGB8 || colorMode != splashModeBGR8) {
3587f3
+          for (y = 0; y < h; ++y) {
3587f3
 	    if (useBinary) {
3587f3
-	      // Binary color image
3587f3
+	      // Binary gray image
3587f3
 	      for (x = 0; x < w; ++x) {
3587f3
-	        col[comp] |= p[4*x + comp];
3587f3
-	        hexBuf[i++] = p[4*x + comp];
3587f3
+	        hexBuf[i++] = (Guchar) p[3*x];
3587f3
 	        if (i >= 64) {
3587f3
 	          writePSBuf(hexBuf, i);
3587f3
 	          i = 0;
3587f3
 	        }
3587f3
 	      }
3587f3
 	    } else {
3587f3
-	      // Gray color image
3587f3
+	      // Hex gray image
3587f3
 	      for (x = 0; x < w; ++x) {
3587f3
-	        col[comp] |= p[4*x + comp];
3587f3
-	        digit = p[4*x + comp] / 16;
3587f3
+	        digit = p[3*x] / 16;
3587f3
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
-	        digit = p[4*x + comp] % 16;
3587f3
+	        digit = p[3*x] % 16;
3587f3
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
 	        if (i >= 64) {
3587f3
 	          hexBuf[i++] = '\n';
3587f3
@@ -3485,29 +3459,31 @@ GBool PSOutputDev::checkPageSlice(Page *
3587f3
 	          i = 0;
3587f3
 	        }
3587f3
 	      }
3587f3
-	    }
3587f3
+            }
3587f3
           }
3587f3
-          p -= bitmap->getRowSize();
3587f3
-        }
3587f3
-      } else {
3587f3
-	// Color image, do not need to check color flags
3587f3
-        for (y = 0; y < h; ++y) {
3587f3
-          for (comp = 0; comp < 4; ++comp) {
3587f3
+        } else {
3587f3
+          for (y = 0; y < h; ++y) {
3587f3
 	    if (useBinary) {
3587f3
-	      // Binary color image
3587f3
+	      // Binary gray image
3587f3
 	      for (x = 0; x < w; ++x) {
3587f3
-	        hexBuf[i++] = p[4*x + comp];
3587f3
+	        g = p[4*x] + p[4*x + 3];
3587f3
+	        g = 255 - g;
3587f3
+	        if (g < 0) g = 0;
3587f3
+	        hexBuf[i++] = (Guchar) g;
3587f3
 	        if (i >= 64) {
3587f3
 	          writePSBuf(hexBuf, i);
3587f3
 	          i = 0;
3587f3
 	        }
3587f3
 	      }
3587f3
 	    } else {
3587f3
-	      // Hex color image
3587f3
+	      // Hex gray image
3587f3
 	      for (x = 0; x < w; ++x) {
3587f3
-	        digit = p[4*x + comp] / 16;
3587f3
+	        g = p[4*x] + p[4*x + 3];
3587f3
+	        g = 255 - g;
3587f3
+	        if (g < 0) g = 0;
3587f3
+	        digit = g / 16;
3587f3
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
-	        digit = p[4*x + comp] % 16;
3587f3
+	        digit = g % 16;
3587f3
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
 	        if (i >= 64) {
3587f3
 	          hexBuf[i++] = '\n';
3587f3
@@ -3515,9 +3491,207 @@ GBool PSOutputDev::checkPageSlice(Page *
3587f3
 	          i = 0;
3587f3
 	        }
3587f3
 	      }
3587f3
-	    }
3587f3
+            }
3587f3
           }
3587f3
-          p -= bitmap->getRowSize();
3587f3
+        }
3587f3
+        p -= bitmap->getRowSize();
3587f3
+      } else if (((psProcessCyan | psProcessMagenta | psProcessYellow | psProcessBlack) & ~processColors) != 0) {
3587f3
+	// Color image, need to check color flags for each dot
3587f3
+        switch (colorMode) {
3587f3
+          case splashModeRGB8:
3587f3
+          case splashModeBGR8:
3587f3
+            rgbCS = new GfxDeviceRGBColorSpace();
3587f3
+            for (y = 0; y < h; ++y) {
3587f3
+              for (comp = 0; comp < 4; ++comp) {
3587f3
+	        if (useBinary) {
3587f3
+	          // Binary color image
3587f3
+	          for (x = 0; x < w; ++x) {
3587f3
+	            if (likely(colorMode == splashModeRGB8)) {
3587f3
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
3587f3
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
3587f3
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
3587f3
+                    } else {
3587f3
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
3587f3
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
3587f3
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
3587f3
+                    }
3587f3
+                    rgbCS->getCMYK(&inputColor, &cmyk);
3587f3
+                    cmykColor[0] = colToByte(cmyk.c);
3587f3
+                    cmykColor[1] = colToByte(cmyk.m);
3587f3
+                    cmykColor[2] = colToByte(cmyk.y);
3587f3
+                    cmykColor[3] = colToByte(cmyk.k);
3587f3
+
3587f3
+	            col[comp] |= cmykColor[comp];
3587f3
+	            hexBuf[i++] = cmykColor[comp];
3587f3
+	            if (i >= 64) {
3587f3
+	              writePSBuf(hexBuf, i);
3587f3
+	              i = 0;
3587f3
+	            }
3587f3
+	          }
3587f3
+	        } else {
3587f3
+	          // Gray color image
3587f3
+	          for (x = 0; x < w; ++x) {
3587f3
+	            if (likely(colorMode == splashModeRGB8)) {
3587f3
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
3587f3
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
3587f3
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
3587f3
+                    } else {
3587f3
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
3587f3
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
3587f3
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
3587f3
+                    }
3587f3
+                    rgbCS->getCMYK(&inputColor, &cmyk);
3587f3
+                    cmykColor[0] = colToByte(cmyk.c);
3587f3
+                    cmykColor[1] = colToByte(cmyk.m);
3587f3
+                    cmykColor[2] = colToByte(cmyk.y);
3587f3
+                    cmykColor[3] = colToByte(cmyk.k);
3587f3
+
3587f3
+	            col[comp] |= cmykColor[comp];
3587f3
+	            digit = cmykColor[comp] / 16;
3587f3
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
+	            digit = cmykColor[comp] % 16;
3587f3
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
+	            if (i >= 64) {
3587f3
+	              hexBuf[i++] = '\n';
3587f3
+	              writePSBuf(hexBuf, i);
3587f3
+	              i = 0;
3587f3
+	            }
3587f3
+	          }
3587f3
+	        }
3587f3
+              }
3587f3
+              p -= bitmap->getRowSize();
3587f3
+            }
3587f3
+            delete rgbCS;
3587f3
+            break;
3587f3
+          default:
3587f3
+            for (y = 0; y < h; ++y) {
3587f3
+              for (comp = 0; comp < 4; ++comp) {
3587f3
+	        if (useBinary) {
3587f3
+	          // Binary color image
3587f3
+	          for (x = 0; x < w; ++x) {
3587f3
+	            col[comp] |= p[4*x + comp];
3587f3
+	            hexBuf[i++] = p[4*x + comp];
3587f3
+	            if (i >= 64) {
3587f3
+	              writePSBuf(hexBuf, i);
3587f3
+	              i = 0;
3587f3
+	            }
3587f3
+	          }
3587f3
+	        } else {
3587f3
+	          // Gray color image
3587f3
+	          for (x = 0; x < w; ++x) {
3587f3
+	            col[comp] |= p[4*x + comp];
3587f3
+	            digit = p[4*x + comp] / 16;
3587f3
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
+	            digit = p[4*x + comp] % 16;
3587f3
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
+	            if (i >= 64) {
3587f3
+	              hexBuf[i++] = '\n';
3587f3
+	              writePSBuf(hexBuf, i);
3587f3
+	              i = 0;
3587f3
+	            }
3587f3
+	          }
3587f3
+	        }
3587f3
+              }
3587f3
+              p -= bitmap->getRowSize();
3587f3
+            }
3587f3
+          break;
3587f3
+        }
3587f3
+      } else {
3587f3
+	// Color image, do not need to check color flags
3587f3
+        switch (colorMode) {
3587f3
+          case splashModeRGB8:
3587f3
+          case splashModeBGR8:
3587f3
+            rgbCS = new GfxDeviceRGBColorSpace();
3587f3
+            for (y = 0; y < h; ++y) {
3587f3
+              for (comp = 0; comp < 4; ++comp) {
3587f3
+	        if (useBinary) {
3587f3
+	          // Binary color image
3587f3
+	          for (x = 0; x < w; ++x) {
3587f3
+	            if (likely(colorMode == splashModeRGB8)) {
3587f3
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
3587f3
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
3587f3
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
3587f3
+                    } else {
3587f3
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
3587f3
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
3587f3
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
3587f3
+                    }
3587f3
+                    rgbCS->getCMYK(&inputColor, &cmyk);
3587f3
+                    cmykColor[0] = colToByte(cmyk.c);
3587f3
+                    cmykColor[1] = colToByte(cmyk.m);
3587f3
+                    cmykColor[2] = colToByte(cmyk.y);
3587f3
+                    cmykColor[3] = colToByte(cmyk.k);
3587f3
+
3587f3
+	            hexBuf[i++] = cmykColor[comp];
3587f3
+	            if (i >= 64) {
3587f3
+	              writePSBuf(hexBuf, i);
3587f3
+	              i = 0;
3587f3
+	            }
3587f3
+	          }
3587f3
+	        } else {
3587f3
+	          // Hex color image
3587f3
+	          for (x = 0; x < w; ++x) {
3587f3
+	            if (likely(colorMode == splashModeRGB8)) {
3587f3
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
3587f3
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
3587f3
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
3587f3
+                    } else {
3587f3
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
3587f3
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
3587f3
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
3587f3
+                    }
3587f3
+                    rgbCS->getCMYK(&inputColor, &cmyk);
3587f3
+                    cmykColor[0] = colToByte(cmyk.c);
3587f3
+                    cmykColor[1] = colToByte(cmyk.m);
3587f3
+                    cmykColor[2] = colToByte(cmyk.y);
3587f3
+                    cmykColor[3] = colToByte(cmyk.k);
3587f3
+
3587f3
+	            digit = cmykColor[comp] / 16;
3587f3
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
+	            digit = cmykColor[comp] % 16;
3587f3
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
+	            if (i >= 64) {
3587f3
+	              hexBuf[i++] = '\n';
3587f3
+	              writePSBuf(hexBuf, i);
3587f3
+	              i = 0;
3587f3
+	            }
3587f3
+	          }
3587f3
+	        }
3587f3
+              }
3587f3
+              p -= bitmap->getRowSize();
3587f3
+            }
3587f3
+            delete rgbCS;
3587f3
+            break;
3587f3
+          default:
3587f3
+            for (y = 0; y < h; ++y) {
3587f3
+              for (comp = 0; comp < 4; ++comp) {
3587f3
+	        if (useBinary) {
3587f3
+	          // Binary color image
3587f3
+	          for (x = 0; x < w; ++x) {
3587f3
+	            hexBuf[i++] = p[4*x + comp];
3587f3
+	            if (i >= 64) {
3587f3
+	              writePSBuf(hexBuf, i);
3587f3
+	              i = 0;
3587f3
+	            }
3587f3
+	          }
3587f3
+	        } else {
3587f3
+	          // Hex color image
3587f3
+	          for (x = 0; x < w; ++x) {
3587f3
+	            digit = p[4*x + comp] / 16;
3587f3
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
+	            digit = p[4*x + comp] % 16;
3587f3
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
3587f3
+	            if (i >= 64) {
3587f3
+	              hexBuf[i++] = '\n';
3587f3
+	              writePSBuf(hexBuf, i);
3587f3
+	              i = 0;
3587f3
+	            }
3587f3
+	          }
3587f3
+	        }
3587f3
+              }
3587f3
+              p -= bitmap->getRowSize();
3587f3
+            }
3587f3
+          break;
3587f3
         }
3587f3
       }
3587f3
       if (i != 0) {