f8a0a1
From 64aa150a92ccb082db6a3383fa734a6ac91cf1bf Mon Sep 17 00:00:00 2001
f8a0a1
From: Marek Kasik <mkasik@redhat.com>
f8a0a1
Date: Tue, 30 Apr 2019 18:47:44 +0200
f8a0a1
Subject: [PATCH] PSOutputDev: Don't read outside of image buffer
f8a0a1
f8a0a1
Check whether input image is RGB or BGR to not treat
f8a0a1
it as CMYK in those cases in PSOutputDev::checkPageSlice().
f8a0a1
f8a0a1
Fixes #751
f8a0a1
---
f8a0a1
 poppler/PSOutputDev.cc | 248 ++++++++++++++++++++++++++++++++---------
f8a0a1
 1 file changed, 196 insertions(+), 52 deletions(-)
f8a0a1
f8a0a1
diff --git a/poppler/PSOutputDev.cc b/poppler/PSOutputDev.cc
f8a0a1
index 0d201835..155a8cbe 100644
f8a0a1
--- a/poppler/PSOutputDev.cc
f8a0a1
+++ b/poppler/PSOutputDev.cc
f8a0a1
@@ -3385,13 +3385,21 @@ GBool PSOutputDev::checkPageSlice(Page *
f8a0a1
       }
f8a0a1
       break;
f8a0a1
     case psLevel1Sep:
f8a0a1
+      GfxColor inputColor;
f8a0a1
+      GfxCMYK cmyk;
f8a0a1
+      unsigned char cmykColor[4];
f8a0a1
+      GfxDeviceRGBColorSpace *rgbCS;
f8a0a1
+      SplashColorMode colorMode;
f8a0a1
+
f8a0a1
+      colorMode = bitmap->getMode();
f8a0a1
+
f8a0a1
       p = bitmap->getDataPtr();
f8a0a1
       // Check for an all gray image
f8a0a1
       if (getOptimizeColorSpace()) {
f8a0a1
         isGray = gTrue;
f8a0a1
         for (y = 0; y < h; ++y) {
f8a0a1
 	  for (x = 0; x < w; ++x) {
f8a0a1
-	    if (p[4*x] != p[4*x + 1] || p[4*x] != p[4*x + 2]) {
f8a0a1
+	    if (p[numComps*x] != p[numComps*x + 1] || p[numComps*x] != p[numComps*x + 2]) {
f8a0a1
 	      isGray = gFalse;
f8a0a1
 	      y = h;
f8a0a1
 	      break;
f8a0a1
@@ -3411,7 +3419,9 @@ GBool PSOutputDev::checkPageSlice(Page *
f8a0a1
       col[0] = col[1] = col[2] = col[3] = 0;
f8a0a1
       if (isGray) {
f8a0a1
         int g;
f8a0a1
-        if ((psProcessBlack & processColors) == 0) {
f8a0a1
+        if ((psProcessBlack & processColors) == 0 &&
f8a0a1
+            colorMode != splashModeRGB8 &&
f8a0a1
+            colorMode != splashModeBGR8) {
f8a0a1
 	  // Check if the image uses black
f8a0a1
 	  for (y = 0; y < h; ++y) {
f8a0a1
 	    for (x = 0; x < w; ++x) {
f8a0a1
@@ -3425,59 +3435,23 @@ GBool PSOutputDev::checkPageSlice(Page *
f8a0a1
 	  }
f8a0a1
           p = bitmap->getDataPtr() + (h - 1) * bitmap->getRowSize();
f8a0a1
         }
f8a0a1
-        for (y = 0; y < h; ++y) {
f8a0a1
-	  if (useBinary) {
f8a0a1
-	    // Binary gray image
f8a0a1
-	    for (x = 0; x < w; ++x) {
f8a0a1
-	      g = p[4*x] + p[4*x + 3];
f8a0a1
-	      g = 255 - g;
f8a0a1
-	      if (g < 0) g = 0;
f8a0a1
-	      hexBuf[i++] = (Guchar) g;
f8a0a1
-	      if (i >= 64) {
f8a0a1
-	        writePSBuf(hexBuf, i);
f8a0a1
-	        i = 0;
f8a0a1
-	      }
f8a0a1
-	    }
f8a0a1
-	  } else {
f8a0a1
-	    // Hex gray image
f8a0a1
-	    for (x = 0; x < w; ++x) {
f8a0a1
-	      g = p[4*x] + p[4*x + 3];
f8a0a1
-	      g = 255 - g;
f8a0a1
-	      if (g < 0) g = 0;
f8a0a1
-	      digit = g / 16;
f8a0a1
-	      hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
-	      digit = g % 16;
f8a0a1
-	      hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
-	      if (i >= 64) {
f8a0a1
-	        hexBuf[i++] = '\n';
f8a0a1
-	        writePSBuf(hexBuf, i);
f8a0a1
-	        i = 0;
f8a0a1
-	      }
f8a0a1
-	    }
f8a0a1
-          }
f8a0a1
-          p -= bitmap->getRowSize();
f8a0a1
-        }
f8a0a1
-      } else if (((psProcessCyan | psProcessMagenta | psProcessYellow | psProcessBlack) & ~processColors) != 0) {
f8a0a1
-	// Color image, need to check color flags for each dot
f8a0a1
-        for (y = 0; y < h; ++y) {
f8a0a1
-          for (comp = 0; comp < 4; ++comp) {
f8a0a1
+        if (colorMode == splashModeRGB8 || colorMode != splashModeBGR8) {
f8a0a1
+          for (y = 0; y < h; ++y) {
f8a0a1
 	    if (useBinary) {
f8a0a1
-	      // Binary color image
f8a0a1
+	      // Binary gray image
f8a0a1
 	      for (x = 0; x < w; ++x) {
f8a0a1
-	        col[comp] |= p[4*x + comp];
f8a0a1
-	        hexBuf[i++] = p[4*x + comp];
f8a0a1
+	        hexBuf[i++] = (Guchar) p[3*x];
f8a0a1
 	        if (i >= 64) {
f8a0a1
 	          writePSBuf(hexBuf, i);
f8a0a1
 	          i = 0;
f8a0a1
 	        }
f8a0a1
 	      }
f8a0a1
 	    } else {
f8a0a1
-	      // Gray color image
f8a0a1
+	      // Hex gray image
f8a0a1
 	      for (x = 0; x < w; ++x) {
f8a0a1
-	        col[comp] |= p[4*x + comp];
f8a0a1
-	        digit = p[4*x + comp] / 16;
f8a0a1
+	        digit = p[3*x] / 16;
f8a0a1
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
-	        digit = p[4*x + comp] % 16;
f8a0a1
+	        digit = p[3*x] % 16;
f8a0a1
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
 	        if (i >= 64) {
f8a0a1
 	          hexBuf[i++] = '\n';
f8a0a1
@@ -3485,29 +3459,31 @@ GBool PSOutputDev::checkPageSlice(Page *
f8a0a1
 	          i = 0;
f8a0a1
 	        }
f8a0a1
 	      }
f8a0a1
-	    }
f8a0a1
+            }
f8a0a1
           }
f8a0a1
-          p -= bitmap->getRowSize();
f8a0a1
-        }
f8a0a1
-      } else {
f8a0a1
-	// Color image, do not need to check color flags
f8a0a1
-        for (y = 0; y < h; ++y) {
f8a0a1
-          for (comp = 0; comp < 4; ++comp) {
f8a0a1
+        } else {
f8a0a1
+          for (y = 0; y < h; ++y) {
f8a0a1
 	    if (useBinary) {
f8a0a1
-	      // Binary color image
f8a0a1
+	      // Binary gray image
f8a0a1
 	      for (x = 0; x < w; ++x) {
f8a0a1
-	        hexBuf[i++] = p[4*x + comp];
f8a0a1
+	        g = p[4*x] + p[4*x + 3];
f8a0a1
+	        g = 255 - g;
f8a0a1
+	        if (g < 0) g = 0;
f8a0a1
+	        hexBuf[i++] = (Guchar) g;
f8a0a1
 	        if (i >= 64) {
f8a0a1
 	          writePSBuf(hexBuf, i);
f8a0a1
 	          i = 0;
f8a0a1
 	        }
f8a0a1
 	      }
f8a0a1
 	    } else {
f8a0a1
-	      // Hex color image
f8a0a1
+	      // Hex gray image
f8a0a1
 	      for (x = 0; x < w; ++x) {
f8a0a1
-	        digit = p[4*x + comp] / 16;
f8a0a1
+	        g = p[4*x] + p[4*x + 3];
f8a0a1
+	        g = 255 - g;
f8a0a1
+	        if (g < 0) g = 0;
f8a0a1
+	        digit = g / 16;
f8a0a1
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
-	        digit = p[4*x + comp] % 16;
f8a0a1
+	        digit = g % 16;
f8a0a1
 	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
 	        if (i >= 64) {
f8a0a1
 	          hexBuf[i++] = '\n';
f8a0a1
@@ -3515,9 +3491,207 @@ GBool PSOutputDev::checkPageSlice(Page *
f8a0a1
 	          i = 0;
f8a0a1
 	        }
f8a0a1
 	      }
f8a0a1
-	    }
f8a0a1
+            }
f8a0a1
           }
f8a0a1
-          p -= bitmap->getRowSize();
f8a0a1
+        }
f8a0a1
+        p -= bitmap->getRowSize();
f8a0a1
+      } else if (((psProcessCyan | psProcessMagenta | psProcessYellow | psProcessBlack) & ~processColors) != 0) {
f8a0a1
+	// Color image, need to check color flags for each dot
f8a0a1
+        switch (colorMode) {
f8a0a1
+          case splashModeRGB8:
f8a0a1
+          case splashModeBGR8:
f8a0a1
+            rgbCS = new GfxDeviceRGBColorSpace();
f8a0a1
+            for (y = 0; y < h; ++y) {
f8a0a1
+              for (comp = 0; comp < 4; ++comp) {
f8a0a1
+	        if (useBinary) {
f8a0a1
+	          // Binary color image
f8a0a1
+	          for (x = 0; x < w; ++x) {
f8a0a1
+	            if (likely(colorMode == splashModeRGB8)) {
f8a0a1
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
f8a0a1
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
f8a0a1
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
f8a0a1
+                    } else {
f8a0a1
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
f8a0a1
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
f8a0a1
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
f8a0a1
+                    }
f8a0a1
+                    rgbCS->getCMYK(&inputColor, &cmyk);
f8a0a1
+                    cmykColor[0] = colToByte(cmyk.c);
f8a0a1
+                    cmykColor[1] = colToByte(cmyk.m);
f8a0a1
+                    cmykColor[2] = colToByte(cmyk.y);
f8a0a1
+                    cmykColor[3] = colToByte(cmyk.k);
f8a0a1
+
f8a0a1
+	            col[comp] |= cmykColor[comp];
f8a0a1
+	            hexBuf[i++] = cmykColor[comp];
f8a0a1
+	            if (i >= 64) {
f8a0a1
+	              writePSBuf(hexBuf, i);
f8a0a1
+	              i = 0;
f8a0a1
+	            }
f8a0a1
+	          }
f8a0a1
+	        } else {
f8a0a1
+	          // Gray color image
f8a0a1
+	          for (x = 0; x < w; ++x) {
f8a0a1
+	            if (likely(colorMode == splashModeRGB8)) {
f8a0a1
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
f8a0a1
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
f8a0a1
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
f8a0a1
+                    } else {
f8a0a1
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
f8a0a1
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
f8a0a1
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
f8a0a1
+                    }
f8a0a1
+                    rgbCS->getCMYK(&inputColor, &cmyk);
f8a0a1
+                    cmykColor[0] = colToByte(cmyk.c);
f8a0a1
+                    cmykColor[1] = colToByte(cmyk.m);
f8a0a1
+                    cmykColor[2] = colToByte(cmyk.y);
f8a0a1
+                    cmykColor[3] = colToByte(cmyk.k);
f8a0a1
+
f8a0a1
+	            col[comp] |= cmykColor[comp];
f8a0a1
+	            digit = cmykColor[comp] / 16;
f8a0a1
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
+	            digit = cmykColor[comp] % 16;
f8a0a1
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
+	            if (i >= 64) {
f8a0a1
+	              hexBuf[i++] = '\n';
f8a0a1
+	              writePSBuf(hexBuf, i);
f8a0a1
+	              i = 0;
f8a0a1
+	            }
f8a0a1
+	          }
f8a0a1
+	        }
f8a0a1
+              }
f8a0a1
+              p -= bitmap->getRowSize();
f8a0a1
+            }
f8a0a1
+            delete rgbCS;
f8a0a1
+            break;
f8a0a1
+          default:
f8a0a1
+            for (y = 0; y < h; ++y) {
f8a0a1
+              for (comp = 0; comp < 4; ++comp) {
f8a0a1
+	        if (useBinary) {
f8a0a1
+	          // Binary color image
f8a0a1
+	          for (x = 0; x < w; ++x) {
f8a0a1
+	            col[comp] |= p[4*x + comp];
f8a0a1
+	            hexBuf[i++] = p[4*x + comp];
f8a0a1
+	            if (i >= 64) {
f8a0a1
+	              writePSBuf(hexBuf, i);
f8a0a1
+	              i = 0;
f8a0a1
+	            }
f8a0a1
+	          }
f8a0a1
+	        } else {
f8a0a1
+	          // Gray color image
f8a0a1
+	          for (x = 0; x < w; ++x) {
f8a0a1
+	            col[comp] |= p[4*x + comp];
f8a0a1
+	            digit = p[4*x + comp] / 16;
f8a0a1
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
+	            digit = p[4*x + comp] % 16;
f8a0a1
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
+	            if (i >= 64) {
f8a0a1
+	              hexBuf[i++] = '\n';
f8a0a1
+	              writePSBuf(hexBuf, i);
f8a0a1
+	              i = 0;
f8a0a1
+	            }
f8a0a1
+	          }
f8a0a1
+	        }
f8a0a1
+              }
f8a0a1
+              p -= bitmap->getRowSize();
f8a0a1
+            }
f8a0a1
+          break;
f8a0a1
+        }
f8a0a1
+      } else {
f8a0a1
+	// Color image, do not need to check color flags
f8a0a1
+        switch (colorMode) {
f8a0a1
+          case splashModeRGB8:
f8a0a1
+          case splashModeBGR8:
f8a0a1
+            rgbCS = new GfxDeviceRGBColorSpace();
f8a0a1
+            for (y = 0; y < h; ++y) {
f8a0a1
+              for (comp = 0; comp < 4; ++comp) {
f8a0a1
+	        if (useBinary) {
f8a0a1
+	          // Binary color image
f8a0a1
+	          for (x = 0; x < w; ++x) {
f8a0a1
+	            if (likely(colorMode == splashModeRGB8)) {
f8a0a1
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
f8a0a1
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
f8a0a1
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
f8a0a1
+                    } else {
f8a0a1
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
f8a0a1
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
f8a0a1
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
f8a0a1
+                    }
f8a0a1
+                    rgbCS->getCMYK(&inputColor, &cmyk);
f8a0a1
+                    cmykColor[0] = colToByte(cmyk.c);
f8a0a1
+                    cmykColor[1] = colToByte(cmyk.m);
f8a0a1
+                    cmykColor[2] = colToByte(cmyk.y);
f8a0a1
+                    cmykColor[3] = colToByte(cmyk.k);
f8a0a1
+
f8a0a1
+	            hexBuf[i++] = cmykColor[comp];
f8a0a1
+	            if (i >= 64) {
f8a0a1
+	              writePSBuf(hexBuf, i);
f8a0a1
+	              i = 0;
f8a0a1
+	            }
f8a0a1
+	          }
f8a0a1
+	        } else {
f8a0a1
+	          // Hex color image
f8a0a1
+	          for (x = 0; x < w; ++x) {
f8a0a1
+	            if (likely(colorMode == splashModeRGB8)) {
f8a0a1
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
f8a0a1
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
f8a0a1
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
f8a0a1
+                    } else {
f8a0a1
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
f8a0a1
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
f8a0a1
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
f8a0a1
+                    }
f8a0a1
+                    rgbCS->getCMYK(&inputColor, &cmyk);
f8a0a1
+                    cmykColor[0] = colToByte(cmyk.c);
f8a0a1
+                    cmykColor[1] = colToByte(cmyk.m);
f8a0a1
+                    cmykColor[2] = colToByte(cmyk.y);
f8a0a1
+                    cmykColor[3] = colToByte(cmyk.k);
f8a0a1
+
f8a0a1
+	            digit = cmykColor[comp] / 16;
f8a0a1
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
+	            digit = cmykColor[comp] % 16;
f8a0a1
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
+	            if (i >= 64) {
f8a0a1
+	              hexBuf[i++] = '\n';
f8a0a1
+	              writePSBuf(hexBuf, i);
f8a0a1
+	              i = 0;
f8a0a1
+	            }
f8a0a1
+	          }
f8a0a1
+	        }
f8a0a1
+              }
f8a0a1
+              p -= bitmap->getRowSize();
f8a0a1
+            }
f8a0a1
+            delete rgbCS;
f8a0a1
+            break;
f8a0a1
+          default:
f8a0a1
+            for (y = 0; y < h; ++y) {
f8a0a1
+              for (comp = 0; comp < 4; ++comp) {
f8a0a1
+	        if (useBinary) {
f8a0a1
+	          // Binary color image
f8a0a1
+	          for (x = 0; x < w; ++x) {
f8a0a1
+	            hexBuf[i++] = p[4*x + comp];
f8a0a1
+	            if (i >= 64) {
f8a0a1
+	              writePSBuf(hexBuf, i);
f8a0a1
+	              i = 0;
f8a0a1
+	            }
f8a0a1
+	          }
f8a0a1
+	        } else {
f8a0a1
+	          // Hex color image
f8a0a1
+	          for (x = 0; x < w; ++x) {
f8a0a1
+	            digit = p[4*x + comp] / 16;
f8a0a1
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
+	            digit = p[4*x + comp] % 16;
f8a0a1
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
f8a0a1
+	            if (i >= 64) {
f8a0a1
+	              hexBuf[i++] = '\n';
f8a0a1
+	              writePSBuf(hexBuf, i);
f8a0a1
+	              i = 0;
f8a0a1
+	            }
f8a0a1
+	          }
f8a0a1
+	        }
f8a0a1
+              }
f8a0a1
+              p -= bitmap->getRowSize();
f8a0a1
+            }
f8a0a1
+          break;
f8a0a1
         }
f8a0a1
       }
f8a0a1
       if (i != 0) {