From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001

From: Frediano Ziglio <fziglio@redhat.com>

Date: Mon, 29 Feb 2016 14:24:03 +0000

Subject: [PATCH] create a function to validate surface parameters

Make possible to reuse it outside red-parse-qxl.c

Signed-off-by: Frediano Ziglio <fziglio@redhat.com>

---

 server/red_parse_qxl.c | 50 ++++++++++++++++++++++++++++++++------------------

 server/red_parse_qxl.h |  5 +++++

 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/server/red_parse_qxl.c b/server/red_parse_qxl.c

index bd0c408..7dc6a4d 100644

--- a/server/red_parse_qxl.c

+++ b/server/red_parse_qxl.c

@@ -19,7 +19,6 @@

 #include <config.h>

 #endif

-#include <stdbool.h>

 #include <inttypes.h>

 #include <glib.h>

 #include "common/lz_common.h"

@@ -1306,13 +1305,41 @@ static unsigned int surface_format_to_bpp(uint32_t format)

     return 0;

 }

+bool red_validate_surface(uint32_t width, uint32_t height,

+                          int32_t stride, uint32_t format)

+{

+    unsigned int bpp;

+    uint64_t size;

+

+    bpp = surface_format_to_bpp(format);

+

+    /* check if format is valid */

+    if (!bpp) {

+        return false;

+    }

+

+    /* check stride is larger than required bytes */

+    size = ((uint64_t) width * bpp + 7u) / 8u;

+    /* the uint32_t conversion is here to avoid problems with -2^31 value */

+    if (stride == G_MININT32 || size > (uint32_t) abs(stride)) {

+        return false;

+    }

+

+    /* the multiplication can overflow, also abs(-2^31) may return a negative value */

+    size = (uint64_t) height * abs(stride);

+    if (size > MAX_DATA_CHUNK) {

+        return false;

+    }

+

+    return true;

+}

+

 int red_get_surface_cmd(RedMemSlotInfo *slots, int group_id,

                         RedSurfaceCmd *red, QXLPHYSICAL addr)

 {

     QXLSurfaceCmd *qxl;

     uint64_t size;

     int error;

-    unsigned int bpp;

     qxl = (QXLSurfaceCmd *)get_virt(slots, addr, sizeof(*qxl), group_id,

                                     &error);

@@ -1331,26 +1358,13 @@ int red_get_surface_cmd(RedMemSlotInfo *slots, int group_id,

         red->u.surface_create.width  = qxl->u.surface_create.width;

         red->u.surface_create.height = qxl->u.surface_create.height;

         red->u.surface_create.stride = qxl->u.surface_create.stride;

-        bpp = surface_format_to_bpp(red->u.surface_create.format);

-        /* check if format is valid */

-        if (!bpp) {

+        if (!red_validate_surface(red->u.surface_create.width, red->u.surface_create.height,

+                                  red->u.surface_create.stride, red->u.surface_create.format)) {

             return 1;

         }

-        /* check stride is larger than required bytes */

-        size = ((uint64_t) red->u.surface_create.width * bpp + 7u) / 8u;

-        /* the uint32_t conversion is here to avoid problems with -2^31 value */

-        if (red->u.surface_create.stride == G_MININT32

-            || size > (uint32_t) abs(red->u.surface_create.stride)) {

-            return 1;

-        }

-

-        /* the multiplication can overflow, also abs(-2^31) may return a negative value */

-        size = (uint64_t) red->u.surface_create.height * abs(red->u.surface_create.stride);

-        if (size > MAX_DATA_CHUNK) {

-            return 1;

-        }

+        size = red->u.surface_create.height * abs(red->u.surface_create.stride);

         red->u.surface_create.data =

             (uint8_t*)get_virt(slots, qxl->u.surface_create.data, size, group_id, &error);

         if (error) {

diff --git a/server/red_parse_qxl.h b/server/red_parse_qxl.h

index 3adc9fa..e18d8d0 100644

--- a/server/red_parse_qxl.h

+++ b/server/red_parse_qxl.h

@@ -19,6 +19,8 @@

 #ifndef RED_ABI_TRANSLATE_H

 #define RED_ABI_TRANSLATE_H

+#include <stdbool.h>

+

 #include <spice/qxl_dev.h>

 #include "red_common.h"

 #include "red_memslots.h"

@@ -128,6 +130,9 @@ int red_get_message(RedMemSlotInfo *slots, int group_id,

                     RedMessage *red, QXLPHYSICAL addr);

 void red_put_message(RedMessage *red);

+bool red_validate_surface(uint32_t width, uint32_t height,

+                          int32_t stride, uint32_t format);

+

 int red_get_surface_cmd(RedMemSlotInfo *slots, int group_id,

                         RedSurfaceCmd *red, QXLPHYSICAL addr);

 void red_put_surface_cmd(RedSurfaceCmd *red);