From e28c08d63490a2fb6b8cc07bf968eb16243e9c63 Mon Sep 17 00:00:00 2001
From: Frediano Ziglio <fziglio@redhat.com>
Date: Tue, 8 Sep 2015 10:13:24 +0100
Subject: [PATCH 49/57] Fix integer overflow computing glyph_size in
 red_get_string

If bpp is int the formula can lead to weird overflows. width and height
are uint16_t so the formula is:

  size_t = u16 * (u16 * int + const_int) / const_int;

so it became

  size_t = (int) u16 * ((int) u16 * int + const_int) / const_int;

However the (int) u16 * (int) u16 can then became negative to overflow.
Under 64 bit architectures size_t is 64 and int usually 32 so converting
this negative 32 bit number to a unsigned 64 bit lead to a very big
number as the signed is extended and then converted to unsigned.
Using unsigned arithmetic prevent extending the sign.

Signed-off-by: Frediano Ziglio <fziglio@redhat.com>
Acked-by: Christophe Fergeau <cfergeau@redhat.com>
---
 server/red_parse_qxl.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

Index: spice-0.12.5/server/red_parse_qxl.c
===================================================================
--- spice-0.12.5.orig/server/red_parse_qxl.c	2015-10-01 07:18:16.641721919 -0400
+++ spice-0.12.5/server/red_parse_qxl.c	2015-10-01 07:18:16.637721952 -0400
@@ -808,7 +808,9 @@
     uint8_t *data;
     bool free_data;
     size_t chunk_size, qxl_size, red_size, glyph_size;
-    int glyphs, bpp = 0, i;
+    int glyphs, i;
+    /* use unsigned to prevent integer overflow in multiplication below */
+    unsigned int bpp = 0;
     int error;
     uint16_t qxl_flags, qxl_length;
 
@@ -847,7 +849,7 @@
     while (start < end) {
         spice_assert((QXLRasterGlyph*)(&start->data[0]) <= end);
         glyphs++;
-        glyph_size = start->height * ((start->width * bpp + 7) / 8);
+        glyph_size = start->height * ((start->width * bpp + 7u) / 8u);
         red_size += sizeof(SpiceRasterGlyph *) + SPICE_ALIGN(sizeof(SpiceRasterGlyph) + glyph_size, 4);
         start = (QXLRasterGlyph*)(&start->data[glyph_size]);
     }
@@ -868,7 +870,7 @@
         glyph->height = start->height;
         red_get_point_ptr(&glyph->render_pos, &start->render_pos);
         red_get_point_ptr(&glyph->glyph_origin, &start->glyph_origin);
-        glyph_size = glyph->height * ((glyph->width * bpp + 7) / 8);
+        glyph_size = glyph->height * ((glyph->width * bpp + 7u) / 8u);
         spice_assert((QXLRasterGlyph*)(&start->data[glyph_size]) <= end);
         memcpy(glyph->data, start->data, glyph_size);
         start = (QXLRasterGlyph*)(&start->data[glyph_size]);