Integer overflow on illegal server response

The 32 bit field "rep.length" is not checked for validity, which allows
an integer overflow on 32 bit systems.

A malicious server could send INT_MAX as length, which gets multiplied
by the size of XRectangle. In that case the client won't read the whole
data from server, getting out of sync.

From Tobias Stoeckmann / X.Org security advisory Oct 4, 2016

1 files changed, 12 insertions, 3 deletions

diff --git a/lib/libXfixes/src/Region.c b/lib/libXfixes/src/Region.c
index cb0cf6e09..59bcc1aa6 100644
--- a/lib/libXfixes/src/Region.c
+++ b/lib/libXfixes/src/Region.c
@@ -23,6 +23,7 @@
 #ifdef HAVE_CONFIG_H
 #include <config.h>
 #endif
+#include <limits.h>
 #include "Xfixesint.h"
 
 XserverRegion
@@ -333,9 +334,17 @@ XFixesFetchRegionAndBounds (Display	    *dpy,
     bounds->y = rep.y;
     bounds->width = rep.width;
     bounds->height = rep.height;
-    nbytes = (long) rep.length << 2;
-    nrects = rep.length >> 1;
-    rects = Xmalloc (nrects * sizeof (XRectangle));
+
+    if (rep.length < (INT_MAX >> 2)) {
+	nbytes = (long) rep.length << 2;
+	nrects = rep.length >> 1;
+	rects = Xmalloc (nrects * sizeof (XRectangle));
+    } else {
+	nbytes = 0;
+	nrects = 0;
+	rects = NULL;
+    }
+
     if (!rects)
     {
 	_XEatDataWords(dpy, rep.length);