Use _XEatDataWords to avoid overflow of rep.length bit shifting

rep.length is a CARD32, so rep.length << 2 could overflow in 32-bit builds

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>

7 files changed, 28 insertions, 17 deletions

diff --git a/src/Xrandrint.h b/src/Xrandrint.h
index aed10e4..1687c29 100644
--- a/src/Xrandrint.h
+++ b/src/Xrandrint.h
@@ -42,6 +42,19 @@ extern char XRRExtensionName[];
 
 XExtDisplayInfo *XRRFindDisplay (Display *dpy);
 
+#ifndef HAVE__XEATDATAWORDS
+#include <X11/Xmd.h>  /* for LONG64 on 64-bit platforms */
+#include <limits.h>
+
+static inline void _XEatDataWords(Display *dpy, unsigned long n)
+{
+# ifndef LONG64
+    if (n >= (ULONG_MAX >> 2))
+        _XIOError(dpy);
+# endif
+    _XEatData (dpy, n << 2);
+}
+#endif
 
 /* deliberately opaque internal data structure; can be extended,
    but not reordered */
diff --git a/src/XrrCrtc.c b/src/XrrCrtc.c
index 04087c5..a704a52 100644
--- a/src/XrrCrtc.c
+++ b/src/XrrCrtc.c
@@ -74,7 +74,7 @@ XRRGetCrtcInfo (Display *dpy, XRRScreenResources *resources, RRCrtc crtc)
 
     xci = (XRRCrtcInfo *) Xmalloc(rbytes);
     if (xci == NULL) {
-	_XEatData (dpy, (unsigned long) nbytes);
+	_XEatDataWords (dpy, rep.length);
 	UnlockDisplay (dpy);
 	SyncHandle ();
 	return NULL;
@@ -203,7 +203,7 @@ XRRGetCrtcGamma (Display *dpy, RRCrtc crtc)
 
     if (!crtc_gamma)
     {
-	_XEatData (dpy, (unsigned long) nbytes);
+	_XEatDataWords (dpy, rep.length);
 	goto out;
     }
     _XRead16 (dpy, crtc_gamma->red, rep.size * 2);
@@ -397,7 +397,7 @@ XRRGetCrtcTransform (Display	*dpy,
 	    int extraBytes = rep.length * 4 - CrtcTransformExtra;
 	    extra = Xmalloc (extraBytes);
 	    if (!extra) {
-		_XEatData (dpy, extraBytes);
+		_XEatDataWords (dpy, rep.length - (CrtcTransformExtra >> 2));
 		UnlockDisplay (dpy);
 		SyncHandle ();
 		return False;
diff --git a/src/XrrOutput.c b/src/XrrOutput.c
index f13a932..4df894e 100644
--- a/src/XrrOutput.c
+++ b/src/XrrOutput.c
@@ -81,7 +81,7 @@ XRRGetOutputInfo (Display *dpy, XRRScreenResources *resources, RROutput output)
 
     xoi = (XRROutputInfo *) Xmalloc(rbytes);
     if (xoi == NULL) {
-	_XEatData (dpy, (unsigned long) nbytes);
+	_XEatDataWords (dpy, rep.length - (OutputInfoExtra >> 2));
 	UnlockDisplay (dpy);
 	SyncHandle ();
 	return NULL;
diff --git a/src/XrrProperty.c b/src/XrrProperty.c
index 4c3fdb0..2b065b2 100644
--- a/src/XrrProperty.c
+++ b/src/XrrProperty.c
@@ -62,7 +62,7 @@ XRRListOutputProperties (Display *dpy, RROutput output, int *nprop)
 
 	props = (Atom *) Xmalloc (rbytes);
 	if (props == NULL) {
-	    _XEatData (dpy, nbytes);
+	    _XEatDataWords (dpy, rep.length);
 	    UnlockDisplay (dpy);
 	    SyncHandle ();
 	    *nprop = 0;
@@ -107,7 +107,7 @@ XRRQueryOutputProperty (Display *dpy, RROutput output, Atom property)
 
     prop_info = (XRRPropertyInfo *) Xmalloc (rbytes);
     if (prop_info == NULL) {
-	_XEatData (dpy, nbytes);
+	_XEatDataWords(dpy, rep.length);
 	UnlockDisplay (dpy);
 	SyncHandle ();
 	return NULL;
@@ -313,14 +313,13 @@ XRRGetOutputProperty (Display *dpy, RROutput output,
 	     * This part of the code should never be reached.  If it is,
 	     * the server sent back a property with an invalid format.
 	     */
-	    nbytes = rep.length << 2;
-	    _XEatData(dpy, (unsigned long) nbytes);
+	    _XEatDataWords(dpy, rep.length);
 	    UnlockDisplay(dpy);
 	    SyncHandle();
 	    return(BadImplementation);
 	}
 	if (! *prop) {
-	    _XEatData(dpy, (unsigned long) nbytes);
+	    _XEatDataWords(dpy, rep.length);
 	    UnlockDisplay(dpy);
 	    SyncHandle();
 	    return(BadAlloc);
diff --git a/src/XrrProvider.c b/src/XrrProvider.c
index fcd06ff..309e321 100644
--- a/src/XrrProvider.c
+++ b/src/XrrProvider.c
@@ -67,7 +67,7 @@ XRRGetProviderResources(Display *dpy, Window window)
     xrpr = (XRRProviderResources *) Xmalloc(rbytes);
 
     if (xrpr == NULL) {
-       _XEatData (dpy, (unsigned long) nbytes);
+       _XEatDataWords (dpy, rep.length);
        UnlockDisplay (dpy);
        SyncHandle ();
        return NULL;
@@ -136,7 +136,7 @@ XRRGetProviderInfo(Display *dpy, XRRScreenResources *resources, RRProvider provi
 
     xpi = (XRRProviderInfo *)Xmalloc(rbytes);
     if (xpi == NULL) {
-	_XEatData (dpy, (unsigned long) nbytes);
+	_XEatDataWords (dpy, rep.length - (ProviderInfoExtra >> 2));
 	UnlockDisplay (dpy);
 	SyncHandle ();
 	return NULL;
diff --git a/src/XrrProviderProperty.c b/src/XrrProviderProperty.c
index c8c08e9..2d90a0a 100644
--- a/src/XrrProviderProperty.c
+++ b/src/XrrProviderProperty.c
@@ -62,7 +62,7 @@ XRRListProviderProperties (Display *dpy, RRProvider provider, int *nprop)
 
 	props = (Atom *) Xmalloc (rbytes);
 	if (props == NULL) {
-	    _XEatData (dpy, nbytes);
+	    _XEatDataWords (dpy, rep.length);
 	    UnlockDisplay (dpy);
 	    SyncHandle ();
 	    *nprop = 0;
@@ -107,7 +107,7 @@ XRRQueryProviderProperty (Display *dpy, RRProvider provider, Atom property)
 
     prop_info = (XRRPropertyInfo *) Xmalloc (rbytes);
     if (prop_info == NULL) {
-	_XEatData (dpy, nbytes);
+	_XEatDataWords (dpy, rep.length);
 	UnlockDisplay (dpy);
 	SyncHandle ();
 	return NULL;
@@ -313,14 +313,13 @@ XRRGetProviderProperty (Display *dpy, RRProvider provider,
 	     * This part of the code should never be reached.  If it is,
 	     * the server sent back a property with an invalid format.
 	     */
-	    nbytes = rep.length << 2;
-	    _XEatData(dpy, (unsigned long) nbytes);
+	    _XEatDataWords(dpy, rep.length);
 	    UnlockDisplay(dpy);
 	    SyncHandle();
 	    return(BadImplementation);
 	}
 	if (! *prop) {
-	    _XEatData(dpy, (unsigned long) nbytes);
+	    _XEatDataWords(dpy, rep.length);
 	    UnlockDisplay(dpy);
 	    SyncHandle();
 	    return(BadAlloc);
diff --git a/src/XrrScreen.c b/src/XrrScreen.c
index f830913..08710b6 100644
--- a/src/XrrScreen.c
+++ b/src/XrrScreen.c
@@ -129,7 +129,7 @@ doGetScreenResources (Display *dpy, Window window, int poll)
     if (xrsr == NULL || wire_names == NULL) {
 	if (xrsr) Xfree (xrsr);
 	if (wire_names) Xfree (wire_names);
-	_XEatData (dpy, (unsigned long) nbytes);
+	_XEatDataWords (dpy, rep.length);
 	UnlockDisplay (dpy);
 	SyncHandle ();
 	return NULL;