integer overflow in _xvmc_create_*()

rep.length is a CARD32 and should be bounds checked before left-shifting
by 2 bits to come up with the total size to allocate, though in these
cases, no buffer overflow should occur here, since the XRead call is passed
the same rep.length << 2 length argument, but the *priv_count returned to
the caller could be interpreted or used to calculate a larger buffer size
than was actually allocated, leading them to go out of bounds.

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>

1 files changed, 6 insertions, 3 deletions

diff --git a/src/XvMC.c b/src/XvMC.c
index 8d602ec..d8bc59d 100644
--- a/src/XvMC.c
+++ b/src/XvMC.c
@@ -285,7 +285,8 @@ Status _xvmc_create_context (
     context->flags = rep.flags_return;
 
     if(rep.length) {
-	*priv_data = Xmalloc(rep.length << 2);
+	if (rep.length < (INT_MAX >> 2))
+	    *priv_data = Xmalloc(rep.length << 2);
 	if(*priv_data) {
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
 	    *priv_count = rep.length;
@@ -366,7 +367,8 @@ Status _xvmc_create_surface (
     }
 
     if(rep.length) {
-        *priv_data = Xmalloc(rep.length << 2);
+        if (rep.length < (INT_MAX >> 2))
+            *priv_data = Xmalloc(rep.length << 2);
         if(*priv_data) {
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
             *priv_count = rep.length;
@@ -456,7 +458,8 @@ Status _xvmc_create_subpicture (
     subpicture->component_order[3] = rep.component_order[3];
 
     if(rep.length) {
-        *priv_data = Xmalloc(rep.length << 2);
+        if (rep.length < (INT_MAX >> 2))
+            *priv_data = Xmalloc(rep.length << 2);
         if(*priv_data) {
             _XRead(dpy, (char*)(*priv_data), rep.length << 2);
             *priv_count = rep.length;