Avoid CVE-2023-43786: stack exhaustion in XPutImage()

This doesn't fix the CVE - that has to happen in libX11, this
just tries to avoid triggering it from libXpm, and saves time
in not pretending we can successfully create an X11 pixmap with
dimensions larger than the unsigned 16-bit integers used in the
X11 protocol for the dimensions.

Reported by Yair Mizrahi of the JFrog Vulnerability Research team

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>

6 files changed, 102 insertions, 25 deletions

diff --git a/src/CrPFrBuf.c b/src/CrPFrBuf.c
index 2c28a41..e9e2243 100644
--- a/src/CrPFrBuf.c
+++ b/src/CrPFrBuf.c
@@ -46,7 +46,7 @@ XpmCreatePixmapFromBuffer(
     Pixmap		*shapemask_return,
     XpmAttributes	*attributes)
 {
-    XImage *ximage, *shapeimage;
+    XImage *ximage = NULL, *shapeimage = NULL;
     int ErrorStatus;
 
     /* initialize return values */
@@ -63,16 +63,34 @@ XpmCreatePixmapFromBuffer(
 					   attributes);
 
     if (ErrorStatus < 0)		/* fatal error */
-	return (ErrorStatus);
+	goto cleanup;
 
     /* create the pixmaps and destroy images */
     if (pixmap_return && ximage) {
-	xpmCreatePixmapFromImage(display, d, ximage, pixmap_return);
-	XDestroyImage(ximage);
+	ErrorStatus =
+	    xpmCreatePixmapFromImage(display, d, ximage, pixmap_return);
+	if (ErrorStatus < 0)		/* fatal error */
+	    goto cleanup;
     }
     if (shapemask_return && shapeimage) {
-	xpmCreatePixmapFromImage(display, d, shapeimage, shapemask_return);
+	ErrorStatus =
+	    xpmCreatePixmapFromImage(display, d, shapeimage, shapemask_return);
+    }
+
+  cleanup:
+    if (ximage != NULL)
+	XDestroyImage(ximage);
+    if (shapeimage != NULL)
 	XDestroyImage(shapeimage);
+    if (ErrorStatus < 0) {
+	if (pixmap_return && *pixmap_return) {
+	    XFreePixmap(display, *pixmap_return);
+	    *pixmap_return = 0;
+	}
+	if (shapemask_return && *shapemask_return) {
+	    XFreePixmap(display, *shapemask_return);
+	    *shapemask_return = 0;
+	}
     }
     return (ErrorStatus);
 }
diff --git a/src/CrPFrDat.c b/src/CrPFrDat.c
index b65771d..8622663 100644
--- a/src/CrPFrDat.c
+++ b/src/CrPFrDat.c
@@ -46,7 +46,7 @@ XpmCreatePixmapFromData(
     Pixmap		 *shapemask_return,
     XpmAttributes	 *attributes)
 {
-    XImage *ximage, *shapeimage;
+    XImage *ximage = NULL, *shapeimage = NULL;
     int ErrorStatus;
 
     /* initialize return values */
@@ -63,19 +63,34 @@ XpmCreatePixmapFromData(
 					 attributes);
 
     if (ErrorStatus != XpmSuccess)
-	return (ErrorStatus);
-
-    if (ErrorStatus < 0)		/* fatal error */
-	return (ErrorStatus);
+	goto cleanup;
 
     /* create the pixmaps and destroy images */
     if (pixmap_return && ximage) {
-	xpmCreatePixmapFromImage(display, d, ximage, pixmap_return);
-	XDestroyImage(ximage);
+	ErrorStatus =
+	    xpmCreatePixmapFromImage(display, d, ximage, pixmap_return);
+	if (ErrorStatus < 0)		/* fatal error */
+	    goto cleanup;
     }
     if (shapemask_return && shapeimage) {
-	xpmCreatePixmapFromImage(display, d, shapeimage, shapemask_return);
+	ErrorStatus =
+	    xpmCreatePixmapFromImage(display, d, shapeimage, shapemask_return);
+    }
+
+  cleanup:
+    if (ximage != NULL)
+	XDestroyImage(ximage);
+    if (shapeimage != NULL)
 	XDestroyImage(shapeimage);
+    if (ErrorStatus < 0) {
+	if (pixmap_return && *pixmap_return) {
+	    XFreePixmap(display, *pixmap_return);
+	    *pixmap_return = 0;
+	}
+	if (shapemask_return && *shapemask_return) {
+	    XFreePixmap(display, *shapemask_return);
+	    *shapemask_return = 0;
+	}
     }
     return (ErrorStatus);
 }
diff --git a/src/CrPFrI.c b/src/CrPFrI.c
index 8f6f4aa..f6688c5 100644
--- a/src/CrPFrI.c
+++ b/src/CrPFrI.c
@@ -36,8 +36,9 @@
 #include <config.h>
 #endif
 #include "XpmI.h"
+#include <stdint.h>
 
-void
+int
 xpmCreatePixmapFromImage(
     Display	*display,
     Drawable	 d,
@@ -47,6 +48,11 @@ xpmCreatePixmapFromImage(
     GC gc;
     XGCValues values;
 
+    /* X Pixmaps are limited to unsigned 16-bit height/width */
+    if ((ximage->width > UINT16_MAX) || (ximage->height > UINT16_MAX)) {
+	return XpmNoMemory;
+    }
+
     *pixmap_return = XCreatePixmap(display, d, ximage->width,
 				   ximage->height, ximage->depth);
     /* set fg and bg in case we have an XYBitmap */
@@ -59,4 +65,6 @@ xpmCreatePixmapFromImage(
 	      ximage->width, ximage->height);
 
     XFreeGC(display, gc);
+
+    return XpmSuccess;
 }
diff --git a/src/RdFToP.c b/src/RdFToP.c
index f829757..2c3e7f9 100644
--- a/src/RdFToP.c
+++ b/src/RdFToP.c
@@ -46,7 +46,7 @@ XpmReadFileToPixmap(
     Pixmap		*shapemask_return,
     XpmAttributes	*attributes)
 {
-    XImage *ximage, *shapeimage;
+    XImage *ximage = NULL, *shapeimage = NULL;
     int ErrorStatus;
 
     /* initialize return values */
@@ -62,16 +62,34 @@ XpmReadFileToPixmap(
 				     attributes);
 
     if (ErrorStatus < 0)		/* fatal error */
-	return (ErrorStatus);
+	goto cleanup;
 
     /* create the pixmaps and destroy images */
     if (pixmap_return && ximage) {
-	xpmCreatePixmapFromImage(display, d, ximage, pixmap_return);
-	XDestroyImage(ximage);
+	ErrorStatus =
+	    xpmCreatePixmapFromImage(display, d, ximage, pixmap_return);
+	if (ErrorStatus < 0)		/* fatal error */
+	    goto cleanup;
     }
     if (shapemask_return && shapeimage) {
-	xpmCreatePixmapFromImage(display, d, shapeimage, shapemask_return);
+	ErrorStatus =
+	    xpmCreatePixmapFromImage(display, d, shapeimage, shapemask_return);
+    }
+
+  cleanup:
+    if (ximage != NULL)
+	XDestroyImage(ximage);
+    if (shapeimage != NULL)
 	XDestroyImage(shapeimage);
+    if (ErrorStatus < 0) {
+	if (pixmap_return && *pixmap_return) {
+	    XFreePixmap(display, *pixmap_return);
+	    *pixmap_return = 0;
+	}
+	if (shapemask_return && *shapemask_return) {
+	    XFreePixmap(display, *shapemask_return);
+	    *shapemask_return = 0;
+	}
     }
     return (ErrorStatus);
 }
diff --git a/src/XpmI.h b/src/XpmI.h
index ab7a680..6691693 100644
--- a/src/XpmI.h
+++ b/src/XpmI.h
@@ -195,7 +195,7 @@ HFUNC(xpmSetAttributes, void, (XpmAttributes *attributes, XpmImage *image,
 			      XpmInfo *info));
 
 #if !defined(FOR_MSW) && !defined(AMIGA)
-HFUNC(xpmCreatePixmapFromImage, void, (Display *display, Drawable d,
+HFUNC(xpmCreatePixmapFromImage, int, (Display *display, Drawable d,
 				      XImage *ximage, Pixmap *pixmap_return));
 
 HFUNC(xpmCreateImageFromPixmap, void, (Display *display, Pixmap pixmap,
diff --git a/src/create.c b/src/create.c
index 4921c7d..ec562b2 100644
--- a/src/create.c
+++ b/src/create.c
@@ -1652,7 +1652,7 @@ XpmCreatePixmapFromXpmImage(
     Pixmap		*shapemask_return,
     XpmAttributes	*attributes)
 {
-    XImage *ximage, *shapeimage;
+    XImage *ximage = NULL, *shapeimage = NULL;
     int ErrorStatus;
 
     /* initialize return values */
@@ -1668,16 +1668,34 @@ XpmCreatePixmapFromXpmImage(
 					      &shapeimage : NULL),
 					     attributes);
     if (ErrorStatus < 0)
-	return (ErrorStatus);
+	goto cleanup;
 
     /* create the pixmaps and destroy images */
     if (pixmap_return && ximage) {
-	xpmCreatePixmapFromImage(display, d, ximage, pixmap_return);
-	XDestroyImage(ximage);
+	ErrorStatus =
+	    xpmCreatePixmapFromImage(display, d, ximage, pixmap_return);
+	if (ErrorStatus < 0)		/* fatal error */
+	    goto cleanup;
     }
     if (shapemask_return && shapeimage) {
-	xpmCreatePixmapFromImage(display, d, shapeimage, shapemask_return);
+	ErrorStatus =
+	    xpmCreatePixmapFromImage(display, d, shapeimage, shapemask_return);
+    }
+
+  cleanup:
+    if (ximage != NULL)
+	XDestroyImage(ximage);
+    if (shapeimage != NULL)
 	XDestroyImage(shapeimage);
+    if (ErrorStatus < 0) {
+	if (pixmap_return && *pixmap_return) {
+	    XFreePixmap(display, *pixmap_return);
+	    *pixmap_return = 0;
+	}
+	if (shapemask_return && *shapemask_return) {
+	    XFreePixmap(display, *shapemask_return);
+	    *shapemask_return = 0;
+	}
     }
     return (ErrorStatus);
 }