Bug #594: CAN-2005-2495: Fix exploitable integer overflow in pixmap

creation, where we could create a far smaller pixmap than we thought,
    allowing changes to arbitrary chunks of memory. (Søren Sandmann
    Pedersen)

afb/afbpixmap.c

@@ -77,10 +77,14 @@ afbCreatePixmap(pScreen, width, height, depth)
 	int				depth;
 {
 	PixmapPtr pPixmap;
-	int datasize;
-	int paddedWidth;
+	size_t datasize;
+	size_t paddedWidth;
 
 	paddedWidth = BitmapBytePad(width);
+
+	if (paddedWidth > 32767 || height > 32767 || depth > 4)
+	    return NullPixmap;
+	
 	datasize = height * paddedWidth * depth;
 	pPixmap = AllocatePixmap(pScreen, datasize);
 	if (!pPixmap)

cfb/cfbpixmap.c

@@ -72,10 +72,13 @@ cfbCreatePixmap (pScreen, width, height, depth)
     int		depth;
 {
     PixmapPtr pPixmap;
-    int datasize;
-    int paddedWidth;
+    size_t datasize;
+    size_t paddedWidth;
 
     paddedWidth = PixmapBytePad(width, depth);
+
+    if (paddedWidth / 4 > 32767 || height > 32767)
+	return NullPixmap;
     datasize = height * paddedWidth;
     pPixmap = AllocatePixmap(pScreen, datasize);
     if (!pPixmap)

dix/dispatch.c

@@ -1483,6 +1483,23 @@ ProcCreatePixmap(register ClientPtr client)
 	client->errorValue = 0;
         return BadValue;
     }
+    if (stuff->width > 32767 || stuff->height > 32767)
+    {
+	/* It is allowed to try and allocate a pixmap which is larger than
+	 * 32767 in either dimension. However, all of the framebuffer code
+	 * is buggy and does not reliably draw to such big pixmaps, basically
+	 * because the Region data structure operates with signed shorts
+	 * for the rectangles in it.
+	 *
+	 * Furthermore, several places in the X server computes the
+	 * size in bytes of the pixmap and tries to store it in an
+	 * integer. This integer can overflow and cause the allocated size
+	 * to be much smaller.
+	 *
+	 * So, such big pixmaps are rejected here with a BadAlloc
+	 */
+	return BadAlloc;
+    }
     if (stuff->depth != 1)
     {
         pDepth = pDraw->pScreen->allowedDepths;

dix/pixmap.c

@@ -118,6 +118,9 @@ AllocatePixmap(ScreenPtr pScreen, int pixDataSize)
     unsigned size;
     int i;
 
+    if (pScreen->totalPixmapSize > ((size_t)-1) - pixDataSize)
+	return NullPixmap;
+    
     pPixmap = (PixmapPtr)xalloc(pScreen->totalPixmapSize + pixDataSize);
     if (!pPixmap)
 	return NullPixmap;

exa/exa.c

@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {

exa/exa_accel.c

@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {

exa/exa_migration.c

@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {

fb/fbpixmap.c

@@ -36,12 +36,14 @@ PixmapPtr
 fbCreatePixmapBpp (ScreenPtr pScreen, int width, int height, int depth, int bpp)
 {
     PixmapPtr	pPixmap;
-    int		datasize;
-    int		paddedWidth;
+    size_t	datasize;
+    size_t	paddedWidth;
     int		adjust;
     int		base;
 
     paddedWidth = ((width * bpp + FB_MASK) >> FB_SHIFT) * sizeof (FbBits);
+    if (paddedWidth / 4 > 32767 || height > 32767)
+	return NullPixmap;
     datasize = height * paddedWidth;
 #ifdef PIXPRIV
     base = pScreen->totalPixmapSize;

hw/xfree86/exa/exa.c

@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {

hw/xfree86/exa/exa_accel.c

@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {

hw/xfree86/exa/exa_migration.c

@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {

hw/xfree86/xaa/xaaInit.c

@@ -502,6 +502,9 @@ XAACreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     XAAPixmapPtr pPriv;
     PixmapPtr pPix = NULL;
     int size = w * h;
+
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
     
     if (!infoRec->offscreenDepthsInitialized)
 	XAAInitializeOffscreenDepths (pScreen);

hw/xfree86/xf4bpp/ppcPixmap.c

@@ -89,7 +89,7 @@ xf4bppCreatePixmap( pScreen, width, height, depth )
     int		depth ;
 {
     register PixmapPtr pPixmap  = (PixmapPtr)NULL;
-    int size ;
+    size_t size ;
     
     TRACE(("xf4bppCreatePixmap(pScreen=0x%x, width=%d, height=%d, depth=%d)\n", pScreen, width, height, depth)) ;
 
@@ -97,6 +97,10 @@ xf4bppCreatePixmap( pScreen, width, height, depth )
 	return (PixmapPtr) NULL ;
 
     size = PixmapBytePad(width, depth);
+
+    if (size / 4 > 32767 || height > 32767)
+	return (PixmapPtr) NULL ;
+    
     pPixmap = AllocatePixmap (pScreen, (height * size));
     
     if ( !pPixmap )

ilbm/ilbmpixmap.c

@@ -79,10 +79,12 @@ ilbmCreatePixmap(pScreen, width, height, depth)
 	int				depth;
 {
 	PixmapPtr pPixmap;
-	int datasize;
-	int paddedWidth;
+	size_t datasize;
+	size_t paddedWidth;
 
 	paddedWidth = BitmapBytePad(width);
+	if (paddedWidth > 32767 || height > 32767 || depth > 4)
+		return NullPixmap;
 	datasize = height * paddedWidth * depth;
 	pPixmap = AllocatePixmap(pScreen, datasize);
 	if (!pPixmap)

iplan2p4/iplpixmap.c

@@ -78,12 +78,14 @@ iplCreatePixmap (pScreen, width, height, depth)
     int		depth;
 {
     PixmapPtr pPixmap;
-    int datasize;
-    int paddedWidth;
+    size_t datasize;
+    size_t paddedWidth;
     int ipad=INTER_PLANES*2 - 1;
 
     paddedWidth = PixmapBytePad(width, depth);
     paddedWidth = (paddedWidth + ipad) & ~ipad;
+    if (paddedWidth / 4 > 32767 || height > 32767)
+	return NullPixmap;
     datasize = height * paddedWidth;
     pPixmap = AllocatePixmap(pScreen, datasize);
     if (!pPixmap)

mfb/mfbpixmap.c

@@ -75,12 +75,14 @@ mfbCreatePixmap (pScreen, width, height, depth)
     int		depth;
 {
     PixmapPtr pPixmap;
-    int datasize;
-    int paddedWidth;
+    size_t datasize;
+    size_t paddedWidth;
 
     if (depth != 1)
 	return NullPixmap;
     paddedWidth = BitmapBytePad(width);
+    if (paddedWidth / 4 > 32767 || height > 32767)
+	return NullPixmap;
     datasize = height * paddedWidth;
     pPixmap = AllocatePixmap(pScreen, datasize);
     if (!pPixmap)