xserver: fix RGB mask handling
On FreeBSD 13.0-CURRENT for PowerPC64 big-endian (BE), X was crashing in some cases. For instance, when twm was started and the background was clicked to open its menu, X crashed with a segmentation fault, trying to dereference a null pointer at CreatePicture(). There were 2 issues with xorg-server handling of RGB masks that caused the pointer above to be null and thus the crash: - wrong use of ffs() to get the RGB offsets from the masks - overflow when shifting a 16-bit integer This change fixes both issues. They happen when the system is BE but has a video adapter using a little-endian (LE) ARGB32 framebuffer. In order to display the correct colors, this setup requires a BE RGBA32 color format to be used by X, by setting the RGB masks appropriately, that didn't work properly because of the issues above.
This commit is contained in:
parent
5be3b80b8d
commit
7e142cb2a8
|
@ -729,9 +729,9 @@ xf86SetWeight(ScrnInfoPtr scrp, rgb weight, rgb mask)
|
|||
scrp->mask.red = mask.red;
|
||||
scrp->mask.green = mask.green;
|
||||
scrp->mask.blue = mask.blue;
|
||||
scrp->offset.red = ffs(mask.red);
|
||||
scrp->offset.green = ffs(mask.green);
|
||||
scrp->offset.blue = ffs(mask.blue);
|
||||
scrp->offset.red = ffs(mask.red) - 1;
|
||||
scrp->offset.green = ffs(mask.green) - 1;
|
||||
scrp->offset.blue = ffs(mask.blue) - 1;
|
||||
}
|
||||
return TRUE;
|
||||
}
|
||||
|
|
|
@ -523,12 +523,12 @@ PictureMatchVisual(ScreenPtr pScreen, int depth, VisualPtr pVisual)
|
|||
return format;
|
||||
}
|
||||
else {
|
||||
if (format->direct.redMask << format->direct.red ==
|
||||
pVisual->redMask &&
|
||||
format->direct.greenMask << format->direct.green ==
|
||||
pVisual->greenMask &&
|
||||
format->direct.blueMask << format->direct.blue ==
|
||||
pVisual->blueMask) {
|
||||
if ((unsigned long)format->direct.redMask <<
|
||||
format->direct.red == pVisual->redMask &&
|
||||
(unsigned long)format->direct.greenMask <<
|
||||
format->direct.green == pVisual->greenMask &&
|
||||
(unsigned long)format->direct.blueMask <<
|
||||
format->direct.blue == pVisual->blueMask) {
|
||||
return format;
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue