From 7e142cb2a848acb6af986fa91d254d4c23963b24 Mon Sep 17 00:00:00 2001 From: luporl Date: Wed, 9 Dec 2020 11:11:15 -0300 Subject: [PATCH] xserver: fix RGB mask handling On FreeBSD 13.0-CURRENT for PowerPC64 big-endian (BE), X was crashing in some cases. For instance, when twm was started and the background was clicked to open its menu, X crashed with a segmentation fault, trying to dereference a null pointer at CreatePicture(). There were 2 issues with xorg-server handling of RGB masks that caused the pointer above to be null and thus the crash: - wrong use of ffs() to get the RGB offsets from the masks - overflow when shifting a 16-bit integer This change fixes both issues. They happen when the system is BE but has a video adapter using a little-endian (LE) ARGB32 framebuffer. In order to display the correct colors, this setup requires a BE RGBA32 color format to be used by X, by setting the RGB masks appropriately, that didn't work properly because of the issues above. --- hw/xfree86/common/xf86Helper.c | 6 +++--- render/picture.c | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/hw/xfree86/common/xf86Helper.c b/hw/xfree86/common/xf86Helper.c index 31aa068e0..16ab309e2 100644 --- a/hw/xfree86/common/xf86Helper.c +++ b/hw/xfree86/common/xf86Helper.c @@ -729,9 +729,9 @@ xf86SetWeight(ScrnInfoPtr scrp, rgb weight, rgb mask) scrp->mask.red = mask.red; scrp->mask.green = mask.green; scrp->mask.blue = mask.blue; - scrp->offset.red = ffs(mask.red); - scrp->offset.green = ffs(mask.green); - scrp->offset.blue = ffs(mask.blue); + scrp->offset.red = ffs(mask.red) - 1; + scrp->offset.green = ffs(mask.green) - 1; + scrp->offset.blue = ffs(mask.blue) - 1; } return TRUE; } diff --git a/render/picture.c b/render/picture.c index 876316dc1..afa0d258f 100644 --- a/render/picture.c +++ b/render/picture.c @@ -523,12 +523,12 @@ PictureMatchVisual(ScreenPtr pScreen, int depth, VisualPtr pVisual) return format; } else { - if (format->direct.redMask << format->direct.red == - pVisual->redMask && - format->direct.greenMask << format->direct.green == - pVisual->greenMask && - format->direct.blueMask << format->direct.blue == - pVisual->blueMask) { + if ((unsigned long)format->direct.redMask << + format->direct.red == pVisual->redMask && + (unsigned long)format->direct.greenMask << + format->direct.green == pVisual->greenMask && + (unsigned long)format->direct.blueMask << + format->direct.blue == pVisual->blueMask) { return format; } }