modesetting: Defer crtc gamma size upgrade to drmmode_setup_colormap

Rather than trying to create a gamma ramp array of the appropriate size in
drmmode_crtc_init when the GAMMA_LUT property should be used, just flag the crtc
as wanting to use the GAMMA_LUT property and then replace the gamma ramp later,
right before calling xf86HandleColormaps. This avoids a problem during initial
startup where xf86RandR12CreateObjects12 hard-codes a gamma ramp size of 256,
causing xf86RandR12CrtcSetGamma to read past the end of the DIX layer's RandR
gamma ramp array:

  PreInit
    drmmode_pre_init
      drmmode_crtc_init
        crtc->gamma_size = 1024

  ScreenInit
    xf86CrtcScreenInit
      xf86RandR12Init
        xf86RandR12Init12
          xf86RandR12CreateObjects12
            RRCrtcCreate
              randr_crtc->gammaSize = 0
          xf86RandR12InitGamma(pScrn, 256)
            RRCrtcGammaSetSize
              randr_crtc->gammaSize = 256
          xf86RandR12InitGamma
            xf86RandR12CrtcInitGamma
              RRCrtcGammaSet
                xf86RandR12CrtcSetGamma
                  // crtc->gamma_size is 1024 here, while randr_crtc->gammaRed
                  // is a 256-element array.
                  memcpy(crtc->gamma_red, randr_crtc->gammaRed, crtc->gamma_size * sizeof(crtc->gamma_red[0]));
    drmmode_setup_colormap
      xf86HandleColormaps
        xf86RandR12InitGamma
          RRCrtcGammaSetSize
            randr_crtc->gammaSize = 1024

Fixes: 245b9db0 - modesetting: Use GAMMA_LUT when available
Closes: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1126
Signed-off-by: Aaron Plattner <aplattner@nvidia.com>
Reviewed-by: Robert Morell <rmorell@nvidia.com>
This commit is contained in:
Aaron Plattner 2021-02-19 11:47:13 -08:00
parent ab8f8c9841
commit b75d0cca28

View File

@ -2404,38 +2404,10 @@ drmmode_crtc_init(ScrnInfoPtr pScrn, drmmode_ptr drmmode, drmModeResPtr mode_res
xf86DrvMsgVerb(pScrn->scrnIndex, X_INFO, MS_LOGLEVEL_DEBUG,
"Allocated crtc nr. %d to this screen.\n", num);
/* If the GAMMA_LUT property is available, replace the server's default
* gamma ramps with ones of the appropriate size. */
if (drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].prop_id) {
Bool try_gamma_lut =
xf86ReturnOptValBool(drmmode->Options, OPTION_USE_GAMMA_LUT, TRUE);
uint64_t size = drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].value;
if (try_gamma_lut && size != crtc->gamma_size) {
uint16_t *gamma = malloc(3 * size * sizeof(uint16_t));
if (gamma) {
free(crtc->gamma_red);
crtc->gamma_size = size;
crtc->gamma_red = gamma;
crtc->gamma_green = gamma + size;
crtc->gamma_blue = gamma + size * 2;
drmmode_crtc->use_gamma_lut = TRUE;
xf86DrvMsgVerb(pScrn->scrnIndex, X_INFO, MS_LOGLEVEL_DEBUG,
"Gamma ramp set to %ld entries on CRTC %d\n",
size, num);
} else {
xf86DrvMsg(pScrn->scrnIndex, X_WARNING,
"Failed to allocate memory for %ld gamma ramp "
"entries on CRTC %d. Falling back to legacy "
"%d-entry mode.\n",
size, num, crtc->gamma_size);
}
}
}
drmmode_crtc->use_gamma_lut =
drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].prop_id &&
drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].value &&
xf86ReturnOptValBool(drmmode->Options, OPTION_USE_GAMMA_LUT, TRUE);
if (drmmode_crtc->use_gamma_lut &&
drmmode_crtc->props[DRMMODE_CRTC_CTM].prop_id) {
@ -3906,15 +3878,64 @@ drmmode_load_palette(ScrnInfoPtr pScrn, int numColors,
}
}
static Bool
drmmode_crtc_upgrade_lut(xf86CrtcPtr crtc, int num)
{
drmmode_crtc_private_ptr drmmode_crtc = crtc->driver_private;
uint64_t size;
if (!drmmode_crtc->use_gamma_lut)
return TRUE;
assert(drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].prop_id);
size = drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].value;
if (size != crtc->gamma_size) {
ScrnInfoPtr pScrn = crtc->scrn;
uint16_t *gamma = malloc(3 * size * sizeof(uint16_t));
if (gamma) {
free(crtc->gamma_red);
crtc->gamma_size = size;
crtc->gamma_red = gamma;
crtc->gamma_green = gamma + size;
crtc->gamma_blue = gamma + size * 2;
xf86DrvMsgVerb(pScrn->scrnIndex, X_INFO, MS_LOGLEVEL_DEBUG,
"Gamma ramp set to %ld entries on CRTC %d\n",
size, num);
} else {
xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
"Failed to allocate memory for %ld gamma ramp entries "
"on CRTC %d.\n",
size, num);
return FALSE;
}
}
return TRUE;
}
Bool
drmmode_setup_colormap(ScreenPtr pScreen, ScrnInfoPtr pScrn)
{
xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(pScrn);
int i;
xf86DrvMsg(pScrn->scrnIndex, X_INFO,
"Initializing kms color map for depth %d, %d bpc.\n",
pScrn->depth, pScrn->rgbBits);
if (!miCreateDefColormap(pScreen))
return FALSE;
/* If the GAMMA_LUT property is available, replace the server's default
* gamma ramps with ones of the appropriate size. */
for (i = 0; i < xf86_config->num_crtc; i++)
if (!drmmode_crtc_upgrade_lut(xf86_config->crtc[i], i))
return FALSE;
/* Adapt color map size and depth to color depth of screen. */
if (!xf86HandleColormaps(pScreen, 1 << pScrn->rgbBits, 10,
drmmode_load_palette, NULL,