modesetting: Add support for using RandR shadow buffers

This replaces the stubs for shadow buffer creation/allocation with actual
functions and adds a shadow_destroy function.  With this, we actually get
shadow buffers and RandR now works properly.  Most of this is copied from
the xf86-video-intel driver and modified for modesetting.

v2 Jason Ekstrand <jason.ekstrand@intel.com>:
 - Fix build with --disable-glamor
 - Set the pixel data pointer in the pixmap header for dumb shadow bo's
 - Call drmmode_create_bo with the right bpp

v2 Jason Ekstrand <jason.ekstrand@intel.com>:
 - Make shadow buffers per-crtc and leave shadow_enable alone

Signed-off-by: Jason Ekstrand <jason.ekstrand@intel.com>
Reviewed-by: Keith Packard <keithp@keithp.com>
Signed-off-by: Keith Packard <keithp@keithp.com>

hw/xfree86/drivers/modesetting/drmmode_display.c

@@ -540,13 +540,122 @@ drmmode_set_scanout_pixmap(xf86CrtcPtr crtc, PixmapPtr ppix)
 static void *
 drmmode_shadow_allocate(xf86CrtcPtr crtc, int width, int height)
 {
-    return NULL;
+    drmmode_crtc_private_ptr drmmode_crtc = crtc->driver_private;
+    drmmode_ptr drmmode = drmmode_crtc->drmmode;
+    int ret;
+
+    if (!drmmode_create_bo(drmmode, &drmmode_crtc->rotate_bo,
+                           width, height, crtc->scrn->bitsPerPixel)) {
+        xf86DrvMsg(crtc->scrn->scrnIndex, X_ERROR,
+               "Couldn't allocate shadow memory for rotated CRTC\n");
+        return NULL;
+    }
+
+    ret = drmModeAddFB(drmmode->fd, width, height, crtc->scrn->depth,
+                       crtc->scrn->bitsPerPixel,
+                       drmmode_bo_get_pitch(&drmmode_crtc->rotate_bo),
+                       drmmode_bo_get_handle(&drmmode_crtc->rotate_bo),
+                       &drmmode_crtc->rotate_fb_id);
+
+    if (ret) {
+        ErrorF("failed to add rotate fb\n");
+        drmmode_bo_destroy(drmmode, &drmmode_crtc->rotate_bo);
+        return NULL;
+    }
+
+#ifdef GLAMOR_HAS_GBM
+    if (drmmode->gbm)
+        return drmmode_crtc->rotate_bo.gbm;
+#endif
+    return drmmode_crtc->rotate_bo.dumb;
 }
 
+static PixmapPtr
+drmmode_create_pixmap_header(ScreenPtr pScreen, int width, int height,
+                             int depth, int bitsPerPixel, int devKind,
+                             void *pPixData)
+{
+    PixmapPtr pixmap;
+
+    /* width and height of 0 means don't allocate any pixmap data */
+    pixmap = (*pScreen->CreatePixmap)(pScreen, 0, 0, depth, 0);
+
+    if (pixmap) {
+        if ((*pScreen->ModifyPixmapHeader)(pixmap, width, height, depth,
+                                           bitsPerPixel, devKind, pPixData))
+            return pixmap;
+        (*pScreen->DestroyPixmap)(pixmap);
+    }
+    return NullPixmap;
+}
+
+static Bool
+drmmode_set_pixmap_bo(drmmode_ptr drmmode, PixmapPtr pixmap, drmmode_bo *bo);
+
 static PixmapPtr
 drmmode_shadow_create(xf86CrtcPtr crtc, void *data, int width, int height)
 {
-    return NULL;
+    ScrnInfoPtr scrn = crtc->scrn;
+    drmmode_crtc_private_ptr drmmode_crtc = crtc->driver_private;
+    drmmode_ptr drmmode = drmmode_crtc->drmmode;
+    uint32_t rotate_pitch;
+    PixmapPtr rotate_pixmap;
+    void *pPixData = NULL;
+
+    if (!data) {
+        data = drmmode_shadow_allocate(crtc, width, height);
+        if (!data) {
+            xf86DrvMsg(scrn->scrnIndex, X_ERROR,
+                       "Couldn't allocate shadow pixmap for rotated CRTC\n");
+            return NULL;
+        }
+    }
+
+    if (!drmmode_bo_has_bo(&drmmode_crtc->rotate_bo)) {
+        xf86DrvMsg(scrn->scrnIndex, X_ERROR,
+                   "Couldn't allocate shadow pixmap for rotated CRTC\n");
+        return NULL;
+    }
+
+    pPixData = drmmode_bo_map(drmmode, &drmmode_crtc->rotate_bo);
+    rotate_pitch = drmmode_bo_get_pitch(&drmmode_crtc->rotate_bo),
+
+    rotate_pixmap = drmmode_create_pixmap_header(scrn->pScreen,
+                                                 width, height,
+                                                 scrn->depth,
+                                                 scrn->bitsPerPixel,
+                                                 rotate_pitch,
+                                                 pPixData);
+
+    if (rotate_pixmap == NULL) {
+        xf86DrvMsg(scrn->scrnIndex, X_ERROR,
+                   "Couldn't allocate shadow pixmap for rotated CRTC\n");
+        return NULL;
+    }
+
+    drmmode_set_pixmap_bo(drmmode, rotate_pixmap, &drmmode_crtc->rotate_bo);
+
+    return rotate_pixmap;
+}
+
+static void
+drmmode_shadow_destroy(xf86CrtcPtr crtc, PixmapPtr rotate_pixmap, void *data)
+{
+    drmmode_crtc_private_ptr drmmode_crtc = crtc->driver_private;
+    drmmode_ptr drmmode = drmmode_crtc->drmmode;
+
+    if (rotate_pixmap) {
+        drmmode_set_pixmap_bo(drmmode, rotate_pixmap, NULL);
+        rotate_pixmap->drawable.pScreen->DestroyPixmap(rotate_pixmap);
+    }
+
+    if (data) {
+        drmModeRmFB(drmmode->fd, drmmode_crtc->rotate_fb_id);
+        drmmode_crtc->rotate_fb_id = 0;
+
+        drmmode_bo_destroy(drmmode, &drmmode_crtc->rotate_bo);
+        memset(&drmmode_crtc->rotate_bo, 0, sizeof drmmode_crtc->rotate_bo);
+    }
 }
 
 static const xf86CrtcFuncsRec drmmode_crtc_funcs = {
@@ -563,6 +672,7 @@ static const xf86CrtcFuncsRec drmmode_crtc_funcs = {
     .set_scanout_pixmap = drmmode_set_scanout_pixmap,
     .shadow_allocate = drmmode_shadow_allocate,
     .shadow_create = drmmode_shadow_create,
+    .shadow_destroy = drmmode_shadow_destroy,
 };
 
 static uint32_t

hw/xfree86/drivers/modesetting/drmmode_display.h

@@ -89,10 +89,12 @@ typedef struct {
     int dpms_mode;
     struct dumb_bo *cursor_bo;
     Bool cursor_up;
-    unsigned rotate_fb_id;
     uint16_t lut_r[256], lut_g[256], lut_b[256];
     DamagePtr slave_damage;
 
+    drmmode_bo rotate_bo;
+    unsigned rotate_fb_id;
+
     /**
      * @{ MSC (vblank count) handling for the PRESENT extension.
      *