fbdevhw: Consolidate modeset ioctl calling, report failure if it modifies mode.

The fbdev API allows the driver to 'accept' modes it doesn't really support by
modifying it to the nearest supported mode. Without this check, e.g. vesafb
would appear to accept all modes, even though it actually can't set any modes
other than the bootup mode at all.

hw/xfree86/fbdevhw/fbdevhw.c

@@ -227,6 +227,26 @@ xfree2fbdev_timing(DisplayModePtr mode, struct fb_var_screeninfo *var)
 		var->vmode = FB_VMODE_NONINTERLACED;
 }
 
+static Bool
+fbdev_modes_equal(struct fb_var_screeninfo *one, struct fb_var_screeninfo *two)
+{
+	return (one->xres_virtual == two->xres_virtual &&
+		one->yres_virtual == two->yres_virtual &&
+		one->bits_per_pixel == two->bits_per_pixel &&
+		one->red.length == two->red.length &&
+		one->green.length == two->green.length &&
+		one->blue.length == two->blue.length &&
+		one->xres == two->xres && one->yres == two->yres &&
+		one->pixclock == two->pixclock &&
+		one->right_margin == two->right_margin &&
+		one->hsync_len == two->hsync_len &&
+		one->left_margin == two->left_margin &&
+		one->lower_margin == two->lower_margin &&
+		one->vsync_len == two->vsync_len &&
+		one->upper_margin == two->upper_margin &&
+		one->sync == two->sync && one->vmode == two->vmode);
+}
+
 static void
 fbdev2xfree_timing(struct fb_var_screeninfo *var, DisplayModePtr mode)
 {
@@ -470,13 +490,52 @@ fbdevHWGetVidmem(ScrnInfoPtr pScrn)
 	return fPtr->fix.smem_len;
 }
 
+static Bool
+fbdevHWSetMode(ScrnInfoPtr pScrn, DisplayModePtr mode, Bool check)
+{
+	fbdevHWPtr fPtr = FBDEVHWPTR(pScrn);
+	struct fb_var_screeninfo req_var = fPtr->var, set_var;
+	
+	TRACE_ENTER("ModeInit");
+
+	xfree2fbdev_fblayout(pScrn, &req_var);
+	xfree2fbdev_timing(mode, &req_var);
+
+#if DEBUG
+	print_xfree_mode("init", mode);
+	print_fbdev_mode("init", &req_var);
+#endif
+
+	set_var = req_var;
+
+	if (check)
+		set_var.activate = FB_ACTIVATE_TEST;
+
+	if (0 != ioctl(fPtr->fd, FBIOPUT_VSCREENINFO, (void*)(&set_var))) {
+		xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
+			   "FBIOPUT_VSCREENINFO: %s\n", strerror(errno));
+		return FALSE;
+	}
+
+	if (!fbdev_modes_equal(&set_var, &req_var)) {
+		if (!check)
+			xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
+				   "FBIOPUT_VSCREENINFO succeeded but modified "
+				   "mode\n");
+#if DEBUG
+		print_fbdev_mode("returned", &set_var);
+#endif
+		return FALSE;
+	}
+
+	fPtr->var = set_var;
+
+	return TRUE;
+}
+
 void
 fbdevHWSetVideoModes(ScrnInfoPtr pScrn)
 {
-	fbdevHWPtr fPtr = FBDEVHWPTR(pScrn);
-	int virtX = pScrn->display->virtualX;
-	int virtY = pScrn->display->virtualY;
-	struct fb_var_screeninfo var;
 	char **modename;
 	DisplayModePtr mode,this,last = pScrn->modes;
 
@@ -484,6 +543,9 @@ fbdevHWSetVideoModes(ScrnInfoPtr pScrn)
 	if (NULL == pScrn->display->modes)
 		return;
 
+	pScrn->virtualX = pScrn->display->virtualX;
+	pScrn->virtualY = pScrn->display->virtualY;
+
 	for (modename = pScrn->display->modes; *modename != NULL; modename++) {
 		for (mode = pScrn->monitor->Modes; mode != NULL; mode = mode->next)
 			if (0 == strcmp(mode->name,*modename))
@@ -493,27 +555,20 @@ fbdevHWSetVideoModes(ScrnInfoPtr pScrn)
 				   "\tmode \"%s\" not found\n", *modename);
 			continue;
 		}
-		memset(&var,0,sizeof(var));
-		xfree2fbdev_timing(mode,&var);
-		var.xres_virtual = virtX;
-		var.yres_virtual = virtY;
-		var.bits_per_pixel = pScrn->bitsPerPixel;
-		var.red.length = pScrn->weight.red;
-		var.green.length = pScrn->weight.green;
-		var.blue.length = pScrn->weight.blue;
 
-		var.activate = FB_ACTIVATE_TEST;
+		if (!fbdevHWSetMode(pScrn, mode, TRUE)) {
-		if (var.xres_virtual < var.xres) var.xres_virtual = var.xres;
-		if (var.yres_virtual < var.yres) var.yres_virtual = var.yres;
-		if (-1 == ioctl(fPtr->fd,FBIOPUT_VSCREENINFO,(void*)(&var))) {
 			xf86DrvMsg(pScrn->scrnIndex, X_INFO,
 				   "\tmode \"%s\" test failed\n", *modename);
 			continue;
 		}
 		xf86DrvMsg(pScrn->scrnIndex, X_INFO,
 			   "\tmode \"%s\" ok\n", *modename);
-		if (virtX < var.xres) virtX = var.xres;
+
-		if (virtY < var.yres) virtY = var.yres;
+		if (pScrn->virtualX < mode->HDisplay)
+			pScrn->virtualX = mode->HDisplay;
+		if (pScrn->virtualY < mode->VDisplay)
+			pScrn->virtualY = mode->VDisplay;
+
 		if (NULL == pScrn->modes) {
 			pScrn->modes = xnfalloc(sizeof(DisplayModeRec));
 			this = pScrn->modes;
@@ -530,8 +585,6 @@ fbdevHWSetVideoModes(ScrnInfoPtr pScrn)
 		}
 		last = this;
 	}
-	pScrn->virtualX     = virtX;
-	pScrn->virtualY     = virtY;
 }
 
 DisplayModePtr
@@ -673,21 +726,12 @@ fbdevHWModeInit(ScrnInfoPtr pScrn, DisplayModePtr mode)
 {	
 	fbdevHWPtr fPtr = FBDEVHWPTR(pScrn);
 	
-	TRACE_ENTER("ModeInit");
-	xfree2fbdev_fblayout(pScrn, &fPtr->var);
-	xfree2fbdev_timing(mode,  &fPtr->var);
-#if DEBUG
-	print_xfree_mode("init",mode);
-	print_fbdev_mode("init",&fPtr->var);
-#endif
 	pScrn->vtSema = TRUE;
 
 	/* set */
-	if (0 != ioctl(fPtr->fd,FBIOPUT_VSCREENINFO,(void*)(&fPtr->var))) {
+	if (!fbdevHWSetMode(pScrn, mode, FALSE))
-		xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
-			   "FBIOPUT_VSCREENINFO: %s\n", strerror(errno));
 		return FALSE;
-	}
+
 	/* read back */
 	if (0 != ioctl(fPtr->fd,FBIOGET_FSCREENINFO,(void*)(&fPtr->fix))) {
 		xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
@@ -767,18 +811,12 @@ ModeStatus
 fbdevHWValidMode(int scrnIndex, DisplayModePtr mode, Bool verbose, int flags)
 {
 	ScrnInfoPtr pScrn = xf86Screens[scrnIndex];
-	fbdevHWPtr fPtr = FBDEVHWPTR(pScrn);
-	struct fb_var_screeninfo var;
 
 	TRACE_ENTER("ValidMode");
-	memcpy(&var,&fPtr->var,sizeof(var));
+
-	xfree2fbdev_timing(mode, &var);
+	if (!fbdevHWSetMode(pScrn, mode, TRUE))
-	var.activate = FB_ACTIVATE_TEST;
-	if (0 != ioctl(fPtr->fd,FBIOPUT_VSCREENINFO,(void*)(&fPtr->var))) {
-		xf86DrvMsg(scrnIndex, X_ERROR,
-			   "FBIOPUT_VSCREENINFO: %s\n", strerror(errno));
 		return MODE_BAD;
-	}
+
 	return MODE_OK;
 }
 
@@ -786,15 +824,12 @@ Bool
 fbdevHWSwitchMode(int scrnIndex, DisplayModePtr mode, int flags)
 {
 	ScrnInfoPtr pScrn = xf86Screens[scrnIndex];
-	fbdevHWPtr fPtr = FBDEVHWPTR(pScrn);
 
 	TRACE_ENTER("SwitchMode");
-	xfree2fbdev_timing(mode, &fPtr->var);
+
-	if (0 != ioctl(fPtr->fd,FBIOPUT_VSCREENINFO,(void*)(&fPtr->var))) {
+	if (!fbdevHWSetMode(pScrn, mode, FALSE))
-		xf86DrvMsg(scrnIndex, X_ERROR,
-			   "FBIOPUT_VSCREENINFO: %s\n", strerror(errno));
 		return FALSE;
-	}
+
 	return TRUE;
 }