Make the dedicated scanout mechanism work with arbitrary transforms

This makes TearFree work with arbitrary transforms, and makes transforms
work better even without TearFree, with xserver >= 1.12.

(Ported from amdgpu commit bf000ea7ef91f5ecb59fc3c1ab8ed9eddcc0841d)

2 files changed, 17 insertions, 35 deletions

diff --git a/src/drmmode_display.c b/src/drmmode_display.c
index 849dc3c4..1b1b3e64 100644
--- a/src/drmmode_display.c
+++ b/src/drmmode_display.c
@@ -687,15 +687,15 @@ drmmode_handle_transform(xf86CrtcPtr crtc)
 	Bool ret;
 
 #if XF86_CRTC_VERSION >= 7
-	if (!crtc->transformPresent && crtc->rotation != RR_Rotate_0)
+	if (crtc->transformPresent || crtc->rotation != RR_Rotate_0)
 	    crtc->driverIsPerformingTransform = XF86DriverTransformOutput;
 	else
 	    crtc->driverIsPerformingTransform = XF86DriverTransformNone;
 #else
 	RADEONInfoPtr info = RADEONPTR(crtc->scrn);
 
-	crtc->driverIsPerformingTransform = info->tear_free &&
-		!crtc->transformPresent && crtc->rotation != RR_Rotate_0;
+	crtc->driverIsPerformingTransform = crtc->transformPresent ||
+		(info->tear_free && crtc->rotation != RR_Rotate_0);
 #endif
 
 	ret = xf86CrtcRotate(crtc);
@@ -815,19 +815,10 @@ drmmode_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
 				RegionUninit(pRegion);
 				pRegion->data = NULL;
 				pBox = RegionExtents(pRegion);
-				pBox->x1 = min(pBox->x1, x);
-				pBox->y1 = min(pBox->y1, y);
-
-				switch (crtc->rotation & 0xf) {
-				case RR_Rotate_90:
-				case RR_Rotate_270:
-					pBox->x2 = max(pBox->x2, x + mode->VDisplay);
-					pBox->y2 = max(pBox->y2, y + mode->HDisplay);
-					break;
-				default:
-					pBox->x2 = max(pBox->x2, x + mode->HDisplay);
-					pBox->y2 = max(pBox->y2, y + mode->VDisplay);
-				}
+				pBox->x1 = 0;
+				pBox->y1 = 0;
+				pBox->x2 = max(pBox->x2, pScrn->virtualX);
+				pBox->y2 = max(pBox->y2, pScrn->virtualY);
 
 				drmmode_crtc->scanout_id = 0;
 				fb_id = drmmode_crtc->scanout[0].fb_id;
diff --git a/src/radeon_kms.c b/src/radeon_kms.c
index b64c6367..9d968117 100644
--- a/src/radeon_kms.c
+++ b/src/radeon_kms.c
@@ -386,19 +386,16 @@ static Bool
 radeon_scanout_extents_intersect(xf86CrtcPtr xf86_crtc, BoxPtr extents, int w,
 				 int h)
 {
-    extents->x1 = max(extents->x1 - xf86_crtc->x, 0);
-    extents->y1 = max(extents->y1 - xf86_crtc->y, 0);
-
-    switch (xf86_crtc->rotation & 0xf) {
-    case RR_Rotate_90:
-    case RR_Rotate_270:
-	extents->x2 = min(extents->x2 - xf86_crtc->x, h);
-	extents->y2 = min(extents->y2 - xf86_crtc->y, w);
-	break;
-    default:
-	extents->x2 = min(extents->x2 - xf86_crtc->x, w);
-	extents->y2 = min(extents->y2 - xf86_crtc->y, h);
-    }
+    extents->x1 -= xf86_crtc->filter_width >> 1;
+    extents->x2 += xf86_crtc->filter_width >> 1;
+    extents->y1 -= xf86_crtc->filter_height >> 1;
+    extents->y2 += xf86_crtc->filter_height >> 1;
+    pixman_f_transform_bounds(&xf86_crtc->f_framebuffer_to_crtc, extents);
+
+    extents->x1 = max(extents->x1, 0);
+    extents->y1 = max(extents->y1, 0);
+    extents->x2 = min(extents->x2, w);
+    extents->y2 = min(extents->y2, h);
 
     return (extents->x1 < extents->x2 && extents->y1 < extents->y2);
 }
@@ -928,12 +925,6 @@ radeon_scanout_do_update(xf86CrtcPtr xf86_crtc, int scanout_id)
 	    SetPicturePictFilter(src, xf86_crtc->filter, xf86_crtc->params,
 				 xf86_crtc->nparams);
 
-	extents.x1 += xf86_crtc->x - (xf86_crtc->filter_width >> 1);
-	extents.x2 += xf86_crtc->x + (xf86_crtc->filter_width >> 1);
-	extents.y1 += xf86_crtc->y - (xf86_crtc->filter_height >> 1);
-	extents.y2 += xf86_crtc->y + (xf86_crtc->filter_height >> 1);
-	pixman_f_transform_bounds(&xf86_crtc->f_framebuffer_to_crtc, &extents);
-
 	pScreen->SourceValidate = NULL;
 	CompositePicture(PictOpSrc,
 			 src, NULL, dst,