sna: flatten source alphamaps

Replace the source picture+alpha with a bo that contains the RGB
channels from source and A from the alpha map.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

10 files changed, 120 insertions, 8 deletions

diff --git a/src/sna/gen2_render.c b/src/sna/gen2_render.c
index 53accbd1..519ba189 100644
--- a/src/sna/gen2_render.c
+++ b/src/sna/gen2_render.c
@@ -1155,6 +1155,12 @@ gen2_composite_picture(struct sna *sna,
 						x, y, w, h, dst_x, dst_y);
 	}
 
+	if (picture->alphaMap) {
+		DBG(("%s -- fallback, alphamap\n", __FUNCTION__));
+		return sna_render_picture_fixup(sna, picture, channel,
+						x, y, w, h, dst_x, dst_y);
+	}
+
 	if (!gen2_check_repeat(picture)) {
 		DBG(("%s -- fallback, unhandled repeat %d\n",
 		     __FUNCTION__, picture->repeat));
diff --git a/src/sna/gen3_render.c b/src/sna/gen3_render.c
index 1dec2b3a..12eebaab 100644
--- a/src/sna/gen3_render.c
+++ b/src/sna/gen3_render.c
@@ -2099,6 +2099,12 @@ gen3_composite_picture(struct sna *sna,
 		return ret;
 	}
 
+	if (picture->alphaMap) {
+		DBG(("%s -- fallback, alphamap\n", __FUNCTION__));
+		return sna_render_picture_fixup(sna, picture, channel,
+						x, y, w, h, dst_x, dst_y);
+	}
+
 	if (sna_picture_is_solid(picture, &color))
 		return gen3_init_solid(channel, color);
 
diff --git a/src/sna/gen4_render.c b/src/sna/gen4_render.c
index 150b7c83..6ccddd20 100644
--- a/src/sna/gen4_render.c
+++ b/src/sna/gen4_render.c
@@ -1745,6 +1745,12 @@ gen4_composite_picture(struct sna *sna,
 						x, y, w, h, dst_x, dst_y);
 	}
 
+	if (picture->alphaMap) {
+		DBG(("%s -- fallback, alphamap\n", __FUNCTION__));
+		return sna_render_picture_fixup(sna, picture, channel,
+						x, y, w, h, dst_x, dst_y);
+	}
+
 	if (!gen4_check_repeat(picture)) {
 		DBG(("%s: unknown repeat mode fixup\n", __FUNCTION__));
 		return sna_render_picture_fixup(sna, picture, channel,
diff --git a/src/sna/gen5_render.c b/src/sna/gen5_render.c
index e1117cc6..180b561a 100644
--- a/src/sna/gen5_render.c
+++ b/src/sna/gen5_render.c
@@ -1785,6 +1785,12 @@ gen5_composite_picture(struct sna *sna,
 		return sna_render_picture_fixup(sna, picture, channel,
 						x, y, w, h, dst_x, dst_y);
 
+	if (picture->alphaMap) {
+		DBG(("%s -- fallback, alphamap\n", __FUNCTION__));
+		return sna_render_picture_fixup(sna, picture, channel,
+						x, y, w, h, dst_x, dst_y);
+	}
+
 	if (!gen5_check_repeat(picture))
 		return sna_render_picture_fixup(sna, picture, channel,
 						x, y, w, h, dst_x, dst_y);
diff --git a/src/sna/gen6_render.c b/src/sna/gen6_render.c
index 7d481374..8e7ea156 100644
--- a/src/sna/gen6_render.c
+++ b/src/sna/gen6_render.c
@@ -1962,6 +1962,12 @@ gen6_composite_picture(struct sna *sna,
 		return sna_render_picture_fixup(sna, picture, channel,
 						x, y, w, h, dst_x, dst_y);
 
+	if (picture->alphaMap) {
+		DBG(("%s -- fallback, alphamap\n", __FUNCTION__));
+		return sna_render_picture_fixup(sna, picture, channel,
+						x, y, w, h, dst_x, dst_y);
+	}
+
 	if (!gen6_check_repeat(picture))
 		return sna_render_picture_fixup(sna, picture, channel,
 						x, y, w, h, dst_x, dst_y);
diff --git a/src/sna/gen7_render.c b/src/sna/gen7_render.c
index 885a5dcd..58647f74 100644
--- a/src/sna/gen7_render.c
+++ b/src/sna/gen7_render.c
@@ -2075,6 +2075,12 @@ gen7_composite_picture(struct sna *sna,
 		return sna_render_picture_fixup(sna, picture, channel,
 						x, y, w, h, dst_x, dst_y);
 
+	if (picture->alphaMap) {
+		DBG(("%s -- fallback, alphamap\n", __FUNCTION__));
+		return sna_render_picture_fixup(sna, picture, channel,
+						x, y, w, h, dst_x, dst_y);
+	}
+
 	if (!gen7_check_repeat(picture))
 		return sna_render_picture_fixup(sna, picture, channel,
 						x, y, w, h, dst_x, dst_y);
diff --git a/src/sna/sna_composite.c b/src/sna/sna_composite.c
index fcc0070a..7adbbfb5 100644
--- a/src/sna/sna_composite.c
+++ b/src/sna/sna_composite.c
@@ -437,7 +437,7 @@ sna_composite(CARD8 op,
 		goto fallback;
 	}
 
-	if (dst->alphaMap || src->alphaMap || (mask && mask->alphaMap)) {
+	if (dst->alphaMap) {
 		DBG(("%s: fallback due to unhandled alpha-map\n", __FUNCTION__));
 		goto fallback;
 	}
diff --git a/src/sna/sna_glyphs.c b/src/sna/sna_glyphs.c
index 4fd47529..811b8663 100644
--- a/src/sna/sna_glyphs.c
+++ b/src/sna/sna_glyphs.c
@@ -1225,8 +1225,8 @@ sna_glyphs(CARD8 op,
 		goto fallback;
 	}
 
-	if (dst->alphaMap || src->alphaMap) {
-		DBG(("%s: fallback -- alpha maps\n", __FUNCTION__));
+	if (dst->alphaMap) {
+		DBG(("%s: fallback -- dst alpha map\n", __FUNCTION__));
 		goto fallback;
 	}
 
diff --git a/src/sna/sna_render.c b/src/sna/sna_render.c
index 72b3f78f..89d1d812 100644
--- a/src/sna/sna_render.c
+++ b/src/sna/sna_render.c
@@ -1014,6 +1014,73 @@ sna_render_picture_convolve(struct sna *sna,
 	return 1;
 }
 
+static int
+sna_render_picture_flatten(struct sna *sna,
+			   PicturePtr picture,
+			   struct sna_composite_channel *channel,
+			   int16_t x, int16_t y,
+			   int16_t w, int16_t h,
+			   int16_t dst_x, int16_t dst_y)
+{
+	ScreenPtr screen = picture->pDrawable->pScreen;
+	PixmapPtr pixmap;
+	PicturePtr tmp, alpha;
+	int old_format, error;
+
+	assert(picture->pDrawable);
+	assert(picture->alphaMap);
+	assert(w <= sna->render.max_3d_size && h <= sna->render.max_3d_size);
+
+	/* XXX shortcut a8? */
+
+	pixmap = screen->CreatePixmap(screen, w, h, 32, SNA_CREATE_SCRATCH);
+	if (pixmap == NullPixmap)
+		return 0;
+
+	tmp = CreatePicture(0, &pixmap->drawable,
+			    PictureMatchFormat(screen, 32, PICT_a8r8g8b8),
+			    0, NULL, serverClient, &error);
+	screen->DestroyPixmap(pixmap);
+	if (tmp == NULL)
+		return 0;
+
+	old_format = picture->format;
+	picture->format = PICT_FORMAT(PICT_FORMAT_BPP(picture->format),
+				      PICT_FORMAT_TYPE(picture->format),
+				      0,
+				      PICT_FORMAT_R(picture->format),
+				      PICT_FORMAT_G(picture->format),
+				      PICT_FORMAT_B(picture->format));
+
+	alpha = picture->alphaMap;
+	picture->alphaMap = NULL;
+
+	sna_composite(PictOpSrc, picture, alpha, tmp,
+		      x, y,
+		      x + picture->alphaOrigin.x, y + picture->alphaOrigin.y,
+		      0, 0,
+		      w, h);
+
+	picture->format = old_format;
+	picture->alphaMap = alpha;
+
+	channel->height = h;
+	channel->width  = w;
+	channel->filter = PictFilterNearest;
+	channel->repeat = RepeatNone;
+	channel->pict_format = PIXMAN_a8r8g8b8;
+	channel->is_affine = TRUE;
+	channel->transform = NULL;
+	channel->scale[0] = 1.f / w;
+	channel->scale[1] = 1.f / h;
+	channel->offset[0] = -dst_x;
+	channel->offset[1] = -dst_y;
+	channel->bo = kgem_bo_reference(sna_pixmap_get_bo(pixmap));
+	FreePicture(tmp, 0);
+
+	return 1;
+}
+
 int
 sna_render_picture_fixup(struct sna *sna,
 			 PicturePtr picture,
@@ -1042,6 +1109,16 @@ sna_render_picture_fixup(struct sna *sna,
 		return -1;
 	}
 
+	if (picture->alphaMap) {
+		DBG(("%s: alphamap\n", __FUNCTION__));
+		if ((is_gpu(picture->pDrawable) || is_gpu(picture->alphaMap->pDrawable))) {
+			return sna_render_picture_flatten(sna, picture, channel,
+							  x, y, w, y, dst_x, dst_y);
+		}
+
+		goto do_fixup;
+	}
+
 	if (picture->filter == PictFilterConvolution) {
 		DBG(("%s: convolution\n", __FUNCTION__));
 		if (picture->pDrawable && is_gpu(picture->pDrawable)) {
diff --git a/src/sna/sna_trapezoids.c b/src/sna/sna_trapezoids.c
index 287f8833..3837532a 100644
--- a/src/sna/sna_trapezoids.c
+++ b/src/sna/sna_trapezoids.c
@@ -3094,8 +3094,8 @@ trapezoid_span_inplace(CARD8 op, PicturePtr src, PicturePtr dst,
 		     __FUNCTION__));
 		return false;
 	}
-	if (dst->alphaMap || src->alphaMap) {
-		DBG(("%s: fallback -- alphamaps\n",
+	if (dst->alphaMap) {
+		DBG(("%s: fallback -- dst alphamap\n",
 		     __FUNCTION__));
 		return false;
 	}
@@ -3374,9 +3374,8 @@ sna_composite_trapezoids(CARD8 op,
 		goto fallback;
 	}
 
-	if (dst->alphaMap || src->alphaMap) {
-		DBG(("%s: fallback -- alpha maps=(dst=%p, src=%p)\n",
-		     __FUNCTION__, dst->alphaMap, src->alphaMap));
+	if (dst->alphaMap) {
+		DBG(("%s: fallback -- dst alpha map\n", __FUNCTION__));
 		goto fallback;
 	}