sna: Wrap inplace trapezoid operators with SIGBUS protection

For the moment, this still leaves open the vexing question of how to
protect the multi-threaded variants, but it should provide more shelter
for extreme OOM.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

4 files changed, 165 insertions, 126 deletions

diff --git a/src/sna/sna_trapezoids_boxes.c b/src/sna/sna_trapezoids_boxes.c
index b462de53..6d81da68 100644
--- a/src/sna/sna_trapezoids_boxes.c
+++ b/src/sna/sna_trapezoids_boxes.c
@@ -253,13 +253,16 @@ composite_aligned_boxes(struct sna *sna,
 				RegionIntersect(&region, &region, &clip);
 				b = REGION_RECTS(&region);
 				count = REGION_NUM_RECTS(&region);
-				for (i = 0; i < count; i++) {
-					fbComposite(op, src, NULL, dst,
-						    src_x + b[i].x1 - boxes[0].x1,
-						    src_y + b[i].y1 - boxes[0].y1,
-						    0, 0,
-						    b[i].x1, b[i].y1,
-						    b[i].x2 - b[i].x1, b[i].y2 - b[i].y1);
+				if (sigtrap_get() == 0) {
+					for (i = 0; i < count; i++) {
+						fbComposite(op, src, NULL, dst,
+							    src_x + b[i].x1 - boxes[0].x1,
+							    src_y + b[i].y1 - boxes[0].y1,
+							    0, 0,
+							    b[i].x1, b[i].y1,
+							    b[i].x2 - b[i].x1, b[i].y2 - b[i].y1);
+					}
+					sigtrap_put();
 				}
 				pixman_region_fini(&region);
 				pixman_region_fini(&region);
@@ -789,52 +792,55 @@ composite_unaligned_boxes_inplace__solid(struct sna *sna,
 			continue;
 		}
 
-		RegionTranslate(&clip, dx, dy);
-		count = REGION_NUM_RECTS(&clip);
-		extents = REGION_RECTS(&clip);
-		while (count--) {
-			int16_t y1 = dy + pixman_fixed_to_int(t->top);
-			uint16_t fy1 = pixman_fixed_frac(t->top);
-			int16_t y2 = dy + pixman_fixed_to_int(t->bottom);
-			uint16_t fy2 = pixman_fixed_frac(t->bottom);
-
-			DBG(("%s: t=(%d, %d), (%d, %d), extents (%d, %d), (%d, %d)\n",
-			     __FUNCTION__,
-			     pixman_fixed_to_int(t->left.p1.x),
-			     pixman_fixed_to_int(t->top),
-			     pixman_fixed_to_int(t->right.p2.x),
-			     pixman_fixed_to_int(t->bottom),
-			     extents->x1, extents->y1,
-			     extents->x2, extents->y2));
-
-			if (y1 < extents->y1)
-				y1 = extents->y1, fy1 = 0;
-			if (y2 >= extents->y2)
-				y2 = extents->y2, fy2 = 0;
-
-			if (y1 < y2) {
-				if (fy1) {
+		if (sigtrap_get() == 0) {
+			RegionTranslate(&clip, dx, dy);
+			count = REGION_NUM_RECTS(&clip);
+			extents = REGION_RECTS(&clip);
+			while (count--) {
+				int16_t y1 = dy + pixman_fixed_to_int(t->top);
+				uint16_t fy1 = pixman_fixed_frac(t->top);
+				int16_t y2 = dy + pixman_fixed_to_int(t->bottom);
+				uint16_t fy2 = pixman_fixed_frac(t->bottom);
+
+				DBG(("%s: t=(%d, %d), (%d, %d), extents (%d, %d), (%d, %d)\n",
+				     __FUNCTION__,
+				     pixman_fixed_to_int(t->left.p1.x),
+				     pixman_fixed_to_int(t->top),
+				     pixman_fixed_to_int(t->right.p2.x),
+				     pixman_fixed_to_int(t->bottom),
+				     extents->x1, extents->y1,
+				     extents->x2, extents->y2));
+
+				if (y1 < extents->y1)
+					y1 = extents->y1, fy1 = 0;
+				if (y2 >= extents->y2)
+					y2 = extents->y2, fy2 = 0;
+
+				if (y1 < y2) {
+					if (fy1) {
+						lerp32_unaligned_box_row(pixmap, color, extents,
+									 t, dx, y1, 1,
+									 SAMPLES_Y - grid_coverage(SAMPLES_Y, fy1));
+						y1++;
+					}
+
+					if (y2 > y1)
+						lerp32_unaligned_box_row(pixmap, color, extents,
+									 t, dx, y1, y2 - y1,
+									 SAMPLES_Y);
+
+					if (fy2)
+						lerp32_unaligned_box_row(pixmap, color,  extents,
+									 t, dx, y2, 1,
+									 grid_coverage(SAMPLES_Y, fy2));
+				} else if (y1 == y2 && fy2 > fy1) {
 					lerp32_unaligned_box_row(pixmap, color, extents,
 								 t, dx, y1, 1,
-								 SAMPLES_Y - grid_coverage(SAMPLES_Y, fy1));
-					y1++;
+								 grid_coverage(SAMPLES_Y, fy2) - grid_coverage(SAMPLES_Y, fy1));
 				}
-
-				if (y2 > y1)
-					lerp32_unaligned_box_row(pixmap, color, extents,
-								 t, dx, y1, y2 - y1,
-								 SAMPLES_Y);
-
-				if (fy2)
-					lerp32_unaligned_box_row(pixmap, color,  extents,
-								 t, dx, y2, 1,
-								 grid_coverage(SAMPLES_Y, fy2));
-			} else if (y1 == y2 && fy2 > fy1) {
-				lerp32_unaligned_box_row(pixmap, color, extents,
-							 t, dx, y1, 1,
-							 grid_coverage(SAMPLES_Y, fy2) - grid_coverage(SAMPLES_Y, fy1));
+				extents++;
 			}
-			extents++;
+			sigtrap_put();
 		}
 
 		RegionUninit(&clip);
@@ -877,37 +883,40 @@ pixman:
 		pi.color = color;
 		pi.op = op;
 
-		count = REGION_NUM_RECTS(&clip);
-		extents = REGION_RECTS(&clip);
-		while (count--) {
-			int16_t y1 = pixman_fixed_to_int(t->top);
-			uint16_t fy1 = pixman_fixed_frac(t->top);
-			int16_t y2 = pixman_fixed_to_int(t->bottom);
-			uint16_t fy2 = pixman_fixed_frac(t->bottom);
-
-			if (y1 < extents->y1)
-				y1 = extents->y1, fy1 = 0;
-			if (y2 >= extents->y2)
-				y2 = extents->y2, fy2 = 0;
-			if (y1 < y2) {
-				if (fy1) {
-					pixsolid_unaligned_box_row(&pi, extents, t, y1, 1,
-								   SAMPLES_Y - grid_coverage(SAMPLES_Y, fy1));
-					y1++;
-				}
+		if (sigtrap_get() == 0) {
+			count = REGION_NUM_RECTS(&clip);
+			extents = REGION_RECTS(&clip);
+			while (count--) {
+				int16_t y1 = pixman_fixed_to_int(t->top);
+				uint16_t fy1 = pixman_fixed_frac(t->top);
+				int16_t y2 = pixman_fixed_to_int(t->bottom);
+				uint16_t fy2 = pixman_fixed_frac(t->bottom);
 
-				if (y2 > y1)
-					pixsolid_unaligned_box_row(&pi, extents, t, y1, y2 - y1,
-								   SAMPLES_Y);
+				if (y1 < extents->y1)
+					y1 = extents->y1, fy1 = 0;
+				if (y2 >= extents->y2)
+					y2 = extents->y2, fy2 = 0;
+				if (y1 < y2) {
+					if (fy1) {
+						pixsolid_unaligned_box_row(&pi, extents, t, y1, 1,
+									   SAMPLES_Y - grid_coverage(SAMPLES_Y, fy1));
+						y1++;
+					}
 
-				if (fy2)
-					pixsolid_unaligned_box_row(&pi, extents, t, y2, 1,
-								   grid_coverage(SAMPLES_Y, fy2));
-			} else if (y1 == y2 && fy2 > fy1) {
-				pixsolid_unaligned_box_row(&pi, extents, t, y1, 1,
-							   grid_coverage(SAMPLES_Y, fy2) - grid_coverage(SAMPLES_Y, fy1));
+					if (y2 > y1)
+						pixsolid_unaligned_box_row(&pi, extents, t, y1, y2 - y1,
+									   SAMPLES_Y);
+
+					if (fy2)
+						pixsolid_unaligned_box_row(&pi, extents, t, y2, 1,
+									   grid_coverage(SAMPLES_Y, fy2));
+				} else if (y1 == y2 && fy2 > fy1) {
+					pixsolid_unaligned_box_row(&pi, extents, t, y1, 1,
+								   grid_coverage(SAMPLES_Y, fy2) - grid_coverage(SAMPLES_Y, fy1));
+				}
+				extents++;
 			}
-			extents++;
+			sigtrap_put();
 		}
 
 		RegionUninit(&clip);
@@ -1115,37 +1124,40 @@ composite_unaligned_boxes_inplace(struct sna *sna,
 			pi.bits = pixman_image_get_data(pi.mask);
 			pi.op = op;
 
-			count = REGION_NUM_RECTS(&clip);
-			extents = REGION_RECTS(&clip);
-			while (count--) {
-				int16_t y1 = pixman_fixed_to_int(t->top);
-				uint16_t fy1 = pixman_fixed_frac(t->top);
-				int16_t y2 = pixman_fixed_to_int(t->bottom);
-				uint16_t fy2 = pixman_fixed_frac(t->bottom);
-
-				if (y1 < extents->y1)
-					y1 = extents->y1, fy1 = 0;
-				if (y2 > extents->y2)
-					y2 = extents->y2, fy2 = 0;
-				if (y1 < y2) {
-					if (fy1) {
+			if (sigtrap_get() == 0) {
+				count = REGION_NUM_RECTS(&clip);
+				extents = REGION_RECTS(&clip);
+				while (count--) {
+					int16_t y1 = pixman_fixed_to_int(t->top);
+					uint16_t fy1 = pixman_fixed_frac(t->top);
+					int16_t y2 = pixman_fixed_to_int(t->bottom);
+					uint16_t fy2 = pixman_fixed_frac(t->bottom);
+
+					if (y1 < extents->y1)
+						y1 = extents->y1, fy1 = 0;
+					if (y2 > extents->y2)
+						y2 = extents->y2, fy2 = 0;
+					if (y1 < y2) {
+						if (fy1) {
+							pixmask_unaligned_box_row(&pi, extents, t, y1, 1,
+										  SAMPLES_Y - grid_coverage(SAMPLES_Y, fy1));
+							y1++;
+						}
+
+						if (y2 > y1)
+							pixmask_unaligned_box_row(&pi, extents, t, y1, y2 - y1,
+										  SAMPLES_Y);
+
+						if (fy2)
+							pixmask_unaligned_box_row(&pi, extents, t, y2, 1,
+										  grid_coverage(SAMPLES_Y, fy2));
+					} else if (y1 == y2 && fy2 > fy1) {
 						pixmask_unaligned_box_row(&pi, extents, t, y1, 1,
-									  SAMPLES_Y - grid_coverage(SAMPLES_Y, fy1));
-						y1++;
+									  grid_coverage(SAMPLES_Y, fy2) - grid_coverage(SAMPLES_Y, fy1));
 					}
-
-					if (y2 > y1)
-						pixmask_unaligned_box_row(&pi, extents, t, y1, y2 - y1,
-									  SAMPLES_Y);
-
-					if (fy2)
-						pixmask_unaligned_box_row(&pi, extents, t, y2, 1,
-									  grid_coverage(SAMPLES_Y, fy2));
-				} else if (y1 == y2 && fy2 > fy1) {
-					pixmask_unaligned_box_row(&pi, extents, t, y1, 1,
-								  grid_coverage(SAMPLES_Y, fy2) - grid_coverage(SAMPLES_Y, fy1));
+					extents++;
 				}
-				extents++;
+				sigtrap_put();
 			}
 
 			pixman_image_unref(pi.image);
diff --git a/src/sna/sna_trapezoids_imprecise.c b/src/sna/sna_trapezoids_imprecise.c
index a354f280..ddf52c2c 100644
--- a/src/sna/sna_trapezoids_imprecise.c
+++ b/src/sna/sna_trapezoids_imprecise.c
@@ -2759,8 +2759,11 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
 			DBG(("%s: render inplace op=%d, color=%08x\n",
 			     __FUNCTION__, op, color));
 
-			tor_render(NULL, &tor, (void*)&inplace,
-				   dst->pCompositeClip, span, false);
+			if (sigtrap_get() == 0) {
+				tor_render(NULL, &tor, (void*)&inplace,
+					   dst->pCompositeClip, span, false);
+				sigtrap_put();
+			}
 		} else if (is_solid) {
 			struct pixman_inplace pi;
 
@@ -2778,9 +2781,12 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
 			else
 				span = pixmask_span_solid;
 
-			tor_render(NULL, &tor, (void*)&pi,
-				   dst->pCompositeClip, span,
-				   false);
+			if (sigtrap_get() == 0) {
+				tor_render(NULL, &tor, (void*)&pi,
+					   dst->pCompositeClip, span,
+					   false);
+				sigtrap_put();
+			}
 
 			pixman_image_unref(pi.source);
 			pixman_image_unref(pi.image);
@@ -2804,9 +2810,12 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
 			else
 				span = pixmask_span;
 
-			tor_render(NULL, &tor, (void*)&pi,
-				   dst->pCompositeClip, span,
-				   false);
+			if (sigtrap_get() == 0) {
+				tor_render(NULL, &tor, (void*)&pi,
+					   dst->pCompositeClip, span,
+					   false);
+				sigtrap_put();
+			}
 
 			pixman_image_unref(pi.mask);
 			pixman_image_unref(pi.source);
@@ -3083,8 +3092,11 @@ imprecise_trapezoid_span_inplace(struct sna *sna,
 			tor_add_edge(&tor, &t, &t.right, -1);
 		}
 
-		tor_render(NULL, &tor, (void*)&inplace,
-			   dst->pCompositeClip, span, unbounded);
+		if (sigtrap_get() == 0) {
+			tor_render(NULL, &tor, (void*)&inplace,
+				   dst->pCompositeClip, span, unbounded);
+			sigtrap_put();
+		}
 
 		tor_fini(&tor);
 	} else {
diff --git a/src/sna/sna_trapezoids_mono.c b/src/sna/sna_trapezoids_mono.c
index 49296a36..ca316d7e 100644
--- a/src/sna/sna_trapezoids_mono.c
+++ b/src/sna/sna_trapezoids_mono.c
@@ -1168,7 +1168,10 @@ unbounded_pass:
 		mono.span = mono_span__fast;
 	else
 		mono.span = mono_span;
-	mono_render(&mono);
+	if (sigtrap_get() == 0) {
+		mono_render(&mono);
+		sigtrap_put();
+	}
 	mono_fini(&mono);
 
 	if (op) {
diff --git a/src/sna/sna_trapezoids_precise.c b/src/sna/sna_trapezoids_precise.c
index de95dfdc..e5bab16b 100644
--- a/src/sna/sna_trapezoids_precise.c
+++ b/src/sna/sna_trapezoids_precise.c
@@ -2757,8 +2757,11 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
 			DBG(("%s: render inplace op=%d, color=%08x\n",
 			     __FUNCTION__, op, color));
 
-			tor_render(NULL, &tor, (void*)&inplace,
-				   dst->pCompositeClip, span, false);
+			if (sigtrap_get() == 0) {
+				tor_render(NULL, &tor, (void*)&inplace,
+					   dst->pCompositeClip, span, false);
+				sigtrap_put();
+			}
 		} else if (is_solid) {
 			struct pixman_inplace pi;
 
@@ -2776,9 +2779,12 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
 			else
 				span = pixmask_span_solid;
 
-			tor_render(NULL, &tor, (void*)&pi,
-				   dst->pCompositeClip, span,
-				   false);
+			if (sigtrap_get() == 0) {
+				tor_render(NULL, &tor, (void*)&pi,
+					   dst->pCompositeClip, span,
+					   false);
+				sigtrap_put();
+			}
 
 			pixman_image_unref(pi.source);
 			pixman_image_unref(pi.image);
@@ -2802,9 +2808,12 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
 			else
 				span = pixmask_span;
 
-			tor_render(NULL, &tor, (void*)&pi,
-				   dst->pCompositeClip, span,
-				   false);
+			if (sigtrap_get() == 0) {
+				tor_render(NULL, &tor, (void*)&pi,
+					   dst->pCompositeClip, span,
+					   false);
+				sigtrap_put();
+			}
 
 			pixman_image_unref(pi.mask);
 			pixman_image_unref(pi.source);
@@ -3082,8 +3091,11 @@ precise_trapezoid_span_inplace(struct sna *sna,
 			tor_add_edge(&tor, &t, &t.right, -1);
 		}
 
-		tor_render(NULL, &tor, (void*)&inplace,
-			   dst->pCompositeClip, span, unbounded);
+		if (sigtrap_get() == 0) {
+			tor_render(NULL, &tor, (void*)&inplace,
+				   dst->pCompositeClip, span, unbounded);
+			sigtrap_put();
+		}
 
 		tor_fini(&tor);
 	} else {