sna: Rearrange damage allocation to handle malloc failure more gracefully

If we fail to allocate new damage boxes, first try collescing the
existing boxes to free up memory.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

1 files changed, 201 insertions, 151 deletions

diff --git a/src/sna/sna_damage.c b/src/sna/sna_damage.c
index 51b15d34..47e88835 100644
--- a/src/sna/sna_damage.c
+++ b/src/sna/sna_damage.c
@@ -191,6 +191,143 @@ struct sna_damage *sna_damage_create(void)
 	return _sna_damage_create();
 }
 
+static void free_list(struct list *head)
+{
+	while (!list_is_empty(head)) {
+		struct list *l = head->next;
+		list_del(l);
+		free(l);
+	}
+}
+
+static void __sna_damage_reduce(struct sna_damage *damage)
+{
+	int n, nboxes;
+	BoxPtr boxes, free_boxes = NULL;
+	pixman_region16_t *region = &damage->region;
+	struct sna_damage_box *iter;
+
+	assert(damage->mode != DAMAGE_ALL);
+	assert(damage->dirty);
+
+	DBG(("    reduce: before region.n=%ld\n", (long)REGION_NUM_RECTS(region)));
+
+	nboxes = damage->embedded_box.size;
+	list_for_each_entry(iter, &damage->embedded_box.list, list)
+		nboxes += iter->size;
+	DBG(("   nboxes=%d, residual=%d\n", nboxes, damage->remain));
+	nboxes -= damage->remain;
+	if (nboxes == 0)
+		goto done;
+	if (nboxes == 1) {
+		pixman_region16_t tmp;
+
+		tmp.extents = damage->embedded_box.box[0];
+		tmp.data = NULL;
+
+		if (damage->mode == DAMAGE_ADD)
+			pixman_region_union(region, region, &tmp);
+		else
+			pixman_region_subtract(region, region, &tmp);
+		damage->extents = region->extents;
+
+		goto done;
+	}
+
+	if (damage->mode == DAMAGE_ADD)
+		nboxes += REGION_NUM_RECTS(region);
+
+	iter = list_entry(damage->embedded_box.list.prev,
+			  struct sna_damage_box,
+			  list);
+	n = iter->size - damage->remain;
+	boxes = (BoxRec *)(iter+1);
+	DBG(("   last box count=%d/%d, need=%d\n", n, iter->size, nboxes));
+	if (nboxes > iter->size) {
+		boxes = malloc(sizeof(BoxRec)*nboxes);
+		if (boxes == NULL)
+			goto done;
+
+		free_boxes = boxes;
+	}
+
+	if (boxes != damage->embedded_box.box) {
+		if (list_is_empty(&damage->embedded_box.list)) {
+			DBG(("   copying embedded boxes\n"));
+			memcpy(boxes,
+			       damage->embedded_box.box,
+			       n*sizeof(BoxRec));
+		} else {
+			if (boxes != (BoxPtr)(iter+1)) {
+				DBG(("   copying %d boxes from last\n", n));
+				memcpy(boxes, iter+1, n*sizeof(BoxRec));
+			}
+
+			iter = list_entry(iter->list.prev,
+					  struct sna_damage_box,
+					  list);
+			while (&iter->list != &damage->embedded_box.list) {
+				DBG(("   copy %d boxes from %d\n",
+				     iter->size, n));
+				memcpy(boxes + n, iter+1,
+				       iter->size * sizeof(BoxRec));
+				n += iter->size;
+
+				iter = list_entry(iter->list.prev,
+						  struct sna_damage_box,
+						  list);
+			}
+
+			DBG(("   copying embedded boxes to %d\n", n));
+			memcpy(boxes + n,
+			       damage->embedded_box.box,
+			       sizeof(damage->embedded_box.box));
+			n += damage->embedded_box.size;
+		}
+	}
+
+	if (damage->mode == DAMAGE_ADD) {
+		memcpy(boxes + n,
+		       REGION_RECTS(region),
+		       REGION_NUM_RECTS(region)*sizeof(BoxRec));
+		assert(n + REGION_NUM_RECTS(region) == nboxes);
+		pixman_region_fini(region);
+		pixman_region_init_rects(region, boxes, nboxes);
+
+		assert(pixman_region_not_empty(region));
+		assert(damage->extents.x1 == region->extents.x1 &&
+		       damage->extents.y1 == region->extents.y1 &&
+		       damage->extents.x2 == region->extents.x2 &&
+		       damage->extents.y2 == region->extents.y2);
+	} else {
+		pixman_region16_t tmp;
+
+		assert(n == nboxes);
+		pixman_region_init_rects(&tmp, boxes, nboxes);
+		pixman_region_subtract(region, region, &tmp);
+		pixman_region_fini(&tmp);
+
+		assert(damage->extents.x1 <= region->extents.x1 &&
+		       damage->extents.y1 <= region->extents.y1 &&
+		       damage->extents.x2 >= region->extents.x2 &&
+		       damage->extents.y2 >= region->extents.y2);
+		if (pixman_region_not_empty(region))
+			damage->extents = region->extents;
+		else
+			reset_extents(damage);
+	}
+
+	free(free_boxes);
+
+done:
+	damage->mode = DAMAGE_ADD;
+	free_list(&damage->embedded_box.list);
+	reset_embedded_box(damage);
+
+	DBG(("    reduce: after region.n=%ld\n", (long)REGION_NUM_RECTS(region)));
+}
+
+
 static bool _sna_damage_create_boxes(struct sna_damage *damage,
 				     int count)
 {
@@ -206,7 +343,7 @@ static bool _sna_damage_create_boxes(struct sna_damage *damage,
 
 	DBG(("    %s(%d->%d): new\n", __FUNCTION__, count, n));
 
-	if (n > (INT_MAX - sizeof(*box)) / sizeof(BoxRec))
+	if (n >= (INT_MAX - sizeof(*box)) / sizeof(BoxRec))
 		return false;
 
 	box = malloc(sizeof(*box) + sizeof(BoxRec)*n);
@@ -229,7 +366,7 @@ _sna_damage_create_elt(struct sna_damage *damage,
 	DBG(("    %s: prev=(remain %d), count=%d\n",
 	     __FUNCTION__, damage->remain, count));
 
-	damage->dirty = true;
+restart:
 	n = count;
 	if (n > damage->remain)
 		n = damage->remain;
@@ -237,6 +374,7 @@ _sna_damage_create_elt(struct sna_damage *damage,
 		memcpy(damage->box, boxes, n * sizeof(BoxRec));
 		damage->box += n;
 		damage->remain -= n;
+		damage->dirty = true;
 
 		count -= n;
 		boxes += n;
@@ -246,11 +384,23 @@ _sna_damage_create_elt(struct sna_damage *damage,
 
 	DBG(("    %s(): new elt\n", __FUNCTION__));
 
-	if (_sna_damage_create_boxes(damage, count)) {
-		memcpy(damage->box, boxes, count * sizeof(BoxRec));
-		damage->box += count;
-		damage->remain -= count;
+	if (!_sna_damage_create_boxes(damage, count)) {
+		unsigned mode;
+
+		if (!damage->dirty)
+			return damage;
+
+		mode = damage->mode;
+		__sna_damage_reduce(damage);
+		damage->mode = mode;
+
+		goto restart;
 	}
+
+	memcpy(damage->box, boxes, count * sizeof(BoxRec));
+	damage->box += count;
+	damage->remain -= count;
+	damage->dirty = true;
 	assert(damage->remain >= 0);
 
 	return damage;
@@ -265,7 +415,7 @@ _sna_damage_create_elt_from_boxes(struct sna_damage *damage,
 
 	DBG(("    %s: prev=(remain %d)\n", __FUNCTION__, damage->remain));
 
-	damage->dirty = true;
+restart:
 	n = count;
 	if (n > damage->remain)
 		n = damage->remain;
@@ -278,6 +428,7 @@ _sna_damage_create_elt_from_boxes(struct sna_damage *damage,
 		}
 		damage->box += n;
 		damage->remain -= n;
+		damage->dirty = true;
 
 		count -= n;
 		boxes += n;
@@ -287,8 +438,18 @@ _sna_damage_create_elt_from_boxes(struct sna_damage *damage,
 
 	DBG(("    %s(): new elt\n", __FUNCTION__));
 
-	if (!_sna_damage_create_boxes(damage, count))
-		return damage;
+	if (!_sna_damage_create_boxes(damage, count)) {
+		unsigned mode;
+
+		if (!damage->dirty)
+			return damage;
+
+		mode = damage->mode;
+		__sna_damage_reduce(damage);
+		damage->mode = mode;
+
+		goto restart;
+	}
 
 	for (i = 0; i < count; i++) {
 		damage->box[i].x1 = boxes[i].x1 + dx;
@@ -298,6 +459,7 @@ _sna_damage_create_elt_from_boxes(struct sna_damage *damage,
 	}
 	damage->box += count;
 	damage->remain -= count;
+	damage->dirty = true;
 	assert(damage->remain >= 0);
 
 	return damage;
@@ -313,7 +475,7 @@ _sna_damage_create_elt_from_rectangles(struct sna_damage *damage,
 	DBG(("    %s: prev=(remain %d), count=%d\n",
 	     __FUNCTION__, damage->remain, count));
 
-	damage->dirty = true;
+restart:
 	n = count;
 	if (n > damage->remain)
 		n = damage->remain;
@@ -326,6 +488,7 @@ _sna_damage_create_elt_from_rectangles(struct sna_damage *damage,
 		}
 		damage->box += n;
 		damage->remain -= n;
+		damage->dirty = true;
 
 		count -= n;
 		r += n;
@@ -335,8 +498,18 @@ _sna_damage_create_elt_from_rectangles(struct sna_damage *damage,
 
 	DBG(("    %s(): new elt\n", __FUNCTION__));
 
-	if (!_sna_damage_create_boxes(damage, count))
-		return damage;
+	if (!_sna_damage_create_boxes(damage, count)) {
+		unsigned mode;
+
+		if (!damage->dirty)
+			return damage;
+
+		mode = damage->mode;
+		__sna_damage_reduce(damage);
+		damage->mode = mode;
+
+		goto restart;
+	}
 
 	for (i = 0; i < count; i++) {
 		damage->box[i].x1 = r[i].x + dx;
@@ -346,6 +519,7 @@ _sna_damage_create_elt_from_rectangles(struct sna_damage *damage,
 	}
 	damage->box += count;
 	damage->remain -= count;
+	damage->dirty = true;
 	assert(damage->remain >= 0);
 
 	return damage;
@@ -361,7 +535,7 @@ _sna_damage_create_elt_from_points(struct sna_damage *damage,
 	DBG(("    %s: prev=(remain %d), count=%d\n",
 	     __FUNCTION__, damage->remain, count));
 
-	damage->dirty = true;
+restart:
 	n = count;
 	if (n > damage->remain)
 		n = damage->remain;
@@ -374,6 +548,7 @@ _sna_damage_create_elt_from_points(struct sna_damage *damage,
 		}
 		damage->box += n;
 		damage->remain -= n;
+		damage->dirty = true;
 
 		count -= n;
 		p += n;
@@ -383,8 +558,18 @@ _sna_damage_create_elt_from_points(struct sna_damage *damage,
 
 	DBG(("    %s(): new elt\n", __FUNCTION__));
 
-	if (!_sna_damage_create_boxes(damage, count))
-		return damage;
+	if (!_sna_damage_create_boxes(damage, count)) {
+		unsigned mode;
+
+		if (!damage->dirty)
+			return damage;
+
+		mode = damage->mode;
+		__sna_damage_reduce(damage);
+		damage->mode = mode;
+
+		goto restart;
+	}
 
 	for (i = 0; i < count; i++) {
 		damage->box[i].x1 = p[i].x + dx;
@@ -394,147 +579,12 @@ _sna_damage_create_elt_from_points(struct sna_damage *damage,
 	}
 	damage->box += count;
 	damage->remain -= count;
+	damage->dirty = true;
 	assert(damage->remain >= 0);
 
 	return damage;
 }
 
-static void free_list(struct list *head)
-{
-	while (!list_is_empty(head)) {
-		struct list *l = head->next;
-		list_del(l);
-		free(l);
-	}
-}
-
-static void __sna_damage_reduce(struct sna_damage *damage)
-{
-	int n, nboxes;
-	BoxPtr boxes, free_boxes = NULL;
-	pixman_region16_t *region = &damage->region;
-	struct sna_damage_box *iter;
-
-	assert(damage->mode != DAMAGE_ALL);
-	assert(damage->dirty);
-
-	DBG(("    reduce: before region.n=%ld\n", (long)REGION_NUM_RECTS(region)));
-
-	nboxes = damage->embedded_box.size;
-	list_for_each_entry(iter, &damage->embedded_box.list, list)
-		nboxes += iter->size;
-	DBG(("   nboxes=%d, residual=%d\n", nboxes, damage->remain));
-	nboxes -= damage->remain;
-	if (nboxes == 0)
-		goto done;
-	if (nboxes == 1) {
-		pixman_region16_t tmp;
-
-		tmp.extents = damage->embedded_box.box[0];
-		tmp.data = NULL;
-
-		if (damage->mode == DAMAGE_ADD)
-			pixman_region_union(region, region, &tmp);
-		else
-			pixman_region_subtract(region, region, &tmp);
-		damage->extents = region->extents;
-
-		goto done;
-	}
-
-	if (damage->mode == DAMAGE_ADD)
-		nboxes += REGION_NUM_RECTS(region);
-
-	iter = list_entry(damage->embedded_box.list.prev,
-			  struct sna_damage_box,
-			  list);
-	n = iter->size - damage->remain;
-	boxes = (BoxRec *)(iter+1);
-	DBG(("   last box count=%d/%d, need=%d\n", n, iter->size, nboxes));
-	if (nboxes > iter->size) {
-		boxes = malloc(sizeof(BoxRec)*nboxes);
-		if (boxes == NULL)
-			goto done;
-
-		free_boxes = boxes;
-	}
-
-	if (boxes != damage->embedded_box.box) {
-		if (list_is_empty(&damage->embedded_box.list)) {
-			DBG(("   copying embedded boxes\n"));
-			memcpy(boxes,
-			       damage->embedded_box.box,
-			       n*sizeof(BoxRec));
-		} else {
-			if (boxes != (BoxPtr)(iter+1)) {
-				DBG(("   copying %d boxes from last\n", n));
-				memcpy(boxes, iter+1, n*sizeof(BoxRec));
-			}
-
-			iter = list_entry(iter->list.prev,
-					  struct sna_damage_box,
-					  list);
-			while (&iter->list != &damage->embedded_box.list) {
-				DBG(("   copy %d boxes from %d\n",
-				     iter->size, n));
-				memcpy(boxes + n, iter+1,
-				       iter->size * sizeof(BoxRec));
-				n += iter->size;
-
-				iter = list_entry(iter->list.prev,
-						  struct sna_damage_box,
-						  list);
-			}
-
-			DBG(("   copying embedded boxes to %d\n", n));
-			memcpy(boxes + n,
-			       damage->embedded_box.box,
-			       sizeof(damage->embedded_box.box));
-			n += damage->embedded_box.size;
-		}
-	}
-
-	if (damage->mode == DAMAGE_ADD) {
-		memcpy(boxes + n,
-		       REGION_RECTS(region),
-		       REGION_NUM_RECTS(region)*sizeof(BoxRec));
-		assert(n + REGION_NUM_RECTS(region) == nboxes);
-		pixman_region_fini(region);
-		pixman_region_init_rects(region, boxes, nboxes);
-
-		assert(pixman_region_not_empty(region));
-		assert(damage->extents.x1 == region->extents.x1 &&
-		       damage->extents.y1 == region->extents.y1 &&
-		       damage->extents.x2 == region->extents.x2 &&
-		       damage->extents.y2 == region->extents.y2);
-	} else {
-		pixman_region16_t tmp;
-
-		assert(n == nboxes);
-		pixman_region_init_rects(&tmp, boxes, nboxes);
-		pixman_region_subtract(region, region, &tmp);
-		pixman_region_fini(&tmp);
-
-		assert(damage->extents.x1 <= region->extents.x1 &&
-		       damage->extents.y1 <= region->extents.y1 &&
-		       damage->extents.x2 >= region->extents.x2 &&
-		       damage->extents.y2 >= region->extents.y2);
-		if (pixman_region_not_empty(region))
-			damage->extents = region->extents;
-		else
-			reset_extents(damage);
-	}
-
-	free(free_boxes);
-
-done:
-	damage->mode = DAMAGE_ADD;
-	free_list(&damage->embedded_box.list);
-	reset_embedded_box(damage);
-
-	DBG(("    reduce: after region.n=%ld\n", (long)REGION_NUM_RECTS(region)));
-}
-
 static void damage_union(struct sna_damage *damage, const BoxRec *box)
 {
 	DBG(("%s: extending damage (%d, %d), (%d, %d) by (%d, %d), (%d, %d)\n",