sna/gen3+: Flush vertex threads before touching global state

We need to be careful not just when finishing the current vbo to
synchronize with the sharing threads, but also before we emit the batch
state that no other thread will try and do the same.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

6 files changed, 57 insertions, 60 deletions

diff --git a/src/sna/gen3_render.c b/src/sna/gen3_render.c
index 06c4e784..6b9a2f81 100644
--- a/src/sna/gen3_render.c
+++ b/src/sna/gen3_render.c
@@ -1900,6 +1900,9 @@ static bool gen3_rectangle_begin(struct sna *sna,
 	struct gen3_render_state *state = &sna->render_state.gen3;
 	int ndwords, i1_cmd = 0, i1_len = 0;
 
+	if (sna_vertex_wait__locked(&sna->render) && sna->render.vertex_offset)
+		return true;
+
 	ndwords = 2;
 	if (op->need_magic_ca_pass)
 		ndwords += 100;
@@ -1939,6 +1942,13 @@ static bool gen3_rectangle_begin(struct sna *sna,
 static int gen3_get_rectangles__flush(struct sna *sna,
 				      const struct sna_composite_op *op)
 {
+	/* Preventing discarding new vbo after lock contention */
+	if (sna_vertex_wait__locked(&sna->render)) {
+		int rem = vertex_space(sna);
+		if (rem > op->floats_per_rect)
+			return rem;
+	}
+
 	if (!kgem_check_batch(&sna->kgem, op->need_magic_ca_pass ? 105: 5))
 		return 0;
 	if (!kgem_check_reloc_and_exec(&sna->kgem, 1))
@@ -1955,17 +1965,6 @@ static int gen3_get_rectangles__flush(struct sna *sna,
 		}
 	}
 
-	/* Preventing discarding new vbo after lock contention */
-	if (sna->render.active) {
-		int rem;
-
-		sna_vertex_wait__locked(&sna->render);
-
-		rem = vertex_space(sna);
-		if (rem > op->floats_per_rect)
-			return rem;
-	}
-
 	return gen3_vertex_finish(sna);
 }
 
diff --git a/src/sna/gen4_render.c b/src/sna/gen4_render.c
index 829f4e9a..7fc85209 100644
--- a/src/sna/gen4_render.c
+++ b/src/sna/gen4_render.c
@@ -587,6 +587,9 @@ static bool gen4_rectangle_begin(struct sna *sna,
 	int id = op->u.gen4.ve_id;
 	int ndwords;
 
+	if (sna_vertex_wait__locked(&sna->render) && sna->render.vertex_offset)
+		return true;
+
 	/* 7xpipelined pointers + 6xprimitive + 1xflush */
 	ndwords = op->need_magic_ca_pass? 20 : 6;
 	if ((sna->render.vb_id & (1 << id)) == 0)
@@ -606,6 +609,13 @@ static bool gen4_rectangle_begin(struct sna *sna,
 static int gen4_get_rectangles__flush(struct sna *sna,
 				      const struct sna_composite_op *op)
 {
+	/* Preventing discarding new vbo after lock contention */
+	if (sna_vertex_wait__locked(&sna->render)) {
+		int rem = vertex_space(sna);
+		if (rem > op->floats_per_rect)
+			return rem;
+	}
+
 	if (!kgem_check_batch(&sna->kgem, op->need_magic_ca_pass ? 25 : 6))
 		return 0;
 	if (!kgem_check_reloc_and_exec(&sna->kgem, 2))
@@ -621,17 +631,6 @@ static int gen4_get_rectangles__flush(struct sna *sna,
 						     op->u.gen4.wm_kernel);
 	}
 
-	/* Preventing discarding new vbo after lock contention */
-	if (sna->render.active) {
-		int rem;
-
-		sna_vertex_wait__locked(&sna->render);
-
-		rem = vertex_space(sna);
-		if (rem > op->floats_per_rect)
-			return rem;
-	}
-
 	return gen4_vertex_finish(sna);
 }
 
@@ -644,7 +643,7 @@ inline static int gen4_get_rectangles(struct sna *sna,
 
 start:
 	rem = vertex_space(sna);
-	if (rem < op->floats_per_rect) {
+	if (unlikely(rem < op->floats_per_rect)) {
 		DBG(("flushing vbo for %s: %d < %d\n",
 		     __FUNCTION__, rem, op->floats_per_rect));
 		rem = gen4_get_rectangles__flush(sna, op);
diff --git a/src/sna/gen5_render.c b/src/sna/gen5_render.c
index 07e95712..35f70f83 100644
--- a/src/sna/gen5_render.c
+++ b/src/sna/gen5_render.c
@@ -577,6 +577,9 @@ static bool gen5_rectangle_begin(struct sna *sna,
 	int id = op->u.gen5.ve_id;
 	int ndwords;
 
+	if (sna_vertex_wait__locked(&sna->render) && sna->render.vertex_offset)
+		return true;
+
 	ndwords = op->need_magic_ca_pass ? 20 : 6;
 	if ((sna->render.vb_id & (1 << id)) == 0)
 		ndwords += 5;
@@ -595,6 +598,13 @@ static bool gen5_rectangle_begin(struct sna *sna,
 static int gen5_get_rectangles__flush(struct sna *sna,
 				      const struct sna_composite_op *op)
 {
+	/* Preventing discarding new vbo after lock contention */
+	if (sna_vertex_wait__locked(&sna->render)) {
+		int rem = vertex_space(sna);
+		if (rem > op->floats_per_rect)
+			return rem;
+	}
+
 	if (!kgem_check_batch(&sna->kgem, op->need_magic_ca_pass ? 20 : 6))
 		return 0;
 	if (!kgem_check_reloc_and_exec(&sna->kgem, 2))
@@ -607,17 +617,6 @@ static int gen5_get_rectangles__flush(struct sna *sna,
 						     op->u.gen5.wm_kernel);
 	}
 
-	/* Preventing discarding new vbo after lock contention */
-	if (sna->render.active) {
-		int rem;
-
-		sna_vertex_wait__locked(&sna->render);
-
-		rem = vertex_space(sna);
-		if (rem > op->floats_per_rect)
-			return rem;
-	}
-
 	return gen4_vertex_finish(sna);
 }
 
@@ -631,7 +630,7 @@ inline static int gen5_get_rectangles(struct sna *sna,
 
 start:
 	rem = vertex_space(sna);
-	if (rem < op->floats_per_rect) {
+	if (unlikely(rem < op->floats_per_rect)) {
 		DBG(("flushing vbo for %s: %d < %d\n",
 		     __FUNCTION__, rem, op->floats_per_rect));
 		rem = gen5_get_rectangles__flush(sna, op);
diff --git a/src/sna/gen6_render.c b/src/sna/gen6_render.c
index a9d81114..ec642504 100644
--- a/src/sna/gen6_render.c
+++ b/src/sna/gen6_render.c
@@ -1125,6 +1125,9 @@ static bool gen6_rectangle_begin(struct sna *sna,
 	int id = 1 << GEN6_VERTEX(op->u.gen6.flags);
 	int ndwords;
 
+	if (sna_vertex_wait__locked(&sna->render) && sna->render.vertex_offset)
+		return true;
+
 	ndwords = op->need_magic_ca_pass ? 60 : 6;
 	if ((sna->render.vb_id & id) == 0)
 		ndwords += 5;
@@ -1141,6 +1144,13 @@ static bool gen6_rectangle_begin(struct sna *sna,
 static int gen6_get_rectangles__flush(struct sna *sna,
 				      const struct sna_composite_op *op)
 {
+	/* Preventing discarding new vbo after lock contention */
+	if (sna_vertex_wait__locked(&sna->render)) {
+		int rem = vertex_space(sna);
+		if (rem > op->floats_per_rect)
+			return rem;
+	}
+
 	if (!kgem_check_batch(&sna->kgem, op->need_magic_ca_pass ? 65 : 5))
 		return 0;
 	if (!kgem_check_reloc_and_exec(&sna->kgem, 2))
@@ -1157,17 +1167,6 @@ static int gen6_get_rectangles__flush(struct sna *sna,
 		}
 	}
 
-	/* Preventing discarding new vbo after lock contention */
-	if (sna->render.active) {
-		int rem;
-
-		sna_vertex_wait__locked(&sna->render);
-
-		rem = vertex_space(sna);
-		if (rem > op->floats_per_rect)
-			return rem;
-	}
-
 	return gen4_vertex_finish(sna);
 }
 
@@ -1180,7 +1179,7 @@ inline static int gen6_get_rectangles(struct sna *sna,
 
 start:
 	rem = vertex_space(sna);
-	if (rem < op->floats_per_rect) {
+	if (unlikely(rem < op->floats_per_rect)) {
 		DBG(("flushing vbo for %s: %d < %d\n",
 		     __FUNCTION__, rem, op->floats_per_rect));
 		rem = gen6_get_rectangles__flush(sna, op);
diff --git a/src/sna/gen7_render.c b/src/sna/gen7_render.c
index dc704dde..59b38f6c 100644
--- a/src/sna/gen7_render.c
+++ b/src/sna/gen7_render.c
@@ -1270,6 +1270,9 @@ static bool gen7_rectangle_begin(struct sna *sna,
 	int id = 1 << GEN7_VERTEX(op->u.gen7.flags);
 	int ndwords;
 
+	if (sna_vertex_wait__locked(&sna->render) && sna->render.vertex_offset)
+		return true;
+
 	ndwords = op->need_magic_ca_pass ? 60 : 6;
 	if ((sna->render.vb_id & id) == 0)
 		ndwords += 5;
@@ -1286,6 +1289,13 @@ static bool gen7_rectangle_begin(struct sna *sna,
 static int gen7_get_rectangles__flush(struct sna *sna,
 				      const struct sna_composite_op *op)
 {
+	/* Preventing discarding new vbo after lock contention */
+	if (sna_vertex_wait__locked(&sna->render)) {
+		int rem = vertex_space(sna);
+		if (rem > op->floats_per_rect)
+			return rem;
+	}
+
 	if (!kgem_check_batch(&sna->kgem, op->need_magic_ca_pass ? 65 : 6))
 		return 0;
 	if (!kgem_check_reloc_and_exec(&sna->kgem, 2))
@@ -1300,17 +1310,6 @@ static int gen7_get_rectangles__flush(struct sna *sna,
 		}
 	}
 
-	/* Preventing discarding new vbo after lock contention */
-	if (sna->render.active) {
-		int rem;
-
-		sna_vertex_wait__locked(&sna->render);
-
-		rem = vertex_space(sna);
-		if (rem > op->floats_per_rect)
-			return rem;
-	}
-
 	return gen4_vertex_finish(sna);
 }
 
@@ -1323,7 +1322,7 @@ inline static int gen7_get_rectangles(struct sna *sna,
 
 start:
 	rem = vertex_space(sna);
-	if (rem < op->floats_per_rect) {
+	if (unlikely(rem < op->floats_per_rect)) {
 		DBG(("flushing vbo for %s: %d < %d\n",
 		     __FUNCTION__, rem, op->floats_per_rect));
 		rem = gen7_get_rectangles__flush(sna, op);
diff --git a/src/sna/sna_render.h b/src/sna/sna_render.h
index c953e50c..6cf90eb9 100644
--- a/src/sna/sna_render.h
+++ b/src/sna/sna_render.h
@@ -763,10 +763,12 @@ static inline void sna_vertex_release__locked(struct sna_render *r)
 		pthread_cond_signal(&r->wait);
 }
 
-static inline void sna_vertex_wait__locked(struct sna_render *r)
+static inline bool sna_vertex_wait__locked(struct sna_render *r)
 {
+	bool was_active = r->active;
 	while (r->active)
 		pthread_cond_wait(&r->wait, &r->lock);
+	return was_active;
 }
 
 #endif /* SNA_RENDER_H */