KMS CS fixes.

* Flush CS before running out of space.
* Don't flush CS every time when switching between 2D and 3D engine, but make
  sure to re-initialize 2D/3D state after flushing.

Doesn't seem quite perfect yet (some rendering seems to get dropped on the floor
sometimes), but this should fix a lot of CS failures people have been seeing.

5 files changed, 132 insertions, 118 deletions

diff --git a/src/r600_exa.c b/src/r600_exa.c
index 18831f7f..3673064c 100644
--- a/src/r600_exa.c
+++ b/src/r600_exa.c
@@ -1166,7 +1166,7 @@ static Bool R600TextureSetup(PicturePtr pPict, PixmapPtr pPix,
     }
 
     if (unit == 0) {
-	if (!accel_state->has_mask) {
+	if (!accel_state->msk_pic) {
 	    if (PICT_FORMAT_RGB(pPict->format) == 0) {
 		pix_r = SQ_SEL_0;
 		pix_g = SQ_SEL_0;
@@ -1397,7 +1397,7 @@ static Bool R600PrepareComposite(int op, PicturePtr pSrcPicture,
     /* return FALSE; */
 
     if (pMask) {
-	accel_state->has_mask = TRUE;
+	accel_state->msk_pic = pMaskPicture;
 	if (pMaskPicture->componentAlpha) {
 	    accel_state->component_alpha = TRUE;
 	    if (R600BlendOp[op].src_alpha)
@@ -1409,7 +1409,7 @@ static Bool R600PrepareComposite(int op, PicturePtr pSrcPicture,
 	    accel_state->src_alpha = FALSE;
 	}
     } else {
-	accel_state->has_mask = FALSE;
+	accel_state->msk_pic = NULL;
 	accel_state->component_alpha = FALSE;
 	accel_state->src_alpha = FALSE;
     }
@@ -1598,7 +1598,7 @@ static void R600Composite(PixmapPtr pDst,
     srcBottomRight.x = IntToxFixed(srcX + w);
     srcBottomRight.y = IntToxFixed(srcY + h);
 
-    if (accel_state->has_mask) {
+    if (accel_state->msk_pic) {
 	xPointFixed maskTopLeft, maskTopRight, maskBottomLeft, maskBottomRight;
 
 	if (((accel_state->vb_index + 3) * 24) > (accel_state->ib->total / 2)) {
@@ -1693,7 +1693,7 @@ static void R600DoneComposite(PixmapPtr pDst)
 
 
     /* Vertex buffer setup */
-    if (accel_state->has_mask) {
+    if (accel_state->msk_pic) {
 	accel_state->vb_size = accel_state->vb_index * 24;
 	vtx_res.id              = SQ_VTX_RESOURCE_vs;
 	vtx_res.vtx_size_dw     = 24 / 4;
diff --git a/src/radeon.h b/src/radeon.h
index 82136c83..ad62a613 100644
--- a/src/radeon.h
+++ b/src/radeon.h
@@ -672,9 +672,15 @@ struct radeon_accel_state {
 #define EXA_ENGINEMODE_2D      1
 #define EXA_ENGINEMODE_3D      2
 
+    int               composite_op;
+    PicturePtr        dst_pic;
+    PicturePtr        msk_pic;
+    PicturePtr        src_pic;
+    PixmapPtr         dst_pix;
+    PixmapPtr         msk_pix;
+    PixmapPtr         src_pix;
     Bool              is_transform[2];
     PictTransform     *transform[2];
-    Bool              has_mask;
     /* Whether we are tiling horizontally and vertically */
     Bool              need_src_tile_x;
     Bool              need_src_tile_y;
@@ -1513,8 +1519,10 @@ do {									\
 	uint32_t flush = 0;                                             \
 	switch (info->accel_state->engineMode) {			\
 	case EXA_ENGINEMODE_UNKNOWN:					\
-	case EXA_ENGINEMODE_3D:						\
 	    flush = 1;                                                  \
+	    break;							\
+	case EXA_ENGINEMODE_3D:						\
+	    flush = info->cs == NULL || info->cs->cdw > 15 * 1024;      \
 	case EXA_ENGINEMODE_2D:						\
 	    break;							\
 	}								\
@@ -1532,8 +1540,10 @@ do {									\
 	uint32_t flush = 0;						\
 	switch (info->accel_state->engineMode) {			\
 	case EXA_ENGINEMODE_UNKNOWN:					\
-	case EXA_ENGINEMODE_2D:						\
 	    flush = 1;                                                  \
+	    break;							\
+	case EXA_ENGINEMODE_2D:						\
+	    flush = info->cs == NULL || info->cs->cdw > 15 * 1024;      \
 	case EXA_ENGINEMODE_3D:						\
 	    break;							\
 	}								\
@@ -1542,8 +1552,9 @@ do {									\
 	        radeon_cs_flush_indirect(pScrn);			\
 	    else if (info->directRenderingEnabled)				\
 	        RADEONCPFlushIndirect(pScrn, 1);                        \
-	    RADEONInit3DEngine(pScrn);                                  \
 	}                                                               \
+	if (!info->accel_state->XInited3D)				\
+	    RADEONInit3DEngine(pScrn);                                  \
         info->accel_state->engineMode = EXA_ENGINEMODE_3D;              \
 } while (0);
 #else
diff --git a/src/radeon_accel.c b/src/radeon_accel.c
index 79052a15..16d88e4e 100644
--- a/src/radeon_accel.c
+++ b/src/radeon_accel.c
@@ -667,6 +667,13 @@ void radeon_cs_flush_indirect(ScrnInfoPtr pScrn)
 	return;
     radeon_cs_emit(info->cs);
     radeon_cs_erase(info->cs);
+
+    /* Need to re-initialize 3D state in each CS */
+    info->accel_state->XInited3D = FALSE;
+
+#ifdef USE_EXA
+    info->accel_state->engineMode = EXA_ENGINEMODE_UNKNOWN;
+#endif
 #endif
 }
 
diff --git a/src/radeon_exa_funcs.c b/src/radeon_exa_funcs.c
index 5b723609..fb33b6fe 100644
--- a/src/radeon_exa_funcs.c
+++ b/src/radeon_exa_funcs.c
@@ -213,6 +213,17 @@ FUNC_NAME(RADEONSolid)(PixmapPtr pPix, int x1, int y1, int x2, int y2)
 
     TRACE;
 
+#ifdef ACCEL_CP
+    if (info->cs && info->cs->cdw > 15 * 1024) {
+	struct radeon_cs_space_check bos[1];
+
+	radeon_cs_flush_indirect(pScrn);
+	radeon_add_pixmap(bos, 0, pPix, 0, RADEON_GEM_DOMAIN_VRAM);
+	radeon_cs_space_check(info->cs, bos, 1);
+	FUNC_NAME(Emit2DState)(pScrn, RADEON_2D_EXA_SOLID);
+    }
+#endif
+
     if (info->accel_state->vsync)
 	FUNC_NAME(RADEONWaitForVLine)(pScrn, pPix, RADEONBiggerCrtcArea(pPix), y1, y2);
 
@@ -342,6 +353,21 @@ FUNC_NAME(RADEONCopy)(PixmapPtr pDst,
 
     TRACE;
 
+#ifdef ACCEL_CP
+    if (info->cs && info->cs->cdw > 15 * 1024) {
+	struct radeon_cs_space_check bos[2];
+
+	radeon_cs_flush_indirect(pScrn);
+	bos[0].bo = info->state_2d.src_bo;
+	bos[0].read_domains = RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM;
+	bos[0].write_domain = 0;
+	bos[0].new_accounted = 0;
+	radeon_add_pixmap(bos, 1, pDst, 0, RADEON_GEM_DOMAIN_VRAM);
+	radeon_cs_space_check(info->cs, bos, 2);
+	FUNC_NAME(Emit2DState)(pScrn, RADEON_2D_EXA_COPY);
+    }
+#endif
+
     if (info->accel_state->xdir < 0) {
 	srcX += w - 1;
 	dstX += w - 1;
diff --git a/src/radeon_exa_render.c b/src/radeon_exa_render.c
index 60c40a23..ee82d85f 100644
--- a/src/radeon_exa_render.c
+++ b/src/radeon_exa_render.c
@@ -542,6 +542,55 @@ static Bool R100CheckComposite(int op, PicturePtr pSrcPicture,
 
     return TRUE;
 }
+
+static Bool
+RADEONPrepareCompositeState(int op, PicturePtr pSrcPicture, PicturePtr pMaskPicture,
+			    PicturePtr pDstPicture, PixmapPtr pSrc, PixmapPtr pMask,
+			    PixmapPtr pDst)
+{
+    RINFO_FROM_SCREEN(pDst->drawable.pScreen);
+    struct radeon_cs_space_check bos[3];
+    int i, ret, retry_count = 0;
+
+    /* switch to 3D before doing buffer space checks as it may flush */
+    RADEON_SWITCH_TO_3D();
+
+ retry:
+    if (info->cs) {
+      
+	i = 0;
+	radeon_add_pixmap(bos, i++, pSrc, RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
+
+	if (pMask)
+	    radeon_add_pixmap(bos, i++, pMask, RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
+
+	radeon_add_pixmap(bos, i++, pDst, 0, RADEON_GEM_DOMAIN_VRAM);
+
+	ret = radeon_cs_space_check(info->cs, bos, i);
+	if (ret == RADEON_CS_SPACE_OP_TO_BIG) {
+	    RADEON_FALLBACK(("Not enough RAM to hw accel composite operation\n"));
+	}
+	if (ret == RADEON_CS_SPACE_FLUSH) {
+	    radeon_cs_flush_indirect(pScrn);
+	    retry_count++;
+	    if (retry_count == 2)
+	        RADEON_FALLBACK(("Not enough Video RAM for src\n"));
+	    RADEON_SWITCH_TO_3D();
+	    goto retry;
+	}
+    }
+
+    info->accel_state->composite_op = op;
+    info->accel_state->dst_pic = pDstPicture;
+    info->accel_state->msk_pic = pMaskPicture;
+    info->accel_state->src_pic = pSrcPicture;
+    info->accel_state->dst_pix = pDst;
+    info->accel_state->msk_pix = pMask;
+    info->accel_state->src_pix = pSrc;
+
+    return TRUE;
+}
+
 #endif /* ONLY_ONCE */
 
 static Bool FUNC_NAME(R100PrepareComposite)(int op,
@@ -557,9 +606,6 @@ static Bool FUNC_NAME(R100PrepareComposite)(int op,
     uint32_t pp_cntl, blendcntl, cblend, ablend;
     int pixel_shift;
     struct radeon_exa_pixmap_priv *driver_priv;
-    int retry_count = 0;
-    struct radeon_cs_space_check bos[3];
-    int i, ret;
     ACCEL_PREAMBLE();
 
     TRACE;
@@ -570,11 +616,6 @@ static Bool FUNC_NAME(R100PrepareComposite)(int op,
     if (pDstPicture->format == PICT_a8 && RadeonBlendOp[op].dst_alpha)
 	RADEON_FALLBACK(("Can't dst alpha blend A8\n"));
 
-    if (pMask)
-	info->accel_state->has_mask = TRUE;
-    else
-	info->accel_state->has_mask = FALSE;
-
     pixel_shift = pDst->drawable.bitsPerPixel >> 4;
 
     dst_pitch = exaGetPixmapPitch(pDst);
@@ -587,35 +628,13 @@ static Bool FUNC_NAME(R100PrepareComposite)(int op,
     if (((dst_pitch >> pixel_shift) & 0x7) != 0)
 	RADEON_FALLBACK(("Bad destination pitch 0x%x\n", (int)dst_pitch));
 
-    /* switch to 3D before doing buffer space checks as it may flush */
-    RADEON_SWITCH_TO_3D();
- retry:
-    if (info->cs) {
-      
-	i = 0;
-	radeon_add_pixmap(bos, i++, pSrc, RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
-
-	if (pMask)
-	    radeon_add_pixmap(bos, i++, pMask, RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
-
-	radeon_add_pixmap(bos, i++, pDst, 0, RADEON_GEM_DOMAIN_VRAM);
-
-	ret = radeon_cs_space_check(info->cs, bos, i);
-	if (ret == RADEON_CS_SPACE_OP_TO_BIG) {
-	    RADEON_FALLBACK(("Not enough RAM to hw accel composite operation\n"));
-	}
-	if (ret == RADEON_CS_SPACE_FLUSH) {
-	    radeon_cs_flush_indirect(pScrn);
-	    retry_count++;
-	    if (retry_count == 2)
-	        RADEON_FALLBACK(("Not enough Video RAM for src\n"));
-	    goto retry;
-	}
-    }
-
     if (!RADEONSetupSourceTile(pSrcPicture, pSrc, FALSE, TRUE))
 	return FALSE;
 
+    if (!RADEONPrepareCompositeState(op, pSrcPicture, pMaskPicture, pDstPicture,
+				     pSrc, pMask, pDst))
+	return FALSE;
+
     if (!FUNC_NAME(R100TextureSetup)(pSrcPicture, pSrc, 0))
 	return FALSE;
     pp_cntl = RADEON_TEX_0_ENABLE | RADEON_TEX_BLEND_0_ENABLE;
@@ -914,9 +933,6 @@ static Bool FUNC_NAME(R200PrepareComposite)(int op, PicturePtr pSrcPicture,
     uint32_t pp_cntl, blendcntl, cblend, ablend, colorpitch;
     int pixel_shift;
     struct radeon_exa_pixmap_priv *driver_priv;
-    int retry_count = 0;
-    struct radeon_cs_space_check bos[3];
-    int i, ret;
     ACCEL_PREAMBLE();
 
     TRACE;
@@ -927,11 +943,6 @@ static Bool FUNC_NAME(R200PrepareComposite)(int op, PicturePtr pSrcPicture,
     if (pDstPicture->format == PICT_a8 && RadeonBlendOp[op].dst_alpha)
 	RADEON_FALLBACK(("Can't dst alpha blend A8\n"));
 
-    if (pMask)
-	info->accel_state->has_mask = TRUE;
-    else
-	info->accel_state->has_mask = FALSE;
-
     pixel_shift = pDst->drawable.bitsPerPixel >> 4;
 
     dst_pitch = exaGetPixmapPitch(pDst);
@@ -944,35 +955,13 @@ static Bool FUNC_NAME(R200PrepareComposite)(int op, PicturePtr pSrcPicture,
     if (((dst_pitch >> pixel_shift) & 0x7) != 0)
 	RADEON_FALLBACK(("Bad destination pitch 0x%x\n", (int)dst_pitch));
 
-    /* switch to 3D before doing buffer space checks as it may flush */
-    RADEON_SWITCH_TO_3D();
-
- retry:
-    if (info->cs) {
-      
-	i = 0;
-	radeon_add_pixmap(bos, i++, pSrc, RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
-
-	if (pMask)
-	    radeon_add_pixmap(bos, i++, pMask, RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
-
-	radeon_add_pixmap(bos, i++, pDst, 0, RADEON_GEM_DOMAIN_VRAM);
-
-	ret = radeon_cs_space_check(info->cs, bos, i);
-	if (ret == RADEON_CS_SPACE_OP_TO_BIG) {
-	    RADEON_FALLBACK(("Not enough RAM to hw accel composite operation\n"));
-	}
-	if (ret == RADEON_CS_SPACE_FLUSH) {
-	    radeon_cs_flush_indirect(pScrn);
-	    retry_count++;
-	    if (retry_count == 2)
-	        RADEON_FALLBACK(("Not enough Video RAM for src\n"));
-	    goto retry;
-	}
-    }
     if (!RADEONSetupSourceTile(pSrcPicture, pSrc, FALSE, TRUE))
 	return FALSE;
 
+    if (!RADEONPrepareCompositeState(op, pSrcPicture, pMaskPicture, pDstPicture,
+				     pSrc, pMask, pDst))
+	return FALSE;
+
     if (!FUNC_NAME(R200TextureSetup)(pSrcPicture, pSrc, 0))
 	return FALSE;
     pp_cntl = RADEON_TEX_0_ENABLE | RADEON_TEX_BLEND_0_ENABLE;
@@ -1165,7 +1154,7 @@ static Bool FUNC_NAME(R300TextureSetup)(PicturePtr pPict, PixmapPtr pPix,
     txformat1 = R300TexFormats[i].card_fmt;
 
     if (IS_R300_3D) {
-	if ((unit == 0) && info->accel_state->has_mask)
+	if ((unit == 0) && info->accel_state->msk_pic)
 	    txformat1 |= R300_TX_FORMAT_CACHE_HALF_REGION_0;
 	else if (unit == 1)
 	    txformat1 |= R300_TX_FORMAT_CACHE_HALF_REGION_1;
@@ -1403,22 +1392,13 @@ static Bool FUNC_NAME(R300PrepareComposite)(int op, PicturePtr pSrcPicture,
     uint32_t src_color, src_alpha;
     uint32_t mask_color, mask_alpha;
     int pixel_shift;
-    int ret;
-    int retry_count = 0;
     struct radeon_exa_pixmap_priv *driver_priv;
-    struct radeon_cs_space_check bos[3];
-    int i;
     ACCEL_PREAMBLE();
     TRACE;
 
     if (!R300GetDestFormat(pDstPicture, &dst_format))
 	return FALSE;
 
-    if (pMask)
-	info->accel_state->has_mask = TRUE;
-    else
-	info->accel_state->has_mask = FALSE;
-
     pixel_shift = pDst->drawable.bitsPerPixel >> 4;
 
     dst_pitch = exaGetPixmapPitch(pDst);
@@ -1434,36 +1414,13 @@ static Bool FUNC_NAME(R300PrepareComposite)(int op, PicturePtr pSrcPicture,
     if (((dst_pitch >> pixel_shift) & 0x7) != 0)
 	RADEON_FALLBACK(("Bad destination pitch 0x%x\n", (int)dst_pitch));
 
-    /* have to execute switch before doing buffer sizing check as it flushes */
-    RADEON_SWITCH_TO_3D();
- retry:
-    if (info->cs) {
-      
-	i = 0;
-	driver_priv = exaGetPixmapDriverPrivate(pSrc);
-	radeon_add_pixmap(bos, i++, pSrc, RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
-
-	if (pMask)
-	    radeon_add_pixmap(bos, i++, pMask, RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM, 0);
-
-	radeon_add_pixmap(bos, i++, pDst, 0, RADEON_GEM_DOMAIN_VRAM);
-
-	ret = radeon_cs_space_check(info->cs, bos, i);
-	if (ret == RADEON_CS_SPACE_OP_TO_BIG) {
-	    RADEON_FALLBACK(("Not enough RAM to hw accel composite operation\n"));
-	}
-	if (ret == RADEON_CS_SPACE_FLUSH) {
-            radeon_cs_flush_indirect(pScrn);
-	    retry_count++;
-	    if (retry_count == 2)
-	        RADEON_FALLBACK(("Not enough Video RAM - this really shouldn't happen\nm"));
-	    goto retry;
-	}
-    }
-
     if (!RADEONSetupSourceTile(pSrcPicture, pSrc, TRUE, FALSE))
 	return FALSE;
 
+    if (!RADEONPrepareCompositeState(op, pSrcPicture, pMaskPicture, pDstPicture,
+				     pSrc, pMask, pDst))
+	return FALSE;
+
     if (!FUNC_NAME(R300TextureSetup)(pSrcPicture, pSrc, 0))
 	return FALSE;
     txenable = R300_TEX_0_ENABLE;
@@ -2060,7 +2017,7 @@ static Bool FUNC_NAME(R300PrepareComposite)(int op, PicturePtr pSrcPicture,
     FINISH_ACCEL();
 
     BEGIN_ACCEL(1);
-    if (info->accel_state->has_mask)
+    if (pMask)
 	OUT_ACCEL_REG(R300_VAP_VTX_SIZE, 6);
     else
 	OUT_ACCEL_REG(R300_VAP_VTX_SIZE, 4);
@@ -2143,6 +2100,19 @@ static void FUNC_NAME(RadeonCompositeTile)(ScrnInfoPtr pScrn,
     /* ErrorF("RadeonComposite (%d,%d) (%d,%d) (%d,%d) (%d,%d)\n",
        srcX, srcY, maskX, maskY,dstX, dstY, w, h); */
 
+#ifdef ACCEL_CP
+    if (info->cs && info->cs->cdw > 15 * 1024) {
+	radeon_cs_flush_indirect(pScrn);
+	info->accel_state->exa->PrepareComposite(info->accel_state->composite_op,
+						 info->accel_state->dst_pic,
+						 info->accel_state->msk_pic,
+						 info->accel_state->src_pic,
+						 info->accel_state->dst_pix,
+						 info->accel_state->msk_pix,
+						 info->accel_state->src_pix);
+    }
+#endif
+
     srcTopLeft.x     = IntToxFixed(srcX);
     srcTopLeft.y     = IntToxFixed(srcY);
     srcTopRight.x    = IntToxFixed(srcX + w);
@@ -2161,7 +2131,7 @@ static void FUNC_NAME(RadeonCompositeTile)(ScrnInfoPtr pScrn,
 	}
     }
 
-    if (info->accel_state->has_mask) {
+    if (info->accel_state->msk_pic) {
 	maskTopLeft.x     = IntToxFixed(maskX);
 	maskTopLeft.y     = IntToxFixed(maskY);
 	maskTopRight.x    = IntToxFixed(maskX + w);
@@ -2192,7 +2162,7 @@ static void FUNC_NAME(RadeonCompositeTile)(ScrnInfoPtr pScrn,
 	BEGIN_RING(3 * vtx_count + 3);
 	OUT_RING(CP_PACKET3(RADEON_CP_PACKET3_3D_DRAW_IMMD,
 			    3 * vtx_count + 1));
-	if (info->accel_state->has_mask)
+	if (info->accel_state->msk_pic)
 	    OUT_RING(RADEON_CP_VC_FRMT_XY |
 		     RADEON_CP_VC_FRMT_ST0 |
 		     RADEON_CP_VC_FRMT_ST1);
@@ -2242,7 +2212,7 @@ static void FUNC_NAME(RadeonCompositeTile)(ScrnInfoPtr pScrn,
 
 #endif
 
-    if (info->accel_state->has_mask) {
+    if (info->accel_state->msk_pic) {
 	if (IS_R300_3D || IS_R500_3D) {
 	    VTX_OUT_MASK((float)dstX,                                      (float)dstY,
 			 xFixedToFloat(srcTopLeft.x) / info->accel_state->texW[0],      xFixedToFloat(srcTopLeft.y) / info->accel_state->texH[0],