Use batchbuffers instead of ring emits for general commands.

The batchbuffers are managed using libdrm and bufmgr_fake, and dispatched from
the ring from userland.

10 files changed, 346 insertions, 42 deletions

diff --git a/src/Makefile.am b/src/Makefile.am
index 0784c064..7ab1f086 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -89,6 +89,8 @@ intel_drv_la_SOURCES = \
          i830_accel.c \
          i830_bios.c \
          i830_bios.h \
+         i830_batchbuffer.c \
+         i830_batchbuffer.h \
          i830_common.h \
 	 i830_crt.c \
          i830_cursor.c \
diff --git a/src/i810_reg.h b/src/i810_reg.h
index d8e0cb2c..6a82c19b 100644
--- a/src/i810_reg.h
+++ b/src/i810_reg.h
@@ -2379,6 +2379,13 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 #define STATE3D_COLOR_FACTOR	((0x3<<29)|(0x1d<<24)|(0x01<<16))
 
+/* Batch */
+#define MI_BATCH_BUFFER		((0x30 << 23) | 1)
+#define MI_BATCH_BUFFER_START	(0x31 << 23)
+#define MI_BATCH_BUFFER_END	(0xA << 23)
+#define MI_BATCH_NON_SECURE		(1)
+#define MI_BATCH_NON_SECURE_I965	(1 << 8)
+
 /* STATE3D_FOG_MODE stuff */
 #define ENABLE_FOG_SOURCE	(1<<27)
 #define ENABLE_FOG_CONST	(1<<24)
diff --git a/src/i830.h b/src/i830.h
index 9cce5bc9..ccf91a04 100644
--- a/src/i830.h
+++ b/src/i830.h
@@ -81,6 +81,8 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include "damage.h"
 #endif
 #endif
+#include "dri_bufmgr.h"
+#include "intel_bufmgr.h"
 
 #ifdef I830_USE_EXA
 #include "exa.h"
@@ -95,7 +97,6 @@ Bool I830XAAInit(ScreenPtr pScreen);
 typedef struct _I830OutputRec I830OutputRec, *I830OutputPtr;
 
 #include "common.h"
-#include "i830_ring.h"
 #include "i830_sdvo.h"
 #include "i2c_vid.h"
 
@@ -401,6 +402,8 @@ typedef struct _I830Rec {
    i830_memory *exa_offscreen;
    i830_memory *gen4_render_state_mem;
 #endif
+   i830_memory *fake_bufmgr_mem;
+
    /* Regions allocated either from the above pools, or from agpgart. */
    I830RingBuffer *LpRing;
 
@@ -411,6 +414,17 @@ typedef struct _I830Rec {
    /** Offset in the ring for the next DWORD emit */
    uint32_t ring_next;
 
+   dri_bufmgr *bufmgr;
+
+   uint8_t *batch_ptr;
+   /** Byte offset in batch_ptr for the next dword to be emitted. */
+   unsigned int batch_used;
+   /** Position in batch_ptr at the start of the current BEGIN_BATCH */
+   unsigned int batch_emit_start;
+   /** Number of bytes to be emitted in the current BEGIN_BATCH. */
+   uint32_t batch_emitting;
+   dri_bo *batch_bo;
+
 #ifdef I830_XV
    /* For Xvideo */
    i830_memory *overlay_regs;
@@ -674,6 +688,9 @@ typedef struct _I830Rec {
 #define I830_SELECT_DEPTH	2
 #define I830_SELECT_THIRD	3
 
+/* Batchbuffer support macros and functions */
+#include "i830_batchbuffer.h"
+
 /* I830 specific functions */
 extern int I830WaitLpRing(ScrnInfoPtr pScrn, int n, int timeout_millis);
 extern void I830SetPIOAccess(I830Ptr pI830);
@@ -898,13 +915,6 @@ Bool i830_pixmap_tiled(PixmapPtr p);
     if (pitch > KB(8)) I830FALLBACK("pitch exceeds 3d limit 8K\n");\
 } while(0)
 
-/* Batchbuffer compatibility handling */
-#define BEGIN_BATCH(n) BEGIN_LP_RING(n)
-#define ENSURE_BATCH(n)
-#define OUT_BATCH(d) OUT_RING(d)
-#define OUT_BATCH_F(x) OUT_RING_F(x)
-#define ADVANCE_BATCH() ADVANCE_LP_RING()
-
 extern const int I830PatternROP[16];
 extern const int I830CopyROP[16];
 
diff --git a/src/i830_accel.c b/src/i830_accel.c
index 7784c62a..a71ea475 100644
--- a/src/i830_accel.c
+++ b/src/i830_accel.c
@@ -59,6 +59,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include "i830.h"
 #include "i810_reg.h"
 #include "i830_debug.h"
+#include "i830_ring.h"
 
 unsigned long
 intel_get_pixmap_offset(PixmapPtr pPix)
@@ -168,7 +169,6 @@ void
 I830Sync(ScrnInfoPtr pScrn)
 {
    I830Ptr pI830 = I830PTR(pScrn);
-   int flags = MI_WRITE_DIRTY_STATE | MI_INVALIDATE_MAP_CACHE;
 
    if (I810_DEBUG & (DEBUG_VERBOSE_ACCEL | DEBUG_VERBOSE_SYNC))
       ErrorF("I830Sync\n");
@@ -186,24 +186,12 @@ I830Sync(ScrnInfoPtr pScrn)
 
    if (pI830->entityPrivate && !pI830->entityPrivate->RingRunning) return;
 
-   if (IS_I965G(pI830))
-      flags = 0;
+   I830EmitFlush(pScrn);
 
-   /* Send a flush instruction and then wait till the ring is empty.
-    * This is stronger than waiting for the blitter to finish as it also
-    * flushes the internal graphics caches.
-    */
-   
-   {
-      BEGIN_BATCH(2);
-      OUT_BATCH(MI_FLUSH | flags);
-      OUT_BATCH(MI_NOOP);		/* pad to quadword */
-      ADVANCE_BATCH();
-   }
+   intel_batch_flush(pScrn);
 
    i830_wait_ring_idle(pScrn);
 
-   pI830->LpRing->space = pI830->LpRing->mem->size - 8;
    pI830->nextColorExpandBuf = 0;
 }
 
diff --git a/src/i830_batchbuffer.c b/src/i830_batchbuffer.c
new file mode 100644
index 00000000..19d642d5
--- /dev/null
+++ b/src/i830_batchbuffer.c
@@ -0,0 +1,119 @@
+/* -*- c-basic-offset: 4 -*- */
+/*
+ * Copyright © 2006 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eric Anholt <eric@anholt.net>
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <assert.h>
+#include <stdlib.h>
+
+#include "xf86.h"
+#include "i830.h"
+#include "i830_ring.h"
+
+static void
+intel_next_batch(ScrnInfoPtr pScrn)
+{
+    I830Ptr pI830 = I830PTR(pScrn);
+
+    /* The 865 has issues with larger-than-page-sized batch buffers. */
+    if (IS_I865G(pI830))
+	pI830->batch_bo = dri_bo_alloc(pI830->bufmgr, "batch", 4096, 4096);
+    else
+	pI830->batch_bo = dri_bo_alloc(pI830->bufmgr, "batch", 4096 * 4, 4096);
+
+    dri_bo_map(pI830->batch_bo, 1);
+    pI830->batch_used = 0;
+    pI830->batch_ptr = pI830->batch_bo->virtual;
+}
+
+void
+intel_batch_init(ScrnInfoPtr pScrn)
+{
+    I830Ptr pI830 = I830PTR(pScrn);
+
+    pI830->batch_emit_start = 0;
+    pI830->batch_emitting = 0;
+
+    intel_next_batch(pScrn);
+}
+
+void
+intel_batch_teardown(ScrnInfoPtr pScrn)
+{
+    I830Ptr pI830 = I830PTR(pScrn);
+
+    if (pI830->batch_ptr != NULL) {
+	dri_bo_unmap(pI830->batch_bo);
+	pI830->batch_ptr = NULL;
+    }
+}
+
+void
+intel_batch_flush(ScrnInfoPtr pScrn)
+{
+    I830Ptr pI830 = I830PTR(pScrn);
+
+    if (pI830->batch_used == 0)
+	return;
+
+    /* Emit a padding dword if we aren't going to be quad-word aligned. */
+    if ((pI830->batch_used & 4) == 0) {
+	*(uint32_t *)(pI830->batch_ptr + pI830->batch_used) = MI_NOOP;
+	pI830->batch_used += 4;
+    }
+
+    /* Mark the end of the batchbuffer. */
+    *(uint32_t *)(pI830->batch_ptr + pI830->batch_used) = MI_BATCH_BUFFER_END;
+    pI830->batch_used += 4;
+
+    dri_bo_unmap(pI830->batch_bo);
+    pI830->batch_ptr = NULL;
+
+    dri_process_relocs(pI830->batch_bo);
+
+    if (!IS_I830(pI830) && !IS_845G(pI830)) {
+	BEGIN_LP_RING(2);
+	OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
+	OUT_RING(pI830->batch_bo->offset);
+	ADVANCE_LP_RING();
+    } else {
+	BEGIN_LP_RING(4);
+	OUT_RING(MI_BATCH_BUFFER);
+	OUT_RING(pI830->batch_bo->offset);
+	OUT_RING(pI830->batch_bo->offset + pI830->batch_used - 4);
+	OUT_RING(MI_NOOP);
+	ADVANCE_LP_RING();
+    }
+
+    dri_post_submit(pI830->batch_bo);
+
+    dri_bo_unreference(pI830->batch_bo);
+    intel_next_batch(pScrn);
+}
diff --git a/src/i830_batchbuffer.h b/src/i830_batchbuffer.h
new file mode 100644
index 00000000..91793264
--- /dev/null
+++ b/src/i830_batchbuffer.h
@@ -0,0 +1,106 @@
+/**************************************************************************
+
+Copyright 1998-1999 Precision Insight, Inc., Cedar Park, Texas.
+Copyright © 2002 David Dawes
+
+All Rights Reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sub license, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR
+ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+**************************************************************************/
+
+#ifndef _INTEL_BATCHBUFFER_H
+#define _INTEL_BATCHBUFFER_H
+
+#define BATCH_RESERVED		16
+
+void intel_batch_init(ScrnInfoPtr pScrn);
+void intel_batch_teardown(ScrnInfoPtr pScrn);
+void intel_batch_flush(ScrnInfoPtr pScrn);
+
+static inline int
+intel_batch_space(I830Ptr pI830)
+{
+    return (pI830->batch_bo->size - BATCH_RESERVED) - (pI830->batch_used);
+}
+
+static inline void
+intel_batch_require_space(ScrnInfoPtr pScrn, I830Ptr pI830, GLuint sz)
+{
+    assert(sz < pI830->batch_bo->size - 8);
+    if (intel_batch_space(pI830) < sz)
+	intel_batch_flush(pScrn);
+}
+
+static inline void
+intel_batch_emit_dword(I830Ptr pI830, uint32_t dword)
+{
+    assert(pI830->batch_ptr != NULL);
+    assert(intel_batch_space(pI830) >= 4);
+    *(uint32_t *)(pI830->batch_ptr + pI830->batch_used) = dword;
+    pI830->batch_used += 4;
+}
+
+#define OUT_BATCH(dword) intel_batch_emit_dword(pI830, dword)
+
+union intfloat {
+	float f;
+	unsigned int ui;
+};
+
+#define OUT_BATCH_F(x) do {			\
+	union intfloat tmp;			\
+	tmp.f = (float)(x);			\
+	OUT_BATCH(tmp.ui);			\
+} while(0)
+
+#define BEGIN_BATCH(n)							\
+do {									\
+    if (pI830->batch_emitting != 0)					\
+	FatalError("%s: BEGIN_BATCH called without closing "		\
+		   "ADVANCE_BATCH\n", __FUNCTION__);			\
+    pI830->batch_emitting = (n) * 4;					\
+    intel_batch_require_space(pScrn, pI830, pI830->batch_emitting);	\
+    pI830->batch_emit_start = pI830->batch_used;			\
+} while (0)
+
+#define ADVANCE_BATCH() do {						\
+    if (pI830->batch_emitting == 0)					\
+	FatalError("%s: ADVANCE_BATCH called with no matching "		\
+		   "BEGIN_BATCH\n", __FUNCTION__);			\
+    if (pI830->batch_used > pI830->batch_emit_start + pI830->batch_emitting) \
+	FatalError("%s: ADVANCE_BATCH: exceeded allocation %d/%d\n ",	\
+		   __FUNCTION__,					\
+		   pI830->batch_used - pI830->batch_emit_start,		\
+		   pI830->batch_emitting);				\
+    if (pI830->batch_used < pI830->batch_emit_start + pI830->batch_emitting) \
+	FatalError("%s: ADVANCE_BATCH: under-used allocation %d/%d\n ", \
+		   __FUNCTION__,					\
+		   pI830->batch_used - pI830->batch_emit_start,		\
+		   pI830->batch_emitting);				\
+    if ((pI830->batch_emitting > 8) && (I810_DEBUG & DEBUG_ALWAYS_SYNC)) { \
+	/* Note: not actually syncing, just flushing each batch. */	\
+	intel_batch_flush(pScrn);					\
+    }									\
+    pI830->batch_emitting = 0;						\
+} while (0)
+
+#endif /* _INTEL_BATCHBUFFER_H */
diff --git a/src/i830_driver.c b/src/i830_driver.c
index 35df0c75..3a9d4252 100644
--- a/src/i830_driver.c
+++ b/src/i830_driver.c
@@ -1838,7 +1838,7 @@ i830_stop_ring(ScrnInfoPtr pScrn, Bool flush)
       temp = INREG(LP_RING + RING_LEN);
       if (temp & RING_VALID) {
 	 i830_refresh_ring(pScrn);
-	 I830Sync(pScrn);
+	 i830_wait_ring_idle(pScrn);
       }
 
       OUTREG(LP_RING + RING_LEN, 0);
@@ -2493,14 +2493,21 @@ I830BlockHandler(int i,
     pI830->BlockHandler = pScreen->BlockHandler;
     pScreen->BlockHandler = I830BlockHandler;
 
-    /* Emit a flush of the rendering cache, or on the 965 and beyond
-     * rendering results may not hit the framebuffer until significantly
-     * later.  In the direct rendering case this is already done just
-     * after the page flipping updates, so there's no need to duplicate
-     * the effort here.
-     */
-    if (pScrn->vtSema && !pI830->noAccel && !pI830->directRenderingEnabled)
-	I830EmitFlush(pScrn);
+    if (pScrn->vtSema && !pI830->noAccel) {
+       /* Emit a flush of the rendering cache, or on the 965 and beyond
+	* rendering results may not hit the framebuffer until significantly
+	* later.  In the direct rendering case this is already done just
+	* after the page flipping updates, so there's no need to duplicate
+	* the effort here.
+	*/
+       if (!pI830->noAccel && !pI830->directRenderingEnabled)
+	  I830EmitFlush(pScrn);
+
+       /* Flush the batch, so that any rendering is executed in a timely
+	* fashion.
+	*/
+       intel_batch_flush(pScrn);
+    }
 
     /*
      * Check for FIFO underruns at block time (which amounts to just
@@ -2704,6 +2711,55 @@ i830_memory_init(ScrnInfoPtr pScrn)
     return FALSE;
 }
 
+/**
+ * Returns a cookie to be waited on.  This is just a stub implementation, and
+ * should be hooked up to the emit/wait irq functions when available (DRI
+ * enabled).
+ */
+static unsigned int
+i830_fake_fence_emit(void *priv)
+{
+   static unsigned int fence = 0;
+
+   /* Match DRM in not using half the range. The fake bufmgr relies on this. */
+   if (++fence >= 0x8000000)
+      fence = 1;
+
+   return fence;
+}
+
+/**
+ * Waits on a cookie representing a request to be passed.
+ *
+ * Stub implementation that should be replaced with DRM functions when
+ * available.
+ */
+static int
+i830_fake_fence_wait(void *priv, unsigned int fence)
+{
+   ScrnInfoPtr pScrn = priv;
+
+   i830_wait_ring_idle(pScrn);
+
+   return 0;
+}
+
+static void
+i830_init_bufmgr(ScrnInfoPtr pScrn)
+{
+   I830Ptr pI830 = I830PTR(pScrn);
+
+   assert(pI830->FbBase != NULL);
+   pI830->bufmgr = intel_bufmgr_fake_init(pI830->fake_bufmgr_mem->offset,
+					  pI830->FbBase +
+					  pI830->fake_bufmgr_mem->offset,
+					  pI830->fake_bufmgr_mem->size,
+					  i830_fake_fence_emit,
+					  i830_fake_fence_wait,
+					  pScrn);
+}
+
+
 static Bool
 I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
 {
@@ -3033,6 +3089,8 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
    pI830->xoffset = (pScrn->fbOffset / pI830->cpp) % pScrn->displayWidth;
    pI830->yoffset = (pScrn->fbOffset / pI830->cpp) / pScrn->displayWidth;
 
+   i830_init_bufmgr(pScrn);
+
    vgaHWSetMmioFuncs(hwp, pI830->MMIOBase, 0);
    vgaHWGetIOBase(hwp);
    DPRINTF(PFX, "assert( if(!vgaHWMapMem(pScrn)) )\n");
@@ -3276,8 +3334,13 @@ I830LeaveVT(int scrnIndex, int flags)
 
    xf86_hide_cursors (pScrn);
 
+   I830Sync(pScrn);
+
    RestoreHWState(pScrn);
 
+   intel_bufmgr_fake_evict_all(pI830->bufmgr);
+   intel_batch_teardown(pScrn);
+
    i830_stop_ring(pScrn, TRUE);
 
    if (pI830->debug_modes) {
@@ -3351,6 +3414,8 @@ I830EnterVT(int scrnIndex, int flags)
    /* Update the screen pixmap in case the buffer moved */
    i830_update_front_offset(pScrn);
 
+   intel_batch_init(pScrn);
+
    if (IS_I965G(pI830))
       gen4_render_state_init(pScrn);
 
@@ -3478,6 +3543,9 @@ I830CloseScreen(int scrnIndex, ScreenPtr pScreen)
       I830LeaveVT(scrnIndex, 0);
    }
 
+   dri_bufmgr_destroy(pI830->bufmgr);
+   pI830->bufmgr = NULL;
+
    if (pI830->devicesTimer)
       TimerCancel(pI830->devicesTimer);
    pI830->devicesTimer = NULL;
diff --git a/src/i830_memory.c b/src/i830_memory.c
index dc48967f..57e4f2e2 100644
--- a/src/i830_memory.c
+++ b/src/i830_memory.c
@@ -350,6 +350,7 @@ i830_reset_allocations(ScrnInfoPtr pScrn)
     pI830->textures = NULL;
 #endif
     pI830->LpRing->mem = NULL;
+    pI830->fake_bufmgr_mem = NULL;
 }
 
 void
@@ -1374,6 +1375,14 @@ i830_allocate_2d_memory(ScrnInfoPtr pScrn)
 	return FALSE;
     }
 
+    pI830->fake_bufmgr_mem = i830_allocate_memory(pScrn, "fake bufmgr",
+						  MB(1), GTT_PAGE_SIZE, 0);
+    if (pI830->fake_bufmgr_mem == NULL) {
+	xf86DrvMsg(pScrn->scrnIndex, X_WARNING,
+		   "Failed to allocate fake bufmgr space.\n");
+	return FALSE;
+    }
+
     /* even in XAA, 965G needs state mem buffer for rendering */
     if (IS_I965G(pI830) && !pI830->noAccel &&
 	pI830->gen4_render_state_mem == NULL)
diff --git a/src/i830_ring.h b/src/i830_ring.h
index c2078fb4..c296d41a 100644
--- a/src/i830_ring.h
+++ b/src/i830_ring.h
@@ -42,11 +42,6 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
     pI830->ring_next &= pI830->LpRing->tail_mask;			\
 } while (0)
 
-union intfloat {
-	float f;
-	unsigned int ui;
-};
-
 #define OUT_RING_F(x) do {			\
 	union intfloat tmp;			\
 	tmp.f = (float)(x);			\
diff --git a/src/i915_3d.h b/src/i915_3d.h
index 1a0bd45b..d3330e5b 100644
--- a/src/i915_3d.h
+++ b/src/i915_3d.h
@@ -446,12 +446,12 @@ do {									\
 #define FS_END()							\
 do {									\
     int _i, _pad = (_cur_shader_commands & 0x1) ? 0 : 1;		\
-    BEGIN_LP_RING(_cur_shader_commands * 3 + 1 + _pad);			\
-    OUT_RING(_3DSTATE_PIXEL_SHADER_PROGRAM |				\
+    BEGIN_BATCH(_cur_shader_commands * 3 + 1 + _pad);			\
+    OUT_BATCH(_3DSTATE_PIXEL_SHADER_PROGRAM |				\
 	     (_cur_shader_commands * 3 - 1));				\
     for (_i = 0; _i < _cur_shader_commands * 3; _i++)			\
-	OUT_RING(_shader_buf[_i]);					\
+	OUT_BATCH(_shader_buf[_i]);					\
     if (_pad != 0)							\
-	OUT_RING(MI_NOOP);						\
-    ADVANCE_LP_RING();							\
+	OUT_BATCH(MI_NOOP);						\
+    ADVANCE_BATCH();							\
 } while (0);