Years ago, I fixed an R5000 O2 instability by implementing a workaround for

a chip bug, which was supposed to be fixed in that particular revision of
the die but wasn't (tlbhandler.S 1.16).

Being lazy, I did not write a runtime selection of the appropriate TLB
handler code, although this was on my list.

It turns out that this fix confuses the hell of R10000 processors revision 3
(but not earlier 2.x revisions), to the point of making the Origin 200 here
hang so hard it would not even enter the NMI handler (don't ask me how I
figured this was the cause).

So it's time to choose the appropriate TLB handling flavour at runtime,
building the trampoline code from the fixed exception handler location
jumping to the handler address at runtime. As a bonus, kernels linked in
KSEG0 get the address computation optimized and thus a smaller trampoline
than before.

2 files changed, 123 insertions, 62 deletions

diff --git a/sys/arch/mips64/mips64/tlbhandler.S b/sys/arch/mips64/mips64/tlbhandler.S
index c8141037618..3dce155233e 100644
--- a/sys/arch/mips64/mips64/tlbhandler.S
+++ b/sys/arch/mips64/mips64/tlbhandler.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tlbhandler.S,v 1.17 2009/05/22 20:37:53 miod Exp $ */
+/*	$OpenBSD: tlbhandler.S,v 1.18 2009/05/25 17:10:38 miod Exp $ */
 
 /*
  * Copyright (c) 1995-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -37,8 +37,6 @@
 #include <machine/regnum.h>
 #include <machine/cpustate.h>
 
-#define	HAIRY_R5000_ERRATA
-
 #include "assym.h"
 
 	.set	mips3
@@ -53,12 +51,11 @@
  *	available storage. If the startup code finds out that it
  *	is larger, the trampoline code is copied instead of panicing.
  */
-/*****************************  Start of code copied to exception vector */
-	.globl	tlb_miss	/* 0xffffffff80000000 */
 	.set	noat
-	.ent	tlb_miss, 0
-tlb_miss:
-#ifdef HAIRY_R5000_ERRATA
+
+	.globl	tlb_miss_err_r5k
+	.ent	tlb_miss_err_r5k, 0
+tlb_miss_err_r5k:
 	/*
 	 * R5000 errata: edge cases can trigger a TLB miss exception
 	 * instead of an invalid TLB exception. Servicing the TLB miss
@@ -75,12 +72,15 @@ tlb_miss:
 	 */
 	tlbp
 	mfc0	k1, COP_0_TLB_INDEX
-	bltz	k1, 1f			# missing!
+	bltz	k1, tlb_miss		# missing!
 	 nop
 	j	k_tlb_inv
 	 nop
-1:
-#endif
+	.end	tlb_miss_err_r5k
+
+	.globl	tlb_miss
+	.ent	tlb_miss, 0
+tlb_miss:
 	PTR_L	k1, curprocpaddr
 	dmfc0	k0, COP_0_BAD_VADDR
 	bltz	k0, _k_miss		# kernel address space
@@ -115,26 +115,24 @@ tlb_load:
 	eret				# RM7000 need 4 for JTLB usage.
 	.end	tlb_miss
 
-	.globl	e_tlb_miss
-e_tlb_miss:
-
 /*---------------------------------------------------------------- xtlb_miss
  *	Low level XTLB exception handler.
  */
-	.globl	xtlb_miss	/* 0xffffffff80000080 */
-	.set	noat
-	.ent	xtlb_miss, 0
-xtlb_miss:
-#ifdef HAIRY_R5000_ERRATA
+	.globl	xtlb_miss_err_r5k
+	.ent	xtlb_miss_err_r5k, 0
+xtlb_miss_err_r5k:
 	/* See errata comments in tlb_miss above */
 	tlbp
 	mfc0	k1, COP_0_TLB_INDEX
-	bltz	k1, 1f			# missing!
+	bltz	k1, xtlb_miss		# missing!
 	 nop
 	j	k_tlb_inv
 	 nop
-1:
-#endif
+	.end	xtlb_miss_err_r5k
+
+	.globl	xtlb_miss
+	.ent	xtlb_miss, 0
+xtlb_miss:
 	dmfc0	k0, COP_0_BAD_VADDR
 	bltz	k0, _k_miss		# kernel address space
 	PTR_SRL	k0, k0, SEGSHIFT
@@ -162,15 +160,9 @@ _k_miss:
 	dmfc0	k0, COP_0_BAD_VADDR	# must reload.
 	.end	xtlb_miss
 
-	.globl	e_xtlb_miss
-e_xtlb_miss:
-	.set	at
-/*****************************  End of code copied to exception vector */
-
 	.globl	tlb_miss_nopt
 	.ent	tlb_miss_nopt, 0
 tlb_miss_nopt:
-	.set	noat
 	mfc0	k0, COP_0_STATUS_REG
 	andi	k0, SR_KSU_USER
 	bne	k0, zero, go_u_general
@@ -178,35 +170,8 @@ tlb_miss_nopt:
 	j	k_general
 	nop
 	.end	tlb_miss_nopt
-	.set	at
 
-/*
- * Trampolines copied to exception vectors when code is too big.
- */
-	.globl	tlb_miss_tramp
-	.ent	tlb_miss_tramp, 0
-tlb_miss_tramp:
-	.set	noat
-	LA	k0, tlb_miss
-	jr	k0
-	nop
-	.end	tlb_miss_tramp
 	.set	at
-	.globl	e_tlb_miss_tramp
-e_tlb_miss_tramp:
-
-	.globl	xtlb_miss_tramp
-	.ent	xtlb_miss_tramp, 0
-xtlb_miss_tramp:
-	.set	noat
-	LA	k0, xtlb_miss
-	jr	k0
-	nop
-	.end	xtlb_miss_tramp
-	.set	at
-	.globl	e_xtlb_miss_tramp
-e_xtlb_miss_tramp:
-
 
 /*---------------------------------------------------------------- k_tlb_inv
  *	Handle a TLB invalid exception from kernel mode in kernel
diff --git a/sys/arch/sgi/sgi/machdep.c b/sys/arch/sgi/sgi/machdep.c
index c17009ce11a..503f421585e 100644
--- a/sys/arch/sgi/sgi/machdep.c
+++ b/sys/arch/sgi/sgi/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.67 2009/05/22 20:37:54 miod Exp $ */
+/*	$OpenBSD: machdep.c,v 1.68 2009/05/25 17:10:40 miod Exp $ */
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -137,6 +137,8 @@ caddr_t	allocsys(caddr_t);
 static void dobootopts(int, void *);
 static int atoi(const char *, int, const char **);
 
+void	build_trampoline(vaddr_t, vaddr_t);
+
 /*
  * Do all the stuff that locore normally does before calling main().
  * Reset mapping and set up mapping to hardware and init "wired" reg.
@@ -149,11 +151,14 @@ mips_init(int argc, void *argv, caddr_t boot_esym)
 	int i;
 	caddr_t sd;
 	u_int cputype;
+	vaddr_t tlb_handler, xtlb_handler;
 	extern char start[], edata[], end[];
-	extern char tlb_miss_tramp[], e_tlb_miss_tramp[];
-	extern char xtlb_miss_tramp[], e_xtlb_miss_tramp[];
 	extern char exception[], e_exception[];
 	extern char *hw_vendor, *hw_prod;
+	extern void tlb_miss;
+	extern void tlb_miss_err_r5k;
+	extern void xtlb_miss;
+	extern void xtlb_miss_err_r5k;
 
 	/*
 	 * Make sure we can access the extended address space.
@@ -527,14 +532,38 @@ mips_init(int argc, void *argv, caddr_t boot_esym)
 	/*
 	 * Copy down exception vector code.
 	 */
-	bcopy(tlb_miss_tramp, (char *)TLB_MISS_EXC_VEC,
-	    e_tlb_miss_tramp - tlb_miss_tramp);
-	bcopy(xtlb_miss_tramp, (char *)XTLB_MISS_EXC_VEC,
-	    e_xtlb_miss_tramp - xtlb_miss_tramp);
 	bcopy(exception, (char *)CACHE_ERR_EXC_VEC, e_exception - exception);
 	bcopy(exception, (char *)GEN_EXC_VEC, e_exception - exception);
 
 	/*
+	 * Build proper TLB refill handler trampolines.
+	 */
+	switch (cputype) {
+	case MIPS_R5000:
+		/*
+		 * R5000 processors need a specific chip bug workaround
+		 * in their tlb handlers.  Theoretically only revision 1
+		 * of the processor need it, but there is evidence
+		 * later versions also need it.
+		 *
+		 * This is also necessary on RM52x0; we test on the `rounded'
+		 * cputype value instead of sys_config.cpu[0].type; this
+		 * causes RM7k and RM9k to be included, just to be on the
+		 * safe side.
+		 */
+		tlb_handler = (vaddr_t)&tlb_miss_err_r5k;
+		xtlb_handler = (vaddr_t)&xtlb_miss_err_r5k;
+		break;
+	default:
+		tlb_handler = (vaddr_t)&tlb_miss;
+		xtlb_handler = (vaddr_t)&xtlb_miss;
+		break;
+	}
+
+	build_trampoline(TLB_MISS_EXC_VEC, tlb_handler);
+	build_trampoline(XTLB_MISS_EXC_VEC, xtlb_handler);
+
+	/*
 	 * Turn off bootstrap exception vectors.
 	 */
 	setsr(getsr() & ~SR_BOOT_EXC_VEC);
@@ -580,6 +609,73 @@ allocsys(caddr_t v)
 }
 
 /*
+ * Build a tlb trampoline
+ */
+void
+build_trampoline(vaddr_t addr, vaddr_t dest)
+{
+	const uint32_t insns[] = {
+		0x3c1a0000,	/* lui k0, imm16 */
+		0x675a0000,	/* daddiu k0, k0, imm16 */
+		0x001ad438,	/* dsll k0, k0, 0x10 */
+		0x675a0000,	/* daddiu k0, k0, imm16 */
+		0x001ad438,	/* dsll k0, k0, 0x10 */
+		0x675a0000,	/* daddiu k0, k0, imm16 */
+		0x03400008,	/* jr k0 */
+		0x00000000	/* nop */
+	};
+	uint32_t *dst = (uint32_t *)addr;
+	const uint32_t *src = insns;
+	uint32_t a, b, c, d;
+
+	/*
+	 * Decompose the handler address in the four components which,
+	 * added with sign extension, will produce the correct address.
+	 */
+	d = dest & 0xffff;
+	dest >>= 16;
+	if (d & 0x8000)
+		dest++;
+	c = dest & 0xffff;
+	dest >>= 16;
+	if (c & 0x8000)
+		dest++;
+	b = dest & 0xffff;
+	dest >>= 16;
+	if (b & 0x8000)
+		dest++;
+	a = dest & 0xffff;
+
+	/*
+	 * Build the trampoline, skipping noop computations.
+	 */
+	*dst++ = *src++ | a;
+	if (b != 0)
+		*dst++ = *src++ | b;
+	else
+		src++;
+	*dst++ = *src++;
+	if (c != 0)
+		*dst++ = *src++ | c;
+	else
+		src++;
+	*dst++ = *src++;
+	if (d != 0)
+		*dst++ = *src++ | d;
+	else
+		src++;
+	*dst++ = *src++;
+	*dst++ = *src++;
+
+	/*
+	 * Note that we keep the delay slot instruction a nop, instead
+	 * of branching to the second instruction of the handler and
+	 * having its first instruction in the delay slot, so that the
+	 * tlb handler is free to use k0 immediately.
+	 */
+}
+
+/*
  * Decode boot options.
  */
 static void