Introduce assembly macros for specific processor hazards: tlb update, status

register update, status register update causing a change to the interrupt
enable flag, and a few other arcane ones. <mips64/asm.h> will provide
(supposedly sane) defaults, and <machine/asm.h> may override these with
better tuned versions.

Use these macros instead of random strings of nop in the various .S files
requiring hazard workarounds.

10 files changed, 394 insertions, 463 deletions

diff --git a/sys/arch/mips64/include/asm.h b/sys/arch/mips64/include/asm.h
index 19d7f292250..8de2128eb92 100644
--- a/sys/arch/mips64/include/asm.h
+++ b/sys/arch/mips64/include/asm.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: asm.h,v 1.16 2012/09/29 18:56:23 miod Exp $ */
+/*	$OpenBSD: asm.h,v 1.17 2012/09/29 19:02:25 miod Exp $ */
 
 /*
  * Copyright (c) 2001-2002 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -30,17 +30,6 @@
 
 #include <machine/regdef.h>
 
-/*
- * Due to a flaw in RM7000 1.x processors a pipeline 'drain' is
- * required after some mtc0 instructions.
- * Ten nops in sequence does the trick.
- */
-#ifdef CPU_RM7000
-#define ITLBNOPFIX      nop;nop;nop;nop;nop;nop;nop;nop;nop;nop
-#else
-#define ITLBNOPFIX      nop;nop;nop;nop
-#endif
-
 #define	_MIPS_ISA_MIPS1	1	/* R2000/R3000 */
 #define	_MIPS_ISA_MIPS2	2	/* R4000/R6000 */
 #define	_MIPS_ISA_MIPS3	3	/* R4000 */
@@ -323,4 +312,49 @@ x: ;				\
 	LA	ci, cpu_info_primary
 #endif /* MULTIPROCESSOR */
 
+/*
+ * Hazards
+ */
+
+#ifdef CPU_RM7000
+/*
+ * Due to a flaw in RM7000 1.x processors a pipeline 'drain' is
+ * required after some mtc0 instructions.
+ * Ten nops in sequence does the trick.
+ */
+#define	MTC0_HAZARD		NOP;NOP;NOP;NOP;NOP;NOP;NOP;NOP;NOP;NOP
+#define	MTC0_SR_IE_HAZARD	MTC0_HAZARD
+/*
+ * The RM7000 needs twice as much nops around tlb* instructions.
+ */
+#define	TLB_HAZARD		NOP; NOP; NOP; NOP
+#endif
+
+/* Hazard between {d,}mfc0 of COP_0_VADDR */
+#ifndef	PRE_MFC0_ADDR_HAZARD
+#define	PRE_MFC0_ADDR_HAZARD	/* nothing */
+#endif
+
+/* Hazard after {d,}mfc0 from any register */
+#ifndef	MFC0_HAZARD
+#define	MFC0_HAZARD     	/* nothing */
+#endif
+/* Hazard after {d,}mtc0 to any register */
+#ifndef	MTC0_HAZARD
+#define	MTC0_HAZARD     	NOP; NOP; NOP; NOP
+#endif
+/* Hazard after {d,}mtc0 to COP_0_SR affecting the state of interrupts */
+#ifndef	MTC0_SR_IE_HAZARD
+#define	MTC0_SR_IE_HAZARD	MTC0_HAZARD
+#endif
+/* Hazard after {d,}mtc0 to COP_0_SR affecting the state of coprocessors */
+#ifndef	MTC0_SR_CU_HAZARD
+#define	MTC0_SR_CU_HAZARD	NOP; NOP
+#endif
+
+/* Hazard before and after a tlbp, tlbr, tlbwi or tlbwr instruction */
+#ifndef	TLB_HAZARD
+#define	TLB_HAZARD		NOP; NOP
+#endif
+
 #endif /* !_MIPS64_ASM_H_ */
diff --git a/sys/arch/mips64/include/cpustate.h b/sys/arch/mips64/include/cpustate.h
index 80f642bcc1e..f237d641c53 100644
--- a/sys/arch/mips64/include/cpustate.h
+++ b/sys/arch/mips64/include/cpustate.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpustate.h,v 1.8 2009/10/22 22:08:52 miod Exp $ */
+/*	$OpenBSD: cpustate.h,v 1.9 2012/09/29 19:02:25 miod Exp $ */
 
 /*
  * Copyright (c) 2002-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -53,12 +53,17 @@
 	SAVE_REG(a1, A1, frame, bo)		;\
 	SAVE_REG(a2, A2, frame, bo)		;\
 	SAVE_REG(a3, A3, frame, bo)		;\
+	MFC0	a0, COP_0_CAUSE_REG		;\
 	SAVE_REG($8, T0, frame, bo)		;\
 	SAVE_REG($9, T1, frame, bo)		;\
+	MFC0	a1, COP_0_STATUS_REG		;\
 	SAVE_REG($10, T2, frame, bo)		;\
 	SAVE_REG($11, T3, frame, bo)		;\
+	PRE_MFC0_ADDR_HAZARD			;\
+	DMFC0	a2, COP_0_BAD_VADDR		;\
 	SAVE_REG($12, T4, frame, bo)		;\
 	SAVE_REG($13, T5, frame, bo)		;\
+	DMFC0	a3, COP_0_EXC_PC		;\
 	SAVE_REG($14, T6, frame, bo)		;\
 	SAVE_REG($15, T7, frame, bo)		;\
 	SAVE_REG(t8, T8, frame, bo)		;\
@@ -67,10 +72,6 @@
 	SAVE_REG(ra, RA, frame, bo)		;\
 	mflo	v0				;\
 	mfhi	v1				;\
-	mfc0	a0, COP_0_CAUSE_REG		;\
-	mfc0	a1, COP_0_STATUS_REG		;\
-	dmfc0	a2, COP_0_BAD_VADDR		;\
-	dmfc0	a3, COP_0_EXC_PC		;\
 	SAVE_REG(v0, MULLO, frame, bo)		;\
 	SAVE_REG(v1, MULHI, frame, bo)		;\
 	SAVE_REG(a0, CAUSE, frame, bo)		;\
@@ -104,10 +105,12 @@
 	RESTORE_REG(t1, SR, frame, bo)		;\
 	RESTORE_REG(t2, MULLO, frame, bo)	;\
 	RESTORE_REG(t3, MULHI, frame, bo)	;\
-	mtc0	t1, COP_0_STATUS_REG		;\
+	MTC0	t1, COP_0_STATUS_REG		;\
+	MTC0_SR_IE_HAZARD			;\
 	mtlo	t2				;\
 	mthi	t3				;\
-	dmtc0	a0, COP_0_EXC_PC		;\
+	DMTC0	a0, COP_0_EXC_PC		;\
+	MTC0_HAZARD				;\
 	RESTORE_REG(AT, AST, frame, bo)		;\
 	RESTORE_REG(v0, V0, frame, bo)		;\
 	RESTORE_REG(v1, V1, frame, bo)		;\
@@ -141,4 +144,3 @@
 	RESTORE_REG(s6, S6, frame, bo)		;\
 	RESTORE_REG(s7, S7, frame, bo)		;\
 	RESTORE_REG(s8, S8, frame, bo)
-
diff --git a/sys/arch/mips64/mips64/cache_r5k.c b/sys/arch/mips64/mips64/cache_r5k.c
index 06e0f6cb31f..51c48d4e913 100644
--- a/sys/arch/mips64/mips64/cache_r5k.c
+++ b/sys/arch/mips64/mips64/cache_r5k.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cache_r5k.c,v 1.4 2012/09/29 18:54:38 miod Exp $	*/
+/*	$OpenBSD: cache_r5k.c,v 1.5 2012/09/29 19:02:26 miod Exp $	*/
 
 /*
  * Copyright (c) 2012 Miodrag Vallat.
@@ -132,7 +132,7 @@ static __inline__ void
 set_config(uint32_t cfg)
 {
 	__asm__ __volatile__ ("mtc0 %0, $16" :: "r"(cfg)); /* COP_0_CONFIG */
-	/* ITLBNOPFIX */
+	/* MTC0_HAZARD */
 #ifdef CPU_RM7000
 	nop10();
 #else
diff --git a/sys/arch/mips64/mips64/context.S b/sys/arch/mips64/mips64/context.S
index be7c1b37477..f2e74a8c224 100644
--- a/sys/arch/mips64/mips64/context.S
+++ b/sys/arch/mips64/mips64/context.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: context.S,v 1.47 2012/09/29 18:58:30 miod Exp $ */
+/*	$OpenBSD: context.S,v 1.48 2012/09/29 19:02:26 miod Exp $ */
 
 /*
  * Copyright (c) 2002-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -41,7 +41,6 @@
 #include "assym.h"
 
 	.set	mips3
-
 	.set	noreorder		# Noreorder is default style!
 
 /*
@@ -52,7 +51,8 @@ LEAF(savectx, 0)
 	REG_S	s1, PCB_CONTEXT+1*REGSZ(a0)
 	REG_S	s2, PCB_CONTEXT+2*REGSZ(a0)
 	REG_S	s3, PCB_CONTEXT+3*REGSZ(a0)
-	mfc0	v0, COP_0_STATUS_REG
+	MFC0	v0, COP_0_STATUS_REG
+	MFC0_HAZARD
 	REG_S	s4, PCB_CONTEXT+4*REGSZ(a0)
 	REG_S	s5, PCB_CONTEXT+5*REGSZ(a0)
 	REG_S	s6, PCB_CONTEXT+6*REGSZ(a0)
@@ -66,22 +66,22 @@ LEAF(savectx, 0)
 	REG_S	t1, PCB_CONTEXT+12*REGSZ(a0)	# save status register
 #endif
 	j	ra
-	move	v0, zero
+	 move	v0, zero
 END(savectx)
 
 LEAF(cpu_idle_enter, 0)
 	j	ra
-	nop
+	 NOP
 END(cpu_idle_enter)
 
 LEAF(cpu_idle_leave, 0)
 	j	ra
-	nop
+	 NOP
 END(cpu_idle_leave)
 
 LEAF(cpu_idle_cycle, 0)
 	j	ra
-	nop
+	 NOP
 END(cpu_idle_cycle)
 
 /*
@@ -97,7 +97,7 @@ NON_LEAF(cpu_switchto_asm, FRAMESZ(CF_SZ), ra)
 	.mask	0x80000000, (CF_RA_OFFS - FRAMESZ(CF_SZ))
 
 	beqz	a0, 1f
-	mfc0	v0, COP_0_STATUS_REG
+	 MFC0	v0, COP_0_STATUS_REG
 
 	REG_S	s0, PCB_CONTEXT+0*REGSZ(t3)	# do a 'savectx()'
 	REG_S	s1, PCB_CONTEXT+1*REGSZ(t3)
@@ -119,10 +119,10 @@ NON_LEAF(cpu_switchto_asm, FRAMESZ(CF_SZ), ra)
 	/*
 	 * Disable interrupts
 	 */
-	li	v1, ~SR_INT_ENAB
-	and	v0, v0, v1
-	mtc0	v0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	ori	v0, SR_INT_ENAB
+	xori	v0, SR_INT_ENAB
+	MTC0	v0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 
 	/*
 	 * Switch to new context
@@ -153,7 +153,7 @@ NON_LEAF(cpu_switchto_asm, FRAMESZ(CF_SZ), ra)
 #endif
 	lw	v0, PM_ASID(t1)			# ->pm_asid[cpuid].pma_asid
 
-#if UPAGES > 1
+#if UPAGES > 1	/* { */
 	or	v0, t3
 	dmtc0	v0, COP_0_TLB_HI		# init high entry (tlbid)
 
@@ -172,7 +172,9 @@ NON_LEAF(cpu_switchto_asm, FRAMESZ(CF_SZ), ra)
 	bltz	t2, ctx3			# not mapped.
 	PTR_SRL	t2, PGSHIFT+1
 	PTR_L	t1, Sysmap
+	TLB_HAZARD
 	tlbp
+	TLB_HAZARD			# necessary?
 	PTR_SLL	t2, 3
 	PTR_ADDU t1, t2				# t1 now points at ptes.
 	mfc0	t0, COP_0_TLB_INDEX
@@ -183,14 +185,9 @@ NON_LEAF(cpu_switchto_asm, FRAMESZ(CF_SZ), ra)
 	dmtc0	t2, COP_0_TLB_HI		# invalidate it.
 	dmtc0	zero, COP_0_TLB_LO0
 	dmtc0	zero, COP_0_TLB_LO1
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwi
-	nop
-	nop
-	nop
+	TLB_HAZARD
 
 ctx1:
 	mtc0	zero, COP_0_TLB_INDEX
@@ -203,20 +200,20 @@ ctx1:
 	dsrl	ta1, ta1, (64 - PG_FRAMEBITS)
 	dmtc0	ta0, COP_0_TLB_LO0
 	dmtc0	ta1, COP_0_TLB_LO1
-	nop
-	PTR_ADDU v0, 2*NBPG
-	nop
-	nop
+	PTR_ADDU v0, 2*PAGE_SIZE
+	TLB_HAZARD
 	tlbwi
+	TLB_HAZARD
 
-#if UPAGES > 2
+#if UPAGES > 2	/* { */
 	dmtc0	v0, COP_0_TLB_HI		# init high entry (tlbid)
 	lw	ta0, 8(t1)
 	lw	ta1, 12(t1)
 	dsll	ta0, ta0, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
 	dsrl	ta0, ta0, (64 - PG_FRAMEBITS)
+	TLB_HAZARD
 	tlbp
-	nop
+	TLB_HAZARD			# necessary?
 	dsll	ta1, ta1, (64 - PG_FRAMEBITS)
 	dsrl	ta1, ta1, (64 - PG_FRAMEBITS)
 	mfc0	t0, COP_0_TLB_INDEX
@@ -227,33 +224,22 @@ ctx1:
 	dmtc0	t2, COP_0_TLB_HI		# invalidate it.
 	dmtc0	zero, COP_0_TLB_LO0
 	dmtc0	zero, COP_0_TLB_LO1
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwi
-	nop
-	nop
-	nop
+	TLB_HAZARD
 
 ctx2:
 	mtc0	t2, COP_0_TLB_INDEX
 	dmtc0	v0, COP_0_TLB_HI
 	dmtc0	ta0, COP_0_TLB_LO0
 	dmtc0	ta1, COP_0_TLB_LO1
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwi
-#endif	/* UPAGES > 2 */
-	nop
-	nop
-	nop
-	nop
-#else	/* UPAGES > 1 */
+	TLB_HAZARD
+#endif	/* } UPAGES > 2 */
+#else	/* } UPAGES > 1 { */
 	dmtc0	v0, COP_0_TLB_HI		# init high entry (tlbid)
-#endif	/* UPAGES > 1 */
+#endif	/* } UPAGES > 1 */
 
 ctx3:
 
@@ -283,10 +269,10 @@ ctx3:
 	ctc0	v1, COP_0_ICR		# XXX RM7000
 #endif
 	ori	v0, v0, SR_INT_ENAB
-	mtc0	v0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	v0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 	j	ra
-	nop
+	 NOP
 END(cpu_switchto_asm)
 
 /*-------------------------------------------------------------- proc_trampoline
diff --git a/sys/arch/mips64/mips64/cp0access.S b/sys/arch/mips64/mips64/cp0access.S
index aa2d67f780f..c0918228142 100644
--- a/sys/arch/mips64/mips64/cp0access.S
+++ b/sys/arch/mips64/mips64/cp0access.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cp0access.S,v 1.16 2012/06/23 21:53:38 miod Exp $ */
+/*	$OpenBSD: cp0access.S,v 1.17 2012/09/29 19:02:26 miod Exp $ */
 
 /*
  * Copyright (c) 2001-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -40,6 +40,7 @@
 
 #include "assym.h"
 
+	.set	mips3
 	.set	noreorder		# Noreorder is default style!
 
 /*
@@ -47,39 +48,47 @@
  */
 
 LEAF(setsoftintr0, 0)
-	mfc0	v0, COP_0_CAUSE_REG	# read cause register
-	nop
+	MFC0	v0, COP_0_CAUSE_REG	# read cause register
+	MFC0_HAZARD
+	NOP
 	or	v0, v0, SOFT_INT_MASK_0	# set soft clock interrupt
-	mtc0	v0, COP_0_CAUSE_REG	# save it
+	MTC0	v0, COP_0_CAUSE_REG	# save it
+	MTC0_HAZARD
 	j	ra
-	nop
+	 NOP
 END(setsoftintr0)
 
 LEAF(clearsoftintr0, 0)
-	mfc0	v0, COP_0_CAUSE_REG	# read cause register
-	nop
+	MFC0	v0, COP_0_CAUSE_REG	# read cause register
+	MFC0_HAZARD
+	NOP
 	and	v0, v0, ~SOFT_INT_MASK_0	# clear soft clock interrupt
-	mtc0	v0, COP_0_CAUSE_REG	# save it
+	MTC0	v0, COP_0_CAUSE_REG	# save it
+	MTC0_HAZARD
 	j	ra
-	nop
+	 NOP
 END(clearsoftintr0)
 
 LEAF(setsoftintr1, 0)
-	mfc0	v0, COP_0_CAUSE_REG	# read cause register
-	nop
+	MFC0	v0, COP_0_CAUSE_REG	# read cause register
+	MFC0_HAZARD
+	NOP
 	or	v0, v0, SOFT_INT_MASK_1	# set soft net interrupt
-	mtc0	v0, COP_0_CAUSE_REG	# save it
+	MTC0	v0, COP_0_CAUSE_REG	# save it
+	MTC0_HAZARD
 	j	ra
-	nop
+	 NOP
 END(setsoftintr1)
 
 LEAF(clearsoftintr1, 0)
-	mfc0	v0, COP_0_CAUSE_REG	# read cause register
-	nop
+	MFC0	v0, COP_0_CAUSE_REG	# read cause register
+	MFC0_HAZARD
+	NOP
 	and	v0, v0, ~SOFT_INT_MASK_1	# clear soft net interrupt
-	mtc0	v0, COP_0_CAUSE_REG	# save it
+	MTC0	v0, COP_0_CAUSE_REG	# save it
+	MTC0_HAZARD
 	j	ra
-	nop
+	 NOP
 END(clearsoftintr1)
 
 /*
@@ -88,23 +97,25 @@ END(clearsoftintr1)
  */
 
 LEAF(enableintr, 0)
-	mfc0	v0, COP_0_STATUS_REG	# read status register
-	nop
+	MFC0	v0, COP_0_STATUS_REG	# read status register
+	MFC0_HAZARD
+	NOP
 	or	v1, v0, SR_INT_ENAB
-	mtc0	v1, COP_0_STATUS_REG	# enable all interrupts
-	ITLBNOPFIX
+	MTC0	v1, COP_0_STATUS_REG	# enable all interrupts
+	MTC0_SR_IE_HAZARD
 	j	ra
-	nop
+	 NOP
 END(enableintr)
 
 LEAF(disableintr, 0)
-	mfc0	v0, COP_0_STATUS_REG	# read status register
-	nop
+	MFC0	v0, COP_0_STATUS_REG	# read status register
+	MFC0_HAZARD
+	NOP
 	and	v1, v0, ~SR_INT_ENAB
-	mtc0	v1, COP_0_STATUS_REG	# disable all interrupts
-	ITLBNOPFIX			# Propagate new status
+	MTC0	v1, COP_0_STATUS_REG	# disable all interrupts
+	MTC0_SR_IE_HAZARD
 	j	ra
-	nop
+	 NOP
 END(disableintr)
 
 LEAF(updateimask, 0)
@@ -121,82 +132,91 @@ LEAF(updateimask, 0)
 	or	v1, v0
 	ctc0	v1, COP_0_ICR
 #endif
-	mfc0	v0, COP_0_STATUS_REG
-	li	v1, ~SR_INT_MASK
+	MFC0	v0, COP_0_STATUS_REG
+	MFC0_HAZARD
+	LI	v1, ~SR_INT_MASK
 	and	v1, v0
 	and	v0, a0, SR_INT_MASK
 	or	v1, v0
-	mtc0	v1, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	v1, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 	ori	v1, SR_INT_ENAB		# enable interrupts
-	mtc0	v1, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	v1, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 	jr	ra
-	move	v0, v1
+	 move	v0, v1
 END(updateimask)
 
 LEAF(setsr, 0)
-	mtc0	a0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	a0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 	jr	ra
-	move	v0, a0
+	 move	v0, a0
 END(setsr)
 
 LEAF(getsr, 0)
-	mfc0	v0, COP_0_STATUS_REG
-	jr	ra
-	nop
+	MFC0	v0, COP_0_STATUS_REG
+	MFC0_HAZARD
+	j	ra
+	 NOP
 END(getsr)
 
 LEAF(cp0_get_config, 0)
-	mfc0	v0, COP_0_CONFIG
+	MFC0	v0, COP_0_CONFIG
+	MFC0_HAZARD
 	j	ra
-	nop
+	 NOP
 END(cp0_get_config)
 
-/* WARNING! Needs to be invoked from uncached address. */
+/* WARNING!
+   Needs to be invoked from uncached address if changing cache settings. */
 LEAF(cp0_set_config, 0)
-	mtc0	a0, COP_0_CONFIG
-	ITLBNOPFIX
+	MTC0	a0, COP_0_CONFIG
+	MTC0_HAZARD
 	j	ra
-	nop
+	 NOP
 END(cp0_set_config)
 
 LEAF(cp0_get_prid, 0)
-	mfc0	v0, COP_0_PRID
+	MFC0	v0, COP_0_PRID
+	MFC0_HAZARD
 	j	ra
-	nop
+	NOP
 END(cp0_get_prid)
 
 LEAF(cp0_get_count, 0)
-	mfc0	v0, COP_0_COUNT
+	MFC0	v0, COP_0_COUNT
+	MFC0_HAZARD
 	j	ra
-	nop
+	NOP
 END(cp0_get_count)
 
 LEAF(cp0_set_compare, 0)
-	mtc0	a0, COP_0_COMPARE
+	MTC0	a0, COP_0_COMPARE
+	MTC0_HAZARD
 	j	ra
-	nop
+	NOP
 END(cp0_set_compare)
 
+#ifdef notused
 LEAF(cp0_getperfcount, 0)
-	mfc0	v0, COP_0_PC_COUNT
-	nop; nop
+	MFC0	v0, COP_0_PC_COUNT
+	MFC0_HAZARD
 	j	ra
-	nop
+	NOP
 END(cp0_getperfcount)
 
 LEAF(cp0_setperfcount, 0)
-	mtc0	a0, COP_0_PC_COUNT
-	nop; nop
+	MTC0	a0, COP_0_PC_COUNT
+	MTC0_HAZARD
 	j	ra
-	nop
+	NOP
 END(cp0_setperfcount)
 
 LEAF(cp0_setperfctrl, 0)
-	mtc0	a0, COP_0_PC_CTRL
-	nop; nop
+	MTC0	a0, COP_0_PC_CTRL
+	MTC0_HAZARD
 	j	ra
-	nop
+	NOP
 END(cp0_setperfctrl)
+#endif
diff --git a/sys/arch/mips64/mips64/exception.S b/sys/arch/mips64/mips64/exception.S
index 94bf49f0a27..e01a9a8073b 100644
--- a/sys/arch/mips64/mips64/exception.S
+++ b/sys/arch/mips64/mips64/exception.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: exception.S,v 1.34 2012/09/29 18:58:30 miod Exp $ */
+/*	$OpenBSD: exception.S,v 1.35 2012/09/29 19:02:26 miod Exp $ */
 
 /*
  * Copyright (c) 2002-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -146,8 +146,8 @@ exception:
 	li	k0, COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
 	dmtc0	k0, COP_0_DIAG
 #endif
-	mfc0	k0, COP_0_STATUS_REG
-	mfc0	k1, COP_0_CAUSE_REG
+	MFC0	k0, COP_0_STATUS_REG
+	MFC0	k1, COP_0_CAUSE_REG
 	and	k0, k0, SR_KSU_USER
 	beqz	k0, k_exception		# Kernel mode mode
 	and	k1, k1, CR_EXC_CODE
@@ -194,9 +194,8 @@ NNON_LEAF(k_intr, FRAMESZ(KERN_EXC_FRAME_SIZE), ra)
 	move	sp, k0			# Already on kernel stack
 	LA	gp, _gp
 	and	t0, a1, ~(SR_COP_1_BIT | SR_EXL | SR_INT_ENAB | SR_KSU_MASK)
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
-	ITLBNOPFIX			# XXX necessary?
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 	PTR_S	a0, 0(sp)
 	jal	interrupt
 	PTR_S	a3, CF_RA_OFFS + KERN_REG_SIZE(sp)
@@ -209,8 +208,7 @@ NNON_LEAF(k_intr, FRAMESZ(KERN_EXC_FRAME_SIZE), ra)
 #endif
 	RESTORE_CPU(sp, CF_RA_OFFS)
 	PTR_ADDU sp, sp, FRAMESZ(KERN_EXC_FRAME_SIZE)
-	sync
-	eret
+	ERET
 	.set	at
 END(k_intr)
 
@@ -234,17 +232,16 @@ NNON_LEAF(u_intr, FRAMESZ(CF_SZ), ra)
 	LA	gp, _gp
 	.set	at
 	and	t0, a1, ~(SR_COP_1_BIT | SR_EXL | SR_INT_ENAB | SR_KSU_MASK)
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
-	ITLBNOPFIX			# XXX necessary?
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 	PTR_S	a0, 0(sp)
 	jal	interrupt
 	PTR_S	a3, CF_RA_OFFS(sp)	# for debugging
 
-	mfc0	t0, COP_0_STATUS_REG	# enable interrupts before checking
+	MFC0	t0, COP_0_STATUS_REG	# enable interrupts before checking
 	ori	t0, SR_INT_ENAB		# for ast.
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 
 	GET_CPU_INFO(t1, t0)
 0:
@@ -312,15 +309,15 @@ NNON_LEAF(u_intr, FRAMESZ(CF_SZ), ra)
 	nop
 
 4:
-	mfc0	t0, COP_0_STATUS_REG	# disable interrupts
-	li	v0, ~SR_INT_ENAB
+	MFC0	t0, COP_0_STATUS_REG	# disable interrupts
+	LI	v0, ~SR_INT_ENAB
 	and	t0, t0, v0
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 
 	ori	t0, SR_EXL		# restoring to user mode.
-	mtc0	t0, COP_0_STATUS_REG	# must set exception level bit.
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG	# must set exception level bit.
+	MTC0_SR_IE_HAZARD
 
 	# t1 is curcpu() from earlier
 	move	k1, t1
@@ -337,8 +334,7 @@ NNON_LEAF(u_intr, FRAMESZ(CF_SZ), ra)
 	RESTORE_REG(sp, SP, k0, 0)
 	LI	k0, 0
 	LI	k1, 0
-	sync
-	eret
+	ERET
 	.set	at
 END(u_intr)
 
@@ -363,17 +359,17 @@ NNON_LEAF(k_general, FRAMESZ(KERN_EXC_FRAME_SIZE), ra)
 	move	sp, k0			# Already on kernel stack
 	LA	gp, _gp
 	and	t0, a1, ~(SR_COP_1_BIT | SR_EXL | SR_INT_ENAB | SR_KSU_MASK)
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 	PTR_S	a0, 0(sp)
 	jal	trap
 	PTR_S	a3, CF_RA_OFFS + KERN_REG_SIZE(sp)
 
-	mfc0	t0, COP_0_STATUS_REG	# disable interrupts
-	li	t1, ~SR_INT_ENAB
+	MFC0	t0, COP_0_STATUS_REG	# disable interrupts
+	LI	t1, ~SR_INT_ENAB
 	and	t0, t0, t1
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 
 	.set	noat
 #ifdef RM7000_ICR
@@ -383,8 +379,7 @@ NNON_LEAF(k_general, FRAMESZ(KERN_EXC_FRAME_SIZE), ra)
 	RESTORE_REG(a0, PC, sp, CF_RA_OFFS)
 	RESTORE_CPU(sp, CF_RA_OFFS)
 	PTR_ADDU sp, sp, FRAMESZ(KERN_EXC_FRAME_SIZE)
-	sync
-	eret
+	ERET
 	.set	at
 END(k_general)
 
@@ -407,8 +402,8 @@ NNON_LEAF(u_general, FRAMESZ(CF_SZ), ra)
 	LA	gp, _gp
 	.set	at
 	and	t0, a1, ~(SR_COP_1_BIT | SR_EXL | SR_INT_ENAB | SR_KSU_MASK)
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 
 #ifdef PERFCNTRS
 	lw	t0, cpu_is_rm7k
@@ -451,7 +446,8 @@ NNON_LEAF(u_general, FRAMESZ(CF_SZ), ra)
 	lw	t0, cpu_is_rm7k
 	beqz	t0, 1f			# not an RM7K. Don't do perf setup.
 
-	LOAD	t0, CI_CURPROC(k1)	# set up rm7k.
+	GET_CPU_INFO(t1, t0)
+	PTR_L	t0, CI_CURPROC(k1)	# set up rm7k.
 	ld	v0, P_WATCH_1(t0)
 	dmtc0	v0, COP_0_WATCH_1
 	ld	v0, P_WATCH_2(t0)
@@ -467,15 +463,15 @@ NNON_LEAF(u_general, FRAMESZ(CF_SZ), ra)
 	nop;nop;nop;nop
 1:
 #endif
-	mfc0	t0, COP_0_STATUS_REG	# disable interrupts
-	li	t1, ~SR_INT_ENAB
+	MFC0	t0, COP_0_STATUS_REG	# disable interrupts
+	LI	t1, ~SR_INT_ENAB
 	and	t0, t0, t1
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 
 	ori	t0, SR_EXL		# restoring to user mode.
-	mtc0	t0, COP_0_STATUS_REG	# must set exception level bit.
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG	# must set exception level bit.
+	MTC0_SR_IE_HAZARD
 
 	GET_CPU_INFO(k1, k0)
 	PTR_L	k0, CI_CURPROCPADDR(k1)
@@ -492,7 +488,6 @@ NNON_LEAF(u_general, FRAMESZ(CF_SZ), ra)
 	RESTORE_REG(sp, SP, k0, 0)
 	LI	k0, 0
 	LI	k1, 0
-	sync
-	eret
+	ERET
 	.set	at
 END(u_general)
diff --git a/sys/arch/mips64/mips64/lcore_access.S b/sys/arch/mips64/mips64/lcore_access.S
index 3317c3363c4..1986f1e39db 100644
--- a/sys/arch/mips64/mips64/lcore_access.S
+++ b/sys/arch/mips64/mips64/lcore_access.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: lcore_access.S,v 1.21 2012/04/16 22:23:04 miod Exp $ */
+/*	$OpenBSD: lcore_access.S,v 1.22 2012/09/29 19:02:26 miod Exp $ */
 
 /*
  * Copyright (c) 2001-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -98,17 +98,17 @@ LEAF(copystr, 0)
 	sb	t0, 0(a1)
 	PTR_ADDU a0, a0, 1
 	beq	t0, zero, 2f
-	PTR_ADDU a1, a1, 1
+	 PTR_ADDU a1, a1, 1
 	bne	a2, zero, 1b
-	nop
+	 NOP
 	LI	v0, ENAMETOOLONG	# String is longer than maxlength
 2:
 	beq	a3, zero, 3f
-	PTR_SUBU a2, t2, a2		# Compute length copied
+	 PTR_SUBU a2, t2, a2		# Compute length copied
 	REG_S	a2, 0(a3)
 3:
 	j	ra
-	nop
+	 NOP
 END(copystr)
 
 /*
@@ -116,14 +116,14 @@ END(copystr)
  * mem_zero_page(addr);
  */
 LEAF(mem_zero_page, 0)
-	LI	v0, NBPG
+	LI	v0, PAGE_SIZE
 1:
 	PTR_SUBU v0, 8
 	sd	zero, 0(a0)
 	bne	zero, v0, 1b
-	PTR_ADDU a0, 8
+	 PTR_ADDU a0, 8
 	jr	ra
-	nop
+	 NOP
 END(mem_zero_page)
 
 /*
@@ -141,11 +141,11 @@ NON_LEAF(copyinstr, FRAMESZ(CF_SZ), ra)
 	.mask	0x80000000, (CF_RA_OFFS - FRAMESZ(CF_SZ))
 	PTR_S	ra, CF_RA_OFFS(sp)
 	blt	a0, zero, _copyerr	# make sure address is in user space
-	LI	v0, KT_COPYERR
+	 LI	v0, KT_COPYERR
 	GET_CPU_INFO(t1, t0)
 	PTR_L	t3, CI_CURPROCPADDR(t1)
 	jal	copystr
-	sw	v0, PCB_ONFAULT(t3)
+	 sw	v0, PCB_ONFAULT(t3)
 
 	PTR_L	ra, CF_RA_OFFS(sp)
 	GET_CPU_INFO(t1, t0)
@@ -153,7 +153,7 @@ NON_LEAF(copyinstr, FRAMESZ(CF_SZ), ra)
 	sw	zero, PCB_ONFAULT(t3)
 	PTR_ADDU sp, sp, FRAMESZ(CF_SZ)
 	j	ra
-	nop
+	 NOP
 END(copyinstr)
 
 /*
@@ -171,11 +171,11 @@ NON_LEAF(copyoutstr, FRAMESZ(CF_SZ), ra)
 	.mask	0x80000000, (CF_RA_OFFS - FRAMESZ(CF_SZ))
 	PTR_S	ra, CF_RA_OFFS(sp)
 	blt	a1, zero, _copyerr	# make sure address is in user space
-	LI	v0, KT_COPYERR
+	 LI	v0, KT_COPYERR
 	GET_CPU_INFO(t1, t0)
 	PTR_L	t3, CI_CURPROCPADDR(t1)
 	jal	copystr
-	sw	v0, PCB_ONFAULT(t3)
+	 sw	v0, PCB_ONFAULT(t3)
 
 	PTR_L	ra, CF_RA_OFFS(sp)
 	GET_CPU_INFO(t1, t0)
@@ -183,7 +183,7 @@ NON_LEAF(copyoutstr, FRAMESZ(CF_SZ), ra)
 	sw	zero, PCB_ONFAULT(t3)
 	PTR_ADDU sp, sp, FRAMESZ(CF_SZ)
 	j	ra
-	nop
+	 NOP
 END(copyoutstr)
 
 /*
@@ -198,11 +198,11 @@ NON_LEAF(copyin, FRAMESZ(CF_SZ), ra)
 	.mask	0x80000000, (CF_RA_OFFS - FRAMESZ(CF_SZ))
 	PTR_S	ra, CF_RA_OFFS(sp)
 	blt	a0, zero, _copyerr	# make sure address is in user space
-	li	v0, KT_COPYERR
+	 li	v0, KT_COPYERR
 	GET_CPU_INFO(t1, t0)
 	PTR_L	t3, CI_CURPROCPADDR(t1)
 	jal	bcopy
-	sw	v0, PCB_ONFAULT(t3)
+	 sw	v0, PCB_ONFAULT(t3)
 
 	PTR_L	ra, CF_RA_OFFS(sp)
 	GET_CPU_INFO(t1, t0)
@@ -210,7 +210,7 @@ NON_LEAF(copyin, FRAMESZ(CF_SZ), ra)
 	sw	zero, PCB_ONFAULT(t3)
 	PTR_ADDU sp, sp, FRAMESZ(CF_SZ)
 	j	ra
-	move	v0, zero
+	 move	v0, zero
 END(copyin)
 
 /*
@@ -225,11 +225,11 @@ NON_LEAF(copyout, FRAMESZ(CF_SZ), ra)
 	.mask	0x80000000, (CF_RA_OFFS - FRAMESZ(CF_SZ))
 	PTR_S	ra, CF_RA_OFFS(sp)
 	blt	a1, zero, _copyerr	# make sure address is in user space
-	li	v0, KT_COPYERR
+	 li	v0, KT_COPYERR
 	GET_CPU_INFO(t1, t0)
 	PTR_L	t3, CI_CURPROCPADDR(t1)
 	jal	bcopy
-	sw	v0, PCB_ONFAULT(t3)
+	 sw	v0, PCB_ONFAULT(t3)
 
 	PTR_L	ra, CF_RA_OFFS(sp)
 	GET_CPU_INFO(t1, t0)
@@ -237,7 +237,7 @@ NON_LEAF(copyout, FRAMESZ(CF_SZ), ra)
 	sw	zero, PCB_ONFAULT(t3)
 	PTR_ADDU sp, sp, FRAMESZ(CF_SZ)
 	j	ra
-	move	v0, zero
+	 move	v0, zero
 END(copyout)
 
 _copyerr:
@@ -247,7 +247,7 @@ _copyerr:
 	sw	zero, PCB_ONFAULT(t3)
 	PTR_ADDU sp, sp, FRAMESZ(CF_SZ)
 	j	ra
-	li	v0, EFAULT		# return error
+	 li	v0, EFAULT		# return error
 
 /*
  *  kcopy is a wrapper around bcopy that catches bad memory references.
@@ -262,7 +262,7 @@ NON_LEAF(kcopy, FRAMESZ(CF_SZ + REGSZ), ra)
 	li	v0, KT_KCOPYERR
 	PTR_S	v1, CF_ARGSZ(sp)	# save old pcb_onfault
 	jal	bcopy
-	sw	v0, PCB_ONFAULT(t3)
+	 sw	v0, PCB_ONFAULT(t3)
 
 	PTR_L	v0, CF_ARGSZ(sp)
 	GET_CPU_INFO(t1, t0)
@@ -271,7 +271,7 @@ NON_LEAF(kcopy, FRAMESZ(CF_SZ + REGSZ), ra)
 	sw	v0, PCB_ONFAULT(t3)
 	PTR_ADDU sp, sp, FRAMESZ(CF_SZ + REGSZ)
 	j	ra
-	move	v0, zero
+	 move	v0, zero
 END(kcopy)
 
 _kcopyerr:
@@ -282,7 +282,7 @@ _kcopyerr:
 	sw	v0, PCB_ONFAULT(t3)
 	PTR_ADDU sp, sp, FRAMESZ(CF_SZ + REGSZ)
 	j	ra
-	li	v0, EFAULT		# return error
+	 li	v0, EFAULT		# return error
 
 /*
  * Guarded ``memory'' access routines
@@ -297,7 +297,7 @@ LEAF(guarded_read_1, 0)
 	PTR_L	t3, CI_CURPROCPADDR(t1)
 
 	li	a3, SR_BOOT_EXC_VEC
-	mfc0	a2, COP_0_STATUS_REG
+	MFC0	a2, COP_0_STATUS_REG
 	or	a4, a2, a3
 	xor	a4, a4, a3
 
@@ -305,18 +305,18 @@ LEAF(guarded_read_1, 0)
 	lw	v1, PCB_ONFAULT(t3)
 	sw	v0, PCB_ONFAULT(t3)
 
-	mtc0	a4, COP_0_STATUS_REG
-	NOP
+	MTC0	a4, COP_0_STATUS_REG
+	MTC0_HAZARD
 
 	lb	v0, 0(a0)
 	sb	v0, 0(a1)
 
-	mtc0	a2, COP_0_STATUS_REG
-	NOP
+	MTC0	a2, COP_0_STATUS_REG
+	MTC0_HAZARD
 	
 	sw	v1, PCB_ONFAULT(t3)
 	j	ra
-	move	v0, zero
+	 move	v0, zero
 END(guarded_read_1)
 
 LEAF(guarded_read_2, 0)
@@ -324,7 +324,7 @@ LEAF(guarded_read_2, 0)
 	PTR_L	t3, CI_CURPROCPADDR(t1)
 
 	li	a3, SR_BOOT_EXC_VEC
-	mfc0	a2, COP_0_STATUS_REG
+	MFC0	a2, COP_0_STATUS_REG
 	or	a4, a2, a3
 	xor	a4, a4, a3
 
@@ -332,14 +332,14 @@ LEAF(guarded_read_2, 0)
 	lw	v1, PCB_ONFAULT(t3)
 	sw	v0, PCB_ONFAULT(t3)
 
-	mtc0	a4, COP_0_STATUS_REG
-	NOP
+	MTC0	a4, COP_0_STATUS_REG
+	MTC0_HAZARD
 
 	lh	v0, 0(a0)
 	sh	v0, 0(a1)
 	
-	mtc0	a2, COP_0_STATUS_REG
-	NOP
+	MTC0	a2, COP_0_STATUS_REG
+	MTC0_HAZARD
 	
 	sw	v1, PCB_ONFAULT(t3)
 	j	ra
@@ -351,7 +351,7 @@ LEAF(guarded_read_4, 0)
 	PTR_L	t3, CI_CURPROCPADDR(t1)
 
 	li	a3, SR_BOOT_EXC_VEC
-	mfc0	a2, COP_0_STATUS_REG
+	MFC0	a2, COP_0_STATUS_REG
 	or	a4, a2, a3
 	xor	a4, a4, a3
 
@@ -359,14 +359,14 @@ LEAF(guarded_read_4, 0)
 	lw	v1, PCB_ONFAULT(t3)
 	sw	v0, PCB_ONFAULT(t3)
 
-	mtc0	a4, COP_0_STATUS_REG
-	NOP
+	MTC0	a4, COP_0_STATUS_REG
+	MTC0_HAZARD
 
 	lw	v0, 0(a0)
 	sw	v0, 0(a1)
 	
-	mtc0	a2, COP_0_STATUS_REG
-	NOP
+	MTC0	a2, COP_0_STATUS_REG
+	MTC0_HAZARD
 	
 	sw	v1, PCB_ONFAULT(t3)
 	j	ra
@@ -378,7 +378,7 @@ LEAF(guarded_write_4, 0)
 	PTR_L	t3, CI_CURPROCPADDR(t1)
 
 	li	a3, SR_BOOT_EXC_VEC
-	mfc0	a2, COP_0_STATUS_REG
+	MFC0	a2, COP_0_STATUS_REG
 	or	a4, a2, a3
 	xor	a4, a4, a3
 
@@ -386,13 +386,13 @@ LEAF(guarded_write_4, 0)
 	lw	v1, PCB_ONFAULT(t3)
 	sw	v0, PCB_ONFAULT(t3)
 
-	mtc0	a4, COP_0_STATUS_REG
-	NOP
+	MTC0	a4, COP_0_STATUS_REG
+	MTC0_HAZARD
 
 	sw	a1, 0(a0)
 	
-	mtc0	a2, COP_0_STATUS_REG
-	NOP
+	MTC0	a2, COP_0_STATUS_REG
+	MTC0_HAZARD
 	
 	sw	v1, PCB_ONFAULT(t3)
 	j	ra
@@ -400,8 +400,8 @@ LEAF(guarded_write_4, 0)
 END(guarded_write_4)
 
 _guarderr:
-	mtc0	a2, COP_0_STATUS_REG
-	NOP
+	MTC0	a2, COP_0_STATUS_REG
+	MTC0_HAZARD
 	
 	sw	v1, PCB_ONFAULT(t3)
 	j	ra
diff --git a/sys/arch/mips64/mips64/lcore_ddb.S b/sys/arch/mips64/mips64/lcore_ddb.S
index 82694bdc250..761bfde2271 100644
--- a/sys/arch/mips64/mips64/lcore_ddb.S
+++ b/sys/arch/mips64/mips64/lcore_ddb.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: lcore_ddb.S,v 1.11 2010/01/31 19:39:04 miod Exp $ */
+/*	$OpenBSD: lcore_ddb.S,v 1.12 2012/09/29 19:02:26 miod Exp $ */
 
 /*
  * Copyright (c) 2001-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -37,6 +37,7 @@
 
 #include "assym.h"
 
+	.set	mips3
 	.set	noreorder		# Noreorder is default style!
 
 LEAF(kdbpeekd, 0)
@@ -45,17 +46,17 @@ LEAF(kdbpeekd, 0)
 	li	v0, KT_DDBERR
 	and	v1, a0, 7			# unaligned ?
 	bne	v1, zero, 1f
-	sw	v0, PCB_ONFAULT(t0)
+	 sw	v0, PCB_ONFAULT(t0)
 
 	ld	v0, (a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 
 1:
 	LDHI	v0, 0(a0)
 	LDLO	v0, 7(a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 END(kdbpeekd)
 
 LEAF(kdbpeek, 0)
@@ -64,17 +65,17 @@ LEAF(kdbpeek, 0)
 	li	v0, KT_DDBERR
 	and	v1, a0, 3			# unaligned ?
 	bne	v1, zero, 1f
-	sw	v0, PCB_ONFAULT(t0)
+	 sw	v0, PCB_ONFAULT(t0)
 
-	lw	v0, (a0)
+	lwu	v0, (a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 
 1:
 	LWHI	v0, 0(a0)
 	LWLO	v0, 3(a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 END(kdbpeek)
 
 LEAF(kdbpeekw, 0)
@@ -83,16 +84,16 @@ LEAF(kdbpeekw, 0)
 	li	v0, KT_DDBERR
 	and	v1, a0, 1			# unaligned ?
 	bne	v1, zero, 1f
-	sw	v0, PCB_ONFAULT(t0)
+	 sw	v0, PCB_ONFAULT(t0)
 
 	lh	v0, (a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 
 1:
 	li	v0, -1				# error!
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 END(kdbpeekw)
 
 LEAF(kdbpeekb, 0)
@@ -102,13 +103,13 @@ LEAF(kdbpeekb, 0)
 	sw	v0, PCB_ONFAULT(t0)
 	lb	v0, 0(a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 END(kdbpeekb)
 
 	.globl	kt_ddberr
 kt_ddberr:
 	jr	ra
-	li	v0, -1
+	 li	v0, -1
 
 LEAF(kdbpoked, 0)
 	GET_CPU_INFO(t1, t0)
@@ -116,17 +117,17 @@ LEAF(kdbpoked, 0)
 	li	v0, KT_DDBERR
 	and	v1, a0, 7			# unaligned ?
 	bne	v1, zero, 1f
-	sw	v0, PCB_ONFAULT(t0)
+	 sw	v0, PCB_ONFAULT(t0)
 
 	sd	a1, (a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 
 1:
 	SDHI	a1, 0(a0)
 	SDLO	a1, 7(a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 END(kdbpoked)
 
 LEAF(kdbpoke, 0)
@@ -135,17 +136,17 @@ LEAF(kdbpoke, 0)
 	li	v0, KT_DDBERR
 	and	v1, a0, 3			# unaligned ?
 	bne	v1, zero, 1f
-	sw	v0, PCB_ONFAULT(t0)
+	 sw	v0, PCB_ONFAULT(t0)
 
 	sw	a1, (a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 
 1:
 	SWHI	a1, 0(a0)
 	SWLO	a1, 3(a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 END(kdbpoke)
 
 LEAF(kdbpokew, 0)
@@ -154,15 +155,15 @@ LEAF(kdbpokew, 0)
 	li	v0, KT_DDBERR
 	and	v1, a0, 1			# unaligned ?
 	bne	v1, zero, 1f
-	sw	v0, PCB_ONFAULT(t0)
+	 sw	v0, PCB_ONFAULT(t0)
 
 	sh	a1, (a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 
 1:
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 END(kdbpokew)
 
 LEAF(kdbpokeb, 0)
@@ -172,17 +173,18 @@ LEAF(kdbpokeb, 0)
 	sw	v0, PCB_ONFAULT(t0)
 	sb	a1, 0(a0)
 	jr	ra
-	sw	zero, PCB_ONFAULT(t0)
+	 sw	zero, PCB_ONFAULT(t0)
 END(kdbpokeb)
 
 LEAF(Debugger, 0)
 	break   BREAK_SOVER_VAL
 	jr	ra
-	nop
+	 NOP
 END(Debugger)
 
 LEAF(setjmp, 0)
-	mfc0	v0, COP_0_STATUS_REG	# Later the "real" spl value!
+	MFC0	v0, COP_0_STATUS_REG
+	MFC0_HAZARD
 	REG_S	s0, REGSZ * 0(a0)
 	REG_S	s1, REGSZ * 1(a0)
 	REG_S	s2, REGSZ * 2(a0)
@@ -196,7 +198,7 @@ LEAF(setjmp, 0)
 	REG_S	ra, REGSZ * 10(a0)
 	REG_S	v0, REGSZ * 11(a0)
 	jr	ra
-	li	v0, 0			# setjmp return
+	 li	v0, 0			# setjmp return
 END(setjmp)
 
 LEAF(longjmp, 0)
@@ -212,8 +214,8 @@ LEAF(longjmp, 0)
 	REG_L	s7, REGSZ * 7(a0)
 	REG_L	s8, REGSZ * 8(a0)
 	REG_L	sp, REGSZ * 9(a0)
-	mtc0	v0, COP_0_STATUS_REG	# Later the "real" spl value!
-	ITLBNOPFIX
+	MTC0	v0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 	jr	ra
-	li	v0, 1			# longjmp return
+	 li	v0, 1			# longjmp return
 END(longjmp)
diff --git a/sys/arch/mips64/mips64/lcore_float.S b/sys/arch/mips64/mips64/lcore_float.S
index b89837fe2f3..7b64a514450 100644
--- a/sys/arch/mips64/mips64/lcore_float.S
+++ b/sys/arch/mips64/mips64/lcore_float.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: lcore_float.S,v 1.20 2010/09/21 20:29:17 miod Exp $ */
+/*	$OpenBSD: lcore_float.S,v 1.21 2012/09/29 19:02:26 miod Exp $ */
 
 /*
  * Copyright (c) 2001-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -36,9 +36,8 @@
 
 #include "assym.h"
 
-	.set	noreorder		# Noreorder is default style!
-
 	.set	mips3
+	.set	noreorder		# Noreorder is default style!
 
 /*----------------------------------------------------------------------------
  *
@@ -59,13 +58,14 @@
  *----------------------------------------------------------------------------
  */
 LEAF(MipsSwitchFPState, 0)
-	mfc0	t1, COP_0_STATUS_REG	# Save old SR
+	MFC0	t1, COP_0_STATUS_REG	# Save old SR
+	MFC0_HAZARD
 	or	t0, t1, SR_COP_1_BIT|SR_FR_32	# enable the coprocessor
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_CU_HAZARD
 
 	beq	a0, zero, 1f		# skip save if NULL pointer
-	nop
+	 NOP
 /*
  * First read out the status register to make sure that all FP operations
  * have completed.
@@ -73,11 +73,11 @@ LEAF(MipsSwitchFPState, 0)
 	PTR_L	a0, P_ADDR(a0)		# get pointer to pcb for proc
 	cfc1	t0, FPC_CSR		# stall til FP done
 	cfc1	t0, FPC_CSR		# now get status
-	li	t3, ~SR_COP_1_BIT
-	REG_L	t2, PCB_REGS+(PS * REGSZ)(a0)	# get CPU status register
+	LI	t3, ~SR_COP_1_BIT
+	REG_L	t2, PCB_REGS+(SR * REGSZ)(a0)	# get CPU status register
 	REG_S	t0, PCB_FPREGS+(32 * REGSZ)(a0)	# save FP status
 	and	t2, t2, t3			# clear COP_1 enable bit
-	REG_S	t2, PCB_REGS+(PS * REGSZ)(a0)	# save new status register
+	REG_S	t2, PCB_REGS+(SR * REGSZ)(a0)	# save new status register
 /*
  * Save the floating point registers.
  */
@@ -153,22 +153,23 @@ LEAF(MipsSwitchFPState, 0)
 	ldc1	$f31, PCB_FPREGS+(31 * REGSZ)(a1)
 
 	ctc1	t0, FPC_CSR
-	nop
+	NOP
 
-	mtc0	t1, COP_0_STATUS_REG	# Restore the status register.
-	ITLBNOPFIX
+	MTC0	t1, COP_0_STATUS_REG	# Restore the status register.
+	MTC0_SR_CU_HAZARD
 	j	ra
-	nop
+	 NOP
 END(MipsSwitchFPState)
 
 LEAF(MipsSwitchFPState16, 0)
-	mfc0	t1, COP_0_STATUS_REG	# Save old SR
+	MFC0	t1, COP_0_STATUS_REG	# Save old SR
+	MFC0_HAZARD
 	or	t0, t1, SR_COP_1_BIT	# enable the coprocessor
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_CU_HAZARD
 
 	beq	a0, zero, 1f		# skip save if NULL pointer
-	nop
+	 NOP
 /*
  * First read out the status register to make sure that all FP operations
  * have completed.
@@ -176,11 +177,11 @@ LEAF(MipsSwitchFPState16, 0)
 	PTR_L	a0, P_ADDR(a0)		# get pointer to pcb for proc
 	cfc1	t0, FPC_CSR		# stall til FP done
 	cfc1	t0, FPC_CSR		# now get status
-	li	t3, ~SR_COP_1_BIT
-	REG_L	t2, PCB_REGS+(PS * REGSZ)(a0)	# get CPU status register
+	LI	t3, ~SR_COP_1_BIT
+	REG_L	t2, PCB_REGS+(SR * REGSZ)(a0)	# get CPU status register
 	REG_S	t0, PCB_FPREGS+(32 * REGSZ)(a0)	# save FP status
 	and	t2, t2, t3			# clear COP_1 enable bit
-	REG_S	t2, PCB_REGS+(PS * REGSZ)(a0)	# save new status register
+	REG_S	t2, PCB_REGS+(SR * REGSZ)(a0)	# save new status register
 /*
  * Save the floating point registers.
  */
@@ -256,12 +257,12 @@ LEAF(MipsSwitchFPState16, 0)
 	lwc1	$f31, PCB_FPREGS+(31 * REGSZ)(a1)
 
 	ctc1	t0, FPC_CSR
-	nop
+	NOP
 
-	mtc0	t1, COP_0_STATUS_REG	# Restore the status register.
-	ITLBNOPFIX
+	MTC0	t1, COP_0_STATUS_REG	# Restore the status register.
+	MTC0_SR_CU_HAZARD
 	j	ra
-	nop
+	 NOP
 END(MipsSwitchFPState16)
 
 /*----------------------------------------------------------------------------
@@ -283,22 +284,23 @@ END(MipsSwitchFPState16)
  */
 LEAF(MipsSaveCurFPState, 0)
 	PTR_L	a0, P_ADDR(a0)			# get pointer to pcb for proc
-	mfc0	t1, COP_0_STATUS_REG	# Disable interrupts and
+	MFC0	t1, COP_0_STATUS_REG	# Disable interrupts and
+	MFC0_HAZARD
 	or	t0, t1, SR_COP_1_BIT|SR_FR_32	# enable the coprocessor
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 	GET_CPU_INFO(t2, t3)
 	PTR_S	zero, CI_FPUPROC(t2)		# indicate state has been saved
 /*
  * First read out the status register to make sure that all FP operations
  * have completed.
  */
-	REG_L	t2, PCB_REGS+(PS * REGSZ)(a0)	# get CPU status register
-	li	t3, ~SR_COP_1_BIT
+	REG_L	t2, PCB_REGS+(SR * REGSZ)(a0)	# get CPU status register
+	LI	t3, ~SR_COP_1_BIT
 	and	t2, t2, t3			# clear COP_1 enable bit
 	cfc1	t0, FPC_CSR		# stall til FP done
 	cfc1	t0, FPC_CSR		# now get status
-	REG_S	t2, PCB_REGS+(PS * REGSZ)(a0)	# save new status register
+	REG_S	t2, PCB_REGS+(SR * REGSZ)(a0)	# save new status register
 	REG_S	t0, PCB_FPREGS+(32 * REGSZ)(a0)	# save FP status
 /*
  * Save the floating point registers.
@@ -336,30 +338,31 @@ LEAF(MipsSaveCurFPState, 0)
 	sdc1	$f30, PCB_FPREGS+(30 * REGSZ)(a0)
 	sdc1	$f31, PCB_FPREGS+(31 * REGSZ)(a0)
 
-	mtc0	t1, COP_0_STATUS_REG	# Restore the status register.
-	ITLBNOPFIX
+	MTC0	t1, COP_0_STATUS_REG	# Restore the status register.
+	MTC0_SR_IE_HAZARD
 	j	ra
-	nop
+	 NOP
 END(MipsSaveCurFPState)
 
 LEAF(MipsSaveCurFPState16, 0)
 	PTR_L	a0, P_ADDR(a0)			# get pointer to pcb for proc
-	mfc0	t1, COP_0_STATUS_REG	# Disable interrupts and
+	MFC0	t1, COP_0_STATUS_REG	# Disable interrupts and
+	MFC0_HAZARD
 	or	t0, t1, SR_COP_1_BIT	# enable the coprocessor
-	mtc0	t0, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	t0, COP_0_STATUS_REG
+	MTC0_SR_IE_HAZARD
 	GET_CPU_INFO(t2, t3)
 	PTR_S	zero, CI_FPUPROC(t2)		# indicate state has been saved
 /*
  * First read out the status register to make sure that all FP operations
  * have completed.
  */
-	REG_L	t2, PCB_REGS+(PS * REGSZ)(a0)	# get CPU status register
-	li	t3, ~SR_COP_1_BIT
+	REG_L	t2, PCB_REGS+(SR * REGSZ)(a0)	# get CPU status register
+	LI	t3, ~SR_COP_1_BIT
 	and	t2, t2, t3			# clear COP_1 enable bit
 	cfc1	t0, FPC_CSR		# stall til FP done
 	cfc1	t0, FPC_CSR		# now get status
-	REG_S	t2, PCB_REGS+(PS * REGSZ)(a0)	# save new status register
+	REG_S	t2, PCB_REGS+(SR * REGSZ)(a0)	# save new status register
 	REG_S	t0, PCB_FPREGS+(32 * REGSZ)(a0)	# save FP status
 /*
  * Save the floating point registers.
@@ -397,10 +400,10 @@ LEAF(MipsSaveCurFPState16, 0)
 	swc1	$f30, PCB_FPREGS+(30 * REGSZ)(a0)
 	swc1	$f31, PCB_FPREGS+(31 * REGSZ)(a0)
 
-	mtc0	t1, COP_0_STATUS_REG	# Restore the status register.
-	ITLBNOPFIX
+	MTC0	t1, COP_0_STATUS_REG	# Restore the status register.
+	MTC0_SR_IE_HAZARD
 	j	ra
-	nop
+	 NOP
 END(MipsSaveCurFPState16)
 
 /*----------------------------------------------------------------------------
@@ -420,15 +423,16 @@ END(MipsSaveCurFPState16)
  *----------------------------------------------------------------------------
  */
 LEAF(cp1_get_prid, 0)
-	mfc0	v1, COP_0_STATUS_REG
+	MFC0	v1, COP_0_STATUS_REG
+	MFC0_HAZARD
 	li	a0, SR_COP_1_BIT
 	or	v1, a0
-	mtc0	v1, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	v1, COP_0_STATUS_REG
+	MTC0_SR_CU_HAZARD
 	cfc1	v0, FPC_ID
 	xor	v1, a0
-	mtc0	v1, COP_0_STATUS_REG
-	ITLBNOPFIX
+	MTC0	v1, COP_0_STATUS_REG
+	MTC0_SR_CU_HAZARD
 	jr	ra
-	nop
+	 NOP
 END(cp1_get_prid)
diff --git a/sys/arch/mips64/mips64/tlbhandler.S b/sys/arch/mips64/mips64/tlbhandler.S
index f395e47b387..dc46298b7b4 100644
--- a/sys/arch/mips64/mips64/tlbhandler.S
+++ b/sys/arch/mips64/mips64/tlbhandler.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tlbhandler.S,v 1.35 2012/04/24 20:01:03 miod Exp $ */
+/*	$OpenBSD: tlbhandler.S,v 1.36 2012/09/29 19:02:26 miod Exp $ */
 
 /*
  * Copyright (c) 1995-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -66,7 +66,7 @@ xtlb_miss_err_r4k:
 	mfc0	k1, COP_0_TLB_INDEX
 	bltz	k1, xtlb_miss			# missing!
 	 nop
-	eret
+	ERET
 	 nop
 	.end	xtlb_miss_err_r4k
 
@@ -83,7 +83,7 @@ xtlb_miss_err_r4000SC:
 	mfc0	k1, COP_0_TLB_INDEX
 	bltz	k1, 1f		# missing!
 	 nop
-	eret
+	ERET
 	 nop
 1:
 	LA	k1, xtlb_miss
@@ -148,22 +148,16 @@ xtlb_miss:
 	dsll	k1, k1, (64 - PG_FRAMEBITS)
 	dsrl	k1, k1, (64 - PG_FRAMEBITS)
 	dmtc0	k1, COP_0_TLB_LO1
-	nop				# RM7000 needs 4 nops
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwr				# update TLB
-	nop				# RM7000 need 4 for JTLB usage.
-	nop
-	nop
-	nop
+	TLB_HAZARD
 
 #ifdef CPU_LOONGSON2
 	li	k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
 	dmtc0	k0, COP_0_DIAG
 #endif
 
-	eret
+	ERET
 
 _k_miss:
 	b	k_tlb_miss		# kernel tlbmiss.
@@ -214,7 +208,9 @@ NLEAF(k_tlb_inv, 0)
 
 	PTR_L	k1, Sysmap
 	PTR_SLL	k0, k0, 2			# compute offset from index
+	TLB_HAZARD
 	tlbp					# Probe the invalid entry
+	TLB_HAZARD				# necessary?
 	PTR_ADDU k1, k1, k0
 	and	k0, k0, 4			# check even/odd page
 	bne	k0, zero, k_tlb_inv_odd
@@ -233,22 +229,16 @@ NLEAF(k_tlb_inv, 0)
 	dsll	k0, k0, (64 - PG_FRAMEBITS)
 	dsrl	k0, k0, (64 - PG_FRAMEBITS)
 	dmtc0	k0, COP_0_TLB_LO1		# load PTE entry
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwi					# write TLB
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 
 #ifdef CPU_LOONGSON2
 	li	k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
 	dmtc0	k0, COP_0_DIAG
 #endif
 
-	eret
+	ERET
 
 k_tlb_inv_odd:
 	mfc0	k0, COP_0_TLB_INDEX
@@ -264,22 +254,16 @@ k_tlb_inv_odd:
 	dsll	k0, k0, (64 - PG_FRAMEBITS)
 	dsrl	k0, k0, (64 - PG_FRAMEBITS)
 	dmtc0	k0, COP_0_TLB_LO0		# save PTE entry
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwi					# update TLB
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 
 #ifdef CPU_LOONGSON2
 	li	k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
 	dmtc0	k0, COP_0_DIAG
 #endif
 
-	eret
+	ERET
 END(k_tlb_inv)
 
 /*---------------------------------------------------------------- k_tlb_miss
@@ -313,22 +297,16 @@ NLEAF(k_tlb_miss, 0)
 	dsll	k1, k1, (64 - PG_FRAMEBITS)
 	dsrl	k1, k1, (64 - PG_FRAMEBITS)
 	dmtc0	k1, COP_0_TLB_LO1		# load PTE entry
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwr					# write TLB
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 
 #ifdef CPU_LOONGSON2
 	li	k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
 	dmtc0	k0, COP_0_DIAG
 #endif
 
-	eret
+	ERET
 
 sys_stk_chk:
 	GET_CPU_INFO(k1, k0)
@@ -378,53 +356,6 @@ sys_stk_chk:
 	.set	at
 END(k_tlb_miss)
 
-#if 0	/* currently unused */
-/*---------------------------------------------------------------- tlb_write_i
- *	Write the given entry into the TLB at the given index.
- */
-LEAF(tlb_write_indexed, 0)
-	mfc0	v1, COP_0_STATUS_REG		# Save the status register.
-	ori	v0, v1, SR_INT_ENAB
-	xori	v0, v0, SR_INT_ENAB
-	mtc0	v0, COP_0_STATUS_REG		# Disable interrupts
-	ITLBNOPFIX
-	ld	a2, 16(a1)
-	ld	a3, 24(a1)
-	dmfc0	ta0, COP_0_TLB_HI		# Save the current PID.
-
-	dmtc0	a2, COP_0_TLB_LO0		# Set up entry low0.
-	dmtc0	a3, COP_0_TLB_LO1		# Set up entry low1.
-	ld	a2, 0(a1)
-	ld	a3, 8(a1)
-	mtc0	a0, COP_0_TLB_INDEX		# Set the index.
-	mtc0	a2, COP_0_TLB_PG_MASK		# Set up entry mask.
-	dmtc0	a3, COP_0_TLB_HI		# Set up entry high.
-	nop
-	nop
-	nop
-	nop
-	tlbwi					# Write the TLB
-	nop
-	nop					# Delay for effect
-	nop
-	nop
-
-#ifdef CPU_LOONGSON2
-	li	v0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
-	dmtc0	v0, COP_0_DIAG
-#endif
-
-	dmtc0	ta0, COP_0_TLB_HI		# Restore the PID.
-	nop
-	li	a0, TLB_PAGE_MASK
-	mtc0	a0, COP_0_TLB_PG_MASK		# Restore default mask value.
-	mtc0	v1, COP_0_STATUS_REG		# Restore the status register
-	ITLBNOPFIX
-	j	ra
-	nop
-END(tlb_write_indexed)
-#endif
-
 /*---------------------------------------------------------------- tlb_flush
  *	Flush the "random" entries from the TLB.
  *	Uses "wired" register to determine what register to start with.
@@ -435,7 +366,7 @@ LEAF(tlb_flush, 0)
 	ori	v0, v1, SR_INT_ENAB
 	xori	v0, v0, SR_INT_ENAB
 	mtc0	v0, COP_0_STATUS_REG	# Disable interrupts
-	ITLBNOPFIX
+	MTC0_SR_IE_HAZARD
 	mfc0	ta1, COP_0_TLB_WIRED
 	LA	v0, CKSEG0_BASE		# invalid address
 	dmfc0	ta0, COP_0_TLB_HI	# Save the PID
@@ -459,12 +390,9 @@ LEAF(tlb_flush, 0)
 	addu	v0, v0, 2 * PAGE_SIZE
 #endif
 	addu	ta1, ta1, 1		# Increment index.
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwi				# Write the TLB entry.
-	nop
-	nop
+	TLB_HAZARD
 	bne	ta1, a0, 1b
 	nop
 
@@ -477,7 +405,7 @@ LEAF(tlb_flush, 0)
 	li	a0, TLB_PAGE_MASK
 	mtc0	a0, COP_0_TLB_PG_MASK	# Restore default mask value.
 	mtc0	v1, COP_0_STATUS_REG	# Restore the status register
-	ITLBNOPFIX
+	MTC0_SR_IE_HAZARD
 	j	ra
 	nop
 END(tlb_flush)
@@ -490,19 +418,14 @@ LEAF(tlb_flush_addr, 0)
 	ori	v0, v1, SR_INT_ENAB
 	xori	v0, v0, SR_INT_ENAB
 	mtc0	v0, COP_0_STATUS_REG	# Disable interrupts
-	ITLBNOPFIX
-	dli	v0, (PG_HVPN | PG_ASID)
+	MTC0_SR_IE_HAZARD
+	dli	v0, (PG_HVPN | PG_ASID_MASK)
 	and	a0, a0, v0		# Make sure valid hi value.
 	dmfc0	ta0, COP_0_TLB_HI	# Get current PID
 	dmtc0	a0, COP_0_TLB_HI	# look for addr & PID
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbp				# Probe for the entry.
-	nop
-	nop				# Delay for effect
-	nop
+	TLB_HAZARD			# necessary?
 	LA	ta1, CKSEG0_BASE	# Load invalid entry.
 	mfc0	v0, COP_0_TLB_INDEX	# See what we got
 	bltz	v0, 1f			# index < 0 => !found
@@ -515,15 +438,9 @@ LEAF(tlb_flush_addr, 0)
 
 	dmtc0	zero, COP_0_TLB_LO0	# Zero out low entry.
 	dmtc0	zero, COP_0_TLB_LO1	# Zero out low entry.
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwi
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 
 #ifdef CPU_LOONGSON2
 	li	v0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
@@ -533,7 +450,7 @@ LEAF(tlb_flush_addr, 0)
 1:
 	dmtc0	ta0, COP_0_TLB_HI	# restore PID
 	mtc0	v1, COP_0_STATUS_REG	# Restore the status register
-	ITLBNOPFIX
+	MTC0_SR_IE_HAZARD
 	j	ra
 	nop
 END(tlb_flush_addr)
@@ -546,18 +463,17 @@ LEAF(tlb_update, 0)
 	ori	v0, v1, SR_INT_ENAB
 	xori	v0, v0, SR_INT_ENAB
 	mtc0	v0, COP_0_STATUS_REG	# Disable interrupts
-	ITLBNOPFIX
+	MTC0_SR_IE_HAZARD
 	and	ta1, a0, PG_ODDPG	# ta1 = Even/Odd flag
-	dli	v0, (PG_HVPN | PG_ASID)
+	dli	v0, (PG_HVPN | PG_ASID_MASK)
 	and	a0, a0, v0
 	dmfc0	ta0, COP_0_TLB_HI	# Save current PID
 	dmtc0	a0, COP_0_TLB_HI	# Init high reg
 	and	a2, a1, PG_G		# Copy global bit
 	li	a3, TLB_PAGE_MASK
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbp				# Probe for the entry.
+	TLB_HAZARD			# necessary?
 	dsll	a1, a1, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
 	dsrl	a1, a1, (64 - PG_FRAMEBITS)
 	bne	ta1, zero, 2f		# Decide even odd
@@ -566,16 +482,13 @@ LEAF(tlb_update, 0)
 	bltz	v0, 1f			# index < 0 => !found
 	nop
 
+	TLB_HAZARD
 	tlbr				# update, read entry first
-	nop
-	nop
-	nop
+	TLB_HAZARD			# necessary?
 	dmtc0	a1, COP_0_TLB_LO0	# init low reg0.
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwi				# update slot found
+	TLB_HAZARD
 	b	4f
 	li	v0, 1
 1:
@@ -583,11 +496,9 @@ LEAF(tlb_update, 0)
 	dmtc0	a0, COP_0_TLB_HI	# init high reg.
 	dmtc0	a1, COP_0_TLB_LO0	# init low reg0.
 	dmtc0	a2, COP_0_TLB_LO1	# init low reg1.
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwr				# enter into a random slot
+	TLB_HAZARD
 	b	4f
 	li	v0, 0
 # ODD
@@ -596,16 +507,13 @@ LEAF(tlb_update, 0)
 	bltz	v0, 3f			# index < 0 => !found
 	nop
 
+	TLB_HAZARD
 	tlbr				# read the entry first
-	nop
-	nop
-	nop
+	TLB_HAZARD			# necessary?
 	dmtc0	a1, COP_0_TLB_LO1	# init low reg1.
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwi				# update slot found
+	TLB_HAZARD
 	b	4f
 	li	v0, 1
 3:
@@ -613,18 +521,10 @@ LEAF(tlb_update, 0)
 	dmtc0	a0, COP_0_TLB_HI	# init high reg.
 	dmtc0	a2, COP_0_TLB_LO0	# init low reg0.
 	dmtc0	a1, COP_0_TLB_LO1	# init low reg1.
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbwr				# enter into a random slot
-	nop
-	li	v0, 0
-
-4:					# Make sure pipeline
-	nop				# advances before we
-	nop				# use the tlb.
-
+	TLB_HAZARD
+4:
 #ifdef CPU_LOONGSON2
 	li	v0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE
 	dmtc0	v0, COP_0_DIAG
@@ -632,9 +532,9 @@ LEAF(tlb_update, 0)
 
 	dmtc0	ta0, COP_0_TLB_HI	# restore PID
 	mtc0	v1, COP_0_STATUS_REG	# Restore the status register
-	ITLBNOPFIX
+	MTC0_SR_IE_HAZARD
 	j	ra
-	nop
+	li	v0, 0
 END(tlb_update)
 
 /*---------------------------------------------------------------- tlb_read
@@ -645,19 +545,14 @@ LEAF(tlb_read, 0)
 	ori	v0, v1, SR_INT_ENAB
 	xori	v0, v0, SR_INT_ENAB
 	mtc0	v0, COP_0_STATUS_REG	# Disable interrupts
-	ITLBNOPFIX
+	MTC0_SR_IE_HAZARD
 	dmfc0	v0, COP_0_TLB_HI	# Get current PID
 
 	mtc0	a0, COP_0_TLB_INDEX	# Set the index register
-	nop
-	nop
-	nop
-	nop
+	TLB_HAZARD
 	tlbr				# Read from the TLB
-	nop
-	nop
-	nop
-	dmfc0	ta0, COP_0_TLB_PG_MASK	# fetch the hi entry
+	TLB_HAZARD			# necessary?
+	mfc0	ta0, COP_0_TLB_PG_MASK	# fetch the size
 	dmfc0	ta1, COP_0_TLB_HI	# fetch the hi entry
 	dmfc0	ta2, COP_0_TLB_LO0	# See what we got
 	dmfc0	ta3, COP_0_TLB_LO1	# See what we got
@@ -668,7 +563,7 @@ LEAF(tlb_read, 0)
 	li	a0, TLB_PAGE_MASK
 	mtc0	a0, COP_0_TLB_PG_MASK	# Restore default mask value.
 	mtc0	v1, COP_0_STATUS_REG	# Restore the status register
-	ITLBNOPFIX
+	MTC0_SR_IE_HAZARD
 	sd	ta0, 0(a1)
 	sd	ta1, 8(a1)
 	sd	ta2, 16(a1)
@@ -681,7 +576,7 @@ END(tlb_read)
  */
 LEAF(tlb_get_pid, 0)
 	dmfc0	v0, COP_0_TLB_HI	# get PID
-	li	v1, VMTLB_PID		# mask off PID
+	li	v1, PG_ASID_MASK	# mask off PID
 	j	ra
 	and	v0, v0, v1		# mask off PID
 END(tlb_get_pid)
@@ -691,19 +586,11 @@ END(tlb_get_pid)
  */
 LEAF(tlb_set_pid, 0)
 	dmtc0	a0, COP_0_TLB_HI		# Write the hi reg value
+	TLB_HAZARD
 	j	ra
 	nop
 END(tlb_set_pid)
 
-/*---------------------------------------------------------------- tlb_get_wired
- *	Get the value from the TLB wired reg.
- */
-LEAF(tlb_get_wired, 0)
-	mfc0	v0, COP_0_TLB_WIRED
-	j	ra
-	nop
-END(tlb_get_wired)
-
 /*---------------------------------------------------------------- tlb_set_wired
  *	Write the given value into the TLB wired reg.
  */
@@ -718,6 +605,7 @@ END(tlb_set_wired)
  */
 LEAF(tlb_set_page_mask, 0)
 	mtc0	a0, COP_0_TLB_PG_MASK
+	TLB_HAZARD
 	j	ra
 	nop
 END(tlb_set_page_mask)