Utilize the TLB Execute-Inhibit bit with non-executable mappings on CPUs

that support the Execute-Inhibit exception. This makes user space W^X
effective on Octeon Plus and later Octeon versions.

Feedback from miod@, thanks!
No objection from deraadt@

11 files changed, 195 insertions, 60 deletions

diff --git a/sys/arch/mips64/include/cpu.h b/sys/arch/mips64/include/cpu.h
index c1ba47cc0a4..af67a5aa6ae 100644
--- a/sys/arch/mips64/include/cpu.h
+++ b/sys/arch/mips64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.110 2016/03/06 19:42:27 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.111 2016/08/14 08:23:52 visa Exp $	*/
 
 /*-
  * Copyright (c) 1992, 1993
@@ -480,10 +480,12 @@ register_t cp0_get_config(void);
 uint32_t cp0_get_config_1(void);
 uint32_t cp0_get_config_2(void);
 uint32_t cp0_get_config_3(void);
+uint32_t cp0_get_pagegrain(void);
 register_t cp0_get_prid(void);
 void	cp0_reset_cause(register_t);
 void	cp0_set_compare(u_int);
 void	cp0_set_config(register_t);
+void	cp0_set_pagegrain(uint32_t);
 void	cp0_set_trapbase(register_t);
 u_int	cp1_get_prid(void);
 
diff --git a/sys/arch/mips64/include/mips_cpu.h b/sys/arch/mips64/include/mips_cpu.h
index 9decc90133f..990aa40abdb 100644
--- a/sys/arch/mips64/include/mips_cpu.h
+++ b/sys/arch/mips64/include/mips_cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mips_cpu.h,v 1.3 2016/08/06 09:32:04 visa Exp $	*/
+/*	$OpenBSD: mips_cpu.h,v 1.4 2016/08/14 08:23:52 visa Exp $	*/
 
 /*-
  * Copyright (c) 1992, 1993
@@ -292,6 +292,32 @@
 #define	FPC_CSR			$31
 
 /*
+ * Config3 register
+ */
+#define	CONFIG3_M		0x80000000
+#define	CONFIG3_BPG		0x40000000
+#define	CONFIG3_CMGCR		0x20000000
+#define	CONFIG3_IPLW		0x00600000
+#define	CONFIG3_MMAR		0x001c0000
+#define	CONFIG3_MCU		0x00020000
+#define	CONFIG3_ISAOnExc	0x00010000
+#define	CONFIG3_ISA		0x0000c000
+#define	CONFIG3_ULRI		0x00002000
+#define	CONFIG3_RXI		0x00001000
+#define	CONFIG3_DSP2P		0x00000800
+#define	CONFIG3_DSPP		0x00000400
+#define	CONFIG3_CTXTC		0x00000200
+#define	CONFIG3_ITL		0x00000100
+#define	CONFIG3_LPA		0x00000080
+#define	CONFIG3_VEIC		0x00000040
+#define	CONFIG3_VInt		0x00000020
+#define	CONFIG3_SP		0x00000010
+#define	CONFIG3_CDMM		0x00000008
+#define	CONFIG3_MT		0x00000004
+#define	CONFIG3_SM		0x00000002
+#define	CONFIG3_TL		0x00000001
+
+/*
  * PageGrain register
  */
 #define	PGRAIN_RIE		0x80000000
diff --git a/sys/arch/mips64/include/pte.h b/sys/arch/mips64/include/pte.h
index e4398482564..64a173c6889 100644
--- a/sys/arch/mips64/include/pte.h
+++ b/sys/arch/mips64/include/pte.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pte.h,v 1.18 2014/03/22 00:00:38 miod Exp $	*/
+/*	$OpenBSD: pte.h,v 1.19 2016/08/14 08:23:52 visa Exp $	*/
 
 /*
  * Copyright (c) 1988 University of Utah.
@@ -64,6 +64,27 @@ typedef u_int32_t pt_entry_t;
 
 #endif /* _LOCORE */
 
+#ifdef MIPS_PTE64
+#define	PTE_BITS	64
+#else
+#define	PTE_BITS	32
+#endif
+
+#ifdef CPU_MIPS64R2
+#define	PTE_CLEAR_SWBITS(reg)						\
+	.set	push;							\
+	.set	mips64r2;						\
+	/* Clear SW bits around PG_XI. */				\
+	dins	reg, zero, (PTE_BITS - 1), 1;				\
+	dins	reg, zero, PG_FRAMEBITS, (PTE_BITS - 2 - PG_FRAMEBITS);	\
+	.set	pop
+#else
+#define	PTE_CLEAR_SWBITS(reg)						\
+	/* Clear SW bits left of PG_FRAMEBITS. */			\
+	dsll	reg, reg, (64 - PG_FRAMEBITS);				\
+	dsrl	reg, reg, (64 - PG_FRAMEBITS)
+#endif
+
 /* entryhi values */
 
 #ifndef CPU_R8000
@@ -100,13 +121,23 @@ typedef u_int32_t pt_entry_t;
 #endif
 
 /* software pte bits - not put in entrylo */
-#ifdef CPU_R8000
+#if defined(CPU_R8000)
 #define	PG_WIRED	0x00000010
 #define	PG_RO		0x00000020
-#else
+#elif defined(CPU_R4000)
 #define	PG_WIRED	(1ULL << (PG_FRAMEBITS + 2))
 #define	PG_RO		(1ULL << (PG_FRAMEBITS + 1))
 #define	PG_SP		(1ULL << (PG_FRAMEBITS + 0))	/* ``special'' bit */
+#else
+#define	PG_WIRED	(1ULL << (PG_FRAMEBITS + 2))
+			/* 1ULL << (PG_FRAMEBITS + 1) is PG_XI. */
+#define	PG_RO		(1ULL << (PG_FRAMEBITS + 0))
+#endif
+
+#ifdef CPU_MIPS64R2
+#define	PG_XI		(1ULL << (PTE_BITS - 2))
+#else
+#define	PG_XI		0x00000000
 #endif
 
 #define	PG_NV		0x00000000
diff --git a/sys/arch/mips64/mips64/context.S b/sys/arch/mips64/mips64/context.S
index 40ea1dbbdd3..137b7fc2ae8 100644
--- a/sys/arch/mips64/mips64/context.S
+++ b/sys/arch/mips64/mips64/context.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: context.S,v 1.54 2015/09/29 17:07:53 miod Exp $ */
+/*	$OpenBSD: context.S,v 1.55 2016/08/14 08:23:52 visa Exp $ */
 
 /*
  * Copyright (c) 2002-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -239,10 +239,8 @@ ctx1:
 	dmtc0	v0, COP_0_TLB_HI
 	PTE_LOAD ta0, 0(t1)
 	PTE_LOAD ta1, PTE_OFFS(t1)
-	dsll	ta0, ta0, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
-	dsrl	ta0, ta0, (64 - PG_FRAMEBITS)
-	dsll	ta1, ta1, (64 - PG_FRAMEBITS)
-	dsrl	ta1, ta1, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(ta0)
+	PTE_CLEAR_SWBITS(ta1)
 	dmtc0	ta0, COP_0_TLB_LO0
 	dmtc0	ta1, COP_0_TLB_LO1
 	PTR_ADDU v0, 2*PAGE_SIZE
@@ -254,13 +252,11 @@ ctx1:
 	dmtc0	v0, COP_0_TLB_HI		# init high entry (tlbid)
 	PTE_LOAD ta0, (2*PTE_OFFS)(t1)
 	PTE_LOAD ta1, (3*PTE_OFFS)(t1)
-	dsll	ta0, ta0, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
-	dsrl	ta0, ta0, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(ta0)
 	TLB_HAZARD
 	tlbp
 	TLB_HAZARD			# necessary?
-	dsll	ta1, ta1, (64 - PG_FRAMEBITS)
-	dsrl	ta1, ta1, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(ta1)
 	mfc0	t0, COP_0_TLB_INDEX
 	nop
 	bltz	t0, ctx2			# not in tlb
diff --git a/sys/arch/mips64/mips64/db_machdep.c b/sys/arch/mips64/mips64/db_machdep.c
index fbd82d25edd..3af59557c7d 100644
--- a/sys/arch/mips64/mips64/db_machdep.c
+++ b/sys/arch/mips64/mips64/db_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: db_machdep.c,v 1.45 2016/03/06 19:42:27 mpi Exp $ */
+/*	$OpenBSD: db_machdep.c,v 1.46 2016/08/14 08:23:52 visa Exp $ */
 
 /*
  * Copyright (c) 1998-2003 Opsycon AB (www.opsycon.se)
@@ -400,6 +400,9 @@ db_print_tlb(uint tlbno, uint64_t tlblo)
 #endif
 	if (tlblo & PG_V) {
 		db_printf("%016lx ", pa);
+#ifdef CPU_MIPS64R2
+		db_printf("%c", tlblo & PG_XI ? 'X' : ' ');
+#endif
 		db_printf("%c", tlblo & PG_M ? 'M' : ' ');
 #ifndef CPU_R8000
 		db_printf("%c", tlblo & PG_G ? 'G' : ' ');
diff --git a/sys/arch/mips64/mips64/mips64r2.S b/sys/arch/mips64/mips64/mips64r2.S
index f8169550852..1b0284f6cc3 100644
--- a/sys/arch/mips64/mips64/mips64r2.S
+++ b/sys/arch/mips64/mips64/mips64r2.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mips64r2.S,v 1.2 2012/10/03 11:18:23 miod Exp $	*/
+/*	$OpenBSD: mips64r2.S,v 1.3 2016/08/14 08:23:52 visa Exp $	*/
 
 /*
  * Copyright (c) 2011 Miodrag Vallat.
@@ -40,3 +40,14 @@ LEAF(cp0_get_config_3, 0)
 	j	ra
 	mfc0	v0, COP_0_CONFIG, 3
 END(cp0_get_config_3)
+
+LEAF(cp0_get_pagegrain, 0)
+	j	ra
+	 mfc0	v0, COP_0_TLB_PG_GRAIN
+END(cp0_get_pagegrain)
+
+LEAF(cp0_set_pagegrain, 0)
+	mtc0	a0, COP_0_TLB_PG_GRAIN
+	jr.hb	ra
+	 nop
+END(cp0_set_pagegrain)
diff --git a/sys/arch/mips64/mips64/pmap.c b/sys/arch/mips64/mips64/pmap.c
index d3b9465eed8..d8b96a46910 100644
--- a/sys/arch/mips64/mips64/pmap.c
+++ b/sys/arch/mips64/mips64/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.90 2016/05/11 15:50:29 visa Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.91 2016/08/14 08:23:52 visa Exp $	*/
 
 /*
  * Copyright (c) 2001-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -38,6 +38,7 @@
 #include <sys/atomic.h>
 
 #include <mips64/cache.h>
+#include <mips64/mips_cpu.h>
 #include <machine/autoconf.h>
 #include <machine/cpu.h>
 #include <machine/vmparam.h>
@@ -64,7 +65,7 @@ void	 pmap_do_remove(pmap_t, vaddr_t, vaddr_t);
 int	 pmap_enter_pv(pmap_t, vaddr_t, vm_page_t, pt_entry_t *);
 void	 pmap_remove_pv(pmap_t, vaddr_t, paddr_t);
 void	 pmap_page_remove(struct vm_page *);
-void	 pmap_page_wrprotect(struct vm_page *);
+void	 pmap_page_wrprotect(struct vm_page *, vm_prot_t);
 void	*pmap_pg_alloc(struct pool *, int, int *);
 void	 pmap_pg_free(struct pool *, void *);
 
@@ -156,6 +157,8 @@ pt_entry_t	*Sysmap;		/* kernel pte table */
 u_int		Sysmapsize;		/* number of pte's in Sysmap */
 const vaddr_t	Sysmapbase = VM_MIN_KERNEL_ADDRESS;	/* for libkvm */
 
+pt_entry_t	pg_xi;
+
 void
 pmap_invalidate_user_page(pmap_t pmap, vaddr_t va)
 {
@@ -383,6 +386,11 @@ pmap_bootstrap(void)
 		pmap_asid_info[i].pma_asidgen = 1;
 		pmap_asid_info[i].pma_asid = MIN_USER_ASID + 1;
 	}
+
+#ifdef CPU_MIPS64R2
+	if (cp0_get_pagegrain() & PGRAIN_XIE)
+		pg_xi = PG_XI;
+#endif
 }
 
 /*
@@ -713,12 +721,16 @@ pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
  * Makes all mappings to a given page read-only.
  */
 void
-pmap_page_wrprotect(struct vm_page *pg)
+pmap_page_wrprotect(struct vm_page *pg, vm_prot_t prot)
 {
 	struct cpu_info *ci = curcpu();
-	pt_entry_t *pte, entry;
+	pt_entry_t *pte, entry, p;
 	pv_entry_t pv;
 
+	p = PG_RO;
+	if (!(prot & PROT_EXEC))
+		p |= pg_xi;
+
 	mtx_enter(&pg->mdpage.pv_mtx);
 	for (pv = pg_to_pvh(pg); pv != NULL; pv = pv->pv_next) {
 		if (pv->pv_pmap == pmap_kernel()) {
@@ -735,7 +747,7 @@ pmap_page_wrprotect(struct vm_page *pg)
 			    (entry & PG_CACHEMODE) == PG_CACHED)
 				Mips_HitSyncDCachePage(ci, pv->pv_va,
 				    pfn_to_pad(entry));
-			entry = (entry & ~PG_M) | PG_RO;
+			entry = (entry & ~(PG_M | PG_XI)) | p;
 			*pte = entry;
 			pmap_update_kernel_page(pv->pv_va, entry);
 			pmap_shootdown_page(pmap_kernel(), pv->pv_va);
@@ -750,7 +762,7 @@ pmap_page_wrprotect(struct vm_page *pg)
 			    (entry & PG_CACHEMODE) == PG_CACHED)
 				Mips_SyncDCachePage(ci, pv->pv_va,
 				    pfn_to_pad(entry));
-			entry = (entry & ~PG_M) | PG_RO;
+			entry = (entry & ~(PG_M | PG_XI)) | p;
 			*pte = entry;
 			pmap_update_user_page(pv->pv_pmap, pv->pv_va, entry);
 			pmap_shootdown_page(pv->pv_pmap, pv->pv_va);
@@ -828,7 +840,7 @@ pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
 	/* copy_on_write */
 	case PROT_READ:
 	case PROT_READ | PROT_EXEC:
-		pmap_page_wrprotect(pg);
+		pmap_page_wrprotect(pg, prot);
 		break;
 
 	/* remove_all */
@@ -859,6 +871,8 @@ pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
 	}
 
 	p = (prot & PROT_WRITE) ? PG_M : PG_RO;
+	if (!(prot & PROT_EXEC))
+		p |= pg_xi;
 
 	pmap_lock(pmap);
 
@@ -886,7 +900,7 @@ pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
 				if ((entry & PG_CACHEMODE) == PG_CACHED)
 					Mips_HitSyncDCachePage(ci, sva,
 					    pfn_to_pad(entry));
-			entry = (entry & ~(PG_M | PG_RO)) | p;
+			entry = (entry & ~(PG_M | PG_RO | PG_XI)) | p;
 			*pte = entry;
 			/*
 			 * Update the TLB if the given address is in the cache.
@@ -932,7 +946,7 @@ pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
 					Mips_SyncDCachePage(ci, sva,
 					    pfn_to_pad(entry));
 			}
-			entry = (entry & ~(PG_M | PG_RO)) | p;
+			entry = (entry & ~(PG_M | PG_RO | PG_XI)) | p;
 			*pte = entry;
 			pmap_update_user_page(pmap, sva, entry);
 			pmap_shootdown_page(pmap, sva);
@@ -1026,6 +1040,9 @@ pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
 		}
 	}
 
+	if (!(prot & PROT_EXEC))
+		npte |= pg_xi;
+
 	if (pmap == pmap_kernel()) {
 		if (pg != NULL) {
 			if (pmap_enter_pv(pmap, va, pg, &npte) != 0) {
@@ -1192,6 +1209,8 @@ pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
 		npte |= PG_RWPAGE;
 	else
 		npte |= PG_ROPAGE;
+	if (!(prot & PROT_EXEC))
+		npte |= pg_xi;
 	pte = kvtopte(va);
 	if ((*pte & PG_V) == 0) {
 		atomic_inc_long(&pmap_kernel()->pm_stats.resident_count);
diff --git a/sys/arch/mips64/mips64/tlbhandler.S b/sys/arch/mips64/mips64/tlbhandler.S
index 604a0e5245a..7414e9a12b9 100644
--- a/sys/arch/mips64/mips64/tlbhandler.S
+++ b/sys/arch/mips64/mips64/tlbhandler.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tlbhandler.S,v 1.44 2014/08/12 19:33:59 miod Exp $ */
+/*	$OpenBSD: tlbhandler.S,v 1.45 2016/08/14 08:23:52 visa Exp $ */
 
 /*
  * Copyright (c) 1995-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -160,12 +160,12 @@ xtlb_miss:
 	bltz	k1, _inv_seg
 	dsll	k0, k0, 1
 	dsll	k1, k1, 1
-#else
-	dsll	k0, k0, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
-	dsll	k1, k1, (64 - PG_FRAMEBITS)
-#endif
 	dsrl	k0, k0, (64 - PG_FRAMEBITS)
 	dsrl	k1, k1, (64 - PG_FRAMEBITS)
+#else
+	PTE_CLEAR_SWBITS(k0)
+	PTE_CLEAR_SWBITS(k1)
+#endif
 	dmtc0	k0, COP_0_TLB_LO0
 	dmtc0	k1, COP_0_TLB_LO1
 	TLB_HAZARD
@@ -218,8 +218,21 @@ go_u_general:
  */
 NLEAF(k_tlb_inv, 0)
 	.set	noat
-	LA	k1, (VM_MIN_KERNEL_ADDRESS)	# compute index
+#ifdef CPU_OCTEON
+	/* Check if the fault was caused by an instruction fetch. */
+	dmfc0	k0, COP_0_CAUSE_REG
+	/* XXX bbit0 k0, CR_BR_DELAY_BIT, 1f */
+	li	k1, CR_BR_DELAY
+	and	k0, k0, k1
+	beq	k0, zero, 1f			# fault in a branch delay slot?
+	 dmfc0	k1, COP_0_EXC_PC
+	daddu	k1, k1, 4			# adjust for the slot
+1:	dmfc0	k0, COP_0_BAD_VADDR
+	beq	k0, k1, go_k_general
+#else
 	dmfc0	k0, COP_0_BAD_VADDR		# get the fault address
+#endif
+	LA	k1, (VM_MIN_KERNEL_ADDRESS)	# compute index
 	PTR_SUBU k0, k0, k1
 	lw	k1, Sysmapsize			# index within range?
 	PTR_SRL	k0, k0, PGSHIFT
@@ -241,14 +254,12 @@ NLEAF(k_tlb_inv, 0)
 	bltz	k0, sys_stk_chk			# probe fail
 	PTE_LOAD k0, 0(k1)			# get PTE entry
 
-	dsll	k0, k0, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
-	dsrl	k0, k0, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(k0)
 	dmtc0	k0, COP_0_TLB_LO0		# load PTE entry
 	and	k0, k0, PG_V			# check for valid entry
 	beq	k0, zero, go_k_general		# PTE invalid
 	PTE_LOAD k0, PTE_OFFS(k1)		# get odd PTE entry
-	dsll	k0, k0, (64 - PG_FRAMEBITS)
-	dsrl	k0, k0, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(k0)
 	dmtc0	k0, COP_0_TLB_LO1		# load PTE entry
 	TLB_HAZARD
 	tlbwi					# write TLB
@@ -266,14 +277,12 @@ k_tlb_inv_odd:
 	bltz	k0, sys_stk_chk			# probe fail
 	PTE_LOAD k0, 0(k1)			# get PTE entry
 
-	dsll	k0, k0, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
-	dsrl	k0, k0, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(k0)
 	dmtc0	k0, COP_0_TLB_LO1		# save PTE entry
 	and	k0, k0, PG_V			# check for valid entry
 	beq	k0, zero, go_k_general		# PTE invalid
 	PTE_LOAD k0, -PTE_OFFS(k1)		# get even PTE entry
-	dsll	k0, k0, (64 - PG_FRAMEBITS)
-	dsrl	k0, k0, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(k0)
 	dmtc0	k0, COP_0_TLB_LO0		# save PTE entry
 	TLB_HAZARD
 	tlbwi					# update TLB
@@ -313,11 +322,9 @@ NLEAF(k_tlb_miss, 0)
 	PTR_ADDU k1, k1, k0
 	PTE_LOAD k0, 0(k1)			# get PTE entry
 	PTE_LOAD k1, PTE_OFFS(k1)		# get odd PTE entry
-	dsll	k0, k0, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
-	dsrl	k0, k0, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(k0)
 	dmtc0	k0, COP_0_TLB_LO0		# load PTE entry
-	dsll	k1, k1, (64 - PG_FRAMEBITS)
-	dsrl	k1, k1, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(k1)
 	dmtc0	k1, COP_0_TLB_LO1		# load PTE entry
 	TLB_HAZARD
 	tlbwr					# write TLB
@@ -498,10 +505,10 @@ LEAF(tlb_update, 0)
 	TLB_HAZARD
 	tlbp				# Probe for the entry.
 	TLB_HAZARD			# necessary?
-	dsll	a1, a1, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
+	PTE_CLEAR_SWBITS(a1)
 	mfc0	v0, COP_0_TLB_INDEX	# See what we got
 	bne	ta1, zero, 2f		# Decide even odd
-	dsrl	a1, a1, (64 - PG_FRAMEBITS)
+	nop
 # EVEN
 	bltz	v0, 1f			# index < 0 => !found
 	nop
@@ -604,11 +611,9 @@ LEAF(tlb_update_indexed, 0)
 	mtc0	ta1, COP_0_TLB_PG_MASK	# init mask
 	dmtc0	a0, COP_0_TLB_HI	# init high reg.
 
-	dsll	a1, a1, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
-	dsrl	a1, a1, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(a1)
 	dmtc0	a1, COP_0_TLB_LO0	# init low reg0.
-	dsll	a2, a2, (64 - PG_FRAMEBITS)	# clear bits left of PG_FRAME
-	dsrl	a2, a2, (64 - PG_FRAMEBITS)
+	PTE_CLEAR_SWBITS(a2)
 	dmtc0	a2, COP_0_TLB_LO1	# init low reg1.
 
 	TLB_HAZARD
diff --git a/sys/arch/mips64/mips64/trap.c b/sys/arch/mips64/mips64/trap.c
index 0bd71e562d2..195f914124b 100644
--- a/sys/arch/mips64/mips64/trap.c
+++ b/sys/arch/mips64/mips64/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.116 2016/03/06 19:42:27 mpi Exp $	*/
+/*	$OpenBSD: trap.c,v 1.117 2016/08/14 08:23:52 visa Exp $	*/
 
 /*
  * Copyright (c) 1988 University of Utah.
@@ -304,7 +304,25 @@ itsa(struct trapframe *trapframe, struct cpu_info *ci, struct proc *p,
 
 	case T_TLB_LD_MISS:
 	case T_TLB_ST_MISS:
-		ftype = (type == T_TLB_ST_MISS) ? PROT_WRITE : PROT_READ;
+		if (type == T_TLB_LD_MISS) {
+#ifdef CPU_OCTEON
+			vaddr_t pc;
+
+			/*
+			 * Check if the fault was caused by
+			 * an instruction fetch.
+			 */
+			pc = trapframe->pc;
+			if (trapframe->cause & CR_BR_DELAY)
+				pc += 4;
+			if (pc == trapframe->badvaddr)
+				ftype = PROT_EXEC;
+			else
+#endif
+			ftype = PROT_READ;
+		} else
+			ftype = PROT_WRITE;
+
 		pcb = &p->p_addr->u_pcb;
 		/* check for kernel address */
 		if (trapframe->badvaddr < 0) {
@@ -342,10 +360,22 @@ itsa(struct trapframe *trapframe, struct cpu_info *ci, struct proc *p,
 			goto err;
 		}
 
-	case T_TLB_LD_MISS+T_USER:
+	case T_TLB_LD_MISS+T_USER: {
+#ifdef CPU_OCTEON
+		vaddr_t pc;
+
+		/* Check if the fault was caused by an instruction fetch. */
+		pc = trapframe->pc;
+		if (trapframe->cause & CR_BR_DELAY)
+			pc += 4;
+		if (pc == trapframe->badvaddr)
+			ftype = PROT_EXEC;
+		else
+#endif
 		ftype = PROT_READ;
 		pcb = &p->p_addr->u_pcb;
 		goto fault_common;
+	}
 
 	case T_TLB_ST_MISS+T_USER:
 		ftype = PROT_WRITE;
diff --git a/sys/arch/octeon/octeon/locore.S b/sys/arch/octeon/octeon/locore.S
index 7f77b75131e..2f4491c3299 100644
--- a/sys/arch/octeon/octeon/locore.S
+++ b/sys/arch/octeon/octeon/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.9 2016/08/06 09:32:05 visa Exp $ */
+/*	$OpenBSD: locore.S,v 1.10 2016/08/14 08:23:52 visa Exp $ */
 
 /*
  * Copyright (c) 2001-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -61,10 +61,6 @@ start:
 #endif
 	dmtc0	t0, COP_0_CVMMEMCTL
 
-/* initialize pagegrain */
-	dli	t0, PGRAIN_ELPA
-	dmtc0	t0, COP_0_TLB_PG_GRAIN
-
 	mfc0    v0, COP_0_STATUS_REG
 	li	v1, ~(SR_INT_ENAB | SR_ERL | SR_EXL)
 	and	v0, v1
diff --git a/sys/arch/octeon/octeon/machdep.c b/sys/arch/octeon/octeon/machdep.c
index f323024960a..64a7b260897 100644
--- a/sys/arch/octeon/octeon/machdep.c
+++ b/sys/arch/octeon/octeon/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.74 2016/07/01 15:12:37 visa Exp $ */
+/*	$OpenBSD: machdep.c,v 1.75 2016/08/14 08:23:52 visa Exp $ */
 
 /*
  * Copyright (c) 2009, 2010 Miodrag Vallat.
@@ -130,6 +130,7 @@ vaddr_t		mips_init(__register_t, __register_t, __register_t, __register_t);
 boolean_t 	is_memory_range(paddr_t, psize_t, psize_t);
 void		octeon_memory_init(struct boot_info *);
 int		octeon_cpuspeed(int *);
+void		octeon_tlb_init(void);
 static void	process_bootargs(void);
 static uint64_t	get_ncpusfound(void);
 
@@ -337,7 +338,7 @@ mips_init(__register_t a0, __register_t a1, __register_t a2 __unused,
 	Octeon_ConfigCache(curcpu());
 	Octeon_SyncCache(curcpu());
 
-	tlb_init(bootcpu_hwinfo.tlbsize);
+	octeon_tlb_init();
 
 	/*
 	 * Save the the boot information for future reference since we can't
@@ -593,6 +594,21 @@ octeon_ioclock_speed(void)
 	}
 }
 
+void
+octeon_tlb_init(void)
+{
+	uint32_t pgrain = 0;
+
+#ifdef MIPS_PTE64
+	pgrain |= PGRAIN_ELPA;
+#endif
+	if (cp0_get_config_3() & CONFIG3_RXI)
+		pgrain |= PGRAIN_XIE;
+	cp0_set_pagegrain(pgrain);
+
+	tlb_init(bootcpu_hwinfo.tlbsize);
+}
+
 static u_int64_t
 get_ncpusfound(void)
 {
@@ -825,7 +841,7 @@ hw_cpu_hatch(struct cpu_info *ci)
 	 */
 	setsr(getsr() | SR_KX | SR_UX);
 
-	tlb_init(64);
+	octeon_tlb_init();
 	tlb_set_pid(0);
 
 	/*