Add TLB bypass for instruction emulation

copyinsn() fetches a userland instruction through the direct map.
This lets emulation work with execute-only virtual memory mappings.

OK deraadt@

5 files changed, 76 insertions, 24 deletions

diff --git a/sys/arch/mips64/include/cpu.h b/sys/arch/mips64/include/cpu.h
index 221d4c35d9a..c71b25646ca 100644
--- a/sys/arch/mips64/include/cpu.h
+++ b/sys/arch/mips64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.140 2022/11/19 16:23:48 cheloha Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.141 2023/01/11 03:19:52 visa Exp $	*/
 
 /*-
  * Copyright (c) 1992, 1993
@@ -418,6 +418,7 @@ void	cpu_switchto_asm(struct proc *, struct proc *);
 int	exec_md_map(struct proc *, struct exec_package *);
 void	savectx(struct user *, int);
 
+int	copyinsn(struct proc *, vaddr_t, uint32_t *);
 void	enable_fpu(struct proc *);
 void	save_fpu(void);
 int	fpe_branch_emulate(struct proc *, struct trapframe *, uint32_t,
diff --git a/sys/arch/mips64/include/pmap.h b/sys/arch/mips64/include/pmap.h
index cef8fe7ec33..0352117d422 100644
--- a/sys/arch/mips64/include/pmap.h
+++ b/sys/arch/mips64/include/pmap.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: pmap.h,v 1.51 2023/01/01 19:49:17 miod Exp $ */
+/*      $OpenBSD: pmap.h,v 1.52 2023/01/11 03:19:52 visa Exp $ */
 
 /*
  * Copyright (c) 1987 Carnegie-Mellon University
@@ -158,6 +158,7 @@ extern vaddr_t pmap_prefer_mask;
 #define	PMAP_PREFER_OFFSET(of)		((of) & pmap_prefer_mask)
 
 void	pmap_bootstrap(void);
+int	pmap_copyinsn(pmap_t, vaddr_t, uint32_t *);
 int	pmap_emulate_modify(pmap_t, vaddr_t);
 void	pmap_page_cache(vm_page_t, u_int);
 
diff --git a/sys/arch/mips64/mips64/fp_emulate.c b/sys/arch/mips64/mips64/fp_emulate.c
index f91dd75551a..3b6904e3c60 100644
--- a/sys/arch/mips64/mips64/fp_emulate.c
+++ b/sys/arch/mips64/mips64/fp_emulate.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: fp_emulate.c,v 1.24 2021/03/11 11:16:59 jsg Exp $	*/
+/*	$OpenBSD: fp_emulate.c,v 1.25 2023/01/11 03:19:52 visa Exp $	*/
 
 /*
  * Copyright (c) 2010 Miodrag Vallat.
@@ -210,7 +210,7 @@ MipsFPTrap(struct trapframe *tf)
 	 * if it does, it's probably not your lucky day.
 	 */
 
-	if (copyin32((const void *)pc, &insn) != 0) {
+	if (copyinsn(p, pc, &insn) != 0) {
 		sig = SIGBUS;
 		fault_type = BUS_OBJERR;
 		sv.sival_ptr = (void *)pc;
@@ -219,7 +219,7 @@ MipsFPTrap(struct trapframe *tf)
 	inst = *(InstFmt *)&insn;
 
 	if (tf->cause & CR_BR_DELAY) {
-		if (copyin32((const void *)tf->pc, &branch) != 0) {
+		if (copyinsn(p, tf->pc, &branch) != 0) {
 			sig = SIGBUS;
 			fault_type = BUS_OBJERR;
 			sv.sival_ptr = (void *)tf->pc;
@@ -1639,7 +1639,7 @@ nofpu_emulate_cop1(struct proc *p, struct trapframe *tf, uint32_t insn,
 			 */
 			/* inline MipsEmulateBranch(tf, tf->pc, tf->fsr, insn)*/
 			dest = tf->pc + 4 + ((short)inst.IType.imm << 2);
-			if (copyin32((const void *)(tf->pc + 4), &dinsn) != 0) {
+			if (copyinsn(p, tf->pc + 4, &dinsn) != 0) {
 				sv->sival_ptr = (void *)(tf->pc + 4);
 				return SIGSEGV;
 			}
diff --git a/sys/arch/mips64/mips64/pmap.c b/sys/arch/mips64/mips64/pmap.c
index 69ae0df6b1e..7369e4fa655 100644
--- a/sys/arch/mips64/mips64/pmap.c
+++ b/sys/arch/mips64/mips64/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.123 2023/01/11 03:17:56 visa Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.124 2023/01/11 03:19:52 visa Exp $	*/
 
 /*
  * Copyright (c) 2001-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -2077,3 +2077,38 @@ pmap_update(struct pmap *pmap)
 {
 	Mips_SyncICache(curcpu());
 }
+
+/*
+ * Read an instruction from a given virtual memory address.
+ * TLB read-inhibition is bypassed.
+ */
+int
+pmap_copyinsn(pmap_t pmap, vaddr_t uva, uint32_t *insn)
+{
+	pt_entry_t *pte;
+	paddr_t pa;
+	int found = 0;
+
+	if (uva >= VM_MAXUSER_ADDRESS || pmap == pmap_kernel()) {
+		panic("%s(%p, %p): invalid params", __func__,
+		    pmap, (void *)uva);
+	}
+
+	/*
+	 * Read the instruction through the direct map region.
+	 *
+	 * The pmap lock prevents other threads from changing the mapping
+	 * and repurposing the page frame while this thread accesses the
+	 * direct map.
+	 */
+	pmap_lock(pmap);
+	pte = pmap_pte_lookup(pmap, uva);
+	if (pte != NULL && (*pte & PG_V) != 0 && (*pte & pg_xi) == 0) {
+		pa = pfn_to_pad(*pte) | (uva & PAGE_MASK);
+		*insn = *(uint32_t *)PHYS_TO_XKPHYS(pa, CCA_CACHED);
+		found = 1;
+	}
+	pmap_unlock(pmap);
+
+	return found;
+}
diff --git a/sys/arch/mips64/mips64/trap.c b/sys/arch/mips64/mips64/trap.c
index 3402ebd3d42..784d791d1a7 100644
--- a/sys/arch/mips64/mips64/trap.c
+++ b/sys/arch/mips64/mips64/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.163 2023/01/10 17:04:01 miod Exp $	*/
+/*	$OpenBSD: trap.c,v 1.164 2023/01/11 03:19:52 visa Exp $	*/
 
 /*
  * Copyright (c) 1988 University of Utah.
@@ -412,7 +412,7 @@ fault_common_no_miss:
 		tpc = trapframe->pc; /* Remember if restart */
 		if (trapframe->cause & CR_BR_DELAY) {
 			/* Get the branch instruction. */
-			if (copyin32((const void *)locr0->pc, &branch) != 0) {
+			if (copyinsn(p, locr0->pc, &branch) != 0) {
 				signal = SIGBUS;
 				sicode = BUS_OBJERR;
 				break;
@@ -519,18 +519,17 @@ fault_common_no_miss:
 	case T_BREAK+T_USER:
 	    {
 		struct trapframe *locr0 = p->p_md.md_regs;
-		caddr_t va;
+		vaddr_t va;
 		uint32_t branch = 0;
 		uint32_t instr;
 
 		/* compute address of break instruction */
-		va = (caddr_t)trapframe->pc;
+		va = trapframe->pc;
 		if (trapframe->cause & CR_BR_DELAY) {
 			va += 4;
 
 			/* Read branch instruction. */
-			if (copyin32((const void *)trapframe->pc,
-			    &branch) != 0) {
+			if (copyinsn(p, trapframe->pc, &branch) != 0) {
 				signal = SIGBUS;
 				sicode = BUS_OBJERR;
 				break;
@@ -538,7 +537,7 @@ fault_common_no_miss:
 		}
 
 		/* read break instruction */
-		if (copyin32((const void *)va, &instr) != 0) {
+		if (copyinsn(p, va, &instr) != 0) {
 			signal = SIGBUS;
 			sicode = BUS_OBJERR;
 			break;
@@ -641,18 +640,17 @@ fault_common_no_miss:
 	case T_TRAP+T_USER:
 	    {
 		struct trapframe *locr0 = p->p_md.md_regs;
-		caddr_t va;
+		vaddr_t va;
 		uint32_t branch = 0;
 		uint32_t instr;
 
 		/* compute address of trap instruction */
-		va = (caddr_t)trapframe->pc;
+		va = trapframe->pc;
 		if (trapframe->cause & CR_BR_DELAY) {
 			va += 4;
 
 			/* Read branch instruction. */
-			if (copyin32((const void *)trapframe->pc,
-			    &branch) != 0) {
+			if (copyinsn(p, trapframe->pc, &branch) != 0) {
 				signal = SIGBUS;
 				sicode = BUS_OBJERR;
 				break;
@@ -660,7 +658,7 @@ fault_common_no_miss:
 		}
 
 		/* read break instruction */
-		if (copyin32((const void *)va, &instr) != 0) {
+		if (copyinsn(p, va, &instr) != 0) {
 			signal = SIGBUS;
 			sicode = BUS_OBJERR;
 			break;
@@ -703,18 +701,17 @@ fault_common_no_miss:
 	case T_RES_INST+T_USER:
 	    {
 		register_t *regs = (register_t *)trapframe;
-		caddr_t va;
+		vaddr_t va;
 		uint32_t branch = 0;
 		InstFmt inst;
 
 		/* Compute the instruction's address. */
-		va = (caddr_t)trapframe->pc;
+		va = trapframe->pc;
 		if (trapframe->cause & CR_BR_DELAY) {
 			va += 4;
 
 			/* Get the branch instruction. */
-			if (copyin32((const void *)trapframe->pc,
-			    &branch) != 0) {
+			if (copyinsn(p, trapframe->pc, &branch) != 0) {
 				signal = SIGBUS;
 				sicode = BUS_OBJERR;
 				break;
@@ -722,7 +719,7 @@ fault_common_no_miss:
 		}
 
 		/* Get the faulting instruction. */
-		if (copyin32((const void *)va, &inst.word) != 0) {
+		if (copyinsn(p, va, &inst.word) != 0) {
 			signal = SIGBUS;
 			sicode = BUS_OBJERR;
 			break;
@@ -841,6 +838,24 @@ child_return(void *arg)
 	mi_child_return(p);
 }
 
+int
+copyinsn(struct proc *p, vaddr_t uva, uint32_t *insn)
+{
+	struct vm_map *map = &p->p_vmspace->vm_map;
+	int error = 0;
+
+	if (__predict_false(uva >= VM_MAXUSER_ADDRESS || (uva & 3) != 0))
+		return EFAULT;
+
+	do {
+		if (pmap_copyinsn(map->pmap, uva, insn))
+			break;
+		error = uvm_fault(map, trunc_page(uva), 0, PROT_EXEC);
+	} while (error == 0);
+
+	return error;
+}
+
 #if defined(DDB) || defined(DEBUG)
 void
 trapDump(const char *msg, int (*pr)(const char *, ...))