diff options
Diffstat (limited to 'sys/arch/amd64/amd64/locore.S')
| -rw-r--r-- | sys/arch/amd64/amd64/locore.S | 190 |
1 files changed, 164 insertions, 26 deletions
diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S index 6e00ce3dddf..282a25310c6 100644 --- a/sys/arch/amd64/amd64/locore.S +++ b/sys/arch/amd64/amd64/locore.S @@ -1,4 +1,4 @@ -/* $OpenBSD: locore.S,v 1.93 2018/01/07 19:56:19 mlarkin Exp $ */ +/* $OpenBSD: locore.S,v 1.94 2018/02/21 19:24:15 guenther Exp $ */ /* $NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $ */ /* @@ -113,6 +113,7 @@ #include <sys/syscall.h> #include <machine/param.h> +#include <machine/psl.h> #include <machine/segments.h> #include <machine/specialreg.h> #include <machine/frameasm.h> @@ -176,6 +177,7 @@ _C_LABEL(lapic_isr): .globl _C_LABEL(bootapiver) .globl _C_LABEL(pg_nx) .globl _C_LABEL(pg_g_kern) + .globl _C_LABEL(cpu_meltdown) _C_LABEL(cpu_id): .long 0 # saved from `cpuid' instruction _C_LABEL(cpu_feature): .long 0 # feature flags from 'cpuid' # instruction @@ -210,7 +212,8 @@ _C_LABEL(biosextmem): .long REALEXTMEM _C_LABEL(pg_nx): .quad 0 # NX PTE bit (if CPU supports) _C_LABEL(pg_g_kern): .quad 0 # 0x100 if global pages should be used # in kernel mappings, 0 otherwise (for - # Intel) + # insecure CPUs) +_C_LABEL(cpu_meltdown): .long 0 # 1 if this CPU has Meltdown #define _RELOC(x) ((x) - KERNBASE) #define RELOC(x) _RELOC(_C_LABEL(x)) @@ -236,7 +239,7 @@ gdt64_end: /*****************************************************************************/ /* - * Signal trampoline; copied to top of user stack. + * Signal trampoline; copied to a page mapped into userspace. * gdb's backtrace logic matches against the instructions in this. */ .section .rodata @@ -401,20 +404,34 @@ restore_saved: movq PCB_RSP(%r13),%rsp movq PCB_RBP(%r13),%rbp - movq CPUVAR(TSS),%rcx - movq PCB_KSTACK(%r13),%rdx - movq %rdx,TSS_RSP0(%rcx) - movq PCB_CR3(%r13),%rax - movq %rax,%cr3 + movq %rax,%cr3 /* %rax used below too */ /* Don't bother with the rest if switching to a system process. */ testl $P_SYSTEM,P_FLAG(%r12) jnz switch_restored + /* record the bits needed for future U-->K transition */ + movq PCB_KSTACK(%r13),%rdx + subq $FRAMESIZE,%rdx + movq %rdx,CPUVAR(KERN_RSP) + movq PCB_PMAP(%r13),%rcx + + /* + * Meltdown: iff we're doing separate U+K and U-K page tables, + * then record them in cpu_info for easy access in syscall and + * interrupt trampolines. XXX code patch this + */ + + movq PM_PDIRPA_INTEL(%rcx),%rdx + testq %rdx,%rdx + jz 0f /* yay, no intel suckiness */ + movq %rax,CPUVAR(KERN_CR3) + movq %rdx,CPUVAR(USER_CR3) +0: + /* set the new pmap's bit for the cpu */ movl CPUVAR(CPUID),%edi - movq PCB_PMAP(%r13),%rcx lock btsq %rdi,PM_CPUS(%rcx) #ifdef DIAGNOSTIC @@ -503,8 +520,7 @@ IDTVEC(syscall32) sysret /* go away please */ /* - * syscall insn entry. This currently isn't much faster, but - * it can be made faster in the future. + * syscall insn entry. */ IDTVEC(syscall) /* @@ -514,13 +530,20 @@ IDTVEC(syscall) * the user-space value. * First order of business is to swap to the kernel gs.base so that * we can access our struct cpu_info and use the scratch space there - * to switch to our kernel stack. Once that's in place we can + * to switch to the kernel page tables (thank you, Intel), then + * switch to our kernel stack. Once that's in place we can * unblock interrupts and save the rest of the syscall frame. */ swapgs movq %r15,CPUVAR(SCRATCH) - movq CPUVAR(CURPCB),%r15 - movq PCB_KSTACK(%r15),%r15 + movq CPUVAR(KERN_CR3),%r15 + testq %r15,%r15 + jz Xsyscall_untramp + movq %r15,%cr3 + jmp Xsyscall_untramp + +NENTRY(Xsyscall_untramp) + movq CPUVAR(KERN_RSP),%r15 xchgq %r15,%rsp sti @@ -531,12 +554,11 @@ IDTVEC(syscall) * ss:rsp, etc, so that all GP registers can be * saved. Then, fill in the rest. */ - pushq $(GSEL(GUDATA_SEL, SEL_UPL)) - pushq %r15 - subq $(TF_RSP-TF_TRAPNO),%rsp + movq $(GSEL(GUDATA_SEL, SEL_UPL)),TF_SS(%rsp) + movq %r15,TF_RSP(%rsp) movq CPUVAR(SCRATCH),%r15 - subq $32,%rsp - INTR_SAVE_GPRS + INTR_SAVE_MOST_GPRS_NO_ADJ + movq %rcx,TF_RCX(%rsp) movq %r11, TF_RFLAGS(%rsp) /* old rflags from syscall insn */ movq $(GSEL(GUCODE_SEL, SEL_UPL)), TF_CS(%rsp) movq %rcx,TF_RIP(%rsp) @@ -581,16 +603,45 @@ IDTVEC(syscall) movq TF_RBP(%rsp),%rbp movq TF_RBX(%rsp),%rbx - INTR_RESTORE_SELECTORS + /* Restore FS.base if it's not already in the CPU */ + btsl $CPUF_USERSEGS_BIT,CPUVAR(FLAGS) + jc 99f + movq CPUVAR(CURPCB),%rdx + movq PCB_FSBASE(%rdx),%rax + movq %rax,%rdx + shrq $32,%rdx + movl $MSR_FSBASE,%ecx + wrmsr +99: + /* + * We need to finish reading from the trapframe, then switch + * to the user page tables, swapgs, and return. We need + * to get the final value for the register that was used + * for the mov to %cr3 from somewhere accessible on the + * user page tables, so save it in CPUVAR(SCRATCH) across + * the switch. + */ movq TF_RDX(%rsp),%rdx movq TF_RAX(%rsp),%rax + movq %rax,CPUVAR(SCRATCH) + movq CPUVAR(USER_CR3),%rax movq TF_RIP(%rsp),%rcx movq TF_RFLAGS(%rsp),%r11 movq TF_RSP(%rsp),%rsp + testq %rax,%rax + jz 1f + jmp syscall_trampback + +KUENTRY(syscall_trampback) + movq %rax,%cr3 +1: movq CPUVAR(SCRATCH),%rax + swapgs sysretq + .text + #ifdef DIAGNOSTIC .Lsyscall_spl_not_lowered: movabsq $spl_lowered, %rdi @@ -627,6 +678,12 @@ NENTRY(proc_trampoline) * Return via iretq, for real interrupts and signal returns */ NENTRY(intr_fast_exit) +#ifdef DIAGNOSTIC + pushfq + popq %rdx + testq $PSL_I,%rdx + jnz .Lintr_exit_not_blocked +#endif /* DIAGNOSTIC */ movq TF_RDI(%rsp),%rdi movq TF_RSI(%rsp),%rsi movq TF_R8(%rsp),%r8 @@ -640,11 +697,68 @@ NENTRY(intr_fast_exit) movq TF_RBX(%rsp),%rbx testq $SEL_RPL,TF_CS(%rsp) - je 5f + je intr_exit_recurse /* returning back to kernel? */ + + /* returning to userspace. XXX fix up iret frame here */ + + /* restore FS.base if it's not already in the CPU */ + btsl $CPUF_USERSEGS_BIT,CPUVAR(FLAGS) + jc 99f + movq CPUVAR(CURPCB),%rdx /* for below */ + movq PCB_FSBASE(%rdx),%rax + movq %rax,%rdx + shrq $32,%rdx + movl $MSR_FSBASE,%ecx + wrmsr +99: + /* + * Returning to userspace. We need to go things in this order: + * - update the iret frame from the trapframe + * - finish reading from the trapframe + * - switch to the trampoline stack + * - jump to the .kutext segment + * - switch to the user page tables + * - swapgs + * - iretq + * To get the final value for the register that was used + * for the mov to %cr3, we need access to somewhere accessible + * on the user page tables, so we save it in CPUVAR(SCRATCH) + * across the switch. + */ + /* update iret frame */ + movq CPUVAR(INTR_RSP),%rdx + movq $(GSEL(GUCODE_SEL,SEL_UPL)),IRETQ_CS(%rdx) + movq TF_RIP(%rsp),%rax + movq %rax,IRETQ_RIP(%rdx) + movq TF_RFLAGS(%rsp),%rax + movq %rax,IRETQ_RFLAGS(%rdx) + movq TF_RSP(%rsp),%rax + movq %rax,IRETQ_RSP(%rdx) + movq $(GSEL(GUDATA_SEL,SEL_UPL)),IRETQ_SS(%rdx) + /* finish with the trap frame */ + movq TF_RAX(%rsp),%rax + movq %rax,CPUVAR(SCRATCH) + movq TF_RCX(%rsp),%rcx + movq TF_R11(%rsp),%r11 + /* switch to the trampoline stack */ + xchgq %rdx,%rsp + movq TF_RDX(%rdx),%rdx + movq CPUVAR(USER_CR3),%rax + testq %rax,%rax + jz 1f + jmp iretq_tramp - INTR_RESTORE_SELECTORS +KUENTRY(iretq_tramp) + movq %rax,%cr3 +1: movq CPUVAR(SCRATCH),%rax + swapgs -5: movq TF_RDX(%rsp),%rdx + .globl _C_LABEL(doreti_iret) +_C_LABEL(doreti_iret): + iretq + +NENTRY(intr_exit_recurse) + movq TF_RDX(%rsp),%rdx movq TF_RCX(%rsp),%rcx movq TF_R11(%rsp),%r11 movq TF_RAX(%rsp),%rax @@ -662,9 +776,6 @@ NENTRY(intr_fast_exit) #endif /* !defined(GPROF) && defined(DDBPROF) */ addq $TF_RIP,%rsp - - .globl _C_LABEL(doreti_iret) -_C_LABEL(doreti_iret): iretq @@ -697,6 +808,33 @@ _C_LABEL(doreti_iret): addq $TF_RIP,%rsp iretq #endif /* !defined(GPROF) && defined(DDBPROF) */ + .text + +#ifdef DIAGNOSTIC +.Lintr_exit_not_blocked: + xchgw %bx, %bx + movl warn_once(%rip),%edi + testl %edi,%edi + jnz 1f + incl %edi + movl %edi,warn_once(%rip) + leaq .Lnot_blocked(%rip),%rdi + call _C_LABEL(printf) +#ifdef DDB + int $3 +#endif /* DDB */ +1: cli + jmp intr_fast_exit + + .data +.global warn_once +warn_once: + .long 0 + .section .rodata +.Lnot_blocked: + .asciz "WARNING: INTERRUPTS NOT BLOCKED ON INTERRUPT RETURN: 0x%x 0x%x\n" + .text +#endif ENTRY(xrstor_user) movq %rsi, %rdx |