From 9fbf631ea4d32275e2a0b73042a7f5c42b84dddf Mon Sep 17 00:00:00 2001 From: Martin Pieuchot Date: Mon, 29 May 2017 14:19:51 +0000 Subject: Kill SPINLOCK_SPIN_HOOK, use CPU_BUSY_CYCLE() instead. ok visa@, kettenis@ --- sys/arch/alpha/alpha/lock_machdep.c | 7 +++-- sys/arch/alpha/alpha/machdep.c | 7 ++--- sys/arch/alpha/alpha/mutex.c | 6 ++--- sys/arch/alpha/include/cpu.h | 25 ++++++++++++++++-- sys/arch/alpha/include/lock.h | 25 +----------------- sys/arch/amd64/amd64/fpu.c | 5 ++-- sys/arch/amd64/amd64/lapic.c | 5 ++-- sys/arch/amd64/amd64/lock_machdep.c | 8 +++--- sys/arch/amd64/amd64/pmap.c | 11 ++++---- sys/arch/amd64/include/lock.h | 4 +-- sys/arch/hppa/hppa/lock_machdep.c | 9 +++---- sys/arch/i386/i386/lock_machdep.c | 8 +++--- sys/arch/i386/i386/pmap.c | 13 +++++---- sys/arch/i386/include/lock.h | 4 +-- sys/arch/i386/isa/npx.c | 5 ++-- sys/arch/luna88k/luna88k/machdep.c | 3 +-- sys/arch/m88k/m88k/m8820x_machdep.c | 3 +-- sys/arch/m88k/m88k/m88k_machdep.c | 3 +-- sys/arch/m88k/m88k/mplock.c | 10 +++---- sys/arch/mips64/mips64/lock_machdep.c | 9 +++---- sys/arch/powerpc/include/lock.h | 4 +-- sys/arch/powerpc/powerpc/lock_machdep.c | 8 +++--- sys/arch/powerpc/powerpc/mutex.c | 6 ++--- sys/arch/sparc64/include/cpu.h | 43 ++++++++++++++++++++++++++++-- sys/arch/sparc64/include/lock.h | 47 +-------------------------------- sys/arch/sparc64/sparc64/lock_machdep.c | 8 +++--- 26 files changed, 127 insertions(+), 159 deletions(-) (limited to 'sys/arch') diff --git a/sys/arch/alpha/alpha/lock_machdep.c b/sys/arch/alpha/alpha/lock_machdep.c index 21100c8c0ef..f9292e87ca0 100644 --- a/sys/arch/alpha/alpha/lock_machdep.c +++ b/sys/arch/alpha/alpha/lock_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: lock_machdep.c,v 1.5 2017/04/30 16:45:45 mpi Exp $ */ +/* $OpenBSD: lock_machdep.c,v 1.6 2017/05/29 14:19:49 mpi Exp $ */ /* * Copyright (c) 2007 Artur Grabowski @@ -22,7 +22,6 @@ #include #include -#include #if defined(MP_LOCKDEBUG) #ifndef DDB @@ -69,14 +68,14 @@ __mp_lock_spin(struct __mp_lock *mpl) { #ifndef MP_LOCKDEBUG while (mpl->mpl_count != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); #else int nticks = __mp_lock_spinout; if (!CPU_IS_PRIMARY(curcpu())) nticks += nticks; while (mpl->mpl_count != 0 && --nticks > 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); if (nticks == 0) { db_printf("__mp_lock(%p): lock spun out", mpl); diff --git a/sys/arch/alpha/alpha/machdep.c b/sys/arch/alpha/alpha/machdep.c index 9a0db1004fe..57c7d0f2721 100644 --- a/sys/arch/alpha/alpha/machdep.c +++ b/sys/arch/alpha/alpha/machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: machdep.c,v 1.180 2017/04/30 16:45:45 mpi Exp $ */ +/* $OpenBSD: machdep.c,v 1.181 2017/05/29 14:19:49 mpi Exp $ */ /* $NetBSD: machdep.c,v 1.210 2000/06/01 17:12:38 thorpej Exp $ */ /*- @@ -105,9 +105,6 @@ #ifndef NO_IEEE #include #endif -#ifdef MULTIPROCESSOR -#include -#endif #include @@ -1761,7 +1758,7 @@ fpusave_proc(struct proc *p, int save) alpha_pal_swpipl(s); while (p->p_addr->u_pcb.pcb_fpcpu != NULL) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); #else KASSERT(ci->ci_fpcurproc == p); fpusave_cpu(ci, save); diff --git a/sys/arch/alpha/alpha/mutex.c b/sys/arch/alpha/alpha/mutex.c index ada28fad1cc..04dbf9cdfd3 100644 --- a/sys/arch/alpha/alpha/mutex.c +++ b/sys/arch/alpha/alpha/mutex.c @@ -1,4 +1,4 @@ -/* $OpenBSD: mutex.c,v 1.17 2017/04/20 13:57:29 visa Exp $ */ +/* $OpenBSD: mutex.c,v 1.18 2017/05/29 14:19:49 mpi Exp $ */ /* * Copyright (c) 2004 Artur Grabowski @@ -31,7 +31,7 @@ #include #include -#include +#include #include @@ -48,7 +48,7 @@ void __mtx_enter(struct mutex *mtx) { while (__mtx_enter_try(mtx) == 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } int diff --git a/sys/arch/alpha/include/cpu.h b/sys/arch/alpha/include/cpu.h index fb1dc79e045..bded5b32aa1 100644 --- a/sys/arch/alpha/include/cpu.h +++ b/sys/arch/alpha/include/cpu.h @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.h,v 1.57 2016/03/30 15:39:46 afresh1 Exp $ */ +/* $OpenBSD: cpu.h,v 1.58 2017/05/29 14:19:49 mpi Exp $ */ /* $NetBSD: cpu.h,v 1.45 2000/08/21 02:03:12 thorpej Exp $ */ /*- @@ -246,15 +246,36 @@ void cpu_pause_resume(unsigned long, int); void cpu_pause_resume_all(int); void cpu_unidle(struct cpu_info *); +/* + * On the Alpha, interprocessor interrupts come in at device priority + * level. This can cause some problems while waiting for r/w spinlocks + * from a high'ish priority level: IPIs that come in will not be processed. + * This can lead to deadlock. + * + * This hook allows IPIs to be processed while a spinlock's interlock + * is released. + */ +#define CPU_BUSY_CYCLE() \ +do { \ + struct cpu_info *__ci = curcpu(); \ + int __s; \ + \ + if (__ci->ci_ipis != 0) { \ + __s = splipi(); \ + alpha_ipi_process_with_frame(__ci); \ + splx(__s); \ + } \ +} while (0) + #else /* ! MULTIPROCESSOR */ #define curcpu() (&cpu_info_primary) #define CPU_IS_PRIMARY(ci) 1 #define cpu_unidle(ci) do { /* nothing */ } while (0) +#define CPU_BUSY_CYCLE() do {} while (0) #endif /* MULTIPROCESSOR */ -#define CPU_BUSY_CYCLE() do {} while (0) #define curproc curcpu()->ci_curproc #define fpcurproc curcpu()->ci_fpcurproc diff --git a/sys/arch/alpha/include/lock.h b/sys/arch/alpha/include/lock.h index 3ef8a5cb550..c2214df2f71 100644 --- a/sys/arch/alpha/include/lock.h +++ b/sys/arch/alpha/include/lock.h @@ -1,4 +1,4 @@ -/* $OpenBSD: lock.h,v 1.9 2015/02/11 03:56:00 dlg Exp $ */ +/* $OpenBSD: lock.h,v 1.10 2017/05/29 14:19:49 mpi Exp $ */ /* $NetBSD: lock.h,v 1.16 2001/12/17 23:34:57 thorpej Exp $ */ /*- @@ -38,27 +38,4 @@ #ifndef _MACHINE_LOCK_H_ #define _MACHINE_LOCK_H_ -#if defined(MULTIPROCESSOR) -/* - * On the Alpha, interprocessor interrupts come in at device priority - * level. This can cause some problems while waiting for r/w spinlocks - * from a high'ish priority level: IPIs that come in will not be processed. - * This can lead to deadlock. - * - * This hook allows IPIs to be processed while a spinlock's interlock - * is released. - */ -#define SPINLOCK_SPIN_HOOK \ -do { \ - struct cpu_info *__ci = curcpu(); \ - int __s; \ - \ - if (__ci->ci_ipis != 0) { \ - __s = splipi(); \ - alpha_ipi_process_with_frame(__ci); \ - splx(__s); \ - } \ -} while (0) -#endif /* MULTIPROCESSOR */ - #endif /* _MACHINE_LOCK_H_ */ diff --git a/sys/arch/amd64/amd64/fpu.c b/sys/arch/amd64/amd64/fpu.c index 4b280fb6c9a..4e8365cc694 100644 --- a/sys/arch/amd64/amd64/fpu.c +++ b/sys/arch/amd64/amd64/fpu.c @@ -1,4 +1,4 @@ -/* $OpenBSD: fpu.c,v 1.34 2017/04/27 06:16:39 mlarkin Exp $ */ +/* $OpenBSD: fpu.c,v 1.35 2017/05/29 14:19:49 mpi Exp $ */ /* $NetBSD: fpu.c,v 1.1 2003/04/26 18:39:28 fvdl Exp $ */ /*- @@ -52,7 +52,6 @@ #include #include #include -#include #include @@ -335,7 +334,7 @@ fpusave_proc(struct proc *p, int save) x86_send_ipi(oci, save ? X86_IPI_SYNCH_FPU : X86_IPI_FLUSH_FPU); while (p->p_addr->u_pcb.pcb_fpcpu != NULL) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } #else KASSERT(ci->ci_fpcurproc == p); diff --git a/sys/arch/amd64/amd64/lapic.c b/sys/arch/amd64/amd64/lapic.c index 8ce1462ded9..49363987b23 100644 --- a/sys/arch/amd64/amd64/lapic.c +++ b/sys/arch/amd64/amd64/lapic.c @@ -1,4 +1,4 @@ -/* $OpenBSD: lapic.c,v 1.46 2017/05/25 03:54:10 visa Exp $ */ +/* $OpenBSD: lapic.c,v 1.47 2017/05/29 14:19:49 mpi Exp $ */ /* $NetBSD: lapic.c,v 1.2 2003/05/08 01:04:35 fvdl Exp $ */ /*- @@ -41,7 +41,6 @@ #include #include #include -#include #include #include #include @@ -572,7 +571,7 @@ lapic_delay(int usec) deltat -= otick - tick; otick = tick; - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } } diff --git a/sys/arch/amd64/amd64/lock_machdep.c b/sys/arch/amd64/amd64/lock_machdep.c index 5943c3b4b5e..c3fd540b5a4 100644 --- a/sys/arch/amd64/amd64/lock_machdep.c +++ b/sys/arch/amd64/amd64/lock_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: lock_machdep.c,v 1.15 2017/05/28 08:47:19 mpi Exp $ */ +/* $OpenBSD: lock_machdep.c,v 1.16 2017/05/29 14:19:49 mpi Exp $ */ /* * Copyright (c) 2007 Artur Grabowski @@ -23,7 +23,7 @@ #include #include -#include +#include #include #include @@ -50,12 +50,12 @@ __mp_lock_spin(struct __mp_lock *mpl, u_int me) { #ifndef MP_LOCKDEBUG while (mpl->mpl_ticket != me) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); #else int nticks = __mp_lock_spinout; while (mpl->mpl_ticket != me) { - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); if (--nticks <= 0) { db_printf("__mp_lock(%p): lock spun out", mpl); diff --git a/sys/arch/amd64/amd64/pmap.c b/sys/arch/amd64/amd64/pmap.c index dbc8c1a757c..68580d24877 100644 --- a/sys/arch/amd64/amd64/pmap.c +++ b/sys/arch/amd64/amd64/pmap.c @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.c,v 1.103 2017/01/02 07:41:18 tedu Exp $ */ +/* $OpenBSD: pmap.c,v 1.104 2017/05/29 14:19:49 mpi Exp $ */ /* $NetBSD: pmap.c,v 1.3 2003/05/08 18:13:13 thorpej Exp $ */ /* @@ -114,7 +114,6 @@ #include -#include #include #include #ifdef MULTIPROCESSOR @@ -2473,7 +2472,7 @@ pmap_tlb_shootpage(struct pmap *pm, vaddr_t va, int shootself) while (atomic_cas_ulong(&tlb_shoot_wait, 0, wait) != 0) { while (tlb_shoot_wait != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } tlb_shoot_addr1 = va; CPU_INFO_FOREACH(cii, ci) { @@ -2511,7 +2510,7 @@ pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva, int shootself) while (atomic_cas_ulong(&tlb_shoot_wait, 0, wait) != 0) { while (tlb_shoot_wait != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } tlb_shoot_addr1 = sva; tlb_shoot_addr2 = eva; @@ -2549,7 +2548,7 @@ pmap_tlb_shoottlb(struct pmap *pm, int shootself) while (atomic_cas_ulong(&tlb_shoot_wait, 0, wait) != 0) { while (tlb_shoot_wait != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } CPU_INFO_FOREACH(cii, ci) { @@ -2569,7 +2568,7 @@ void pmap_tlb_shootwait(void) { while (tlb_shoot_wait != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } #else diff --git a/sys/arch/amd64/include/lock.h b/sys/arch/amd64/include/lock.h index d93111b2c14..2ea630a634d 100644 --- a/sys/arch/amd64/include/lock.h +++ b/sys/arch/amd64/include/lock.h @@ -1,10 +1,8 @@ -/* $OpenBSD: lock.h,v 1.12 2017/05/25 03:50:10 visa Exp $ */ +/* $OpenBSD: lock.h,v 1.13 2017/05/29 14:19:49 mpi Exp $ */ /* public domain */ #ifndef _MACHINE_LOCK_H_ #define _MACHINE_LOCK_H_ -#define SPINLOCK_SPIN_HOOK __asm volatile("pause": : :"memory") - #endif /* _MACHINE_LOCK_H_ */ diff --git a/sys/arch/hppa/hppa/lock_machdep.c b/sys/arch/hppa/hppa/lock_machdep.c index 444e2858f64..972b5bb8886 100644 --- a/sys/arch/hppa/hppa/lock_machdep.c +++ b/sys/arch/hppa/hppa/lock_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: lock_machdep.c,v 1.10 2017/04/30 16:45:45 mpi Exp $ */ +/* $OpenBSD: lock_machdep.c,v 1.11 2017/05/29 14:19:49 mpi Exp $ */ /* * Copyright (c) 2007 Artur Grabowski @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -72,19 +73,17 @@ __mp_lock_init(struct __mp_lock *lock) extern int __mp_lock_spinout; #endif -#define SPINLOCK_SPIN_HOOK /**/ - static __inline void __mp_lock_spin(struct __mp_lock *mpl) { #ifndef MP_LOCKDEBUG while (mpl->mpl_count != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); #else int nticks = __mp_lock_spinout; while (mpl->mpl_count != 0 && --nticks > 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); if (nticks == 0) { db_printf("__mp_lock(%p): lock spun out", mpl); diff --git a/sys/arch/i386/i386/lock_machdep.c b/sys/arch/i386/i386/lock_machdep.c index 92e9fe6e84c..0274c6f5461 100644 --- a/sys/arch/i386/i386/lock_machdep.c +++ b/sys/arch/i386/i386/lock_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: lock_machdep.c,v 1.24 2017/05/28 08:47:19 mpi Exp $ */ +/* $OpenBSD: lock_machdep.c,v 1.25 2017/05/29 14:19:49 mpi Exp $ */ /* $NetBSD: lock_machdep.c,v 1.1.2.3 2000/05/03 14:40:30 sommerfeld Exp $ */ /*- @@ -41,7 +41,7 @@ #include #include -#include +#include #include #include @@ -69,12 +69,12 @@ __mp_lock_spin(struct __mp_lock *mpl, u_int me) { #ifndef MP_LOCKDEBUG while (mpl->mpl_ticket != me) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); #else int nticks = __mp_lock_spinout; while (mpl->mpl_ticket != me) { - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); if (--nticks <= 0) { db_printf("__mp_lock(%p): lock spun out", mpl); diff --git a/sys/arch/i386/i386/pmap.c b/sys/arch/i386/i386/pmap.c index 04248baa30d..b2345ce4e89 100644 --- a/sys/arch/i386/i386/pmap.c +++ b/sys/arch/i386/i386/pmap.c @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.c,v 1.195 2016/10/21 06:20:58 mlarkin Exp $ */ +/* $OpenBSD: pmap.c,v 1.196 2017/05/29 14:19:49 mpi Exp $ */ /* $NetBSD: pmap.c,v 1.91 2000/06/02 17:46:37 thorpej Exp $ */ /* @@ -65,7 +65,6 @@ #include -#include #include #include #include @@ -2520,7 +2519,7 @@ pmap_tlb_shootpage(struct pmap *pm, vaddr_t va) while (atomic_cas_uint(&tlb_shoot_wait, 0, wait) != 0) { while (tlb_shoot_wait != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } tlb_shoot_addr1 = va; CPU_INFO_FOREACH(cii, ci) { @@ -2558,7 +2557,7 @@ pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva) while (atomic_cas_uint(&tlb_shoot_wait, 0, wait) != 0) { while (tlb_shoot_wait != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } tlb_shoot_addr1 = sva; tlb_shoot_addr2 = eva; @@ -2596,7 +2595,7 @@ pmap_tlb_shoottlb(void) while (atomic_cas_uint(&tlb_shoot_wait, 0, wait) != 0) { while (tlb_shoot_wait != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } CPU_INFO_FOREACH(cii, ci) { @@ -2632,7 +2631,7 @@ pmap_tlb_droppmap(struct pmap *pm) while (atomic_cas_uint(&tlb_shoot_wait, 0, wait) != 0) { while (tlb_shoot_wait != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } CPU_INFO_FOREACH(cii, ci) { @@ -2654,7 +2653,7 @@ void pmap_tlb_shootwait(void) { while (tlb_shoot_wait != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } #else diff --git a/sys/arch/i386/include/lock.h b/sys/arch/i386/include/lock.h index 6fcd36b1ec6..2ea630a634d 100644 --- a/sys/arch/i386/include/lock.h +++ b/sys/arch/i386/include/lock.h @@ -1,10 +1,8 @@ -/* $OpenBSD: lock.h,v 1.12 2016/04/03 11:05:26 jsg Exp $ */ +/* $OpenBSD: lock.h,v 1.13 2017/05/29 14:19:49 mpi Exp $ */ /* public domain */ #ifndef _MACHINE_LOCK_H_ #define _MACHINE_LOCK_H_ -#define SPINLOCK_SPIN_HOOK __asm volatile("pause": : :"memory") - #endif /* _MACHINE_LOCK_H_ */ diff --git a/sys/arch/i386/isa/npx.c b/sys/arch/i386/isa/npx.c index 2240838cf1d..782e0183670 100644 --- a/sys/arch/i386/isa/npx.c +++ b/sys/arch/i386/isa/npx.c @@ -1,4 +1,4 @@ -/* $OpenBSD: npx.c,v 1.61 2015/02/11 05:54:48 dlg Exp $ */ +/* $OpenBSD: npx.c,v 1.62 2017/05/29 14:19:50 mpi Exp $ */ /* $NetBSD: npx.c,v 1.57 1996/05/12 23:12:24 mycroft Exp $ */ #if 0 @@ -61,7 +61,6 @@ #include #include #include -#include #include #include @@ -875,7 +874,7 @@ npxsave_proc(struct proc *p, int save) i386_send_ipi(oci, save ? I386_IPI_SYNCH_FPU : I386_IPI_FLUSH_FPU); while (p->p_addr->u_pcb.pcb_fpcpu != NULL) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); } #else KASSERT(ci->ci_fpcurproc == p); diff --git a/sys/arch/luna88k/luna88k/machdep.c b/sys/arch/luna88k/luna88k/machdep.c index 1d684f75b27..2c2ecd4f4b0 100644 --- a/sys/arch/luna88k/luna88k/machdep.c +++ b/sys/arch/luna88k/luna88k/machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: machdep.c,v 1.124 2017/04/30 16:45:45 mpi Exp $ */ +/* $OpenBSD: machdep.c,v 1.125 2017/05/29 14:19:50 mpi Exp $ */ /* * Copyright (c) 1998, 1999, 2000, 2001 Steve Murphree, Jr. * Copyright (c) 1996 Nivas Madhur @@ -84,7 +84,6 @@ #include #include #include -#include #include #include #include diff --git a/sys/arch/m88k/m88k/m8820x_machdep.c b/sys/arch/m88k/m88k/m8820x_machdep.c index 32bec51eae8..7281a5de3fa 100644 --- a/sys/arch/m88k/m88k/m8820x_machdep.c +++ b/sys/arch/m88k/m88k/m8820x_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: m8820x_machdep.c,v 1.61 2013/11/16 18:45:20 miod Exp $ */ +/* $OpenBSD: m8820x_machdep.c,v 1.62 2017/05/29 14:19:50 mpi Exp $ */ /* * Copyright (c) 2004, 2007, 2010, 2011, 2013, Miodrag Vallat. * @@ -80,7 +80,6 @@ #include #include #include -#include #include #include diff --git a/sys/arch/m88k/m88k/m88k_machdep.c b/sys/arch/m88k/m88k/m88k_machdep.c index be18179c549..2d1d9fd670e 100644 --- a/sys/arch/m88k/m88k/m88k_machdep.c +++ b/sys/arch/m88k/m88k/m88k_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: m88k_machdep.c,v 1.67 2017/03/19 10:57:29 miod Exp $ */ +/* $OpenBSD: m88k_machdep.c,v 1.68 2017/05/29 14:19:50 mpi Exp $ */ /* * Copyright (c) 1998, 1999, 2000, 2001 Steve Murphree, Jr. * Copyright (c) 1996 Nivas Madhur @@ -60,7 +60,6 @@ #include #include #include -#include #include #include #include diff --git a/sys/arch/m88k/m88k/mplock.c b/sys/arch/m88k/m88k/mplock.c index 91c33ed555c..5422b03edb4 100644 --- a/sys/arch/m88k/m88k/mplock.c +++ b/sys/arch/m88k/m88k/mplock.c @@ -1,4 +1,4 @@ -/* $OpenBSD: mplock.c,v 1.5 2017/04/30 16:45:45 mpi Exp $ */ +/* $OpenBSD: mplock.c,v 1.6 2017/05/29 14:19:50 mpi Exp $ */ /* * Copyright (c) 2004 Niklas Hallqvist. All rights reserved. @@ -28,7 +28,7 @@ #include #include -#include +#include #include @@ -41,19 +41,17 @@ extern int __mp_lock_spinout; #endif -#define SPINLOCK_SPIN_HOOK do { /* nothing */ } while (0) - static __inline void __mp_lock_spin(struct __mp_lock *mpl) { #ifndef MP_LOCKDEBUG while (mpl->mpl_count != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); #else int nticks = __mp_lock_spinout; while (mpl->mpl_count != 0 && nticks-- > 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); if (nticks == 0) { db_printf("__mp_lock(0x%x): lock spun out", mpl); diff --git a/sys/arch/mips64/mips64/lock_machdep.c b/sys/arch/mips64/mips64/lock_machdep.c index e27cf8d209a..4af2c02eeda 100644 --- a/sys/arch/mips64/mips64/lock_machdep.c +++ b/sys/arch/mips64/mips64/lock_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: lock_machdep.c,v 1.4 2017/05/19 00:52:49 visa Exp $ */ +/* $OpenBSD: lock_machdep.c,v 1.5 2017/05/29 14:19:50 mpi Exp $ */ /* * Copyright (c) 2007 Artur Grabowski @@ -22,7 +22,6 @@ #include #include -#include #include @@ -42,19 +41,17 @@ __mp_lock_init(struct __mp_lock *lock) extern int __mp_lock_spinout; #endif -#define SPINLOCK_SPIN_HOOK /**/ - static __inline void __mp_lock_spin(struct __mp_lock *mpl) { #ifndef MP_LOCKDEBUG while (mpl->mpl_count != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); #else int nticks = __mp_lock_spinout; while (mpl->mpl_count != 0 && --nticks > 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); if (nticks == 0) { db_printf("__mp_lock(%p): lock spun out", mpl); diff --git a/sys/arch/powerpc/include/lock.h b/sys/arch/powerpc/include/lock.h index a533cbae429..e3b41ca3188 100644 --- a/sys/arch/powerpc/include/lock.h +++ b/sys/arch/powerpc/include/lock.h @@ -1,4 +1,4 @@ -/* $OpenBSD: lock.h,v 1.7 2015/08/14 06:14:19 dlg Exp $ */ +/* $OpenBSD: lock.h,v 1.8 2017/05/29 14:19:50 mpi Exp $ */ /* $NetBSD: lock.h,v 1.8 2005/12/28 19:09:29 perry Exp $ */ /*- @@ -37,6 +37,4 @@ #ifndef _POWERPC_LOCK_H_ #define _POWERPC_LOCK_H_ -#define SPINLOCK_SPIN_HOOK do { } while (0) - #endif /* _POWERPC_LOCK_H_ */ diff --git a/sys/arch/powerpc/powerpc/lock_machdep.c b/sys/arch/powerpc/powerpc/lock_machdep.c index 3ee3c258afb..b78af743d4d 100644 --- a/sys/arch/powerpc/powerpc/lock_machdep.c +++ b/sys/arch/powerpc/powerpc/lock_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: lock_machdep.c,v 1.5 2017/04/30 16:45:45 mpi Exp $ */ +/* $OpenBSD: lock_machdep.c,v 1.6 2017/05/29 14:19:50 mpi Exp $ */ /* * Copyright (c) 2007 Artur Grabowski @@ -21,7 +21,7 @@ #include #include -#include +#include #include #include @@ -47,12 +47,12 @@ __mp_lock_spin(struct __mp_lock *mpl) { #ifndef MP_LOCKDEBUG while (mpl->mpl_count != 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); #else int nticks = __mp_lock_spinout; while (mpl->mpl_count != 0 && --nticks > 0) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); if (nticks == 0) { db_printf("__mp_lock(%p): lock spun out", mpl); diff --git a/sys/arch/powerpc/powerpc/mutex.c b/sys/arch/powerpc/powerpc/mutex.c index 7559371675b..5a6a8012931 100644 --- a/sys/arch/powerpc/powerpc/mutex.c +++ b/sys/arch/powerpc/powerpc/mutex.c @@ -1,4 +1,4 @@ -/* $OpenBSD: mutex.c,v 1.5 2017/04/30 16:45:45 mpi Exp $ */ +/* $OpenBSD: mutex.c,v 1.6 2017/05/29 14:19:50 mpi Exp $ */ /* * Copyright (c) 2004 Artur Grabowski @@ -31,7 +31,7 @@ #include #include -#include +#include #include @@ -61,7 +61,7 @@ __mtx_enter(struct mutex *mtx) #endif while (__mtx_enter_try(mtx) == 0) { - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); #if defined(MP_LOCKDEBUG) if (--nticks == 0) { diff --git a/sys/arch/sparc64/include/cpu.h b/sys/arch/sparc64/include/cpu.h index d090108b7df..931d9a4caa1 100644 --- a/sys/arch/sparc64/include/cpu.h +++ b/sys/arch/sparc64/include/cpu.h @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.h,v 1.90 2017/04/20 10:03:40 kettenis Exp $ */ +/* $OpenBSD: cpu.h,v 1.91 2017/05/29 14:19:50 mpi Exp $ */ /* $NetBSD: cpu.h,v 1.28 2001/06/14 22:56:58 thorpej Exp $ */ /* @@ -210,7 +210,46 @@ void cpu_unidle(struct cpu_info *); #define curpcb __curcpu->ci_cpcb #define fpproc __curcpu->ci_fpproc -#define CPU_BUSY_CYCLE() do {} while (0) +/* + * On processors with multiple threads we force a thread switch. + * + * On UltraSPARC T2 and its successors, the optimal way to do this + * seems to be to do three nop reads of %ccr. This works on + * UltraSPARC T1 as well, even though three nop casx operations seem + * to be slightly more optimal. Since these instructions are + * effectively nops, executing them on earlier non-CMT processors is + * harmless, so we make this the default. + * + * On SPARC T4 and later, we can use the processor-specific pause + * instruction. + * + * On SPARC64 VI and its successors we execute the processor-specific + * sleep instruction. + */ +#define CPU_BUSY_CYCLE() \ +do { \ + __asm volatile( \ + "999: rd %%ccr, %%g0 \n" \ + " rd %%ccr, %%g0 \n" \ + " rd %%ccr, %%g0 \n" \ + " .section .sun4v_pause_patch, \"ax\" \n" \ + " .word 999b \n" \ + " .word 0xb7802080 ! pause 128 \n" \ + " .word 999b + 4 \n" \ + " nop \n" \ + " .word 999b + 8 \n" \ + " nop \n" \ + " .previous \n" \ + " .section .sun4u_mtp_patch, \"ax\" \n" \ + " .word 999b \n" \ + " .word 0x81b01060 ! sleep \n" \ + " .word 999b + 4 \n" \ + " nop \n" \ + " .word 999b + 8 \n" \ + " nop \n" \ + " .previous \n" \ + : : : "memory"); \ +} while (0) /* * Arguments to hardclock, softclock and gatherstats encapsulate the diff --git a/sys/arch/sparc64/include/lock.h b/sys/arch/sparc64/include/lock.h index 5b3e63db804..ef6820871c7 100644 --- a/sys/arch/sparc64/include/lock.h +++ b/sys/arch/sparc64/include/lock.h @@ -1,4 +1,4 @@ -/* $OpenBSD: lock.h,v 1.11 2017/05/27 15:11:03 mpi Exp $ */ +/* $OpenBSD: lock.h,v 1.12 2017/05/29 14:19:50 mpi Exp $ */ /* * Copyright (c) 2012 Mark Kettenis * @@ -18,49 +18,4 @@ #ifndef _MACHINE_LOCK_H_ #define _MACHINE_LOCK_H_ -#ifdef _KERNEL - -/* - * On processors with multiple threads we force a thread switch. - * - * On UltraSPARC T2 and its successors, the optimal way to do this - * seems to be to do three nop reads of %ccr. This works on - * UltraSPARC T1 as well, even though three nop casx operations seem - * to be slightly more optimal. Since these instructions are - * effectively nops, executing them on earlier non-CMT processors is - * harmless, so we make this the default. - * - * On SPARC T4 and later, we can use the processor-specific pause - * instruction. - * - * On SPARC64 VI and its successors we execute the processor-specific - * sleep instruction. - */ -#define SPINLOCK_SPIN_HOOK \ -do { \ - __asm volatile( \ - "999: rd %%ccr, %%g0 \n" \ - " rd %%ccr, %%g0 \n" \ - " rd %%ccr, %%g0 \n" \ - " .section .sun4v_pause_patch, \"ax\" \n" \ - " .word 999b \n" \ - " .word 0xb7802080 ! pause 128 \n" \ - " .word 999b + 4 \n" \ - " nop \n" \ - " .word 999b + 8 \n" \ - " nop \n" \ - " .previous \n" \ - " .section .sun4u_mtp_patch, \"ax\" \n" \ - " .word 999b \n" \ - " .word 0x81b01060 ! sleep \n" \ - " .word 999b + 4 \n" \ - " nop \n" \ - " .word 999b + 8 \n" \ - " nop \n" \ - " .previous \n" \ - : : : "memory"); \ -} while (0) - - -#endif /* _KERNEL */ #endif /* _MACHINE_LOCK_H_ */ diff --git a/sys/arch/sparc64/sparc64/lock_machdep.c b/sys/arch/sparc64/sparc64/lock_machdep.c index 4680a5fffbe..bd55818c6e4 100644 --- a/sys/arch/sparc64/sparc64/lock_machdep.c +++ b/sys/arch/sparc64/sparc64/lock_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: lock_machdep.c,v 1.16 2017/05/27 15:11:03 mpi Exp $ */ +/* $OpenBSD: lock_machdep.c,v 1.17 2017/05/29 14:19:50 mpi Exp $ */ /* * Copyright (c) 2007 Artur Grabowski @@ -20,7 +20,7 @@ #include #include -#include +#include #include #include @@ -47,12 +47,12 @@ __mp_lock_spin(struct __mp_lock *mpl, u_int me) { #ifndef MP_LOCKDEBUG while (mpl->mpl_ticket != me) - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); #else int nticks = __mp_lock_spinout; while (mpl->mpl_ticket != me) { - SPINLOCK_SPIN_HOOK; + CPU_BUSY_CYCLE(); if (--nticks <= 0) { db_printf("__mp_lock(%p): lock spun out", mpl); -- cgit v1.2.3