MP-safe pmap implemented, enable IPI in interrupt handler to avoid deadlock.

ok miod@

14 files changed, 439 insertions, 124 deletions

diff --git a/sys/arch/mips64/include/atomic.h b/sys/arch/mips64/include/atomic.h
index ea4a2a7f121..7731a67baaa 100644
--- a/sys/arch/mips64/include/atomic.h
+++ b/sys/arch/mips64/include/atomic.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: atomic.h,v 1.5 2009/11/27 00:08:27 syuu Exp $	*/
+/*	$OpenBSD: atomic.h,v 1.6 2009/12/28 06:55:27 syuu Exp $	*/
 
 /* Public Domain */
 
@@ -7,6 +7,24 @@
 
 #if defined(_KERNEL)
 
+/* wait until the bits to set are clear, and set them */
+static __inline void
+atomic_wait_and_setbits_int(__volatile unsigned int *uip, unsigned int v)
+{
+	unsigned int tmp0, tmp1;
+
+	__asm__ __volatile__ (
+	"1:	ll	%0,	0(%2)\n"
+	"	and	%1,	%0,	%3\n"
+	"	bnez	%1,	1b\n"
+	"	or	%0,	%3,	%0\n"
+	"	sc	%0,	0(%2)\n"
+	"	beqz	%0,	1b\n"
+	"	 nop\n" :
+		"=&r"(tmp0), "=&r"(tmp1) :
+		"r"(uip), "r"(v) : "memory");
+}
+
 static __inline void
 atomic_setbits_int(__volatile unsigned int *uip, unsigned int v)
 {
diff --git a/sys/arch/mips64/include/cpu.h b/sys/arch/mips64/include/cpu.h
index 34cb7ecc529..66ddd69cd42 100644
--- a/sys/arch/mips64/include/cpu.h
+++ b/sys/arch/mips64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.48 2009/12/25 21:02:13 miod Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.49 2009/12/28 06:55:27 syuu Exp $	*/
 
 /*-
  * Copyright (c) 1992, 1993
@@ -384,7 +384,7 @@ struct cpu_info {
 	u_int32_t        ci_pendingticks;
 #ifdef MULTIPROCESSOR
 	u_long           ci_flags;		/* flags; see below */
-	struct intrhand  *ci_ipiih;
+	struct intrhand  ci_ipiih;
 #endif
 };
 
@@ -416,12 +416,12 @@ void cpu_boot_secondary_processors(void);
 vaddr_t smp_malloc(size_t);
 
 #define MIPS64_IPI_NOP		0x00000001
-#define MIPS64_NIPIS		1	/* must not exceed 32 */
+#define MIPS64_IPI_RENDEZVOUS	0x00000002
+#define MIPS64_NIPIS		2	/* must not exceed 32 */
 
-void    mips64_ipi_init(void);
-void    mips64_send_ipi(unsigned int, unsigned int);
-void    mips64_broadcast_ipi(unsigned int);
-void    mips64_multicast_ipi(unsigned int, unsigned int);
+void	mips64_ipi_init(void);
+void	mips64_send_ipi(unsigned int, unsigned int);
+void	smp_rendezvous_cpus(unsigned long, void (*)(void *), void *arg);
 
 #include <sys/mplock.h>
 #else
@@ -479,7 +479,11 @@ extern int int_nest_cntr;
  * Notify the current process (p) that it has a signal pending,
  * process as soon as possible.
  */
+#ifdef MULTIPROCESSOR
+#define	signotify(p)		(aston(p), cpu_unidle(p->p_cpu))
+#else
 #define	signotify(p)		aston(p)
+#endif
 
 #define	aston(p)		p->p_md.md_astpending = 1
 
diff --git a/sys/arch/mips64/include/pmap.h b/sys/arch/mips64/include/pmap.h
index 002dcb012d1..98006ad6889 100644
--- a/sys/arch/mips64/include/pmap.h
+++ b/sys/arch/mips64/include/pmap.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: pmap.h,v 1.16 2009/12/07 18:58:32 miod Exp $ */
+/*      $OpenBSD: pmap.h,v 1.17 2009/12/28 06:55:27 syuu Exp $ */
 
 /*
  * Copyright (c) 1987 Carnegie-Mellon University
@@ -98,8 +98,9 @@ typedef struct pmap {
 	int			pm_count;	/* pmap reference count */
 	simple_lock_data_t	pm_lock;	/* lock on pmap */
 	struct pmap_statistics	pm_stats;	/* pmap statistics */
-	u_int			pm_tlbpid;	/* address space tag */
-	u_int			pm_tlbgen;	/* TLB PID generation number */
+	u_int			pm_tlbpid[MAXCPUS];	/* address space tag */
+	u_int			pm_tlbgen[MAXCPUS];	/* TLB PID generation number */
+	int                     pm_active;
 	struct segtab		*pm_segtab;	/* pointers to pages of PTEs */
 } *pmap_t;
 
@@ -136,6 +137,8 @@ void	pmap_page_cache(vm_page_t, int);
 #define pmap_unuse_final(p)		do { /* nothing yet */ } while (0)
 #define	pmap_remove_holes(map)		do { /* nothing */ } while (0)
 
+void pmap_update_user_page(pmap_t, vaddr_t, pt_entry_t);
+void pmap_update_kernel_page(vaddr_t, pt_entry_t);
 #endif	/* _KERNEL */
 
 #endif	/* !_MIPS_PMAP_H_ */
diff --git a/sys/arch/mips64/mips64/clock.c b/sys/arch/mips64/mips64/clock.c
index 0cd58d1559c..02c7de179dc 100644
--- a/sys/arch/mips64/mips64/clock.c
+++ b/sys/arch/mips64/mips64/clock.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: clock.c,v 1.30 2009/11/26 23:32:46 syuu Exp $ */
+/*	$OpenBSD: clock.c,v 1.31 2009/12/28 06:55:27 syuu Exp $ */
 
 /*
  * Copyright (c) 2001-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -117,13 +117,9 @@ clockattach(struct device *parent, struct device *self, void *aux)
 uint32_t
 clock_int5(uint32_t mask, struct trap_frame *tf)
 {
-	u_int32_t clkdiff, sr;
+	u_int32_t clkdiff;
 	struct cpu_info *ci = curcpu();
 
-	/* Enable interrupts at this (hardware) level again */
-	sr = getsr();
-	updateimask(mask);
-
 	/*
 	 * If we got an interrupt before we got ready to process it,
 	 * retrigger it as far as possible. cpu_initclocks() will
@@ -131,7 +127,7 @@ clock_int5(uint32_t mask, struct trap_frame *tf)
 	 */
 	if (ci->ci_clock_started == 0) {
 		cp0_set_compare(cp0_get_count() - 1);
-		setsr(sr);
+
 		return CR_INT_5;
 	}
 
@@ -164,6 +160,10 @@ clock_int5(uint32_t mask, struct trap_frame *tf)
 	 */
 	if (tf->ipl < IPL_CLOCK) {
 #ifdef MULTIPROCESSOR
+		u_int32_t sr;
+		/* Enable interrupts at this (hardware) level again */
+		sr = getsr();
+		ENABLEIPI();
 		__mp_lock(&kernel_lock);
 #endif
 		while (ci->ci_pendingticks) {
@@ -173,9 +173,9 @@ clock_int5(uint32_t mask, struct trap_frame *tf)
 		}
 #ifdef MULTIPROCESSOR
 		__mp_unlock(&kernel_lock);
+		setsr(sr);
 #endif
 	}
-	setsr(sr);
 
 	return CR_INT_5;	/* Clock is always on 5 */
 }
diff --git a/sys/arch/mips64/mips64/context.S b/sys/arch/mips64/mips64/context.S
index 919feb9eaa2..9ffb46bbaae 100644
--- a/sys/arch/mips64/mips64/context.S
+++ b/sys/arch/mips64/mips64/context.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: context.S,v 1.34 2009/12/12 20:07:10 miod Exp $ */
+/*	$OpenBSD: context.S,v 1.35 2009/12/28 06:55:27 syuu Exp $ */
 
 /*
  * Copyright (c) 2002-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -146,7 +146,12 @@ NON_LEAF(cpu_switchto, FRAMESZ(CF_SZ), ra)
 	/* get process ASID */
 	PTR_L	t0, P_VMSPACE(s0)		# p->p_vmspace
 	PTR_L	t1, VMSPACE_PMAP(t0)		# ->vm_map.pmap
-	lw	v0, PM_TLBPID(t1)		# ->pm_tlbpid
+#ifdef MULTIPROCESSOR
+	HW_CPU_NUMBER(v0)			# cpuid
+	PTR_SLL	v0, v0, 0x2
+	PTR_ADDU t1, t1, v0
+#endif
+	lw	v0, PM_TLBPID(t1)		# ->pm_tlbpid[cpuid]
 
 #if UPAGES > 1
 	or	v0, t3
diff --git a/sys/arch/mips64/mips64/cpu.c b/sys/arch/mips64/mips64/cpu.c
index a72bd4c0ac8..7edde7625f5 100644
--- a/sys/arch/mips64/mips64/cpu.c
+++ b/sys/arch/mips64/mips64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.21 2009/12/02 01:42:14 syuu Exp $ */
+/*	$OpenBSD: cpu.c,v 1.22 2009/12/28 06:55:27 syuu Exp $ */
 
 /*
  * Copyright (c) 1997-2004 Opsycon AB (www.opsycon.se)
@@ -113,10 +113,6 @@ cpuattach(struct device *parent, struct device *dev, void *aux)
 		if (ci == NULL)
 			panic("unable to allocate cpu_info\n");
 		bzero((char *)ci, sizeof(*ci));
-		ci->ci_ipiih = 
-			(struct intrhand *)smp_malloc(sizeof(*ci->ci_ipiih));
-		if (ci->ci_ipiih == NULL)
-			panic("unable to allocate ipi intrhand\n");
 		ci->ci_next = cpu_info_list->ci_next;
 		cpu_info_list->ci_next = ci;
 		ci->ci_flags |= CPUF_PRESENT;
@@ -347,6 +343,8 @@ cpu_boot_secondary_processors(void)
 void
 cpu_unidle(struct cpu_info *ci)
 {
+	if (ci != curcpu())
+		mips64_send_ipi(ci->ci_cpuid, MIPS64_IPI_NOP);
 }
 
 vaddr_t 
diff --git a/sys/arch/mips64/mips64/ipifuncs.c b/sys/arch/mips64/mips64/ipifuncs.c
index 7a877545b0e..8b98115c3f1 100644
--- a/sys/arch/mips64/mips64/ipifuncs.c
+++ b/sys/arch/mips64/mips64/ipifuncs.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ipifuncs.c,v 1.1 2009/11/25 17:39:51 syuu Exp $ */
+/* $OpenBSD: ipifuncs.c,v 1.2 2009/12/28 06:55:27 syuu Exp $ */
 /* $NetBSD: ipifuncs.c,v 1.40 2008/04/28 20:23:10 martin Exp $ */
 
 /*-
@@ -41,11 +41,19 @@
 #include <machine/intr.h>
 #include <machine/atomic.h>
 
-static int      mips64_ipi_intr(void *);
+static int	mips64_ipi_intr(void *);
 static void	mips64_ipi_nop(void);
-
+static void	smp_rendezvous_action(void);
+static void	mips64_multicast_ipi(unsigned int, unsigned int);
 static unsigned int ipi_mailbox[MAXCPUS];
 
+/* Variables needed for SMP rendezvous. */
+struct mutex smp_ipi_mtx;
+static volatile unsigned long smp_rv_map;
+static void (*volatile smp_rv_action_func)(void *arg);
+static void * volatile smp_rv_func_arg;
+static volatile unsigned int smp_rv_waiters[2];
+
 /*
  * NOTE: This table must be kept in order with the bit definitions
  * in <machine/intr.h>.
@@ -54,6 +62,7 @@ typedef void (*ipifunc_t)(void);
 
 ipifunc_t ipifuncs[MIPS64_NIPIS] = {
 	mips64_ipi_nop,
+	smp_rendezvous_action
 };
 
 /*
@@ -65,6 +74,9 @@ mips64_ipi_init(void)
 	cpuid_t cpuid = cpu_number();
 	int error;
 
+	if (!cpuid)
+		mtx_init(&smp_ipi_mtx, IPL_IPI);
+
 	hw_ipi_intr_clear(cpuid);
 
 	error = hw_ipi_intr_establish(mips64_ipi_intr, cpuid);
@@ -80,24 +92,22 @@ mips64_ipi_intr(void *arg)
 {
 	unsigned int pending_ipis, bit;
 	unsigned int cpuid = (unsigned int)(unsigned long)arg;
-	int sr;
 
 	KASSERT (cpuid == cpu_number());
 
-	sr = disableintr();
-
-	/* Load the mailbox register to figure out what we're supposed to do */
+	/* figure out which ipi are pending */
 	pending_ipis = ipi_mailbox[cpuid];
+	/* clear ipi interrupt */
+	hw_ipi_intr_clear(cpuid);
+	
 	if (pending_ipis > 0) {
+		/* clear pending ipi, since we're about to handle them */
+		atomic_clearbits_int(&ipi_mailbox[cpuid], pending_ipis);
+
 		for (bit = 0; bit < MIPS64_NIPIS; bit++)
 			if (pending_ipis & (1UL << bit))
 				(*ipifuncs[bit])();
-
-		/* Clear the mailbox to clear the interrupt */
-		atomic_clearbits_int(&ipi_mailbox[cpuid], pending_ipis);
 	}
-	hw_ipi_intr_clear(cpuid);
-	setsr(sr);
 
 	return 1;
 }
@@ -115,28 +125,12 @@ mips64_send_ipi(unsigned int cpuid, unsigned int ipimask)
 	        panic("mips_send_ipi: CPU %ld not running", cpuid);
 #endif
 
-	atomic_setbits_int(&ipi_mailbox[cpuid], ipimask);
+	atomic_wait_and_setbits_int(&ipi_mailbox[cpuid], ipimask);
 
 	hw_ipi_intr_set(cpuid);
 }
 
 /*
- * Broadcast an IPI to all but ourselves.
- */
-void
-mips64_broadcast_ipi(unsigned int ipimask)
-{
-	struct cpu_info *ci;
-	CPU_INFO_ITERATOR cii;
-
-	CPU_INFO_FOREACH(cii, ci) {
-		if (curcpu() == ci || !cpuset_isset(&cpus_running, ci))
-			continue;
-		mips64_send_ipi(ci->ci_cpuid, ipimask);
-	}
-}
-
-/*
  * Send an IPI to all in the list but ourselves.
  */
 void
@@ -145,7 +139,7 @@ mips64_multicast_ipi(unsigned int cpumask, unsigned int ipimask)
 	struct cpu_info *ci;
 	CPU_INFO_ITERATOR cii;
 
-	cpumask &= ~(1UL << cpu_number());
+	cpumask &= ~(1 << cpu_number());
 
 	CPU_INFO_FOREACH(cii, ci) {
 		if (!(cpumask & (1UL << ci->ci_cpuid)) || 
@@ -162,3 +156,69 @@ mips64_ipi_nop(void)
 	printf("mips64_ipi_nop on cpu%d\n", cpu_number());
 #endif
 }
+
+/*
+ * All-CPU rendezvous.  CPUs are signalled, all execute the setup function 
+ * (if specified), rendezvous, execute the action function (if specified),
+ * rendezvous again, execute the teardown function (if specified), and then
+ * resume.
+ *
+ * Note that the supplied external functions _must_ be reentrant and aware
+ * that they are running in parallel and in an unknown lock context.
+ */
+
+void
+smp_rendezvous_action(void)
+{
+	void* local_func_arg = smp_rv_func_arg;
+	void (*local_action_func)(void*) = smp_rv_action_func;
+	unsigned int cpumask = 1 << cpu_number();
+
+	/* Ensure we have up-to-date values. */
+	atomic_setbits_int(&smp_rv_waiters[0], cpumask);
+	while (smp_rv_waiters[0] != smp_rv_map)
+		;
+
+	/* action function */
+	if (local_action_func != NULL)
+		local_action_func(local_func_arg);
+
+	/* spin on exit rendezvous */
+	atomic_setbits_int(&smp_rv_waiters[1], cpumask);
+}
+
+void
+smp_rendezvous_cpus(unsigned long map,
+	void (* action_func)(void *),
+	void *arg)
+{
+	unsigned int cpumask = 1 << cpu_number();
+
+	if (ncpus == 1) {
+		if (action_func != NULL)
+			action_func(arg);
+		return;
+	}
+
+	/* obtain rendezvous lock */
+        mtx_enter(&smp_ipi_mtx);
+
+	/* set static function pointers */
+	smp_rv_map = map;
+	smp_rv_action_func = action_func;
+	smp_rv_func_arg = arg;
+	smp_rv_waiters[0] = 0;
+	smp_rv_waiters[1] = 0;
+
+	/* signal other processors, which will enter the IPI with interrupts off */
+	mips64_multicast_ipi(map & ~cpumask, MIPS64_IPI_RENDEZVOUS);
+
+	/* Check if the current CPU is in the map */
+	if (map & cpumask)
+		smp_rendezvous_action();
+
+	while (smp_rv_waiters[1] != smp_rv_map)
+		;
+	/* release lock */
+	mtx_leave(&smp_ipi_mtx);
+}
diff --git a/sys/arch/mips64/mips64/pmap.c b/sys/arch/mips64/mips64/pmap.c
index 6e2eeecb834..3d1f6c0bd6e 100644
--- a/sys/arch/mips64/mips64/pmap.c
+++ b/sys/arch/mips64/mips64/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.42 2009/12/25 21:02:15 miod Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.43 2009/12/28 06:55:27 syuu Exp $	*/
 
 /*
  * Copyright (c) 2001-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -70,6 +70,15 @@ struct pool_allocator pmap_pg_allocator = {
 	pmap_pg_alloc, pmap_pg_free
 };
 
+static void pmap_invalidate_kernel_page(vaddr_t);
+static void pmap_invalidate_user_page(pmap_t, vaddr_t);
+#ifdef MULTIPROCESSOR
+static void pmap_invalidate_kernel_page_action(void *);
+static void pmap_invalidate_user_page_action(void *);
+static void pmap_update_kernel_page_action(void *);
+static void pmap_update_user_page_action(void *);
+#endif
+
 #ifdef PMAPDEBUG
 struct {
 	int kernel;	/* entering kernel mapping */
@@ -131,14 +140,191 @@ psize_t	mem_size;	/* memory size in bytes */
 vaddr_t	virtual_start;  /* VA of first avail page (after kernel bss)*/
 vaddr_t	virtual_end;	/* VA of last avail page (end of kernel AS) */
 
-u_int		tlbpid_gen = 1;		/* TLB PID generation count */
-u_int		tlbpid_cnt = 2;		/* next available TLB PID */
+u_int		tlbpid_gen[MAXCPUS];	/* TLB PID generation count */
+u_int		tlbpid_cnt[MAXCPUS];	/* next available TLB PID */
 
 pt_entry_t	*Sysmap;		/* kernel pte table */
 u_int		Sysmapsize;		/* number of pte's in Sysmap */
 const vaddr_t	Sysmapbase = VM_MIN_KERNEL_ADDRESS;	/* for libkvm */
 
 
+#ifdef MULTIPROCESSOR
+
+struct pmap_invalidate_page_arg {
+	pmap_t pmap;
+	vaddr_t va;
+};
+
+static void
+pmap_invalidate_kernel_page(vaddr_t va)
+{
+	struct pmap_invalidate_page_arg arg;
+	unsigned int cpumask = 0;
+	struct cpu_info *ci;
+	CPU_INFO_ITERATOR cii;
+
+	CPU_INFO_FOREACH(cii, ci) 
+		if (cpuset_isset(&cpus_running, ci))
+			cpumask |= 1 << ci->ci_cpuid;
+	arg.va = va;
+
+	smp_rendezvous_cpus(cpumask, pmap_invalidate_kernel_page_action, &arg);
+}
+
+static void
+pmap_invalidate_kernel_page_action(void *arg)
+{
+	vaddr_t va = ((struct pmap_invalidate_page_arg *)arg)->va;
+
+	tlb_flush_addr(va);
+}
+
+static void
+pmap_invalidate_user_page(pmap_t pmap, vaddr_t va)
+{
+	struct pmap_invalidate_page_arg arg;
+	unsigned int cpumask = 0;
+	struct cpu_info *ci;
+	CPU_INFO_ITERATOR cii;
+
+	CPU_INFO_FOREACH(cii, ci) 
+		if (cpuset_isset(&cpus_running, ci)) {
+			unsigned int i = ci->ci_cpuid;
+			unsigned int m = 1 << i;
+			if (pmap->pm_tlbgen[i] != tlbpid_gen[i])
+				continue;
+			else if (!(pmap->pm_active & m)) {
+				pmap->pm_tlbgen[i] = 0;
+				continue;
+			}
+			cpumask |= m;
+		}
+
+	arg.pmap = pmap;
+	arg.va = va;
+
+	smp_rendezvous_cpus(cpumask, pmap_invalidate_user_page_action, &arg);
+}
+
+static void
+pmap_invalidate_user_page_action(void *arg)
+{
+	pmap_t pmap = ((struct pmap_invalidate_page_arg *)arg)->pmap;
+	vaddr_t va = ((struct pmap_invalidate_page_arg *)arg)->va;
+	unsigned int cpuid = cpu_number();
+	u_long asid;
+
+	asid = pmap->pm_tlbpid[cpuid] << VMTLB_PID_SHIFT;
+	tlb_flush_addr(va | asid);
+}
+
+struct pmap_update_page_arg {
+	pmap_t pmap;
+	vaddr_t va;
+	pt_entry_t entry;
+};
+
+void
+pmap_update_kernel_page(vaddr_t va, pt_entry_t entry)
+{
+	struct pmap_update_page_arg arg;
+	unsigned long cpumask = 0;
+	struct cpu_info *ci;
+	CPU_INFO_ITERATOR cii;
+
+	CPU_INFO_FOREACH(cii, ci) 
+		if (cpuset_isset(&cpus_running, ci))
+			cpumask |= 1 << ci->ci_cpuid;
+
+	arg.va = va;
+	arg.entry = entry;
+	smp_rendezvous_cpus(cpumask,
+			    pmap_update_kernel_page_action, &arg);
+}
+
+static void
+pmap_update_kernel_page_action(void *arg)
+{
+	vaddr_t va = ((struct pmap_update_page_arg *)arg)->va;
+	pt_entry_t entry = ((struct pmap_update_page_arg *)arg)->entry;
+
+	tlb_update(va, entry);
+}
+
+void
+pmap_update_user_page(pmap_t pmap, vaddr_t va, pt_entry_t entry)
+{
+	struct pmap_update_page_arg arg;
+	unsigned long cpumask = 0;
+	struct cpu_info *ci;
+	CPU_INFO_ITERATOR cii;
+
+	CPU_INFO_FOREACH(cii, ci) 
+		if (cpuset_isset(&cpus_running, ci)) {
+			unsigned int i = ci->ci_cpuid;
+			unsigned int m = 1 << i;
+			if (pmap->pm_tlbgen[i] != tlbpid_gen[i])
+				continue;
+			else if (!(pmap->pm_active & m)) {
+				pmap->pm_tlbgen[i] = 0;
+				continue;
+			}
+			cpumask |= m;
+		}
+
+	arg.pmap = pmap;
+	arg.va = va;
+	arg.entry = entry;
+	smp_rendezvous_cpus(cpumask,
+			    pmap_update_user_page_action, &arg);
+}
+
+static void
+pmap_update_user_page_action(void *arg)
+{
+	pmap_t pmap = ((struct pmap_update_page_arg *)arg)->pmap;
+	vaddr_t va = ((struct pmap_update_page_arg *)arg)->va;
+	pt_entry_t entry = ((struct pmap_update_page_arg *)arg)->entry;
+	unsigned int cpuid = cpu_number();
+	u_long asid;
+
+	asid = pmap->pm_tlbpid[cpuid] << VMTLB_PID_SHIFT;
+	tlb_update(va | asid, entry);
+}
+#else
+static void
+pmap_invalidate_kernel_page(vaddr_t va)
+{
+	tlb_flush_addr(va);
+}
+
+static void
+pmap_invalidate_user_page(pmap_t pmap, vaddr_t va)
+{
+	u_long cpuid = cpu_number();
+	u_long asid = pmap->pm_tlbpid[cpuid] << VMTLB_PID_SHIFT;
+
+	if (pmap->pm_tlbgen[cpuid] == tlbpid_gen[cpuid])
+		tlb_flush_addr(va | asid);
+}
+
+void
+pmap_update_kernel_page(vaddr_t va, pt_entry_t entry)
+{
+	tlb_update(va, entry);
+}
+
+void
+pmap_update_user_page(pmap_t pmap, vaddr_t va, pt_entry_t entry)
+{
+	u_long cpuid = cpu_number();
+	u_long asid = pmap->pm_tlbpid[cpuid] << VMTLB_PID_SHIFT;
+
+	if (pmap->pm_tlbgen[cpuid] == tlbpid_gen[cpuid])
+		tlb_update(va | asid, entry);
+}
+#endif
+
 /*
  *	Bootstrap the system enough to run with virtual memory.
  */
@@ -183,6 +369,11 @@ pmap_bootstrap()
 	 */
 	for (i = Sysmapsize, spte = Sysmap; i != 0; i--, spte++)
 		*spte = PG_G;
+
+	for (i = 0; i < MAXCPUS; i++) {
+		tlbpid_gen[i] = 1;
+		tlbpid_cnt[i] = 2;
+	}
 }
 
 /*
@@ -283,7 +474,8 @@ pmap_t
 pmap_create()
 {
 	pmap_t pmap;
-	int s;
+	vaddr_t va;
+	int i, s;
 
 extern struct vmspace vmspace0;
 extern struct user *proc0paddr;
@@ -305,12 +497,16 @@ extern struct user *proc0paddr;
 		 * The initial process has already been allocated a TLBPID
 		 * in mach_init().
 		 */
-		pmap->pm_tlbpid = 1;
-		pmap->pm_tlbgen = tlbpid_gen;
+		for (i = 0; i < MAXCPUS; i++) {
+			pmap->pm_tlbpid[i] = 1;
+			pmap->pm_tlbgen[i] = tlbpid_gen[i];
+		}
 		proc0paddr->u_pcb.pcb_segtab = pmap->pm_segtab;
 	} else {
-		pmap->pm_tlbpid = 0;
-		pmap->pm_tlbgen = 0;
+		for (i = 0; i < MAXCPUS; i++) {
+			pmap->pm_tlbpid[i] = 0;
+			pmap->pm_tlbgen[i] = 0;
+		}
 	}
 
 	return (pmap);
@@ -391,6 +587,10 @@ void
 pmap_activate(struct proc *p)
 {
 	pmap_t pmap = p->p_vmspace->vm_map.pmap;
+	unsigned int cpuid = cpu_number();
+	unsigned int cpumask = (1 << cpuid);
+
+	atomic_setbits_int(&pmap->pm_active, cpumask);
 	p->p_addr->u_pcb.pcb_segtab = pmap->pm_segtab;
 	pmap_alloc_tlbpid(p);
 }
@@ -401,7 +601,11 @@ pmap_activate(struct proc *p)
 void
 pmap_deactivate(struct proc *p)
 {
-	/* Empty */
+	pmap_t pmap = p->p_vmspace->vm_map.pmap;
+	unsigned int cpuid = cpu_number();
+	unsigned int cpumask = (1 << cpuid);
+
+	atomic_clearbits_int(&pmap->pm_active, cpumask);
 }
 
 /*
@@ -442,7 +646,7 @@ pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
 			/*
 			 * Flush the TLB for the given address.
 			 */
-			tlb_flush_addr(sva);
+			pmap_invalidate_kernel_page(sva);
 			stat_count(remove_stats.flushes);
 		}
 		return;
@@ -478,11 +682,8 @@ pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
 			/*
 			 * Flush the TLB for the given address.
 			 */
-			if (pmap->pm_tlbgen == tlbpid_gen) {
-				tlb_flush_addr(sva |
-				    (pmap->pm_tlbpid << VMTLB_PID_SHIFT));
-				stat_count(remove_stats.flushes);
-			}
+			pmap_invalidate_user_page(pmap, sva);
+			stat_count(remove_stats.flushes);
 		}
 	}
 }
@@ -590,7 +791,7 @@ pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
 			/*
 			 * Update the TLB if the given address is in the cache.
 			 */
-			tlb_update(sva, entry);
+			pmap_update_kernel_page(sva, entry);
 		}
 		return;
 	}
@@ -625,10 +826,7 @@ pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
 					    pfn_to_pad(entry));
 			entry = (entry & ~(PG_M | PG_RO)) | p;
 			*pte = entry;
-			if (pmap->pm_tlbgen == tlbpid_gen)
-				tlb_update(sva |
-				    (pmap->pm_tlbpid << VMTLB_PID_SHIFT),
-				    entry);
+			pmap_update_user_page(pmap, sva, entry);
 		}
 	}
 }
@@ -647,6 +845,7 @@ pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
 {
 	pt_entry_t *pte, npte;
 	vm_page_t pg;
+	u_long cpuid = cpu_number();
 
 	DPRINTF(PDB_FOLLOW|PDB_ENTER,
 		("pmap_enter(%p, %p, %p, %p, %p)\n", pmap, va, pa, prot, flags));
@@ -729,7 +928,7 @@ pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
 		 * Update the same virtual address entry.
 		 */
 		*pte = npte;
-		tlb_update(va, npte);
+		pmap_update_kernel_page(va, npte);
 		return 0;
 	}
 
@@ -766,9 +965,9 @@ pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
 	 * MIPS pages in a OpenBSD page.
 	 */
 	npte |= vad_to_pfn(pa);
-	if (pmap->pm_tlbgen == tlbpid_gen) {
+	if (pmap->pm_tlbgen[cpuid] == tlbpid_gen[cpuid]) {
 		DPRINTF(PDB_ENTER, ("pmap_enter: new pte 0x%08x tlbpid %u\n",
-			npte, pmap->pm_tlbpid));
+			npte, pmap->pm_tlbpid[cpuid]));
 	} else {
 		DPRINTF(PDB_ENTER, ("pmap_enter: new pte 0x%08x\n", npte));
 	}
@@ -782,9 +981,7 @@ pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
 		pmap->pm_stats.resident_count++;
 	}
 	*pte = npte;
-	if (pmap->pm_tlbgen == tlbpid_gen) {
-		tlb_update(va | (pmap->pm_tlbpid << VMTLB_PID_SHIFT), npte);
-	}
+	pmap_update_user_page(pmap, va, npte);
 
 	/*
 	 *  If mapping a memory space address invalidate ICache.
@@ -816,7 +1013,7 @@ pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
 		npte |= PG_ROPAGE;
 	pte = kvtopte(va);
 	*pte = npte;
-	tlb_update(va, npte);
+	pmap_update_kernel_page(va, npte);
 }
 
 /*
@@ -845,7 +1042,7 @@ pmap_kremove(vaddr_t va, vsize_t len)
 			continue;
 		Mips_HitSyncDCache(va, pfn_to_pad(entry), PAGE_SIZE);
 		*pte = PG_NV | PG_G;
-		tlb_flush_addr(va);
+		pmap_invalidate_kernel_page(va);
 	}
 }
 
@@ -1041,7 +1238,7 @@ pmap_clear_modify(struct vm_page *pg)
 				rv = TRUE;
 				entry &= ~PG_M;
 				*pte = entry;
-				tlb_update(pv->pv_va, entry);
+				pmap_update_kernel_page(pv->pv_va, entry);
 			}
 		} else if (pv->pv_pmap != NULL) {
 			if ((pte = pmap_segmap(pv->pv_pmap, pv->pv_va)) == NULL)
@@ -1052,10 +1249,7 @@ pmap_clear_modify(struct vm_page *pg)
 				rv = TRUE;
 				entry &= ~PG_M;
 				*pte = entry;
-				if (pv->pv_pmap->pm_tlbgen == tlbpid_gen)
-					tlb_update(pv->pv_va |
-					    (pv->pv_pmap->pm_tlbpid <<
-					      VMTLB_PID_SHIFT), entry);
+				pmap_update_user_page(pv->pv_pmap, pv->pv_va, entry);
 			}
 		}
 	}
@@ -1155,7 +1349,7 @@ pmap_page_cache(vm_page_t pg, int mode)
 			if (entry & PG_V) {
 				entry = (entry & ~PG_CACHEMODE) | newmode;
 				*pte = entry;
-				tlb_update(pv->pv_va, entry);
+				pmap_update_kernel_page(pv->pv_va, entry);
 			}
 		} else {
 			if ((pte = pmap_segmap(pv->pv_pmap, pv->pv_va))) {
@@ -1164,10 +1358,7 @@ pmap_page_cache(vm_page_t pg, int mode)
 				if (entry & PG_V) {
 					entry = (entry & ~PG_CACHEMODE) | newmode;
 					*pte = entry;
-					if (pv->pv_pmap->pm_tlbgen == tlbpid_gen)
-						tlb_update(pv->pv_va |
-						    (pv->pv_pmap->pm_tlbpid <<
-						     VMTLB_PID_SHIFT), entry);
+					pmap_update_user_page(pv->pv_pmap, pv->pv_va, entry);
 				}
 			}
 		}
@@ -1191,22 +1382,23 @@ pmap_alloc_tlbpid(struct proc *p)
 {
 	pmap_t pmap;
 	uint id;
+	u_long cpuid = cpu_number();
 
 	pmap = p->p_vmspace->vm_map.pmap;
-	if (pmap->pm_tlbgen != tlbpid_gen) {
-		id = tlbpid_cnt;
+	if (pmap->pm_tlbgen[cpuid] != tlbpid_gen[cpuid]) {
+		id = tlbpid_cnt[cpuid];
 		if (id >= VMNUM_PIDS) {
 			tlb_flush(sys_config.cpu[0].tlbsize);
 			/* reserve tlbpid_gen == 0 to alway mean invalid */
-			if (++tlbpid_gen == 0)
-				tlbpid_gen = 1;
+			if (++tlbpid_gen[cpuid] == 0)
+				tlbpid_gen[cpuid] = 1;
 			id = 1;
 		}
-		tlbpid_cnt = id + 1;
-		pmap->pm_tlbpid = id;
-		pmap->pm_tlbgen = tlbpid_gen;
+		tlbpid_cnt[cpuid] = id + 1;
+		pmap->pm_tlbpid[cpuid] = id;
+		pmap->pm_tlbgen[cpuid] = tlbpid_gen[cpuid];
 	} else {
-		id = pmap->pm_tlbpid;
+		id = pmap->pm_tlbpid[cpuid];
 	}
 
 	if (curproc) {
diff --git a/sys/arch/mips64/mips64/softintr.c b/sys/arch/mips64/mips64/softintr.c
index 98f5a0a5cdb..95ced828145 100644
--- a/sys/arch/mips64/mips64/softintr.c
+++ b/sys/arch/mips64/mips64/softintr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: softintr.c,v 1.9 2009/11/28 16:49:30 miod Exp $	*/
+/*	$OpenBSD: softintr.c,v 1.10 2009/12/28 06:55:27 syuu Exp $	*/
 /*	$NetBSD: softintr.c,v 1.2 2003/07/15 00:24:39 lukem Exp $	*/
 
 /*
@@ -204,10 +204,15 @@ dosoftint()
 {
 	struct cpu_info *ci = curcpu();
 	int sir, q, mask;
-
 #ifdef MULTIPROCESSOR
-	if (ci->ci_ipl < IPL_SCHED)
+	u_int32_t sr;
+
+	if (ci->ci_ipl < IPL_SCHED) {
+		/* Enable interrupts */
+		sr = getsr();
+		ENABLEIPI();
 		__mp_lock(&kernel_lock);
+	}
 #endif
 
 	while ((sir = ci->ci_softpending) != 0) {
@@ -221,7 +226,9 @@ dosoftint()
 	}
 
 #ifdef MULTIPROCESSOR
-	if (ci->ci_ipl < IPL_SCHED)
+	if (ci->ci_ipl < IPL_SCHED) {
 		__mp_unlock(&kernel_lock);
+		setsr(sr);
+	}
 #endif
 }
diff --git a/sys/arch/mips64/mips64/trap.c b/sys/arch/mips64/mips64/trap.c
index a5a264f0060..35d33e87fcc 100644
--- a/sys/arch/mips64/mips64/trap.c
+++ b/sys/arch/mips64/mips64/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.52 2009/11/22 00:19:49 syuu Exp $	*/
+/*	$OpenBSD: trap.c,v 1.53 2009/12/28 06:55:27 syuu Exp $	*/
 
 /*
  * Copyright (c) 1988 University of Utah.
@@ -238,12 +238,15 @@ trap(trapframe)
 			}
 			entry |= PG_M;
 			*pte = entry;
-			tlb_update(trapframe->badvaddr & ~PGOFSET, entry);
+			KERNEL_LOCK();
+			pmap_update_kernel_page(trapframe->badvaddr & ~PGOFSET,
+			    entry);
 			pa = pfn_to_pad(entry);
 			pg = PHYS_TO_VM_PAGE(pa);
 			if (pg == NULL)
 				panic("trap: ktlbmod: unmanaged page");
 			pmap_set_modify(pg);
+			KERNEL_UNLOCK();
 			return;
 		}
 		/* FALLTHROUGH */
@@ -271,13 +274,21 @@ trap(trapframe)
 		}
 		entry |= PG_M;
 		*pte = entry;
-		tlb_update((trapframe->badvaddr & ~PGOFSET) |
-		    (pmap->pm_tlbpid << VMTLB_PID_SHIFT), entry);
+		if (USERMODE(trapframe->sr))
+			KERNEL_PROC_LOCK(p);
+		else
+			KERNEL_LOCK();
+		pmap_update_user_page(pmap, (trapframe->badvaddr & ~PGOFSET), 
+		    entry);
 		pa = pfn_to_pad(entry);
 		pg = PHYS_TO_VM_PAGE(pa);
 		if (pg == NULL)
 			panic("trap: utlbmod: unmanaged page");
 		pmap_set_modify(pg);
+		if (USERMODE(trapframe->sr))
+			KERNEL_PROC_UNLOCK(p);
+		else
+			KERNEL_UNLOCK();
 		if (!USERMODE(trapframe->sr))
 			return;
 		goto out;
diff --git a/sys/arch/sgi/include/intr.h b/sys/arch/sgi/include/intr.h
index 03f7706f7f3..b9048294b45 100644
--- a/sys/arch/sgi/include/intr.h
+++ b/sys/arch/sgi/include/intr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr.h,v 1.39 2009/12/02 01:52:28 syuu Exp $ */
+/*	$OpenBSD: intr.h,v 1.40 2009/12/28 06:55:27 syuu Exp $ */
 
 /*
  * Copyright (c) 2001-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -186,6 +186,14 @@ void	set_intr(int, uint32_t, uint32_t(*)(uint32_t, struct trap_frame *));
 uint32_t updateimask(uint32_t);
 void	dosoftint(void);
 
+#ifdef MULTIPROCESSOR
+#if defined (TGT_OCTANE)
+#define ENABLEIPI() updateimask(~CR_INT_2) /* enable IPI interrupt level */
+#else
+#error MULTIPROCESSOR kernel not supported on this configuration
+#endif
+#endif
+
 #endif /* _LOCORE */
 
 #endif /* _MACHINE_INTR_H_ */
diff --git a/sys/arch/sgi/include/mutex.h b/sys/arch/sgi/include/mutex.h
index 1d84c23882d..490d62b4f5c 100644
--- a/sys/arch/sgi/include/mutex.h
+++ b/sys/arch/sgi/include/mutex.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mutex.h,v 1.5 2009/11/04 02:26:24 syuu Exp $	*/
+/*	$OpenBSD: mutex.h,v 1.6 2009/12/28 06:55:27 syuu Exp $	*/
 
 /*
  * Copyright (c) 2004 Artur Grabowski <art@openbsd.org>
@@ -28,9 +28,6 @@
 #ifndef _MACHINE_MUTEX_H_
 #define _MACHINE_MUTEX_H_
 
-/*
- * Simple non-mp implementation.
- */
 struct mutex {
 	int mtx_lock;
 	int mtx_wantipl;
diff --git a/sys/arch/sgi/sgi/intr_template.c b/sys/arch/sgi/sgi/intr_template.c
index 84da344c1ba..f9adce561b2 100644
--- a/sys/arch/sgi/sgi/intr_template.c
+++ b/sys/arch/sgi/sgi/intr_template.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr_template.c,v 1.8 2009/12/02 01:52:28 syuu Exp $	*/
+/*	$OpenBSD: intr_template.c,v 1.9 2009/12/28 06:55:27 syuu Exp $	*/
 
 /*
  * Copyright (c) 2009 Miodrag Vallat.
@@ -150,15 +150,31 @@ INTR_FUNCTIONNAME(uint32_t hwpend, struct trap_frame *frame)
 				rc = 0;
 				for (ih = INTR_HANDLER(bitno); ih != NULL;
 				    ih = ih->ih_next) {
+#ifdef MULTIPROCESSOR
+					u_int32_t sr;
+#endif
 #if defined(INTR_HANDLER_SKIP)
 					if (INTR_HANDLER_SKIP(ih) != 0)
 						continue;
 #endif
 					splraise(ih->ih_level);
+#ifdef MULTIPROCESSOR
+					if (ih->ih_level < IPL_IPI) {
+						sr = getsr();
+						ENABLEIPI();
+						__mp_lock(&kernel_lock);
+					}
+#endif
 					if ((*ih->ih_fun)(ih->ih_arg) != 0) {
 						rc = 1;
 						atomic_add_uint64(&ih->ih_count.ec_count, 1);
 					}
+#ifdef MULTIPROCESSOR
+					if (ih->ih_level < IPL_IPI) {
+						__mp_unlock(&kernel_lock);
+						setsr(sr);
+					}
+#endif
 					__asm__ (".set noreorder\n");
 					ci->ci_ipl = ipl;
 					__asm__ ("sync\n\t.set reorder\n");
diff --git a/sys/arch/sgi/sgi/ip30_machdep.c b/sys/arch/sgi/sgi/ip30_machdep.c
index 6b637e778d9..7367c9bbe10 100644
--- a/sys/arch/sgi/sgi/ip30_machdep.c
+++ b/sys/arch/sgi/sgi/ip30_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip30_machdep.c,v 1.27 2009/12/07 19:05:59 miod Exp $	*/
+/*	$OpenBSD: ip30_machdep.c,v 1.28 2009/12/28 06:55:27 syuu Exp $	*/
 
 /*
  * Copyright (c) 2008, 2009 Miodrag Vallat.
@@ -377,6 +377,7 @@ void
 hw_cpu_hatch(struct cpu_info *ci)
 {
        int cpuid = ci->ci_cpuid;
+       int s;
 
        /*
         * Make sure we can access the extended address space.
@@ -405,7 +406,7 @@ hw_cpu_hatch(struct cpu_info *ci)
        tlb_flush(sys_config.cpu[cpuid].tlbsize);
        tlb_set_wired(sys_config.cpu[cpuid].tlbwired);
 
-       tlb_set_pid(1);
+       tlb_set_pid(0);
 
        /*
         * Turn off bootstrap exception vectors.
@@ -428,19 +429,14 @@ hw_cpu_hatch(struct cpu_info *ci)
        spl0();
        (void)updateimask(0);
 
-#ifdef notyet
        SCHED_LOCK(s);
        cpu_switchto(NULL, sched_chooseproc());
-#else
-       for(;;)
-	       ;
-#endif
 }
 
 int hw_ipi_intr_establish(int (*func)(void *), u_long cpuid)
 {
 	return xheart_intr_establish(func, (void *)cpuid, HEART_ISR_IPI(cpuid), 
-	    IPL_IPI, NULL, curcpu()->ci_ipiih);
+	    IPL_IPI, NULL, &curcpu()->ci_ipiih);
 };
 
 void hw_ipi_intr_set(u_long cpuid)