Try reducing the number of IPIs, by only reinflicting them to us if we are

lowering the spl.

Also, warn and halt in tracks if the interrupt pin of a secondary cpu never
clears (found the hard way in one of the 2P256 modules here), since there is
nothing better we can do.

Last, do not attempt to handle VME interrupts on secondary processors yes
(this confuses the bus, since both processors will perform a VME interrupt
acknowledge sequence, but only one will succeed).

This lets processes correctly run on all the available processors in
single-user mode, but there are still issues to sort out.

2 files changed, 46 insertions, 40 deletions

diff --git a/sys/arch/mvme88k/include/mvme188.h b/sys/arch/mvme88k/include/mvme188.h
index aa13449cf5f..a46a345fbfb 100644
--- a/sys/arch/mvme88k/include/mvme188.h
+++ b/sys/arch/mvme88k/include/mvme188.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mvme188.h,v 1.29 2007/05/14 17:00:40 miod Exp $ */
+/*	$OpenBSD: mvme188.h,v 1.30 2007/05/19 17:03:47 miod Exp $ */
 /*
  * Copyright (c) 1999 Steve Murphree, Jr.
  * All rights reserved.
@@ -58,11 +58,13 @@
  */
 
 /* per-processor interrupt enable registers */
+#define	MVME188_IENBASE	0xfff84000
 #define MVME188_IEN0	0xfff84004	/* interrupt enable CPU 0 */
 #define MVME188_IEN1	0xfff84008	/* interrupt enable CPU 1 */
 #define MVME188_IEN2	0xfff84010	/* interrupt enable CPU 2 */
 #define MVME188_IEN3	0xfff84020	/* interrupt enable CPU 3 */
 #define	MVME188_IENALL	0xfff8403c	/* simultaneous write */
+#define	MVME188_IEN(cpu)	(MVME188_IENBASE + (4 << (cpu)))
 
 #define MVME188_IST	0xfff84040 	/* interrupt status register */
 
@@ -161,7 +163,7 @@
 
 /* groups by interrupt levels */
 
-#define LVL7			(IRQ_ABORT | IRQ_ACF | IRQ_VME7 | IRQ_SF)
+#define LVL7			(IRQ_ABORT | IRQ_ACF /* | IRQ_VME7 */ | IRQ_SF)
 #define LVL6			(IRQ_VME6)
 #define LVL5			(IRQ_VME5 | IRQ_DTI | IRQ_CIOI)
 #define LVL4			(IRQ_VME4)
@@ -171,7 +173,7 @@
 #define LVL0			(0x0)
 
 /* interrupts we want to process on the master CPU only */
-#define SLAVE_MASK		(HW_FAILURE_MASK | OBIO_INTERRUPT_MASK)
+#define SLAVE_MASK (HW_FAILURE_MASK | OBIO_INTERRUPT_MASK | VME_INTERRUPT_MASK)
 
 #define MASK_LVL_0		(LVL7 | LVL6 | LVL5 | LVL4 | LVL3 | LVL2 | LVL1)
 #define MASK_LVL_1		(LVL7 | LVL6 | LVL5 | LVL4 | LVL3 | LVL2)
@@ -184,7 +186,7 @@
 
 #define INT_LEVEL	        8		/* # of interrupt level + 1 */
 #define ISR_GET_CURRENT_MASK(cpu) \
-	(*(volatile u_int *)MVME188_IST & *int_mask_reg[cpu])
+	(*(volatile u_int *)MVME188_IST & int_mask_reg[cpu])
 
 /*
  * Software interrupts 0 to 3 are used to deliver IPIs to cpu0-3.
diff --git a/sys/arch/mvme88k/mvme88k/m188_machdep.c b/sys/arch/mvme88k/mvme88k/m188_machdep.c
index fb57f24eda4..1c030dea0bf 100644
--- a/sys/arch/mvme88k/mvme88k/m188_machdep.c
+++ b/sys/arch/mvme88k/mvme88k/m188_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: m188_machdep.c,v 1.29 2007/05/18 16:38:29 miod Exp $	*/
+/*	$OpenBSD: m188_machdep.c,v 1.30 2007/05/19 17:03:49 miod Exp $	*/
 /*
  * Copyright (c) 1998, 1999, 2000, 2001 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -151,14 +151,9 @@ void	m188_startup(void);
  */
 
 /*
- * interrupt status register for each CPU.
+ * Copy of the interrupt enable register for each CPU.
  */
-unsigned int *volatile int_mask_reg[] = {
-	(unsigned int *)MVME188_IEN0,
-	(unsigned int *)MVME188_IEN1,
-	(unsigned int *)MVME188_IEN2,
-	(unsigned int *)MVME188_IEN3
-};
+unsigned int int_mask_reg[] = { 0, 0, 0, 0 };
 
 unsigned int m188_curspl[] = {0, 0, 0, 0};
 
@@ -283,7 +278,7 @@ m188_getipl(void)
 u_int
 m188_setipl(u_int level)
 {
-	u_int mask, curspl;
+	u_int curspl, mask;
 #ifdef MULTIPROCESSOR
 	struct cpu_info *ci = curcpu();
 	int cpu = ci->ci_cpuid;
@@ -300,12 +295,16 @@ m188_setipl(u_int level)
 	mask |= IPI_BIT(cpu);
 #endif
 
-	*int_mask_reg[cpu] = mask;
+	*(u_int32_t *)MVME188_IEN(cpu) = int_mask_reg[cpu] = mask;
 	m188_curspl[cpu] = level;
 
 #ifdef MULTIPROCESSOR
-	/* need to resend myself the pending ipis */
-	if (ci->ci_ipi != 0)
+	/*
+	 * If we have pending IPIs and we are lowering the spl, inflict
+	 * ourselves an IPI trap so that we have a chance to process this
+	 * now.
+	 */
+	if (level < curspl && ci->ci_ipi != 0 && ci->ci_intrdepth <= 1)
 		*(volatile u_int32_t *)MVME188_SETSWI = IPI_BIT(cpu);
 #endif
 
@@ -332,14 +331,8 @@ m188_raiseipl(u_int level)
 		mask |= IPI_BIT(cpu);
 #endif
 
-		*int_mask_reg[cpu] = mask;
+		*(u_int32_t *)MVME188_IEN(cpu) = int_mask_reg[cpu] = mask;
 		m188_curspl[cpu] = level;
-
-#ifdef MULTIPROCESSOR
-		/* need to resend myself the pending ipis */
-		if (ci->ci_ipi != 0)
-			*(volatile u_int32_t *)MVME188_SETSWI = IPI_BIT(cpu);
-#endif
 	}
 	return curspl;
 }
@@ -351,6 +344,9 @@ m188_send_ipi(int ipi, cpuid_t cpu)
 {
 	struct cpu_info *ci = &m88k_cpus[cpu];
 
+	if (ci->ci_ipi & ipi)
+		return;
+
 	atomic_setbits_int(&ci->ci_ipi, ipi);
 	*(volatile u_int32_t *)MVME188_SETSWI = IPI_BIT(cpu);
 }
@@ -440,7 +436,12 @@ const unsigned int obio_vec[32] = {
 void
 m188_ext_int(u_int v, struct trapframe *eframe)
 {
+#ifdef MULTIPPROCESSOR
+	struct cpu_info *ci = curcpu();
+	int cpu = ci->ci_cpuid;
+#else
 	int cpu = cpu_number();
+#endif
 	unsigned int cur_mask, ign_mask;
 	unsigned int level, old_spl;
 	struct intrhand *intr;
@@ -467,10 +468,22 @@ m188_ext_int(u_int v, struct trapframe *eframe)
 		 * Spurious interrupts - may be caused by debug output clearing
 		 * DUART interrupts.
 		 */
+#ifdef MULTIPROCESSOR
+		if (cpu != master_cpu) {
+			if (++problems >= 10) {
+				printf("cpu%d: interrupt pin won't clear, "
+				    "disabling processor\n");
+				set_psr(get_psr() | PSR_IND);
+				for (;;) ;
+			}
+		}
+#endif
 		flush_pipeline();
 		goto out;
 	}
 
+	uvmexp.intrs++;
+
 #ifdef MULTIPROCESSOR
 	/*
 	 * Clear IPIs immediately, so that we can re enable interrupts
@@ -478,11 +491,11 @@ m188_ext_int(u_int v, struct trapframe *eframe)
 	 * make sure that if we get an IPI, it's really for us and
 	 * no other processor.
 	 */
-	if (IPI_MASK & cur_mask) {
+	if (cur_mask & IPI_MASK) {
 		*(volatile u_int32_t *)MVME188_CLRSWI = cur_mask & IPI_MASK;
+		cur_mask &= ~IPI_MASK;
 	}
 #endif
-	uvmexp.intrs++;
 
 	/*
 	 * We want to service all interrupts marked in the IST register
@@ -491,8 +504,9 @@ m188_ext_int(u_int v, struct trapframe *eframe)
 	 * priority.
 	 */
 	do {
-		level = safe_level(cur_mask & ~IPI_MASK, old_spl);
+		level = safe_level(cur_mask, old_spl);
 		m188_setipl(level);
+
 		if (unmasked == 0) {
 			set_psr(get_psr() & ~PSR_IND);
 			unmasked = 1;
@@ -502,24 +516,14 @@ m188_ext_int(u_int v, struct trapframe *eframe)
 		/*
 		 * Handle IPIs first.
 		 */
-		if (IPI_MASK & cur_mask) {
-			cur_mask &= ~IPI_MASK;
-
-			m188_ipi_handler(eframe);
+		m188_ipi_handler(eframe);
 
-			if (cur_mask == 0)
-				break;
-		}
+		if (cur_mask == 0)
+			break;
 #endif
-		/* generate IACK and get the vector */
 
-		/*
-		 * This is tricky.  If you don't catch all the
-		 * interrupts, you die. Game over. Insert coin...
-		 * XXX smurph
-		 */
+		/* generate IACK and get the vector */
 
-		/* find the first bit set in the current mask */
 		intbit = ff1(cur_mask);
 		if (OBIO_INTERRUPT_MASK & (1 << intbit)) {
 			vec = obio_vec[intbit];