Use two software interrupt sources per processor for IPIs, instead of only

one, so that we can have maskable and unmaskable IPIs. Make the clock ipis
maskable, and masked at IPL_CLOCK and above. This allows us to get rid
of the retrig hack in setipl().

2 files changed, 54 insertions, 30 deletions

diff --git a/sys/arch/mvme88k/include/mvme188.h b/sys/arch/mvme88k/include/mvme188.h
index a46a345fbfb..b064c4c5cd1 100644
--- a/sys/arch/mvme88k/include/mvme188.h
+++ b/sys/arch/mvme88k/include/mvme188.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mvme188.h,v 1.30 2007/05/19 17:03:47 miod Exp $ */
+/*	$OpenBSD: mvme188.h,v 1.31 2007/11/11 13:06:57 miod Exp $ */
 /*
  * Copyright (c) 1999 Steve Murphree, Jr.
  * All rights reserved.
@@ -150,9 +150,8 @@
 
 /* hardware irq bits */
 #define HW_FAILURE_MASK		(IRQ_ABORT | IRQ_ACF | IRQ_ARBTO | IRQ_SF)
-/* software irq bits */
-#define SOFT_INTERRUPT_MASK	(IRQ_SWI7 | IRQ_SWI6 | IRQ_SWI5 | IRQ_SWI4)
 /* IPI bits (see below) */
+#define CLOCK_IPI_MASK		(IRQ_SWI7 | IRQ_SWI6 | IRQ_SWI5 | IRQ_SWI4)
 #define IPI_MASK		(IRQ_SWI3 | IRQ_SWI2 | IRQ_SWI1 | IRQ_SWI0)
 /* VME irq bits */
 #define VME_INTERRUPT_MASK	(IRQ_VME7 | IRQ_VME6 | IRQ_VME5 | IRQ_VME4 | \
@@ -170,7 +169,6 @@
 #define LVL3			(IRQ_VME3 | IRQ_DI)
 #define LVL2			(IRQ_VME2)
 #define LVL1			(IRQ_VME1)
-#define LVL0			(0x0)
 
 /* interrupts we want to process on the master CPU only */
 #define SLAVE_MASK (HW_FAILURE_MASK | OBIO_INTERRUPT_MASK | VME_INTERRUPT_MASK)
@@ -189,11 +187,17 @@
 	(*(volatile u_int *)MVME188_IST & int_mask_reg[cpu])
 
 /*
- * Software interrupts 0 to 3 are used to deliver IPIs to cpu0-3.
+ * Software interrupts 0 to 3, and 4 to 7, are used to deliver IPIs to cpu0-3.
+ * We use two bits because we want clock ipis to be maskable.
  * We rely on the fact that the control bits for these interrupts are
  * the same in the interrupt registers and the set/clear SWI registers.
  */
-#define	IPI_BIT(cpuid)		(1 << (cpuid))
+/* values for SETSWI and CLRSWI registers */
+#define	SWI_IPI_BIT(cpuid)		(0x01 << (cpuid))
+#define	SWI_CLOCK_IPI_BIT(cpuid)	(0x10 << (cpuid))
+/* values for IEN and IST registers */
+#define	SWI_IPI_MASK(cpuid)		(IRQ_SWI0 << (cpuid))
+#define	SWI_CLOCK_IPI_MASK(cpuid)	(IRQ_SWI4 << (cpuid))
 
 /*
  * ISTATE and CLRINT register bits
diff --git a/sys/arch/mvme88k/mvme88k/m188_machdep.c b/sys/arch/mvme88k/mvme88k/m188_machdep.c
index d8aa9d372ac..ba580512ea8 100644
--- a/sys/arch/mvme88k/mvme88k/m188_machdep.c
+++ b/sys/arch/mvme88k/mvme88k/m188_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: m188_machdep.c,v 1.34 2007/11/09 22:50:48 miod Exp $	*/
+/*	$OpenBSD: m188_machdep.c,v 1.35 2007/11/11 13:06:58 miod Exp $	*/
 /*
  * Copyright (c) 1998, 1999, 2000, 2001 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -260,7 +260,7 @@ safe_level(u_int mask, u_int curlevel)
 	int i;
 
 #ifdef MULTIPROCESSOR
-	mask &= ~IPI_MASK;
+	mask &= ~(IPI_MASK | CLOCK_IPI_MASK);
 #endif
 	for (i = curlevel; i < INT_LEVEL; i++)
 		if ((int_mask_val[i] & mask) == 0)
@@ -278,7 +278,7 @@ m188_getipl(void)
 u_int
 m188_setipl(u_int level)
 {
-	u_int curspl, mask;
+	u_int curspl, mask, psr;
 #ifdef MULTIPROCESSOR
 	struct cpu_info *ci = curcpu();
 	int cpu = ci->ci_cpuid;
@@ -286,27 +286,24 @@ m188_setipl(u_int level)
 	int cpu = cpu_number();
 #endif
 
+	psr = get_psr();
+	set_psr(psr | PSR_IND);
+
 	curspl = m188_curspl[cpu];
 
 	mask = int_mask_val[level];
 #ifdef MULTIPROCESSOR
 	if (cpu != master_cpu)
 		mask &= ~SLAVE_MASK;
-	mask |= IPI_BIT(cpu);
+	mask |= SWI_IPI_MASK(cpu);
+	if (level < IPL_CLOCK)
+		mask |= SWI_CLOCK_IPI_MASK(cpu);
 #endif
 
 	*(u_int32_t *)MVME188_IEN(cpu) = int_mask_reg[cpu] = mask;
 	m188_curspl[cpu] = level;
 
-#ifdef MULTIPROCESSOR
-	/*
-	 * If we have pending IPIs and we are lowering the spl, inflict
-	 * ourselves an IPI trap so that we have a chance to process this
-	 * now.
-	 */
-	if (level < curspl && ci->ci_ipi != 0 && ci->ci_intrdepth <= 1)
-		*(volatile u_int32_t *)MVME188_SETSWI = IPI_BIT(cpu);
-#endif
+	set_psr(psr);
 
 	return curspl;
 }
@@ -314,7 +311,7 @@ m188_setipl(u_int level)
 u_int
 m188_raiseipl(u_int level)
 {
-	u_int mask, curspl;
+	u_int mask, curspl, psr;
 #ifdef MULTIPROCESSOR
 	struct cpu_info *ci = curcpu();
 	int cpu = ci->ci_cpuid;
@@ -322,19 +319,26 @@ m188_raiseipl(u_int level)
 	int cpu = cpu_number();
 #endif
 
+	psr = get_psr();
+	set_psr(psr | PSR_IND);
+
 	curspl = m188_curspl[cpu];
 	if (curspl < level) {
 		mask = int_mask_val[level];
 #ifdef MULTIPROCESSOR
 		if (cpu != master_cpu)
 			mask &= ~SLAVE_MASK;
-		mask |= IPI_BIT(cpu);
+		mask |= SWI_IPI_MASK(cpu);
+		if (level < IPL_CLOCK)
+			mask |= SWI_CLOCK_IPI_MASK(cpu);
 #endif
 
 		*(u_int32_t *)MVME188_IEN(cpu) = int_mask_reg[cpu] = mask;
 		m188_curspl[cpu] = level;
 	}
 
+	set_psr(psr);
+
 	return curspl;
 }
 
@@ -349,7 +353,10 @@ m188_send_ipi(int ipi, cpuid_t cpu)
 		return;
 
 	atomic_setbits_int(&ci->ci_ipi, ipi);
-	*(volatile u_int32_t *)MVME188_SETSWI = IPI_BIT(cpu);
+	if (ipi & ~(CI_IPI_HARDCLOCK | CI_IPI_STATCLOCK))
+		*(volatile u_int32_t *)MVME188_SETSWI = SWI_IPI_BIT(cpu);
+	if (ipi & (CI_IPI_HARDCLOCK | CI_IPI_STATCLOCK))
+		*(volatile u_int32_t *)MVME188_SETSWI = SWI_CLOCK_IPI_BIT(cpu);
 }
 
 /*
@@ -360,6 +367,7 @@ m188_ipi_handler(struct trapframe *eframe)
 {
 	struct cpu_info *ci = curcpu();
 	int ipi = ci->ci_ipi;
+	int retrig = 0;
 	int old_spl = eframe->tf_mask;
 	int s;
 
@@ -385,18 +393,29 @@ m188_ipi_handler(struct trapframe *eframe)
 			s = m188_raiseipl(IPL_CLOCK);		/* splclock */
 			hardclock((struct clockframe *)eframe);
 			m188_setipl(s);				/* splx */
-		} else
+		} else {
 			ipi &= ~CI_IPI_HARDCLOCK;	/* leave it pending */
+			retrig |= CI_IPI_HARDCLOCK;
+		}
 	}
 	if (ipi & CI_IPI_STATCLOCK) {
 		if (old_spl < IPL_STATCLOCK) {
 			s = m188_raiseipl(IPL_STATCLOCK);	/* splclock */
 			statclock((struct clockframe *)eframe);
 			m188_setipl(s);				/* splx */
-		} else
+		} else {
 			ipi &= ~CI_IPI_STATCLOCK;	/* leave it pending */
+			retrig |= CI_IPI_STATCLOCK;
+		}
 	}
 
+	/*
+	 * Retrigger the clock ipi if they could not be serviced yet.
+	 */
+	if (retrig != 0)
+		*(volatile u_int32_t *)MVME188_SETSWI =
+		    SWI_CLOCK_IPI_BIT(ci->ci_cpuid);
+
 	atomic_clearbits_int(&ci->ci_ipi, ipi);
 }
 
@@ -451,7 +470,7 @@ const unsigned int obio_vec[32] = {
 void
 m188_ext_int(u_int v, struct trapframe *eframe)
 {
-#ifdef MULTIPPROCESSOR
+#ifdef MULTIPROCESSOR
 	struct cpu_info *ci = curcpu();
 	int cpu = ci->ci_cpuid;
 #else
@@ -503,12 +522,13 @@ m188_ext_int(u_int v, struct trapframe *eframe)
 	/*
 	 * Clear IPIs immediately, so that we can re enable interrupts
 	 * before further processing. We rely on the interrupt mask to
-	 * make sure that if we get an IPI, it's really for us and
-	 * no other processor.
+	 * make sure that if we get an IPI, it's really for us and no
+	 * other processor.
 	 */
-	if (cur_mask & IPI_MASK) {
-		*(volatile u_int32_t *)MVME188_CLRSWI = cur_mask & IPI_MASK;
-		cur_mask &= ~IPI_MASK;
+	if (cur_mask & (IPI_MASK | CLOCK_IPI_MASK)) {
+		*(volatile u_int32_t *)MVME188_CLRSWI =
+		    SWI_IPI_BIT(cpu) | SWI_CLOCK_IPI_BIT(cpu);
+		cur_mask &= ~(IPI_MASK | CLOCK_IPI_MASK);
 	}
 #endif