Implement IPIs.

6 files changed, 202 insertions, 34 deletions

diff --git a/sys/arch/powerpc64/dev/xics.c b/sys/arch/powerpc64/dev/xics.c
index 3a5ec67b00b..36dcd933c83 100644
--- a/sys/arch/powerpc64/dev/xics.c
+++ b/sys/arch/powerpc64/dev/xics.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xics.c,v 1.1 2020/07/14 20:39:40 kettenis Exp $	*/
+/*	$OpenBSD: xics.c,v 1.2 2020/07/22 16:49:13 kettenis Exp $	*/
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
  *
@@ -48,6 +48,8 @@ struct cfdriver xics_cd = {
 
 void	*xics_intr_establish(void *, int *, int,
 	    struct cpu_info *, int (*)(void *), void *, char *);
+void	xics_intr_send_ipi(void *);
+
 int
 xics_match(struct device *parent, void *match, void *aux)
 {
@@ -67,6 +69,7 @@ xics_attach(struct device *parent, struct device *self, void *aux)
 	sc->sc_ic.ic_node = faa->fa_node;
 	sc->sc_ic.ic_cookie = self;
 	sc->sc_ic.ic_establish = xics_intr_establish;
+	sc->sc_ic.ic_send_ipi = xics_intr_send_ipi;
 	interrupt_controller_register(&sc->sc_ic);
 }
 
@@ -77,5 +80,11 @@ xics_intr_establish(void *cookie, int *cell, int level,
 	uint32_t girq = cell[0];
 	int type = cell[1];
 
-	return intr_establish(girq, type, level, func, arg, name);
+	return _intr_establish(girq, type, level, func, arg, name);
+}
+
+void
+xics_intr_send_ipi(void *cookie)
+{
+	return _intr_send_ipi(cookie);
 }
diff --git a/sys/arch/powerpc64/dev/xive.c b/sys/arch/powerpc64/dev/xive.c
index 3a8a20fee44..aa48d56c1b6 100644
--- a/sys/arch/powerpc64/dev/xive.c
+++ b/sys/arch/powerpc64/dev/xive.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xive.c,v 1.7 2020/07/21 20:22:03 kettenis Exp $	*/
+/*	$OpenBSD: xive.c,v 1.8 2020/07/22 16:49:13 kettenis Exp $	*/
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
  *
@@ -94,7 +94,7 @@ struct xive_softc {
 	bus_dma_tag_t		sc_dmat;
 
 	struct intrhand		*sc_handler[XIVE_NUM_IRQS];
-	struct xive_eq		sc_eq[XIVE_NUM_PRIORITIES];
+	struct xive_eq		sc_eq[MAXCPUS][XIVE_NUM_PRIORITIES];
 
 	uint32_t		sc_page_size;
 	uint32_t		sc_lirq;
@@ -152,6 +152,7 @@ struct cfdriver xive_cd = {
 void	xive_hvi(struct trapframe *);
 void 	*xive_intr_establish(uint32_t, int, int,
 	    int (*)(void *), void *, const char *);
+void	xive_intr_send_ipi(void *);
 void	xive_setipl(int);
 
 int
@@ -167,7 +168,8 @@ xive_attach(struct device *parent, struct device *self, void *aux)
 {
 	struct xive_softc *sc = (struct xive_softc *)self;
 	struct fdt_attach_args *faa = aux;
-	struct cpu_info *ci = curcpu();
+	struct cpu_info *ci;
+	CPU_INFO_ITERATOR cii;
 	int64_t error;
 	int i;
 
@@ -194,25 +196,29 @@ xive_attach(struct device *parent, struct device *self, void *aux)
 
 	printf("\n");
 
-	for (i = 0; i < XIVE_NUM_PRIORITIES; i++) {
-		sc->sc_eq[i].eq_queue = xive_dmamem_alloc(sc->sc_dmat,
-		    1 << XIVE_EQ_SIZE, 1 << XIVE_EQ_SIZE);
-		if (sc->sc_eq[i].eq_queue == NULL) {
-			printf("%s: can't allocate event queue\n",
-			    sc->sc_dev.dv_xname);
-			return;
-		}
+	CPU_INFO_FOREACH(cii, ci) {
+		for (i = 0; i < XIVE_NUM_PRIORITIES; i++) {
+			sc->sc_eq[ci->ci_cpuid][i].eq_queue =
+			    xive_dmamem_alloc(sc->sc_dmat,
+			    1 << XIVE_EQ_SIZE, 1 << XIVE_EQ_SIZE);
+			if (sc->sc_eq[ci->ci_cpuid][i].eq_queue == NULL) {
+				printf("%s: can't allocate event queue\n",
+				    sc->sc_dev.dv_xname);
+				return;
+			}
 
-		error = opal_xive_set_queue_info(mfpir(), i,
-		    XIVE_DMA_DVA(sc->sc_eq[i].eq_queue), XIVE_EQ_SIZE,
-		    OPAL_XIVE_EQ_ENABLED | OPAL_XIVE_EQ_ALWAYS_NOTIFY);
-		if (error != OPAL_SUCCESS) {
-			printf("%s: can't enable event queue\n",
-			    sc->sc_dev.dv_xname);
-			return;
-		}
+			error = opal_xive_set_queue_info(ci->ci_pir, i,
+			    XIVE_DMA_DVA(sc->sc_eq[ci->ci_cpuid][i].eq_queue),
+			    XIVE_EQ_SIZE, OPAL_XIVE_EQ_ENABLED |
+			    OPAL_XIVE_EQ_ALWAYS_NOTIFY);
+			if (error != OPAL_SUCCESS) {
+				printf("%s: can't enable event queue\n",
+				    sc->sc_dev.dv_xname);
+				return;
+			}
 
-		sc->sc_eq[i].eq_gen = XIVE_EQ_GEN_MASK;
+			sc->sc_eq[ci->ci_cpuid][i].eq_gen = XIVE_EQ_GEN_MASK;
+		}
 	}
 
 	/* There can be only one. */
@@ -221,10 +227,11 @@ xive_attach(struct device *parent, struct device *self, void *aux)
 
 	_hvi = xive_hvi;
 	_intr_establish = xive_intr_establish;
+	_intr_send_ipi = xive_intr_send_ipi;
 	_setipl = xive_setipl;
 
 	/* Synchronize hardware state to software state. */
-	xive_write_1(sc, XIVE_TM_CPPR_HV, ci->ci_cpl);
+	xive_write_1(sc, XIVE_TM_CPPR_HV, curcpu()->ci_cpl);
 }
 
 int
@@ -307,6 +314,17 @@ xive_intr_establish(uint32_t girq, int type, int level,
 }
 
 void
+xive_intr_send_ipi(void *cookie)
+{
+	struct xive_softc *sc = xive_sc;
+	struct intrhand *ih = cookie;
+
+	if (ih && ih->ih_esb_trig)
+		bus_space_write_8(sc->sc_iot, ih->ih_esb_trig,
+		    XIVE_ESB_STORE_TRIGGER, 0);
+}
+
+void
 xive_eoi(struct xive_softc *sc, struct intrhand *ih)
 {
 	uint64_t eoi;
@@ -375,20 +393,22 @@ xive_hvi(struct trapframe *frame)
 		eieio();
 
 		KASSERT(cppr < XIVE_NUM_PRIORITIES);
-		eq = &sc->sc_eq[cppr];
+		eq = &sc->sc_eq[ci->ci_cpuid][cppr];
 		event = XIVE_DMA_KVA(eq->eq_queue);
 		while ((event[eq->eq_idx] & XIVE_EQ_GEN_MASK) == eq->eq_gen) {
 			lirq = event[eq->eq_idx] & ~XIVE_EQ_GEN_MASK;
 			KASSERT(lirq < XIVE_NUM_IRQS);
 			ih = sc->sc_handler[lirq];
 			if (ih != NULL) {
-				KERNEL_LOCK();
+				if (ih->ih_ipl != IPL_IPI)
+					KERNEL_LOCK();
 				intr_enable();
 				handled = ih->ih_func(ih->ih_arg);
 				intr_disable();
 				if (handled)
 					ih->ih_count.ec_count++;
-				KERNEL_UNLOCK();
+				if (ih->ih_ipl != IPL_IPI)
+					KERNEL_UNLOCK();
 				xive_eoi(sc, ih);
 			}
 			eq->eq_idx = (eq->eq_idx + 1) & XIVE_EQ_IDX_MASK;
diff --git a/sys/arch/powerpc64/include/cpu.h b/sys/arch/powerpc64/include/cpu.h
index 3af27a99a54..be2b7b3e3df 100644
--- a/sys/arch/powerpc64/include/cpu.h
+++ b/sys/arch/powerpc64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.20 2020/07/21 21:36:58 kettenis Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.21 2020/07/22 16:49:13 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -57,6 +57,7 @@ struct cpu_info {
 
 	uint32_t	ci_cpuid;
 	uint32_t	ci_pir;
+	int		ci_node;
 
 	struct proc	*ci_curproc;
 	struct pcb	*ci_curpcb;
@@ -86,6 +87,7 @@ struct cpu_info {
 	struct srp_hazard ci_srp_hazards[SRP_HAZARD_NUM];
 	void		*ci_initstack_end;
 	volatile int		ci_flags;
+	void		*ci_ipi;
 #endif
 };
 
@@ -115,6 +117,8 @@ register struct cpu_info *__curcpu asm("r13");
 #define CPU_INFO_FOREACH(cii, ci) \
 	for (cii = 0, ci = curcpu(); ci != NULL; ci = NULL)
 
+#define cpu_kick(ci)
+
 #else
 
 #define MAXCPUS			16
@@ -125,6 +129,7 @@ register struct cpu_info *__curcpu asm("r13");
 #define CPU_INFO_FOREACH(cii, ci) \
 	for (cii = 0, ci = &cpu_info[0]; cii < ncpus; cii++, ci++)
 
+void	cpu_kick(struct cpu_info *);
 void	cpu_boot_secondary_processors(void);
 void	cpu_startclock(void);
 
@@ -139,7 +144,6 @@ void	cpu_startclock(void);
 #define aston(p)		((p)->p_md.md_astpending = 1)
 #define need_proftick(p)	aston(p)
 
-#define cpu_kick(ci)
 #define cpu_unidle(ci)
 #define CPU_BUSY_CYCLE()	do {} while (0)
 #define signotify(p)		setsoftast()
diff --git a/sys/arch/powerpc64/include/intr.h b/sys/arch/powerpc64/include/intr.h
index fd4b93fedfc..a20ad9cde48 100644
--- a/sys/arch/powerpc64/include/intr.h
+++ b/sys/arch/powerpc64/include/intr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr.h,v 1.7 2020/07/15 10:19:42 kettenis Exp $	*/
+/*	$OpenBSD: intr.h,v 1.8 2020/07/22 16:49:13 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -19,6 +19,8 @@
 #ifndef _MACHINE_INTR_H_
 #define _MACHINE_INTR_H_
 
+struct cpu_info;
+
 #define IPL_NONE	0
 #define IPL_SOFT	1
 #define IPL_SOFTCLOCK	2
@@ -87,29 +89,32 @@ void	intr_init(void);
 
 void	*intr_establish(uint32_t, int, int,
 	    int (*)(void *), void *, const char *);
+void	intr_send_ipi(struct cpu_info *);
 
 extern void (*_hvi)(struct trapframe *);
 extern void *(*_intr_establish)(uint32_t, int, int,
 	    int (*)(void *), void *, const char *);
+extern void (*_intr_send_ipi)(void *);
 extern void (*_setipl)(int);
 
 #include <machine/softintr.h>
 
-struct cpu_info;
-
 struct interrupt_controller {
 	int	ic_node;
 	void	*ic_cookie;
 	void	*(*ic_establish)(void *, int *, int, struct cpu_info *,
 		    int (*)(void *), void *, char *);
+	void	(*ic_send_ipi)(void *);
 
 	LIST_ENTRY(interrupt_controller) ic_list;
 	uint32_t ic_phandle;
 	uint32_t ic_cells;
 };
 
-void	 interrupt_controller_register(struct interrupt_controller *);
+void	interrupt_controller_register(struct interrupt_controller *);
 
+void	*fdt_intr_establish_idx_cpu(int, int, int, struct cpu_info *,
+	    int (*)(void *), void *, char *);
 void	*fdt_intr_establish_imap(int, int *, int, int, int (*)(void *),
 	    void *, char *);
 void	*fdt_intr_establish_imap_cpu(int, int *, int, int,
diff --git a/sys/arch/powerpc64/powerpc64/cpu.c b/sys/arch/powerpc64/powerpc64/cpu.c
index 37b3e11880a..b1e0fbd8648 100644
--- a/sys/arch/powerpc64/powerpc64/cpu.c
+++ b/sys/arch/powerpc64/powerpc64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.12 2020/07/21 21:36:58 kettenis Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.13 2020/07/22 16:49:13 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -72,6 +72,7 @@ struct cfdriver cpu_cd = {
 };
 
 void	cpu_hatch(void);
+int	cpu_intr(void *);
 
 int
 cpu_match(struct device *parent, void *cfdata, void *aux)
@@ -102,6 +103,7 @@ cpu_attach(struct device *parent, struct device *dev, void *aux)
 	ci->ci_dev = dev;
 	ci->ci_cpuid = dev->dv_unit;
 	ci->ci_pir = faa->fa_reg[0].addr;
+	ci->ci_node = faa->fa_node;
 
 	printf(" pir %x", ci->ci_pir);
 
@@ -285,6 +287,10 @@ cpu_boot_secondary_processors(void)
 	CPU_INFO_ITERATOR cii;
 
 	CPU_INFO_FOREACH(cii, ci) {
+		/* Set up IPI handler. */
+		ci->ci_ipi = fdt_intr_establish_idx_cpu(ci->ci_node, 0,
+		    IPL_IPI, ci, cpu_intr, ci, ci->ci_dev->dv_xname);
+
 		if (CPU_IS_PRIMARY(ci))
 			continue;
 
@@ -293,4 +299,17 @@ cpu_boot_secondary_processors(void)
 	}
 }
 
+int
+cpu_intr(void *arg)
+{
+	return 1;
+}
+
+void
+cpu_kick(struct cpu_info *ci)
+{
+	if (ci != curcpu())
+		intr_send_ipi(ci);
+}
+
 #endif
diff --git a/sys/arch/powerpc64/powerpc64/intr.c b/sys/arch/powerpc64/powerpc64/intr.c
index dbe1ffe8abf..25b31e2d438 100644
--- a/sys/arch/powerpc64/powerpc64/intr.c
+++ b/sys/arch/powerpc64/powerpc64/intr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr.c,v 1.3 2020/07/14 20:37:18 kettenis Exp $	*/
+/*	$OpenBSD: intr.c,v 1.4 2020/07/22 16:49:13 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -29,6 +29,7 @@
 void	dummy_hvi(struct trapframe *);
 void	*dummy_intr_establish(uint32_t, int, int,
 	    int (*)(void *), void *, const char *);
+void	dummy_intr_send_ipi(void *);
 void	dummy_setipl(int);
 
 /*
@@ -38,6 +39,7 @@ void	dummy_setipl(int);
 void	(*_hvi)(struct trapframe *) = dummy_hvi;
 void	*(*_intr_establish)(uint32_t, int, int,
 	    int (*)(void *), void *, const char *) = dummy_intr_establish;
+void	(*_intr_send_ipi)(void *) = dummy_intr_send_ipi;
 void	(*_setipl)(int) = dummy_setipl;
 
 void
@@ -176,6 +178,11 @@ dummy_setipl(int new)
 	ci->ci_cpl = new;
 }
 
+void
+dummy_intr_send_ipi(void *cookie)
+{
+}
+
 /*
  * FDT interrupt support.
  */
@@ -202,6 +209,97 @@ interrupt_controller_register(struct interrupt_controller *ic)
 	LIST_INSERT_HEAD(&interrupt_controllers, ic, ic_list);
 }
 
+/*
+ * Find the interrupt parent by walking up the tree.
+ */
+uint32_t
+fdt_intr_get_parent(int node)
+{
+	uint32_t phandle = 0;
+
+	while (node && !phandle) {
+		phandle = OF_getpropint(node, "interrupt-parent", 0);
+		node = OF_parent(node);
+	}
+
+	return phandle;
+}
+
+void *
+fdt_intr_establish_idx_cpu(int node, int idx, int level, struct cpu_info *ci,
+    int (*func)(void *), void *cookie, char *name)
+{
+	struct interrupt_controller *ic;
+	int i, len, ncells, extended = 1;
+	uint32_t *cell, *cells, phandle;
+	struct fdt_intr_handle *ih;
+	void *val = NULL;
+
+	len = OF_getproplen(node, "interrupts-extended");
+	if (len <= 0) {
+		len = OF_getproplen(node, "interrupts");
+		extended = 0;
+	}
+	if (len <= 0 || (len % sizeof(uint32_t) != 0))
+		return NULL;
+
+	/* Old style. */
+	if (!extended) {
+		phandle = fdt_intr_get_parent(node);
+		LIST_FOREACH(ic, &interrupt_controllers, ic_list) {
+			if (ic->ic_phandle == phandle)
+				break;
+		}
+
+		if (ic == NULL)
+			return NULL;
+	}
+
+	cell = cells = malloc(len, M_TEMP, M_WAITOK);
+	if (extended)
+		OF_getpropintarray(node, "interrupts-extended", cells, len);
+	else
+		OF_getpropintarray(node, "interrupts", cells, len);
+	ncells = len / sizeof(uint32_t);
+
+	for (i = 0; i <= idx && ncells > 0; i++) {
+		if (extended) {
+			phandle = cell[0];
+
+			LIST_FOREACH(ic, &interrupt_controllers, ic_list) {
+				if (ic->ic_phandle == phandle)
+					break;
+			}
+
+			if (ic == NULL)
+				break;
+
+			cell++;
+			ncells--;
+		}
+
+		if (i == idx && ncells >= ic->ic_cells && ic->ic_establish) {
+			val = ic->ic_establish(ic->ic_cookie, cell, level,
+			    ci, func, cookie, name);
+			break;
+		}
+
+		cell += ic->ic_cells;
+		ncells -= ic->ic_cells;
+	}
+
+	free(cells, M_TEMP, len);
+
+	if (val == NULL)
+		return NULL;
+
+	ih = malloc(sizeof(*ih), M_DEVBUF, M_WAITOK);
+	ih->ih_ic = ic;
+	ih->ih_ih = val;
+
+	return ih;
+}
+
 void *
 fdt_intr_establish_imap(int node, int *reg, int nreg, int level,
     int (*func)(void *), void *cookie, char *name)
@@ -274,3 +372,16 @@ fdt_intr_establish_imap_cpu(int node, int *reg, int nreg, int level,
 	free(map, M_DEVBUF, len);
 	return ih;
 }
+
+#ifdef MULTIPROCESSOR
+
+void
+intr_send_ipi(struct cpu_info *ci)
+{
+	struct fdt_intr_handle *ih = ci->ci_ipi;
+
+	if (ih && ih->ih_ic)
+		ih->ih_ic->ic_send_ipi(ih->ih_ih);
+}
+
+#endif