pci_intr_establish_cpu() for establishing an interrupt no a specific cpu.

the cpu is specified by a struct cpu_info *, which should generally
come from an intrmap.

this is adapted from a diff that patrick@ sent round a few years
ago for a pci_intr_map_msix_cpuid, where you asked for an msi vector
on a specific cpu, and then called pci_intr_establish with the
handle you get. kettenis pointed out that it's hard on some archs
to carry cpu on a pci interrupt handle, so i tweaked it to turn it
into a pci_intr_establish_cpu instead.

jmatthew@ and i (but mostly jmatthew@ to be honest) have been
experimenting with this api on multiple archs and it is working out
well. i'm putting this diff in now on amd64 so people can kick the
tyres a bit.

tested with hacked up vmx(4), ix(4), and mcx(4)

6 files changed, 37 insertions, 18 deletions

diff --git a/sys/arch/amd64/amd64/acpi_machdep.c b/sys/arch/amd64/amd64/acpi_machdep.c
index 38cfe424770..f2fa5d873e5 100644
--- a/sys/arch/amd64/amd64/acpi_machdep.c
+++ b/sys/arch/amd64/amd64/acpi_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: acpi_machdep.c,v 1.90 2020/04/12 09:21:19 kettenis Exp $	*/
+/*	$OpenBSD: acpi_machdep.c,v 1.91 2020/06/17 06:14:52 dlg Exp $	*/
 /*
  * Copyright (c) 2005 Thorsten Lockert <tholo@sigmasoft.com>
  *
@@ -195,7 +195,7 @@ acpi_intr_establish(int irq, int flags, int level,
 
 	type = (flags & LR_EXTIRQ_MODE) ? IST_EDGE : IST_LEVEL;
 	return (intr_establish(-1, (struct pic *)apic, map->ioapic_pin,
-	    type, level, handler, arg, what));
+	    type, level, NULL, handler, arg, what));
 #else
 	return NULL;
 #endif
diff --git a/sys/arch/amd64/amd64/intr.c b/sys/arch/amd64/amd64/intr.c
index 0eacd25cf3f..ef761f0ce69 100644
--- a/sys/arch/amd64/amd64/intr.c
+++ b/sys/arch/amd64/amd64/intr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr.c,v 1.53 2020/06/16 23:35:10 dlg Exp $	*/
+/*	$OpenBSD: intr.c,v 1.54 2020/06/17 06:14:52 dlg Exp $	*/
 /*	$NetBSD: intr.c,v 1.3 2003/03/03 22:16:20 fvdl Exp $	*/
 
 /*
@@ -282,13 +282,21 @@ duplicate:
 	} else {
 other:
 		/*
-		 * Otherwise, look for a free slot elsewhere. Do the primary
-		 * CPU first.
+		 * Otherwise, look for a free slot elsewhere. If cip is null, it
+		 * means try primary cpu but accept secondary, otherwise we need
+		 * a slot on the requested cpu.
 		 */
-		ci = &cpu_info_primary;
+		if (*cip == NULL)
+			ci = &cpu_info_primary;
+		else
+			ci = *cip;
+
 		error = intr_allocate_slot_cpu(ci, pic, pin, &slot);
 		if (error == 0)
 			goto found;
+		/* Can't alloc on the requested cpu, fail. */
+		if (*cip != NULL)
+			return EBUSY;
 
 		/*
 		 * ..now try the others.
@@ -323,10 +331,9 @@ int	intr_shared_edge;
 
 void *
 intr_establish(int legacy_irq, struct pic *pic, int pin, int type, int level,
-    int (*handler)(void *), void *arg, const char *what)
+    struct cpu_info *ci, int (*handler)(void *), void *arg, const char *what)
 {
 	struct intrhand **p, *q, *ih;
-	struct cpu_info *ci;
 	int slot, error, idt_vec;
 	struct intrsource *source;
 	struct intrstub *stubp;
diff --git a/sys/arch/amd64/include/intr.h b/sys/arch/amd64/include/intr.h
index 633fbfc00c2..c70d05e8c26 100644
--- a/sys/arch/amd64/include/intr.h
+++ b/sys/arch/amd64/include/intr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr.h,v 1.31 2018/12/21 01:51:07 jsg Exp $	*/
+/*	$OpenBSD: intr.h,v 1.32 2020/06/17 06:14:52 dlg Exp $	*/
 /*	$NetBSD: intr.h,v 1.2 2003/05/04 22:01:56 fvdl Exp $	*/
 
 /*-
@@ -201,8 +201,8 @@ void intr_calculatemasks(struct cpu_info *);
 int intr_allocate_slot_cpu(struct cpu_info *, struct pic *, int, int *);
 int intr_allocate_slot(struct pic *, int, int, int, struct cpu_info **, int *,
 	    int *);
-void *intr_establish(int, struct pic *, int, int, int, int (*)(void *),
-	    void *, const char *);
+void *intr_establish(int, struct pic *, int, int, int,
+	    struct cpu_info *, int (*)(void *), void *, const char *);
 void intr_disestablish(struct intrhand *);
 int intr_handler(struct intrframe *, struct intrhand *);
 void cpu_intr_init(struct cpu_info *);
diff --git a/sys/arch/amd64/include/pci_machdep.h b/sys/arch/amd64/include/pci_machdep.h
index 2d0cdf16a6e..bc295cc22db 100644
--- a/sys/arch/amd64/include/pci_machdep.h
+++ b/sys/arch/amd64/include/pci_machdep.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.h,v 1.28 2019/06/25 16:46:32 kettenis Exp $	*/
+/*	$OpenBSD: pci_machdep.h,v 1.29 2020/06/17 06:14:52 dlg Exp $	*/
 /*	$NetBSD: pci_machdep.h,v 1.1 2003/02/26 21:26:11 fvdl Exp $	*/
 
 /*
@@ -87,6 +87,9 @@ int		pci_intr_map(struct pci_attach_args *, pci_intr_handle_t *);
 const char	*pci_intr_string(pci_chipset_tag_t, pci_intr_handle_t);
 void		*pci_intr_establish(pci_chipset_tag_t, pci_intr_handle_t,
 		    int, int (*)(void *), void *, const char *);
+void		*pci_intr_establish_cpu(pci_chipset_tag_t, pci_intr_handle_t,
+		    int, struct cpu_info *,
+		    int (*)(void *), void *, const char *);
 void		pci_intr_disestablish(pci_chipset_tag_t, void *);
 #define	pci_probe_device_hook(c, a)	(0)
 
diff --git a/sys/arch/amd64/isa/isa_machdep.c b/sys/arch/amd64/isa/isa_machdep.c
index e1579d9bc90..4cda54a984c 100644
--- a/sys/arch/amd64/isa/isa_machdep.c
+++ b/sys/arch/amd64/isa/isa_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: isa_machdep.c,v 1.29 2017/10/14 04:44:43 jsg Exp $	*/
+/*	$OpenBSD: isa_machdep.c,v 1.30 2020/06/17 06:14:52 dlg Exp $	*/
 /*	$NetBSD: isa_machdep.c,v 1.22 1997/06/12 23:57:32 thorpej Exp $	*/
 
 #define ISA_DMA_STATS
@@ -312,7 +312,7 @@ isa_intr_establish(isa_chipset_tag_t ic, int irq, int type, int level,
 
 	KASSERT(pic);
 
-	return intr_establish(irq, pic, pin, type, level, ih_fun,
+	return intr_establish(irq, pic, pin, type, level, NULL, ih_fun,
 	    ih_arg, ih_what);
 }
 
diff --git a/sys/arch/amd64/pci/pci_machdep.c b/sys/arch/amd64/pci/pci_machdep.c
index d0fd8d43691..cf4e835de33 100644
--- a/sys/arch/amd64/pci/pci_machdep.c
+++ b/sys/arch/amd64/pci/pci_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.c,v 1.74 2020/05/14 13:07:11 kettenis Exp $	*/
+/*	$OpenBSD: pci_machdep.c,v 1.75 2020/06/17 06:14:52 dlg Exp $	*/
 /*	$NetBSD: pci_machdep.c,v 1.3 2003/05/07 21:33:58 fvdl Exp $	*/
 
 /*-
@@ -669,6 +669,14 @@ void *
 pci_intr_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, int level,
     int (*func)(void *), void *arg, const char *what)
 {
+	return pci_intr_establish_cpu(pc, ih, level, NULL, func, arg, what);
+}
+
+void *
+pci_intr_establish_cpu(pci_chipset_tag_t pc, pci_intr_handle_t ih,
+    int level, struct cpu_info *ci,
+    int (*func)(void *), void *arg, const char *what)
+{
 	int pin, irq;
 	int bus, dev;
 	pcitag_t tag = ih.tag;
@@ -676,11 +684,11 @@ pci_intr_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, int level,
 
 	if (ih.line & APIC_INT_VIA_MSG) {
 		return intr_establish(-1, &msi_pic, tag, IST_PULSE, level,
-		    func, arg, what);
+		    ci, func, arg, what);
 	}
 	if (ih.line & APIC_INT_VIA_MSGX) {
 		return intr_establish(-1, &msix_pic, tag, IST_PULSE, level,
-		    func, arg, what);
+		    ci, func, arg, what);
 	}
 
 	pci_decompose_tag(pc, ih.tag, &bus, &dev, NULL);
@@ -706,7 +714,8 @@ pci_intr_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, int level,
 	}
 #endif
 
-	return intr_establish(irq, pic, pin, IST_LEVEL, level, func, arg, what);
+	return intr_establish(irq, pic, pin, IST_LEVEL, level, ci,
+	    func, arg, what);
 }
 
 void