rework the srp api so it takes an srp_ref struct that the caller provides.

the srp_ref struct is used to track the location of the callers
hazard pointer so later calls to srp_follow and srp_enter already
know what to clear. this in turn means most of the caveats around
using srps go away. specifically, you can now:

- switch cpus while holding an srp ref
  - ie, you can sleep while holding an srp ref
- you can take and release srp refs in any order

the original intent was to simplify use of the api when dealing
with complicated data structures. the caller now no longer has to
track the location of the srp a value was fetched from, the srp_ref
effectively does that for you.

srp lists have been refactored to use srp_refs instead of srpl_iter
structs.

this is in preparation of using srps inside the ART code. ART is a
complicated data structure, and lookups require overlapping holds
of srp references.

ok mpi@ jmatthew@

9 files changed, 121 insertions, 189 deletions

diff --git a/share/man/man9/srp_enter.9 b/share/man/man9/srp_enter.9
index 58ee229fc9f..a6e7cbc0429 100644
--- a/share/man/man9/srp_enter.9
+++ b/share/man/man9/srp_enter.9
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: srp_enter.9,v 1.9 2016/02/12 12:52:28 dlg Exp $
+.\"	$OpenBSD: srp_enter.9,v 1.10 2016/05/18 03:46:03 dlg Exp $
 .\"
 .\" Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
 .\"
@@ -14,7 +14,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: February 12 2016 $
+.Dd $Mdocdate: May 18 2016 $
 .Dt SRP_ENTER 9
 .Os
 .Sh NAME
@@ -45,11 +45,11 @@
 .Ft void
 .Fn srp_update_locked "struct srp_gc *gc" "struct srp *p" "void *v"
 .Ft void *
-.Fn srp_enter "struct srp *p"
+.Fn srp_enter "struct srp_ref *sr" "struct srp *p"
 .Ft void *
-.Fn srp_follow "struct srp *p" "void *v" "struct srp *n"
+.Fn srp_follow "struct srp_ref *sr" "struct srp *n"
 .Ft void
-.Fn srp_leave "struct srp *p" "void *v"
+.Fn srp_leave "struct srp_ref *sr"
 .Ft void *
 .Fn srp_get_locked "struct srp *p"
 .Ft void
@@ -118,39 +118,18 @@ call to
 .Fn srp_leave
 or
 .Fn srp_follow .
+The reference is held via
+.Fa sr .
 .Pp
 .Fn srp_follow
-returns a pointer to the data structure referenced by the srp struct
-.Fa n
-that exists within the structure referenced by
-.Fa v
-via
-.Fa p ,
-while releasing the reference to
-.Fa v
-and making it available for garbage collection.
-It is equivalent to a call to
-.Fn srp_enter
-using
-.Fa n
-as an argument
-followed by a call to
-.Fn srp_leave
-with
-.Fa p
-and
-.Fa v
-as arguments.
-.Fn srp_follow
-is necessary to correctly order the taking and releasing of SRP
-critical sections in situations such as following a chain of data
-structures linked with SRPs.
+replaces the reference held via
+.Fa sr
+with a reference to the data structure represented by
+.Fa p .
 .Pp
 .Fn srp_leave
-releases the reference to
-.Fa v
-by the srp struct
-.Fa p
+releases the reference held via
+.Fa sr
 and makes it available for garbage collection.
 .Pp
 .Fn srp_get_locked
@@ -195,22 +174,6 @@ can be called during autoconf, or from process context.
 and
 .Fn srp_leave
 can be called during autoconf, from process context, or from interrupt context.
-Calling
-.Fn srp_follow
-or
-.Fn srp_leave
-from a different context or on a different CPU to the preceding
-.Fn srp_enter
-or
-.Fn srp_follow
-calls will lead to undefined behaviour.
-.Pp
-SRP critical sections must be released with
-.Fn srp_leave
-in the opposite order in which they were taken with
-.Fn srp_enter
-unless a critical section is exchanged with
-.Fn srp_follow .
 .Sh RETURN VALUES
 .Fn srp_enter ,
 .Fn srp_follow ,
diff --git a/share/man/man9/srpl_rc_init.9 b/share/man/man9/srpl_rc_init.9
index 7009d0b3c81..3c5a385751a 100644
--- a/share/man/man9/srpl_rc_init.9
+++ b/share/man/man9/srpl_rc_init.9
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: srpl_rc_init.9,v 1.9 2015/12/03 16:27:32 mpi Exp $
+.\"	$OpenBSD: srpl_rc_init.9,v 1.10 2016/05/18 03:46:03 dlg Exp $
 .\"
 .\" Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
 .\"
@@ -14,7 +14,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: December 3 2015 $
+.Dd $Mdocdate: May 18 2016 $
 .Dt SRPL_RC_INIT 9
 .Os
 .Sh NAME
@@ -38,7 +38,6 @@
 .Nd singly-linked shared reference pointer list
 .Sh SYNOPSIS
 .In sys/srp.h
-.Vt struct srpl_iter;
 .Vt struct srpl_rc;
 .Ft void
 .Fo "srpl_rc_init"
@@ -51,16 +50,16 @@
 .Fn SRPL_ENTRY "TYPE"
 .Fn "SRPL_INIT" "SRPL_HEAD *sl"
 .Ft void *
-.Fn "SRPL_ENTER" "SRPL_HEAD *sl" "struct srpl_iter *si"
+.Fn "SRPL_ENTER" "struct srp_ref *sr" "SRPL_HEAD *sl"
 .Ft void *
-.Fn "SRPL_NEXT" "struct srpl_iter *si" "struct TYPE *listelm" "FIELDNAME"
+.Fn "SRPL_NEXT" "struct srp_ref *sr" "struct TYPE *listelm" "FIELDNAME"
 .Fo "SRPL_FOREACH"
 .Fa "VARNAME"
+.Fa "struct srp_ref *sr"
 .Fa "SRPL_HEAD *sl"
-.Fa "struct srpl_iter *si"
 .Fa "FIELDNAME"
 .Fc
-.Fn "SRPL_LEAVE" "struct srpl_iter *si" "struct TYPE *listelm"
+.Fn "SRPL_LEAVE" "struct srp_ref *sr"
 .Fn "SRPL_EMPTY_LOCKED" "SRPL_HEAD *sl"
 .Ft void *
 .Fn "SRPL_FIRST_LOCKED" "SRPL_HEAD *sl"
@@ -139,9 +138,8 @@ to an empty state.
 .Fn SRPL_ENTER
 begins iterating over elements in the SRP list
 .Fa sl .
-Local state necessary for iterating over the list is stored in the
-SRP list iterator structure
-.Fa si .
+The reference to the list item is held via
+.Fa sr .
 Every call to
 .Fn SRPL_ENTER
 must have a corresponding call to
@@ -163,7 +161,8 @@ to release references to the list and its elements.
 .Fn SRPL_LEAVE
 releases references to the list and its elements held by previous
 calls to
-.Fn SRPL_ENTER
+.Fn SRPL_ENTER ,
+.Fn SRPL_NEXT ,
 or
 .Fn SRPL_FOREACH .
 .Pp
diff --git a/sys/kern/kern_srp.c b/sys/kern/kern_srp.c
index 324e45bd3b3..4ae2c322898 100644
--- a/sys/kern/kern_srp.c
+++ b/sys/kern/kern_srp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_srp.c,v 1.7 2015/11/23 10:56:19 mpi Exp $ */
+/*	$OpenBSD: kern_srp.c,v 1.8 2016/05/18 03:46:03 dlg Exp $ */
 
 /*
  * Copyright (c) 2014 Jonathan Matthew <jmatthew@openbsd.org>
@@ -207,7 +207,7 @@ srp_v(struct srp_hazard *hzrd, struct srp *srp)
 }
 
 void *
-srp_enter(struct srp *srp)
+srp_enter(struct srp_ref *sr, struct srp *srp)
 {
 	struct cpu_info *ci = curcpu();
 	struct srp_hazard *hzrd;
@@ -215,8 +215,10 @@ srp_enter(struct srp *srp)
 
 	for (i = 0; i < nitems(ci->ci_srp_hazards); i++) {
 		hzrd = &ci->ci_srp_hazards[i];
-		if (hzrd->sh_p == NULL)
+		if (hzrd->sh_p == NULL) {
+			sr->hz = hzrd;
 			return (srp_v(hzrd, srp));
+		}
 	}
 
 	panic("%s: not enough srp hazard records", __func__);
@@ -226,38 +228,15 @@ srp_enter(struct srp *srp)
 }
 
 void *
-srp_follow(struct srp *srp, void *v, struct srp *next)
+srp_follow(struct srp_ref *sr, struct srp *srp)
 {
-	struct cpu_info *ci = curcpu();
-	struct srp_hazard *hzrd;
-
-	hzrd = ci->ci_srp_hazards + nitems(ci->ci_srp_hazards);
-	while (hzrd-- != ci->ci_srp_hazards) {
-		if (hzrd->sh_p == srp && hzrd->sh_v == v)
-			return (srp_v(hzrd, next));
-	}
-
-	panic("%s: unexpected ref %p via %p", __func__, v, srp);
-
-	/* NOTREACHED */
-	return (NULL);
+	return (srp_v(sr->hz, srp));
 }
 
 void
-srp_leave(struct srp *srp, void *v)
+srp_leave(struct srp_ref *sr)
 {
-	struct cpu_info *ci = curcpu();
-	struct srp_hazard *hzrd;
-
-	hzrd = ci->ci_srp_hazards + nitems(ci->ci_srp_hazards);
-	while (hzrd-- != ci->ci_srp_hazards) {
-		if (hzrd->sh_p == srp && hzrd->sh_v == v) {
-			hzrd->sh_p = NULL;
-			return;
-		}
-	}
-
-	panic("%s: unexpected ref %p via %p", __func__, v, srp);
+	sr->hz->sh_p = NULL;
 }
 
 #else /* MULTIPROCESSOR */
diff --git a/sys/net/bpf.c b/sys/net/bpf.c
index 9a8273114bf..31b6ed0c5c2 100644
--- a/sys/net/bpf.c
+++ b/sys/net/bpf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bpf.c,v 1.140 2016/05/10 23:48:07 dlg Exp $	*/
+/*	$OpenBSD: bpf.c,v 1.141 2016/05/18 03:46:03 dlg Exp $	*/
 /*	$NetBSD: bpf.c,v 1.33 1997/02/21 23:59:35 thorpej Exp $	*/
 
 /*
@@ -1134,7 +1134,7 @@ int
 bpf_tap(caddr_t arg, u_char *pkt, u_int pktlen, u_int direction)
 {
 	struct bpf_if *bp = (struct bpf_if *)arg;
-	struct srpl_iter i;
+	struct srp_ref sr;
 	struct bpf_d *d;
 	size_t slen;
 	struct timeval tv;
@@ -1144,20 +1144,21 @@ bpf_tap(caddr_t arg, u_char *pkt, u_int pktlen, u_int direction)
 	if (bp == NULL)
 		return (0);
 
-	SRPL_FOREACH(d, &bp->bif_dlist, &i, bd_next) {
+	SRPL_FOREACH(d, &sr, &bp->bif_dlist, bd_next) {
 		atomic_inc_long(&d->bd_rcount);
 
 		if ((direction & d->bd_dirfilt) != 0)
 			slen = 0;
 		else {
+			struct srp_ref sr;
 			struct bpf_program *bf;
 			struct bpf_insn *fcode = NULL;
 
-			bf = srp_enter(&d->bd_rfilter);
+			bf = srp_enter(&sr, &d->bd_rfilter);
 			if (bf != NULL)
 				fcode = bf->bf_insns;
 			slen = bpf_filter(fcode, pkt, pktlen, pktlen);
-			srp_leave(&d->bd_rfilter, bf);
+			srp_leave(&sr);
 		}
 
 		if (slen > 0) {
@@ -1177,7 +1178,7 @@ bpf_tap(caddr_t arg, u_char *pkt, u_int pktlen, u_int direction)
 				drop = 1;
 		}
 	}
-	SRPL_LEAVE(&i, d);
+	SRPL_LEAVE(&sr);
 
 	return (drop);
 }
@@ -1214,7 +1215,7 @@ _bpf_mtap(caddr_t arg, const struct mbuf *m, u_int direction,
     void (*cpfn)(const void *, void *, size_t))
 {
 	struct bpf_if *bp = (struct bpf_if *)arg;
-	struct srpl_iter i;
+	struct srp_ref sr;
 	struct bpf_d *d;
 	size_t pktlen, slen;
 	const struct mbuf *m0;
@@ -1236,20 +1237,21 @@ _bpf_mtap(caddr_t arg, const struct mbuf *m, u_int direction,
 	for (m0 = m; m0 != NULL; m0 = m0->m_next)
 		pktlen += m0->m_len;
 
-	SRPL_FOREACH(d, &bp->bif_dlist, &i, bd_next) {
+	SRPL_FOREACH(d, &sr, &bp->bif_dlist, bd_next) {
 		atomic_inc_long(&d->bd_rcount);
 
 		if ((direction & d->bd_dirfilt) != 0)
 			slen = 0;
 		else {
+			struct srp_ref bsr;
 			struct bpf_program *bf;
 			struct bpf_insn *fcode = NULL;
 
-			bf = srp_enter(&d->bd_rfilter);
+			bf = srp_enter(&bsr, &d->bd_rfilter);
 			if (bf != NULL)
 				fcode = bf->bf_insns;
 			slen = bpf_mfilter(fcode, m, pktlen);
-			srp_leave(&d->bd_rfilter, bf);
+			srp_leave(&bsr);
 		}
 
 		if (slen > 0) {
@@ -1269,7 +1271,7 @@ _bpf_mtap(caddr_t arg, const struct mbuf *m, u_int direction,
 				drop = 1;
 		}
 	}
-	SRPL_LEAVE(&i, d);
+	SRPL_LEAVE(&sr);
 
 	return (drop);
 }
diff --git a/sys/net/if.c b/sys/net/if.c
index c0e232601c5..9b53bf17ebf 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.432 2016/05/10 06:37:15 dlg Exp $	*/
+/*	$OpenBSD: if.c,v 1.433 2016/05/18 03:46:03 dlg Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -774,7 +774,7 @@ if_input_process(void *xmq)
 	struct mbuf *m;
 	struct ifnet *ifp;
 	struct ifih *ifih;
-	struct srpl_iter i;
+	struct srp_ref sr;
 	int s;
 
 	mq_delist(mq, &ml);
@@ -795,11 +795,11 @@ if_input_process(void *xmq)
 		 * Pass this mbuf to all input handlers of its
 		 * interface until it is consumed.
 		 */
-		SRPL_FOREACH(ifih, &ifp->if_inputs, &i, ifih_next) {
+		SRPL_FOREACH(ifih, &sr, &ifp->if_inputs, ifih_next) {
 			if ((*ifih->ifih_input)(ifp, m, ifih->ifih_cookie))
 				break;
 		}
-		SRPL_LEAVE(&i, ifih);
+		SRPL_LEAVE(&sr);
 
 		if (ifih == NULL)
 			m_freem(m);
@@ -1532,22 +1532,22 @@ ifunit(const char *name)
 struct ifnet *
 if_get(unsigned int index)
 {
+	struct srp_ref sr;
 	struct if_map *if_map;
 	struct srp *map;
 	struct ifnet *ifp = NULL;
 
-	if_map = srp_enter(&if_idxmap.map);
+	if_map = srp_enter(&sr, &if_idxmap.map);
 	if (index < if_map->limit) {
 		map = (struct srp *)(if_map + 1);
 
-		ifp = srp_follow(&if_idxmap.map, if_map, &map[index]);
+		ifp = srp_follow(&sr, &map[index]);
 		if (ifp != NULL) {
 			KASSERT(ifp->if_index == index);
 			if_ref(ifp);
 		}
-		srp_leave(&map[index], ifp);
-	} else
-		srp_leave(&if_idxmap.map, if_map);
+	}
+	srp_leave(&sr);
 
 	return (ifp);
 }
diff --git a/sys/net/if_vlan.c b/sys/net/if_vlan.c
index 7171ca5d8ea..0aa47d77fb0 100644
--- a/sys/net/if_vlan.c
+++ b/sys/net/if_vlan.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_vlan.c,v 1.164 2016/04/19 23:32:15 dlg Exp $	*/
+/*	$OpenBSD: if_vlan.c,v 1.165 2016/05/18 03:46:03 dlg Exp $	*/
 
 /*
  * Copyright 1998 Massachusetts Institute of Technology
@@ -335,7 +335,7 @@ vlan_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 	struct ether_vlan_header	*evl;
 	struct ether_header		*eh;
 	SRPL_HEAD(, ifvlan)		*tagh, *list;
-	struct srpl_iter		 i;
+	struct srp_ref			 sr;
 	u_int				 tag;
 	struct mbuf_list		 ml = MBUF_LIST_INITIALIZER();
 	u_int16_t			 etype;
@@ -370,7 +370,7 @@ vlan_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 		m->m_pkthdr.pf.prio = !m->m_pkthdr.pf.prio;
 
 	list = &tagh[TAG_HASH(tag)];
-	SRPL_FOREACH(ifv, list, &i, ifv_list) {
+	SRPL_FOREACH(ifv, &sr, list, ifv_list) {
 		if (ifp0->if_index == ifv->ifv_ifp0 && tag == ifv->ifv_tag &&
 		    etype == ifv->ifv_type)
 			break;
@@ -400,11 +400,11 @@ vlan_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 
 	ml_enqueue(&ml, m);
 	if_input(&ifv->ifv_if, &ml);
-	SRPL_LEAVE(&i, ifv);
+	SRPL_LEAVE(&sr);
 	return (1);
 
 drop:
-	SRPL_LEAVE(&i, ifv);
+	SRPL_LEAVE(&sr);
 	m_freem(m);
 	return (1);
 }
diff --git a/sys/net/rtable.c b/sys/net/rtable.c
index 0a3a09e0111..aeb5c2777a9 100644
--- a/sys/net/rtable.c
+++ b/sys/net/rtable.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rtable.c,v 1.41 2016/05/02 22:15:49 jmatthew Exp $ */
+/*	$OpenBSD: rtable.c,v 1.42 2016/05/18 03:46:03 dlg Exp $ */
 
 /*
  * Copyright (c) 2014-2015 Martin Pieuchot
@@ -231,14 +231,15 @@ rtable_get(unsigned int rtableid, sa_family_t af)
 {
 	struct rtmap	*map;
 	void		*tbl = NULL;
+	struct srp_ref	 sr;
 
 	if (af >= nitems(af2idx) || af2idx[af] == 0)
 		return (NULL);
 
-	map = srp_enter(&afmap[af2idx[af]]);
+	map = srp_enter(&sr, &afmap[af2idx[af]]);
 	if (rtableid < map->limit)
 		tbl = map->tbl[rtableid];
-	srp_leave(&afmap[af2idx[af]], map);
+	srp_leave(&sr);
 
 	return (tbl);
 }
@@ -267,11 +268,12 @@ rtable_l2(unsigned int rtableid)
 {
 	struct dommp	*dmm;
 	unsigned int	 rdomain = 0;
+	struct srp_ref	 sr;
 
-	dmm = srp_enter(&afmap[0]);
+	dmm = srp_enter(&sr, &afmap[0]);
 	if (rtableid < dmm->limit)
 		rdomain = dmm->dom[rtableid];
-	srp_leave(&afmap[0], dmm);
+	srp_leave(&sr);
 
 	return (rdomain);
 }
@@ -534,7 +536,7 @@ rtable_lookup(unsigned int rtableid, struct sockaddr *dst,
 	struct art_root			*ar;
 	struct art_node			*an;
 	struct rtentry			*rt;
-	struct srpl_iter		 i;
+	struct srp_ref			 sr;
 	uint8_t				*addr;
 	int				 plen;
 
@@ -562,9 +564,9 @@ rtable_lookup(unsigned int rtableid, struct sockaddr *dst,
 	}
 
 #ifdef SMALL_KERNEL
-	rt = SRPL_ENTER(&an->an_rtlist, &i);
+	rt = SRPL_ENTER(&sr, &an->an_rtlist);
 #else
-	SRPL_FOREACH(rt, &an->an_rtlist, &i, rt_next) {
+	SRPL_FOREACH(rt, &sr, &an->an_rtlist, rt_next) {
 		if (prio != RTP_ANY &&
 		    (rt->rt_priority & RTP_MASK) != (prio & RTP_MASK))
 			continue;
@@ -577,13 +579,13 @@ rtable_lookup(unsigned int rtableid, struct sockaddr *dst,
 			break;
 	}
 	if (rt == NULL) {
-		SRPL_LEAVE(&i, rt);
+		SRPL_LEAVE(&sr);
 		return (NULL);
 	}
 #endif /* SMALL_KERNEL */
 
 	rtref(rt);
-	SRPL_LEAVE(&i, rt);
+	SRPL_LEAVE(&sr);
 
 	return (rt);
 }
@@ -594,7 +596,7 @@ rtable_match(unsigned int rtableid, struct sockaddr *dst, uint32_t *src)
 	struct art_root			*ar;
 	struct art_node			*an;
 	struct rtentry			*rt = NULL;
-	struct srpl_iter		 i;
+	struct srp_ref			 sr;
 	uint8_t				*addr;
 #ifndef SMALL_KERNEL
 	int				 hash;
@@ -611,33 +613,32 @@ rtable_match(unsigned int rtableid, struct sockaddr *dst, uint32_t *src)
 	if (an == NULL)
 		goto out;
 
-	rt = SRPL_ENTER(&an->an_rtlist, &i);
+	rt = SRPL_ENTER(&sr, &an->an_rtlist);
 	rtref(rt);
-	SRPL_LEAVE(&i, rt);
+	SRPL_LEAVE(&sr);
 
 #ifndef SMALL_KERNEL
 	/* Gateway selection by Hash-Threshold (RFC 2992) */
 	if ((hash = rt_hash(rt, dst, src)) != -1) {
 		struct rtentry		*mrt;
-		struct srpl_iter	 i;
 		int			 threshold, npaths = 0;
 
 		KASSERT(hash <= 0xffff);
 
-		SRPL_FOREACH(mrt, &an->an_rtlist, &i, rt_next) {
+		SRPL_FOREACH(mrt, &sr, &an->an_rtlist, rt_next) {
 			/* Only count nexthops with the same priority. */
 			if (mrt->rt_priority == rt->rt_priority)
 				npaths++;
 		}
-		SRPL_LEAVE(&i, mrt);
+		SRPL_LEAVE(&sr);
 
 		threshold = (0xffff / npaths) + 1;
 
-		mrt = SRPL_ENTER(&an->an_rtlist, &i);
+		mrt = SRPL_ENTER(&sr, &an->an_rtlist);
 		while (hash > threshold && mrt != NULL) {
 			if (mrt->rt_priority == rt->rt_priority)
 				hash -= threshold;
-			mrt = SRPL_NEXT(&i, mrt, rt_next);
+			mrt = SRPL_NEXT(&sr, mrt, rt_next);
 		}
 
 		if (mrt != NULL) {
@@ -645,7 +646,7 @@ rtable_match(unsigned int rtableid, struct sockaddr *dst, uint32_t *src)
 			rtfree(rt);
 			rt = mrt;
 		}
-		SRPL_LEAVE(&i, mrt);
+		SRPL_LEAVE(&sr);
 	}
 #endif /* SMALL_KERNEL */
 out:
diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index 0ab7d5714c2..954c227d210 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_carp.c,v 1.288 2016/04/13 11:41:15 mpi Exp $	*/
+/*	$OpenBSD: ip_carp.c,v 1.289 2016/05/18 03:46:03 dlg Exp $	*/
 
 /*
  * Copyright (c) 2002 Michael Shalayeff. All rights reserved.
@@ -1321,10 +1321,10 @@ carp_iamatch(struct ifnet *ifp, uint8_t *enaddr)
 {
 	struct carp_softc *sc = ifp->if_softc;
 	struct carp_vhost_entry *vhe;
-	struct srpl_iter i;
+	struct srp_ref sr;
 	int match = 0;
 
-	vhe = SRPL_ENTER(&sc->carp_vhosts, &i); /* head */
+	vhe = SRPL_ENTER(&sr, &sc->carp_vhosts); /* head */
 	if (vhe->state == MASTER) {
 		if (sc->sc_balancing == CARP_BAL_IPSTEALTH ||
 		    sc->sc_balancing == CARP_BAL_IP) {
@@ -1333,7 +1333,7 @@ carp_iamatch(struct ifnet *ifp, uint8_t *enaddr)
 		}
 		match = 1;
 	}
-	SRPL_LEAVE(&i, vhe);
+	SRPL_LEAVE(&sr);
 
 	return (match);
 }
@@ -1379,13 +1379,13 @@ int
 carp_vhe_match(struct carp_softc *sc, uint8_t *ena)
 {
 	struct carp_vhost_entry *vhe;
-	struct srpl_iter i;
+	struct srp_ref sr;
 	int match = 0;
 
-	vhe = SRPL_ENTER(&sc->carp_vhosts, &i); /* head */
+	vhe = SRPL_ENTER(&sr, &sc->carp_vhosts); /* head */
 	match = (vhe->state == MASTER || sc->sc_balancing >= CARP_BAL_IP) &&
 	    !memcmp(ena, sc->sc_ac.ac_enaddr, ETHER_ADDR_LEN);
-	SRPL_LEAVE(&i, vhe);
+	SRPL_LEAVE(&sr);
 
 	return (match);
 }
@@ -1397,13 +1397,13 @@ carp_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
 	struct carp_if *cif;
 	struct carp_softc *sc;
-	struct srpl_iter i;
+	struct srp_ref sr;
 
 	eh = mtod(m, struct ether_header *);
 	cif = (struct carp_if *)cookie;
 	KASSERT(cif == (struct carp_if *)ifp0->if_carp);
 
-	SRPL_FOREACH(sc, &cif->vhif_vrs, &i, sc_list) {
+	SRPL_FOREACH(sc, &sr, &cif->vhif_vrs, sc_list) {
 		if ((sc->sc_if.if_flags & (IFF_UP|IFF_RUNNING)) !=
 		    (IFF_UP|IFF_RUNNING))
 			continue;
@@ -1413,7 +1413,7 @@ carp_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 	}
 
 	if (sc == NULL) {
-		SRPL_LEAVE(&i, sc);
+		SRPL_LEAVE(&sr);
 
 		if (!ETHER_IS_MULTICAST(eh->ether_dhost))
 			return (0);
@@ -1422,7 +1422,7 @@ carp_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 		 * XXX Should really check the list of multicast addresses
 		 * for each CARP interface _before_ copying.
 		 */
-		SRPL_FOREACH(sc, &cif->vhif_vrs, &i, sc_list) {
+		SRPL_FOREACH(sc, &sr, &cif->vhif_vrs, sc_list) {
 			struct mbuf *m0;
 
 			if (!(sc->sc_if.if_flags & IFF_UP))
@@ -1437,7 +1437,7 @@ carp_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 
 			if_input(&sc->sc_if, &ml);
 		}
-		SRPL_LEAVE(&i, sc);
+		SRPL_LEAVE(&sr);
 
 		return (0);
 	}
@@ -1451,7 +1451,7 @@ carp_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 
 	ml_enqueue(&ml, m);
 	if_input(&sc->sc_if, &ml);
-	SRPL_LEAVE(&i, sc);
+	SRPL_LEAVE(&sr);
 
 	return (1);
 }
@@ -2299,15 +2299,15 @@ carp_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
 {
 	struct carp_softc *sc = ((struct carp_softc *)ifp->if_softc);
 	struct carp_vhost_entry *vhe;
-	struct srpl_iter i;
+	struct srp_ref sr;
 	int ismaster;
 
 	KASSERT(sc->sc_carpdev != NULL);
 
 	if (sc->cur_vhe == NULL) {
-		vhe = SRPL_ENTER(&sc->carp_vhosts, &i); /* head */
+		vhe = SRPL_ENTER(&sr, &sc->carp_vhosts); /* head */
 		ismaster = (vhe->state == MASTER);
-		SRPL_LEAVE(&i, vhe);
+		SRPL_LEAVE(&sr);
 	} else {
 		ismaster = (sc->cur_vhe->state == MASTER);
 	}
diff --git a/sys/sys/srp.h b/sys/sys/srp.h
index 716d80f44ca..a952bd6f887 100644
--- a/sys/sys/srp.h
+++ b/sys/sys/srp.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: srp.h,v 1.7 2015/12/03 16:27:32 mpi Exp $ */
+/*	$OpenBSD: srp.h,v 1.8 2016/05/18 03:46:03 dlg Exp $ */
 
 /*
  * Copyright (c) 2014 Jonathan Matthew <jmatthew@openbsd.org>
@@ -21,6 +21,12 @@
 
 #include <sys/refcnt.h>
 
+#ifdef MULTIPROCESSOR
+#define __upunused
+#else
+#define __upunused __attribute__((__unused__))
+#endif
+
 struct srp {
 	void			*ref;
 };
@@ -30,6 +36,10 @@ struct srp_hazard {
 	void			*sh_v;
 };
 
+struct srp_ref {
+	struct srp_hazard	*hz;
+} __upunused;
+
 #define SRP_HAZARD_NUM 16
 	
 struct srp_gc {
@@ -53,14 +63,14 @@ void		 srp_init(struct srp *);
 
 #ifdef MULTIPROCESSOR
 void		 srp_update(struct srp_gc *, struct srp *, void *);
-void		*srp_enter(struct srp *);
-void		*srp_follow(struct srp *, void *, struct srp *);
-void		 srp_leave(struct srp *, void *);
+void		*srp_enter(struct srp_ref *, struct srp *);
+void		*srp_follow(struct srp_ref *, struct srp *);
+void		 srp_leave(struct srp_ref *);
 #else /* MULTIPROCESSOR */
 #define srp_update(_gc, _srp, _v)	srp_update_locked((_gc), (_srp), (_v))
-#define srp_enter(_srp)			((_srp)->ref)
-#define srp_follow(_srp, _v, _next)	((_next)->ref)
-#define srp_leave(_srp, _v)		do { } while (0)
+#define srp_enter(_sr, _srp)		((_srp)->ref)
+#define srp_follow(_sr, _srp)		((_srp)->ref)
+#define srp_leave(_sr)			do { } while (0)
 #endif /* MULTIPROCESSOR */
 
 #endif /* _KERNEL */
@@ -79,10 +89,6 @@ struct srpl {
 	struct srp		sl_head;
 };
 
-struct srpl_iter {
-	struct srp *		si_ref;
-};
-
 #ifdef _KERNEL
 
 void		srpl_rc_init(struct srpl_rc *, void (*)(void *, void *),
@@ -99,41 +105,23 @@ struct {								\
 	struct srp		se_next;				\
 }
 
-static inline void *
-_srpl_enter(struct srpl *sl, struct srpl_iter *si)
-{
-	si->si_ref = &sl->sl_head;
-	return (srp_enter(si->si_ref));
-}
-
-static inline void *
-_srpl_next(struct srpl_iter *si, void *elm, struct srp *nref)
-{
-	void *n;
+#define SRPL_ENTER(_sr, _sl)		srp_enter((_sr), &(_sl)->sl_head)
 
-	n = srp_follow(si->si_ref, elm, nref);
-	si->si_ref = nref;
+#define SRPL_NEXT(_sr, _e, _ENTRY)	\
+	srp_follow((_sr), &(_e)->_ENTRY.se_next)
 
-	return (n);
-}
-
-#define SRPL_ENTER(_sl, _si)		_srpl_enter(_sl, _si)
-
-#define SRPL_NEXT(_si, _e, _ENTRY)					\
-	 _srpl_next(_si, _e, &(_e)->_ENTRY.se_next)
-
-#define SRPL_FOREACH(_c, _sl, _si, _ENTRY)				\
-	for ((_c) = SRPL_ENTER(_sl, _si);				\
+#define SRPL_FOREACH(_c, _sr, _sl, _ENTRY)				\
+	for ((_c) = SRPL_ENTER(_sr, _sl);				\
 	    (_c) != NULL; 						\
-	    (_c) = SRPL_NEXT(_si, _c, _ENTRY))
+	    (_c) = SRPL_NEXT(_sr, _c, _ENTRY))
 
-#define SRPL_LEAVE(_si, _c)		srp_leave((_si)->si_ref, (_c))
+#define SRPL_LEAVE(_sr)			srp_leave((_sr))
 
-#define SRPL_EMPTY_LOCKED(_sl)		(SRPL_FIRST_LOCKED(_sl) == NULL)
 #define SRPL_FIRST_LOCKED(_sl)		srp_get_locked(&(_sl)->sl_head)
+#define SRPL_EMPTY_LOCKED(_sl)		(SRPL_FIRST_LOCKED(_sl) == NULL)
 
 #define SRPL_NEXT_LOCKED(_e, _ENTRY)					\
-    srp_get_locked(&(_e)->_ENTRY.se_next)
+	srp_get_locked(&(_e)->_ENTRY.se_next)
 
 #define SRPL_FOREACH_LOCKED(_c, _sl, _ENTRY)				\
 	for ((_c) = SRPL_FIRST_LOCKED(_sl);				\