Remove all splnet/splx from pipex(4) and pppx(4) and replace some of

them by NET_LOCK/NET_UNLOCK.  Also make the timeout for pipex_timer
run with a thread context and replace pipex softintr by NETISR_PIPEX.

ok mpi

6 files changed, 68 insertions, 112 deletions

diff --git a/sys/net/if.c b/sys/net/if.c
index c6131d0dc30..8532743929e 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.498 2017/05/28 12:47:24 mpi Exp $	*/
+/*	$OpenBSD: if.c,v 1.499 2017/05/28 12:51:33 yasuoka Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -960,6 +960,10 @@ if_netisr(void *unused)
 		if (n & (1 << NETISR_PPPOE))
 			pppoeintr();
 #endif
+#ifdef PIPEX
+		if (n & (1 << NETISR_PIPEX))
+			pipexintr();
+#endif
 		t |= n;
 	}
 
diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c
index ac04b9ce0b4..b1b49434b04 100644
--- a/sys/net/if_ethersubr.c
+++ b/sys/net/if_ethersubr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ethersubr.c,v 1.243 2017/05/22 01:56:22 dlg Exp $	*/
+/*	$OpenBSD: if_ethersubr.c,v 1.244 2017/05/28 12:51:34 yasuoka Exp $	*/
 /*	$NetBSD: if_ethersubr.c,v 1.19 1996/05/07 02:40:30 thorpej Exp $	*/
 
 /*
@@ -416,14 +416,15 @@ decapsulate:
 #ifdef PIPEX
 		if (pipex_enable) {
 			struct pipex_session *session;
+			int s;
 
-			KERNEL_LOCK();
+			NET_LOCK(s);
 			if ((session = pipex_pppoe_lookup_session(m)) != NULL) {
 				pipex_pppoe_input(m, session);
-				KERNEL_UNLOCK();
+				NET_UNLOCK(s);
 				return (1);
 			}
-			KERNEL_UNLOCK();
+			NET_UNLOCK(s);
 		}
 #endif
 		if (etype == ETHERTYPE_PPPOEDISC)
diff --git a/sys/net/if_pppx.c b/sys/net/if_pppx.c
index dacbc678740..3d552ad52e0 100644
--- a/sys/net/if_pppx.c
+++ b/sys/net/if_pppx.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_pppx.c,v 1.58 2017/05/04 15:00:24 bluhm Exp $ */
+/*	$OpenBSD: if_pppx.c,v 1.59 2017/05/28 12:51:34 yasuoka Exp $ */
 
 /*
  * Copyright (c) 2010 Claudio Jeker <claudio@openbsd.org>
@@ -287,10 +287,11 @@ pppxread(dev_t dev, struct uio *uio, int ioflag)
 		if (ISSET(ioflag, IO_NDELAY))
 			return (EWOULDBLOCK);
 
-		s = splnet();
+		NET_LOCK(s);
 		pxd->pxd_waiting = 1;
-		error = tsleep(pxd, (PZERO + 1)|PCATCH, "pppxread", 0);
-		splx(s);
+		error = rwsleep(pxd, &netlock,
+		    (PZERO + 1)|PCATCH, "pppxread", 0);
+		NET_UNLOCK(s);
 		if (error != 0) {
 			return (error);
 		}
@@ -321,9 +322,6 @@ pppxwrite(dev_t dev, struct uio *uio, int ioflag)
 	int tlen;
 	int error = 0;
 	size_t mlen;
-#if NBPFILTER > 0
-	int s;
-#endif
 
 	if (uio->uio_resid < sizeof(*th) + sizeof(uint32_t) ||
 	    uio->uio_resid > MCLBYTES)
@@ -389,11 +387,8 @@ pppxwrite(dev_t dev, struct uio *uio, int ioflag)
 	top->m_pkthdr.ph_ifidx = pxi->pxi_if.if_index;
 
 #if NBPFILTER > 0
-	if (pxi->pxi_if.if_bpf) {
-		s = splnet();
+	if (pxi->pxi_if.if_bpf)
 		bpf_mtap(pxi->pxi_if.if_bpf, top, BPF_DIRECTION_IN);
-		splx(s);
-	}
 #endif
 	/* strip the tunnel header */
 	proto = ntohl(*(uint32_t *)(th + 1));
@@ -423,8 +418,9 @@ int
 pppxioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p)
 {
 	struct pppx_dev *pxd = pppx_dev2pxd(dev);
-	int error = 0;
+	int s, error = 0;
 
+	NET_LOCK(s);
 	switch (cmd) {
 	case PIPEXSMODE:
 		/*
@@ -468,12 +464,13 @@ pppxioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p)
 	case FIONBIO:
 	case FIOASYNC:
 	case FIONREAD:
-		return (0);
+		break;
 
 	default:
 		error = ENOTTY;
 		break;
 	}
+	NET_UNLOCK(s);
 
 	return (error);
 }
@@ -660,7 +657,7 @@ pppx_add_session(struct pppx_dev *pxd, struct pipex_session_req *req)
 	struct pipex_session *session;
 	struct pipex_hash_head *chain;
 	struct ifnet *ifp;
-	int unit, s, error = 0;
+	int unit, error = 0;
 	struct in_ifaddr *ia;
 	struct sockaddr_in ifaddr;
 #ifdef PIPEX_PPPOE
@@ -845,8 +842,6 @@ pppx_add_session(struct pppx_dev *pxd, struct pipex_session_req *req)
 	ifp->if_softc = pxi;
 	/* ifp->if_rdomain = req->pr_rdomain; */
 
-	s = splnet();
-
 	/* hook up pipex context */
 	chain = PIPEX_ID_HASHTABLE(session->session_id);
 	LIST_INSERT_HEAD(chain, session, id_chain);
@@ -864,7 +859,11 @@ pppx_add_session(struct pppx_dev *pxd, struct pipex_session_req *req)
 	if (LIST_NEXT(session, session_list) == NULL)
 		pipex_timer_start();
 
+	/* XXXSMP breaks atomicity */
+	rw_exit_write(&netlock);
 	if_attach(ifp);
+	rw_enter_write(&netlock);
+
 	if_addgroup(ifp, "pppx");
 	if_alloc_sadl(ifp);
 
@@ -913,7 +912,6 @@ pppx_add_session(struct pppx_dev *pxd, struct pipex_session_req *req)
 	} else {
 		dohooks(ifp->if_addrhooks, 0);
 	}
-	splx(s);
 
 out:
 	rw_exit_write(&pppx_ifs_lk);
@@ -957,12 +955,11 @@ pppx_if_destroy(struct pppx_dev *pxd, struct pppx_if *pxi)
 {
 	struct ifnet *ifp;
 	struct pipex_session *session;
-	int s;
 
+	NET_ASSERT_LOCKED();
 	session = &pxi->pxi_session;
 	ifp = &pxi->pxi_if;
 
-	s = splnet();
 	LIST_REMOVE(session, id_chain);
 	LIST_REMOVE(session, session_list);
 	switch (session->protocol) {
@@ -976,9 +973,11 @@ pppx_if_destroy(struct pppx_dev *pxd, struct pppx_if *pxi)
 	/* if final session is destroyed, stop timer */
 	if (LIST_EMPTY(&pipex_session_list))
 		pipex_timer_stop();
-	splx(s);
 
+	/* XXXSMP breaks atomicity */
+	rw_exit_write(&netlock);
 	if_detach(ifp);
+	rw_enter_write(&netlock);
 
 	rw_enter_write(&pppx_ifs_lk);
 	if (RBT_REMOVE(pppx_ifs, &pppx_ifs, pxi) == NULL)
@@ -1023,9 +1022,8 @@ pppx_if_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 	struct pppx_hdr *th;
 	int error = 0;
 	int proto;
-#if NBPFILTER > 0
-	int s;
-#endif
+
+	NET_ASSERT_LOCKED();
 
 	if (!ISSET(ifp->if_flags, IFF_UP)) {
 		m_freem(m);
@@ -1034,11 +1032,8 @@ pppx_if_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 	}
 
 #if NBPFILTER > 0
-	if (ifp->if_bpf) {
-		s = splnet();
+	if (ifp->if_bpf)
 		bpf_mtap_af(ifp->if_bpf, dst->sa_family, m, BPF_DIRECTION_OUT);
-		splx(s);
-	}
 #endif
 	if (pipex_enable) {
 		switch (dst->sa_family) {
@@ -1074,12 +1069,10 @@ pppx_if_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 		rw_enter_read(&pppx_devs_lk);
 		error = mq_enqueue(&pxi->pxi_dev->pxd_svcq, m);
 		if (error == 0) {
-			s = splnet();
 			if (pxi->pxi_dev->pxd_waiting) {
 				wakeup((caddr_t)pxi->pxi_dev);
 				pxi->pxi_dev->pxd_waiting = 0;
 			}
-			splx(s);
 			selwakeup(&pxi->pxi_dev->pxd_rsel);
 		}
 		rw_exit_read(&pppx_devs_lk);
diff --git a/sys/net/netisr.h b/sys/net/netisr.h
index 9f2e34e7950..813ddd6d2bb 100644
--- a/sys/net/netisr.h
+++ b/sys/net/netisr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: netisr.h,v 1.48 2016/11/03 09:04:06 mpi Exp $	*/
+/*	$OpenBSD: netisr.h,v 1.49 2017/05/28 12:51:34 yasuoka Exp $	*/
 /*	$NetBSD: netisr.h,v 1.12 1995/08/12 23:59:24 mycroft Exp $	*/
 
 /*
@@ -48,6 +48,7 @@
 #define	NETISR_IPV6	24		/* same as AF_INET6 */
 #define	NETISR_ISDN	26		/* same as AF_E164 */
 #define	NETISR_PPP	28		/* for PPP processing */
+#define	NETISR_PIPEX	27		/* for pipex processing */
 #define	NETISR_BRIDGE	29		/* for bridge processing */
 #define	NETISR_PPPOE	30		/* for pppoe processing */
 #define	NETISR_SWITCH	31		/* for switch dataplane */
@@ -69,6 +70,7 @@ void	bridgeintr(void);
 void	pppoeintr(void);
 void	switchintr(void);
 void	pfsyncintr(void);
+void	pipexintr(void);
 
 #define	schednetisr(anisr)						\
 do {									\
diff --git a/sys/net/pipex.c b/sys/net/pipex.c
index ef7129a363d..835e0b0f52d 100644
--- a/sys/net/pipex.c
+++ b/sys/net/pipex.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pipex.c,v 1.96 2017/05/27 12:39:12 yasuoka Exp $	*/
+/*	$OpenBSD: pipex.c,v 1.97 2017/05/28 12:51:34 yasuoka Exp $	*/
 
 /*-
  * Copyright (c) 2009 Internet Initiative Japan Inc.
@@ -100,8 +100,6 @@ int pipex_prune = 1;			/* walk list every seconds */
 /* pipex traffic queue */
 struct mbuf_queue pipexinq = MBUF_QUEUE_INITIALIZER(IFQ_MAXLEN, IPL_NET);
 struct mbuf_queue pipexoutq = MBUF_QUEUE_INITIALIZER(IFQ_MAXLEN, IPL_NET);
-void *pipex_softintr = NULL;
-Static void pipex_softintr_handler(void *);
 
 /* borrow an mbuf pkthdr field */
 #define ph_ppp_proto ether_vtag
@@ -123,8 +121,9 @@ void
 pipex_init(void)
 {
 	int		 i;
+	static int	 pipex_init_done = 0;
 
-	if (pipex_softintr != NULL)
+	if (pipex_init_done++)
 		return;
 
 	rn_init(sizeof(struct sockaddr_in6));
@@ -141,21 +140,16 @@ pipex_init(void)
 		LIST_INIT(&pipex_id_hashtable[i]);
 	for (i = 0; i < nitems(pipex_peer_addr_hashtable); i++)
 		LIST_INIT(&pipex_peer_addr_hashtable[i]);
-	/* softintr init */
-	pipex_softintr =
-	    softintr_establish(IPL_SOFTNET, pipex_softintr_handler, NULL);
 }
 
 void
 pipex_iface_init(struct pipex_iface_context *pipex_iface, struct ifnet *ifp)
 {
-	int s;
 	struct pipex_session *session;
 
 	pipex_iface->pipexmode = 0;
 	pipex_iface->ifnet_this = ifp;
 
-	s = splnet();
 	if (pipex_rd_head4 == NULL) {
 		if (!rn_inithead((void **)&pipex_rd_head4,
 		    offsetof(struct sockaddr_in, sin_addr)))
@@ -166,7 +160,6 @@ pipex_iface_init(struct pipex_iface_context *pipex_iface, struct ifnet *ifp)
 		    offsetof(struct sockaddr_in6, sin6_addr)))
 			panic("rn_inithead0() failed on pipex_init()");
 	}
-	splx(s);
 
 	/* virtual pipex_session entry for multicast */
 	session = pool_get(&pipex_session_pool, PR_WAITOK | PR_ZERO);
@@ -203,21 +196,16 @@ pipex_iface_stop(struct pipex_iface_context *pipex_iface)
 void
 pipex_iface_fini(struct pipex_iface_context *pipex_iface)
 {
-	int	 s;
-
-	s = splnet();
-
 	pool_put(&pipex_session_pool, pipex_iface->multicast_session);
 	pipex_iface_stop(pipex_iface);
-
-	splx(s);
 }
 
 int
 pipex_ioctl(struct pipex_iface_context *pipex_iface, u_long cmd, caddr_t data)
 {
-	int pipexmode, ret;
+	int s, pipexmode, ret = 0;
 
+	NET_LOCK(s);
 	switch (cmd) {
 	case PIPEXSMODE:
 		pipexmode = *(int *)data;
@@ -236,31 +224,33 @@ pipex_ioctl(struct pipex_iface_context *pipex_iface, u_long cmd, caddr_t data)
 	case PIPEXASESSION:
 		ret = pipex_add_session((struct pipex_session_req *)data,
 		    pipex_iface);
-		return (ret);
+		break;
 
 	case PIPEXDSESSION:
 		ret = pipex_close_session(
 		    (struct pipex_session_close_req *)data);
-		return (ret);
+		break;
 
 	case PIPEXCSESSION:
 		ret = pipex_config_session(
 		    (struct pipex_session_config_req *)data);
-		return (ret);
+		break;
 
 	case PIPEXGSTAT:
 		ret = pipex_get_stat((struct pipex_session_stat_req *)data);
-		return (ret);
+		break;
 
 	case PIPEXGCLOSED:
 		ret = pipex_get_closed((struct pipex_session_list_req *)data);
-		return (ret);
+		break;
 
 	default:
-		return (ENOTTY);
-
+		ret = ENOTTY;
+		break;
 	}
-	return (0);
+	NET_UNLOCK(s);
+
+	return (ret);
 }
 
 /************************************************************************
@@ -273,7 +263,6 @@ pipex_add_session(struct pipex_session_req *req,
 	struct pipex_session *session;
 	struct pipex_hash_head *chain;
 	struct radix_node *rn;
-	int s;
 #ifdef PIPEX_PPPOE
 	struct ifnet *over_ifp = NULL;
 #endif
@@ -431,12 +420,11 @@ pipex_add_session(struct pipex_session_req *req,
 	}
 #endif
 
+	NET_ASSERT_LOCKED();
 	/* commit the session */
-	s = splnet();
 	if (!in_nullhost(session->ip_address.sin_addr)) {
 		if (pipex_lookup_by_ip_address(session->ip_address.sin_addr)
 		    != NULL) {
-			splx(s);
 			pool_put(&pipex_session_pool, session);
 			return (EADDRINUSE);
 		}
@@ -444,7 +432,6 @@ pipex_add_session(struct pipex_session_req *req,
 		rn = rn_addroute(&session->ip_address, &session->ip_netmask,
 		    pipex_rd_head4, session->ps4_rn, RTP_STATIC);
 		if (rn == NULL) {
-			splx(s);
 			pool_put(&pipex_session_pool, session);
 			return (ENOMEM);
 		}
@@ -453,7 +440,6 @@ pipex_add_session(struct pipex_session_req *req,
 		rn = rn_addroute(&session->ip6_address, &session->ip6_prefixlen,
 		    pipex_rd_head6, session->ps6_rn, RTP_STATIC);
 		if (rn == NULL) {
-			splx(s);
 			pool_put(&pipex_session_pool, session);
 			return (ENOMEM);
 		}
@@ -474,8 +460,6 @@ pipex_add_session(struct pipex_session_req *req,
 	if (LIST_NEXT(session, session_list) == NULL)
 		pipex_timer_start();
 
-	splx(s);
-
 	pipex_session_log(session, LOG_INFO, "PIPEX is ready.");
 
 	return (0);
@@ -484,13 +468,10 @@ pipex_add_session(struct pipex_session_req *req,
 int
 pipex_notify_close_session(struct pipex_session *session)
 {
-	int s;
-
-	s = splnet();
+	NET_ASSERT_LOCKED();
 	session->state = PIPEX_STATE_CLOSE_WAIT;
 	session->stat.idle_time = 0;
 	LIST_INSERT_HEAD(&pipex_close_wait_list, session, state_list);
-	splx(s);
 
 	return (0);
 }
@@ -499,14 +480,11 @@ int
 pipex_notify_close_session_all(void)
 {
 	struct pipex_session *session;
-	int s;
 
-	s = splnet();
+	NET_ASSERT_LOCKED();
 	LIST_FOREACH(session, &pipex_session_list, session_list)
 		if (session->state == PIPEX_STATE_OPENED)
 			pipex_notify_close_session(session);
-	splx(s);
-
 	return (0);
 }
 
@@ -514,15 +492,12 @@ Static int
 pipex_close_session(struct pipex_session_close_req *req)
 {
 	struct pipex_session *session;
-	int s;
 
-	s = splnet();
+	NET_ASSERT_LOCKED();
 	session = pipex_lookup_by_session_id(req->pcr_protocol,
 	    req->pcr_session_id);
-	if (session == NULL) {
-		splx(s);
+	if (session == NULL)
 		return (EINVAL);
-	}
 
 	/* remove from close_wait list */
 	if (session->state == PIPEX_STATE_CLOSE_WAIT)
@@ -531,7 +506,6 @@ pipex_close_session(struct pipex_session_close_req *req)
 	/* get statistics before destroy the session */
 	req->pcr_stat = session->stat;
 	session->state = PIPEX_STATE_CLOSED;
-	splx(s);
 
 	return (0);
 }
@@ -540,17 +514,13 @@ Static int
 pipex_config_session(struct pipex_session_config_req *req)
 {
 	struct pipex_session *session;
-	int s;
 
-	s = splnet();
+	NET_ASSERT_LOCKED();
 	session = pipex_lookup_by_session_id(req->pcr_protocol,
 	    req->pcr_session_id);
-	if (session == NULL) {
-		splx(s);
+	if (session == NULL)
 		return (EINVAL);
-	}
 	session->ip_forward = req->pcr_ip_forward;
-	splx(s);
 
 	return (0);
 }
@@ -559,17 +529,14 @@ Static int
 pipex_get_stat(struct pipex_session_stat_req *req)
 {
 	struct pipex_session *session;
-	int s;
 
-	s = splnet();
+	NET_ASSERT_LOCKED();
 	session = pipex_lookup_by_session_id(req->psr_protocol,
 	    req->psr_session_id);
 	if (session == NULL) {
-		splx(s);
 		return (EINVAL);
 	}
 	req->psr_stat = session->stat;
-	splx(s);
 
 	return (0);
 }
@@ -578,9 +545,8 @@ Static int
 pipex_get_closed(struct pipex_session_list_req *req)
 {
 	struct pipex_session *session;
-	int s;
 
-	s = splnet();
+	NET_ASSERT_LOCKED();
 	bzero(req, sizeof(*req));
 	while (!LIST_EMPTY(&pipex_close_wait_list)) {
 		session = LIST_FIRST(&pipex_close_wait_list);
@@ -593,7 +559,6 @@ pipex_get_closed(struct pipex_session_list_req *req)
 			break;
 		}
 	}
-	splx(s);
 
 	return (0);
 }
@@ -602,10 +567,9 @@ Static int
 pipex_destroy_session(struct pipex_session *session)
 {
 	struct radix_node *rn;
-	int s;
 
 	/* remove from radix tree and hash chain */
-	s = splnet();
+	NET_ASSERT_LOCKED();
 
 	if (!in_nullhost(session->ip_address.sin_addr)) {
 		rn = rn_delete(&session->ip_address, &session->ip_netmask,
@@ -629,8 +593,6 @@ pipex_destroy_session(struct pipex_session *session)
 	if (LIST_EMPTY(&pipex_session_list))
 		pipex_timer_stop();
 
-	splx(s);
-
 	if (session->mppe_recv.old_session_keys)
 		pool_put(&mppe_key_pool, session->mppe_recv.old_session_keys);
 	pool_put(&pipex_session_pool, session);
@@ -695,14 +657,8 @@ pipex_lookup_by_session_id(int protocol, int session_id)
 /***********************************************************************
  * Queue and Software Interrupt Handler
  ***********************************************************************/
-Static void
-pipex_softintr_handler(void *dummy)
-{
-	pipex_ppp_dequeue();
-}
-
-Static void
-pipex_ppp_dequeue(void)
+void
+pipexintr(void)
 {
 	struct pipex_session *pkt_session;
 	u_int16_t proto;
@@ -769,7 +725,8 @@ pipex_ppp_enqueue(struct mbuf *m0, struct pipex_session *session,
 	if (mq_enqueue(mq, m0) != 0)
 		return (1);
 
-	softintr_schedule(pipex_softintr);
+	schednetisr(NETISR_PIPEX);
+
 	return (0);
 }
 
@@ -779,7 +736,7 @@ pipex_ppp_enqueue(struct mbuf *m0, struct pipex_session *session,
 Static void
 pipex_timer_start(void)
 {
-	timeout_set(&pipex_timer_ch, pipex_timer, NULL);
+	timeout_set_proc(&pipex_timer_ch, pipex_timer, NULL);
 	timeout_add_sec(&pipex_timer_ch, pipex_prune);
 }
 
@@ -796,9 +753,9 @@ pipex_timer(void *ignored_arg)
 	struct pipex_session *session;
 	struct pipex_session *session_next;
 
-	s = splnet();
 	timeout_add_sec(&pipex_timer_ch, pipex_prune);
 
+	NET_LOCK(s);
 	/* walk through */
 	for (session = LIST_FIRST(&pipex_session_list); session;
 	    session = session_next) {
@@ -843,7 +800,7 @@ pipex_timer(void *ignored_arg)
 		}
 	}
 
-	splx(s);
+	NET_UNLOCK(s);
 }
 
 /***********************************************************************
diff --git a/sys/net/pipex_local.h b/sys/net/pipex_local.h
index 378b39042c8..25487f3e2ae 100644
--- a/sys/net/pipex_local.h
+++ b/sys/net/pipex_local.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pipex_local.h,v 1.25 2017/05/04 15:00:24 bluhm Exp $	*/
+/*	$OpenBSD: pipex_local.h,v 1.26 2017/05/28 12:51:34 yasuoka Exp $	*/
 
 /*
  * Copyright (c) 2009 Internet Initiative Japan Inc.
@@ -428,7 +428,6 @@ Static void                  pipex_session_log (struct pipex_session *, int, con
 Static uint32_t              pipex_sockaddr_hash_key(struct sockaddr *);
 Static int                   pipex_sockaddr_compar_addr(struct sockaddr *, struct sockaddr *);
 Static int                   pipex_ppp_enqueue (struct mbuf *, struct pipex_session *, struct mbuf_queue *);
-Static void                  pipex_ppp_dequeue (void);
 Static void                  pipex_timer_start (void);
 Static void                  pipex_timer_stop (void);
 Static void                  pipex_timer (void *);