Refactor iked process shutdown and cleanup. Remember configured

addresses and routes in iked_vroute_sc to not depend on ikev2
process for cleanup.

This makes sure that all flows, routes and addresses are deleted
no matter which process is killed first.

ok patrick@

6 files changed, 161 insertions, 31 deletions

diff --git a/sbin/iked/config.c b/sbin/iked/config.c
index 688dba4d8fd..60f6443d3e2 100644
--- a/sbin/iked/config.c
+++ b/sbin/iked/config.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: config.c,v 1.78 2021/02/22 21:58:12 tobhe Exp $	*/
+/*	$OpenBSD: config.c,v 1.79 2021/05/13 15:20:48 tobhe Exp $	*/
 
 /*
  * Copyright (c) 2019 Tobias Heider <tobias.heider@stusta.de>
@@ -524,22 +524,29 @@ config_setreset(struct iked *env, unsigned int mode, enum privsep_procid id)
 int
 config_getreset(struct iked *env, struct imsg *imsg)
 {
-	struct iked_policy	*pol, *poltmp;
-	struct iked_sa		*sa;
-	struct iked_user	*usr;
 	unsigned int		 mode;
 
 	IMSG_SIZE_CHECK(imsg, &mode);
 	memcpy(&mode, imsg->data, sizeof(mode));
 
-	if (mode == RESET_EXIT || mode == RESET_ALL || mode == RESET_POLICY) {
+	return (config_doreset(env, mode));
+}
+
+int
+config_doreset(struct iked *env, unsigned int mode)
+{
+	struct iked_policy	*pol, *poltmp;
+	struct iked_sa		*sa;
+	struct iked_user	*usr;
+
+	if (mode == RESET_ALL || mode == RESET_POLICY) {
 		log_debug("%s: flushing policies", __func__);
 		TAILQ_FOREACH_SAFE(pol, &env->sc_policies, pol_entry, poltmp) {
 			config_free_policy(env, pol);
 		}
 	}
 
-	if (mode == RESET_EXIT || mode == RESET_ALL || mode == RESET_SA) {
+	if (mode == RESET_ALL || mode == RESET_SA) {
 		log_debug("%s: flushing SAs", __func__);
 		while ((sa = RB_MIN(iked_sas, &env->sc_sas))) {
 			/* for RESET_SA we try send a DELETE */
@@ -553,7 +560,7 @@ config_getreset(struct iked *env, struct imsg *imsg)
 		}
 	}
 
-	if (mode == RESET_EXIT || mode == RESET_ALL || mode == RESET_USER) {
+	if (mode == RESET_ALL || mode == RESET_USER) {
 		log_debug("%s: flushing users", __func__);
 		while ((usr = RB_MIN(iked_users, &env->sc_users))) {
 			RB_REMOVE(iked_users, &env->sc_users, usr);
@@ -561,9 +568,6 @@ config_getreset(struct iked *env, struct imsg *imsg)
 		}
 	}
 
-	if (mode == RESET_EXIT)
-		proc_compose(&env->sc_ps, PROC_PARENT, IMSG_CTL_EXIT, NULL, 0);
-
 	return (0);
 }
 
diff --git a/sbin/iked/iked.c b/sbin/iked/iked.c
index a023a304245..777a2281498 100644
--- a/sbin/iked/iked.c
+++ b/sbin/iked/iked.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: iked.c,v 1.56 2021/03/03 22:18:00 tobhe Exp $	*/
+/*	$OpenBSD: iked.c,v 1.57 2021/05/13 15:20:48 tobhe Exp $	*/
 
 /*
  * Copyright (c) 2019 Tobias Heider <tobias.heider@stusta.de>
@@ -207,6 +207,7 @@ main(int argc, char *argv[])
 	event_dispatch();
 
 	log_debug("%d parent exiting", getpid());
+	parent_shutdown(env);
 
 	return (0);
 }
@@ -345,10 +346,8 @@ parent_sig_handler(int sig, short event, void *arg)
 		break;
 	case SIGTERM:
 	case SIGINT:
-		log_info("%s: stopping iked", __func__);
-		config_setreset(ps->ps_env, RESET_EXIT, PROC_IKEV2);
-		config_setreset(ps->ps_env, RESET_ALL, PROC_CERT);
-		break;
+		die = 1;
+		/* FALLTHROUGH */
 	case SIGCHLD:
 		do {
 			int len;
@@ -465,8 +464,6 @@ parent_dispatch_ikev2(int fd, struct privsep_proc *p, struct imsg *imsg)
 		return (vroute_getroute(env, imsg));
 	case IMSG_VROUTE_CLONE:
 		return (vroute_getcloneroute(env, imsg));
-	case IMSG_CTL_EXIT:
-		parent_shutdown(env);
 	default:
 		return (-1);
 	}
@@ -479,6 +476,7 @@ parent_shutdown(struct iked *env)
 {
 	proc_kill(&env->sc_ps);
 
+	vroute_cleanup(env);
 	free(env->sc_vroute);
 	free(env);
 
diff --git a/sbin/iked/iked.h b/sbin/iked/iked.h
index 06c566d1dc2..408472443b7 100644
--- a/sbin/iked/iked.h
+++ b/sbin/iked/iked.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: iked.h,v 1.190 2021/04/20 21:11:56 dv Exp $	*/
+/*	$OpenBSD: iked.h,v 1.191 2021/05/13 15:20:48 tobhe Exp $	*/
 
 /*
  * Copyright (c) 2019 Tobias Heider <tobias.heider@stusta.de>
@@ -851,6 +851,7 @@ int	 config_setmode(struct iked *, unsigned int);
 int	 config_getmode(struct iked *, unsigned int);
 int	 config_setreset(struct iked *, unsigned int, enum privsep_procid);
 int	 config_getreset(struct iked *, struct imsg *);
+int	 config_doreset(struct iked *, unsigned int);
 int	 config_setpolicy(struct iked *, struct iked_policy *,
 	    enum privsep_procid);
 int	 config_getpolicy(struct iked *, struct imsg *);
@@ -971,6 +972,7 @@ ssize_t	 dsa_verify_final(struct iked_dsa *, void *, size_t);
 
 /* vroute.c */
 void vroute_init(struct iked *);
+void vroute_cleanup(struct iked *);
 int vroute_getaddr(struct iked *, struct imsg *);
 int vroute_setaddroute(struct iked *, uint8_t, struct sockaddr *,
     uint8_t, struct sockaddr *);
diff --git a/sbin/iked/ikev2.c b/sbin/iked/ikev2.c
index fd917367799..896f44d51b4 100644
--- a/sbin/iked/ikev2.c
+++ b/sbin/iked/ikev2.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ikev2.c,v 1.319 2021/03/23 21:31:29 tobhe Exp $	*/
+/*	$OpenBSD: ikev2.c,v 1.320 2021/05/13 15:20:48 tobhe Exp $	*/
 
 /*
  * Copyright (c) 2019 Tobias Heider <tobias.heider@stusta.de>
@@ -219,6 +219,7 @@ ikev2_shutdown(struct privsep_proc *p)
 
 	ibuf_release(env->sc_certreq);
 	env->sc_certreq = NULL;
+	config_doreset(env, RESET_ALL);
 }
 
 int
diff --git a/sbin/iked/types.h b/sbin/iked/types.h
index ad599bf85ed..889398db226 100644
--- a/sbin/iked/types.h
+++ b/sbin/iked/types.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: types.h,v 1.42 2021/02/13 16:14:12 tobhe Exp $	*/
+/*	$OpenBSD: types.h,v 1.43 2021/05/13 15:20:48 tobhe Exp $	*/
 
 /*
  * Copyright (c) 2019 Tobias Heider <tobias.heider@stusta.de>
@@ -100,7 +100,6 @@ enum imsg_type {
 	IMSG_CTL_ACTIVE,
 	IMSG_CTL_PASSIVE,
 	IMSG_CTL_RESET_ID,
-	IMSG_CTL_EXIT,
 	IMSG_CTL_SHOW_SA,
 	IMSG_CTL_STATIC,
 	IMSG_COMPILE,
@@ -142,7 +141,6 @@ enum flushmode {
 	RESET_POLICY,
 	RESET_SA,
 	RESET_USER,
-	RESET_EXIT
 };
 
 #ifndef nitems
diff --git a/sbin/iked/vroute.c b/sbin/iked/vroute.c
index daca91deca8..650f6e111e0 100644
--- a/sbin/iked/vroute.c
+++ b/sbin/iked/vroute.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vroute.c,v 1.8 2021/04/03 21:29:14 tobhe Exp $	*/
+/*	$OpenBSD: vroute.c,v 1.9 2021/05/13 15:20:48 tobhe Exp $	*/
 
 /*
  * Copyright (c) 2021 Tobias Heider <tobhe@openbsd.org>
@@ -44,13 +44,36 @@ int vroute_setroute(struct iked *, uint8_t, struct sockaddr *, uint8_t,
 int vroute_doroute(struct iked *, int, int, int, uint8_t, struct sockaddr *,
     struct sockaddr *, struct sockaddr *, int *);
 int vroute_doaddr(struct iked *, char *, struct sockaddr *, struct sockaddr *, int);
+void vroute_cleanup(struct iked *);
+
+void vroute_insertaddr(struct iked *, int, struct sockaddr *, struct sockaddr *);
+void vroute_removeaddr(struct iked *, int, struct sockaddr *, struct sockaddr *);
+void vroute_insertroute(struct iked *, int, struct sockaddr *);
+void vroute_removeroute(struct iked *, int, struct sockaddr *);
+
+struct vroute_addr {
+	int				va_ifidx;
+	struct	sockaddr_storage	va_addr;
+	struct	sockaddr_storage	va_mask;
+	TAILQ_ENTRY(vroute_addr)	va_entry;
+};
+TAILQ_HEAD(vroute_addrs, vroute_addr);
+
+struct vroute_route {
+	int				vr_rdomain;
+	struct	sockaddr_storage	vr_dest;
+	TAILQ_ENTRY(vroute_route)	vr_entry;
+};
+TAILQ_HEAD(vroute_routes, vroute_route);
 
 struct iked_vroute_sc {
-	int	ivr_iosock;
-	int	ivr_iosock6;
-	int	ivr_rtsock;
-	int	ivr_rtseq;
-	pid_t	ivr_pid;
+	struct vroute_addrs	ivr_addrs;
+	struct vroute_routes	ivr_routes;
+	int			ivr_iosock;
+	int			ivr_iosock6;
+	int			ivr_rtsock;
+	int			ivr_rtseq;
+	pid_t			ivr_pid;
 };
 
 struct vroute_msg {
@@ -79,11 +102,40 @@ vroute_init(struct iked *env)
 	if ((ivr->ivr_rtsock = socket(AF_ROUTE, SOCK_RAW, AF_UNSPEC)) == -1)
 		fatal("%s: failed to create routing socket", __func__);
 
+	TAILQ_INIT(&ivr->ivr_addrs);
+	TAILQ_INIT(&ivr->ivr_routes);
+
 	ivr->ivr_pid = getpid();
 
 	env->sc_vroute = ivr;
 }
 
+void
+vroute_cleanup(struct iked *env)
+{
+	char			 ifname[IF_NAMESIZE];
+	struct iked_vroute_sc	*ivr = env->sc_vroute;
+	struct vroute_addr	*addr;
+	struct vroute_route	*route;
+
+	while ((addr = TAILQ_FIRST(&ivr->ivr_addrs))) {
+		if_indextoname(addr->va_ifidx, ifname);
+		vroute_doaddr(env, ifname,
+		    (struct sockaddr *)&addr->va_addr,
+		    (struct sockaddr *)&addr->va_mask, 0);
+		TAILQ_REMOVE(&ivr->ivr_addrs, addr, va_entry);
+		free(addr);
+	}
+
+	while ((route = TAILQ_FIRST(&ivr->ivr_routes))) {
+		vroute_doroute(env, RTF_UP | RTF_GATEWAY | RTF_STATIC,
+		    RTA_DST, route->vr_rdomain, RTM_DELETE,
+		    (struct sockaddr *)&route->vr_dest, NULL, NULL, NULL);
+		TAILQ_REMOVE(&ivr->ivr_routes, route, vr_entry);
+		free(route);
+	}
+}
+
 int
 vroute_getaddr(struct iked *env, struct imsg *imsg)
 {
@@ -91,7 +143,7 @@ vroute_getaddr(struct iked *env, struct imsg *imsg)
 	struct sockaddr	*addr, *mask;
 	uint8_t			*ptr;
 	size_t			 left;
-	int			 af;
+	int			 af, add;
 	unsigned int		 ifidx;
 
 	ptr = imsg->data;
@@ -125,10 +177,82 @@ vroute_getaddr(struct iked *env, struct imsg *imsg)
 	ptr += sizeof(ifidx);
 	left -= sizeof(ifidx);
 
+	add = (imsg->hdr.type == IMSG_IF_ADDADDR);
+	/* Store address for cleanup */
+	if (add)
+		vroute_insertaddr(env, ifidx, addr, mask);
+	else
+		vroute_removeaddr(env, ifidx, addr, mask);
+
 	if_indextoname(ifidx, ifname);
+	return (vroute_doaddr(env, ifname, addr, mask, add));
+}
+
+void
+vroute_insertroute(struct iked *env, int rdomain, struct sockaddr *dest)
+{
+	struct iked_vroute_sc	*ivr = env->sc_vroute;
+	struct vroute_route	*route;
+
+	route = calloc(1, sizeof(*route));
+	if (route == NULL)
+		fatalx("%s: calloc.", __func__);
+
+	memcpy(&route->vr_dest, dest, dest->sa_len);
+	route->vr_rdomain = rdomain;
 
-	return (vroute_doaddr(env, ifname, addr, mask,
-	    imsg->hdr.type == IMSG_IF_ADDADDR));
+	TAILQ_INSERT_TAIL(&ivr->ivr_routes, route, vr_entry);
+}
+
+void
+vroute_removeroute(struct iked *env, int rdomain, struct sockaddr *dest)
+{
+	struct iked_vroute_sc	*ivr = env->sc_vroute;
+	struct vroute_route	*route;
+
+	TAILQ_FOREACH(route, &ivr->ivr_routes, vr_entry) {
+		if (sockaddr_cmp(dest, (struct sockaddr *)&route->vr_dest, -1))
+			continue;
+		if (rdomain != route->vr_rdomain)
+			continue;
+		TAILQ_REMOVE(&ivr->ivr_routes, route, vr_entry);
+	}
+}
+
+void
+vroute_insertaddr(struct iked *env, int ifidx, struct sockaddr *addr,
+    struct sockaddr *mask)
+{
+	struct iked_vroute_sc	*ivr = env->sc_vroute;
+	struct vroute_addr	*vaddr;
+
+	vaddr = calloc(1, sizeof(*vaddr));
+	if (vaddr == NULL)
+		fatalx("%s: calloc.", __func__);
+
+	memcpy(&vaddr->va_addr, addr, addr->sa_len);
+	memcpy(&vaddr->va_mask, mask, mask->sa_len);
+	vaddr->va_ifidx = ifidx;
+
+	TAILQ_INSERT_TAIL(&ivr->ivr_addrs, vaddr, va_entry);
+}
+
+void
+vroute_removeaddr(struct iked *env, int ifidx, struct sockaddr *addr,
+    struct sockaddr *mask)
+{
+	struct iked_vroute_sc	*ivr = env->sc_vroute;
+	struct vroute_addr	*vaddr;
+
+	TAILQ_FOREACH(vaddr, &ivr->ivr_addrs, va_entry) {
+		if (sockaddr_cmp(addr, (struct sockaddr *)&vaddr->va_addr, -1))
+			continue;
+		if (sockaddr_cmp(mask, (struct sockaddr *)&vaddr->va_mask, -1))
+			continue;
+		if (ifidx != vaddr->va_ifidx)
+			continue;
+		TAILQ_REMOVE(&ivr->ivr_addrs, vaddr, va_entry);
+	}
 }
 
 int
@@ -269,6 +393,7 @@ vroute_getroute(struct iked *env, struct imsg *imsg)
 		type = RTM_ADD;
 		break;
 	case IMSG_VROUTE_DEL:
+		vroute_removeroute(env, rdomain, dest);
 		type = RTM_DELETE;
 		break;
 	}
@@ -323,6 +448,8 @@ vroute_getcloneroute(struct iked *env, struct imsg *imsg)
 	if (need_gw)
 		flags |= RTF_GATEWAY;
 
+	vroute_insertroute(env, rdomain, (struct sockaddr *)&dest);
+
 	/* Set explicit route to peer with gateway addr*/
 	addrs = RTA_DST | RTA_GATEWAY | RTA_NETMASK;
 	return (vroute_doroute(env, flags, addrs, rdomain, RTM_ADD,