Add support for autmatically moving traffic between rdomains on ipsec(4)

encryption or decryption.  This allows us to keep plaintext and encrypted
network traffic seperated and reduces the attack surface for network
sidechannel attacks.

The only way to reach the inner rdomain from outside is by successful
decryption and integrity verification through the responsible Security
Association (SA).
The only way for internal traffic to get out is getting encrypted and
moved through the outgoing SA.
Multiple plaintext rdomains can share the same encrypted rdomain while
the unencrypted packets are still kept seperate.
The encrypted and unencrypted rdomains can have different default routes.

The rdomains can be configured with the new SADB_X_EXT_RDOMAIN pfkey
extension.  Each SA (tdb) gets a new attribute 'tdb_rdomain_post'.
If this differs from 'tdb_rdomain' then the packet is moved to
'tdb_rdomain_post' afer IPsec processing.

Flows and outgoing IPsec SAs are installed in the plaintext rdomain,
incoming IPsec SAs are installed in the encrypted rdomain.
IPCOMP SAs are always installed in the plaintext rdomain.
They can be viewed with 'route -T X exec ipsecctl -sa' where X is the
rdomain ID.

As the kernel does not create encX devices automatically when creating
rdomains they have to be added by hand with ifconfig for IPsec to work
in non-default rdomains.

discussed with chris@ and kn@
ok markus@, patrick@

4 files changed, 50 insertions, 47 deletions

diff --git a/sys/netinet/ip_ipsp.c b/sys/netinet/ip_ipsp.c
index c27cabc82d3..8405bb5c18f 100644
--- a/sys/netinet/ip_ipsp.c
+++ b/sys/netinet/ip_ipsp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_ipsp.c,v 1.234 2019/05/11 17:16:21 benno Exp $	*/
+/*	$OpenBSD: ip_ipsp.c,v 1.235 2020/04/23 19:38:08 tobhe Exp $	*/
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr),
@@ -84,7 +84,7 @@ void		tdb_timeout(void *);
 void		tdb_firstuse(void *);
 void		tdb_soft_timeout(void *);
 void		tdb_soft_firstuse(void *);
-int		tdb_hash(u_int, u_int32_t, union sockaddr_union *, u_int8_t);
+int		tdb_hash(u_int32_t, union sockaddr_union *, u_int8_t);
 
 int ipsec_in_use = 0;
 u_int64_t ipsec_last_added = 0;
@@ -185,7 +185,7 @@ static int tdb_count;
  * so we cannot be DoS-attacked via choosing of the data to hash.
  */
 int
-tdb_hash(u_int rdomain, u_int32_t spi, union sockaddr_union *dst,
+tdb_hash(u_int32_t spi, union sockaddr_union *dst,
     u_int8_t proto)
 {
 	SIPHASH_CTX ctx;
@@ -193,7 +193,6 @@ tdb_hash(u_int rdomain, u_int32_t spi, union sockaddr_union *dst,
 	NET_ASSERT_LOCKED();
 
 	SipHash24_Init(&ctx, &tdbkey);
-	SipHash24_Update(&ctx, &rdomain, sizeof(rdomain));
 	SipHash24_Update(&ctx, &spi, sizeof(spi));
 	SipHash24_Update(&ctx, &proto, sizeof(proto));
 	SipHash24_Update(&ctx, dst, dst->sa.sa_len);
@@ -306,7 +305,8 @@ reserve_spi(u_int rdomain, u_int32_t sspi, u_int32_t tspi,
  * is really one of our addresses if we received the packet!
  */
 struct tdb *
-gettdb(u_int rdomain, u_int32_t spi, union sockaddr_union *dst, u_int8_t proto)
+gettdb_dir(u_int rdomain, u_int32_t spi, union sockaddr_union *dst, u_int8_t proto,
+    int reverse)
 {
 	u_int32_t hashval;
 	struct tdb *tdbp;
@@ -316,11 +316,12 @@ gettdb(u_int rdomain, u_int32_t spi, union sockaddr_union *dst, u_int8_t proto)
 	if (tdbh == NULL)
 		return (struct tdb *) NULL;
 
-	hashval = tdb_hash(rdomain, spi, dst, proto);
+	hashval = tdb_hash(spi, dst, proto);
 
 	for (tdbp = tdbh[hashval]; tdbp != NULL; tdbp = tdbp->tdb_hnext)
 		if ((tdbp->tdb_spi == spi) && (tdbp->tdb_sproto == proto) &&
-		    (tdbp->tdb_rdomain == rdomain) &&
+		    ((!reverse && tdbp->tdb_rdomain == rdomain) ||
+		    (reverse && tdbp->tdb_rdomain_post == rdomain)) &&
 		    !memcmp(&tdbp->tdb_dst, dst, dst->sa.sa_len))
 			break;
 
@@ -333,8 +334,8 @@ gettdb(u_int rdomain, u_int32_t spi, union sockaddr_union *dst, u_int8_t proto)
  * matches all SPIs.
  */
 struct tdb *
-gettdbbysrcdst(u_int rdomain, u_int32_t spi, union sockaddr_union *src,
-    union sockaddr_union *dst, u_int8_t proto)
+gettdbbysrcdst_dir(u_int rdomain, u_int32_t spi, union sockaddr_union *src,
+    union sockaddr_union *dst, u_int8_t proto, int reverse)
 {
 	u_int32_t hashval;
 	struct tdb *tdbp;
@@ -345,12 +346,13 @@ gettdbbysrcdst(u_int rdomain, u_int32_t spi, union sockaddr_union *src,
 	if (tdbsrc == NULL)
 		return (struct tdb *) NULL;
 
-	hashval = tdb_hash(rdomain, 0, src, proto);
+	hashval = tdb_hash(0, src, proto);
 
 	for (tdbp = tdbsrc[hashval]; tdbp != NULL; tdbp = tdbp->tdb_snext)
 		if (tdbp->tdb_sproto == proto &&
 		    (spi == 0 || tdbp->tdb_spi == spi) &&
-		    (tdbp->tdb_rdomain == rdomain) &&
+		    ((!reverse && tdbp->tdb_rdomain == rdomain) ||
+		    (reverse && tdbp->tdb_rdomain_post == rdomain)) &&
 		    ((tdbp->tdb_flags & TDBF_INVALID) == 0) &&
 		    (tdbp->tdb_dst.sa.sa_family == AF_UNSPEC ||
 		    !memcmp(&tdbp->tdb_dst, dst, dst->sa.sa_len)) &&
@@ -362,12 +364,13 @@ gettdbbysrcdst(u_int rdomain, u_int32_t spi, union sockaddr_union *src,
 
 	memset(&su_null, 0, sizeof(su_null));
 	su_null.sa.sa_len = sizeof(struct sockaddr);
-	hashval = tdb_hash(rdomain, 0, &su_null, proto);
+	hashval = tdb_hash(0, &su_null, proto);
 
 	for (tdbp = tdbsrc[hashval]; tdbp != NULL; tdbp = tdbp->tdb_snext)
 		if (tdbp->tdb_sproto == proto &&
 		    (spi == 0 || tdbp->tdb_spi == spi) &&
-		    (tdbp->tdb_rdomain == rdomain) &&
+		    ((!reverse && tdbp->tdb_rdomain == rdomain) ||
+		    (reverse && tdbp->tdb_rdomain_post == rdomain)) &&
 		    ((tdbp->tdb_flags & TDBF_INVALID) == 0) &&
 		    (tdbp->tdb_dst.sa.sa_family == AF_UNSPEC ||
 		    !memcmp(&tdbp->tdb_dst, dst, dst->sa.sa_len)) &&
@@ -431,7 +434,7 @@ gettdbbydst(u_int rdomain, union sockaddr_union *dst, u_int8_t sproto,
 	if (tdbdst == NULL)
 		return (struct tdb *) NULL;
 
-	hashval = tdb_hash(rdomain, 0, dst, sproto);
+	hashval = tdb_hash(0, dst, sproto);
 
 	for (tdbp = tdbdst[hashval]; tdbp != NULL; tdbp = tdbp->tdb_dnext)
 		if ((tdbp->tdb_sproto == sproto) &&
@@ -464,7 +467,7 @@ gettdbbysrc(u_int rdomain, union sockaddr_union *src, u_int8_t sproto,
 	if (tdbsrc == NULL)
 		return (struct tdb *) NULL;
 
-	hashval = tdb_hash(rdomain, 0, src, sproto);
+	hashval = tdb_hash(0, src, sproto);
 
 	for (tdbp = tdbsrc[hashval]; tdbp != NULL; tdbp = tdbp->tdb_snext)
 		if ((tdbp->tdb_sproto == sproto) &&
@@ -620,8 +623,7 @@ tdb_rehash(void)
 	for (i = 0; i <= old_hashmask; i++) {
 		for (tdbp = tdbh[i]; tdbp != NULL; tdbp = tdbnp) {
 			tdbnp = tdbp->tdb_hnext;
-			hashval = tdb_hash(tdbp->tdb_rdomain,
-			    tdbp->tdb_spi, &tdbp->tdb_dst,
+			hashval = tdb_hash(tdbp->tdb_spi, &tdbp->tdb_dst,
 			    tdbp->tdb_sproto);
 			tdbp->tdb_hnext = new_tdbh[hashval];
 			new_tdbh[hashval] = tdbp;
@@ -629,18 +631,14 @@ tdb_rehash(void)
 
 		for (tdbp = tdbdst[i]; tdbp != NULL; tdbp = tdbnp) {
 			tdbnp = tdbp->tdb_dnext;
-			hashval = tdb_hash(tdbp->tdb_rdomain,
-			    0, &tdbp->tdb_dst,
-			    tdbp->tdb_sproto);
+			hashval = tdb_hash(0, &tdbp->tdb_dst, tdbp->tdb_sproto);
 			tdbp->tdb_dnext = new_tdbdst[hashval];
 			new_tdbdst[hashval] = tdbp;
 		}
 
 		for (tdbp = tdbsrc[i]; tdbp != NULL; tdbp = tdbnp) {
 			tdbnp = tdbp->tdb_snext;
-			hashval = tdb_hash(tdbp->tdb_rdomain,
-			    0, &tdbp->tdb_src,
-			    tdbp->tdb_sproto);
+			hashval = tdb_hash(0, &tdbp->tdb_src, tdbp->tdb_sproto);
 			tdbp->tdb_snext = new_srcaddr[hashval];
 			new_srcaddr[hashval] = tdbp;
 		}
@@ -676,8 +674,7 @@ puttdb(struct tdb *tdbp)
 		    M_TDB, M_WAITOK | M_ZERO);
 	}
 
-	hashval = tdb_hash(tdbp->tdb_rdomain, tdbp->tdb_spi,
-	    &tdbp->tdb_dst, tdbp->tdb_sproto);
+	hashval = tdb_hash(tdbp->tdb_spi, &tdbp->tdb_dst, tdbp->tdb_sproto);
 
 	/*
 	 * Rehash if this tdb would cause a bucket to have more than
@@ -690,20 +687,18 @@ puttdb(struct tdb *tdbp)
 	if (tdbh[hashval] != NULL && tdbh[hashval]->tdb_hnext != NULL &&
 	    tdb_count * 10 > tdb_hashmask + 1) {
 		tdb_rehash();
-		hashval = tdb_hash(tdbp->tdb_rdomain, tdbp->tdb_spi,
-		    &tdbp->tdb_dst, tdbp->tdb_sproto);
+		hashval = tdb_hash(tdbp->tdb_spi, &tdbp->tdb_dst,
+		    tdbp->tdb_sproto);
 	}
 
 	tdbp->tdb_hnext = tdbh[hashval];
 	tdbh[hashval] = tdbp;
 
-	hashval = tdb_hash(tdbp->tdb_rdomain, 0, &tdbp->tdb_dst,
-	    tdbp->tdb_sproto);
+	hashval = tdb_hash(0, &tdbp->tdb_dst, tdbp->tdb_sproto);
 	tdbp->tdb_dnext = tdbdst[hashval];
 	tdbdst[hashval] = tdbp;
 
-	hashval = tdb_hash(tdbp->tdb_rdomain, 0, &tdbp->tdb_src,
-	    tdbp->tdb_sproto);
+	hashval = tdb_hash(0, &tdbp->tdb_src, tdbp->tdb_sproto);
 	tdbp->tdb_snext = tdbsrc[hashval];
 	tdbsrc[hashval] = tdbp;
 
@@ -727,8 +722,7 @@ tdb_unlink(struct tdb *tdbp)
 	if (tdbh == NULL)
 		return;
 
-	hashval = tdb_hash(tdbp->tdb_rdomain, tdbp->tdb_spi,
-	    &tdbp->tdb_dst, tdbp->tdb_sproto);
+	hashval = tdb_hash(tdbp->tdb_spi, &tdbp->tdb_dst, tdbp->tdb_sproto);
 
 	if (tdbh[hashval] == tdbp) {
 		tdbh[hashval] = tdbp->tdb_hnext;
@@ -744,8 +738,7 @@ tdb_unlink(struct tdb *tdbp)
 
 	tdbp->tdb_hnext = NULL;
 
-	hashval = tdb_hash(tdbp->tdb_rdomain, 0, &tdbp->tdb_dst,
-	    tdbp->tdb_sproto);
+	hashval = tdb_hash(0, &tdbp->tdb_dst, tdbp->tdb_sproto);
 
 	if (tdbdst[hashval] == tdbp) {
 		tdbdst[hashval] = tdbp->tdb_dnext;
@@ -761,8 +754,7 @@ tdb_unlink(struct tdb *tdbp)
 
 	tdbp->tdb_dnext = NULL;
 
-	hashval = tdb_hash(tdbp->tdb_rdomain, 0, &tdbp->tdb_src,
-	    tdbp->tdb_sproto);
+	hashval = tdb_hash(0, &tdbp->tdb_src, tdbp->tdb_sproto);
 
 	if (tdbsrc[hashval] == tdbp) {
 		tdbsrc[hashval] = tdbp->tdb_snext;
@@ -816,6 +808,7 @@ tdb_alloc(u_int rdomain)
 
 	/* Save routing domain */
 	tdbp->tdb_rdomain = rdomain;
+	tdbp->tdb_rdomain_post = rdomain;
 
 	/* Initialize timeouts. */
 	timeout_set_proc(&tdbp->tdb_timer_tmo, tdb_timeout, tdbp);
diff --git a/sys/netinet/ip_ipsp.h b/sys/netinet/ip_ipsp.h
index 49c9d661d57..06e14814b3e 100644
--- a/sys/netinet/ip_ipsp.h
+++ b/sys/netinet/ip_ipsp.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_ipsp.h,v 1.193 2018/08/28 15:15:02 mpi Exp $	*/
+/*	$OpenBSD: ip_ipsp.h,v 1.194 2020/04/23 19:38:08 tobhe Exp $	*/
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr),
@@ -409,6 +409,7 @@ struct tdb {				/* tunnel descriptor block */
 	u_int32_t	tdb_tap;		/* Alternate enc(4) interface */
 
 	u_int		tdb_rdomain;		/* Routing domain */
+	u_int		tdb_rdomain_post;	/* Change domain */
 
 	struct sockaddr_encap   tdb_filter; /* What traffic is acceptable */
 	struct sockaddr_encap   tdb_filtermask; /* And the mask */
@@ -574,15 +575,19 @@ int spd_table_walk(unsigned int,
 /* TDB management routines */
 uint32_t reserve_spi(u_int, u_int32_t, u_int32_t, union sockaddr_union *,
 		union sockaddr_union *, u_int8_t, int *);
-struct	tdb *gettdb(u_int, u_int32_t, union sockaddr_union *, u_int8_t);
+struct	tdb *gettdb_dir(u_int, u_int32_t, union sockaddr_union *, u_int8_t, int);
+#define gettdb(a,b,c,d)		gettdb_dir((a),(b),(c),(d),0)
+#define gettdb_rev(a,b,c,d)	gettdb_dir((a),(b),(c),(d),1)
 struct	tdb *gettdbbydst(u_int, union sockaddr_union *, u_int8_t,
 		struct ipsec_ids *,
 		struct sockaddr_encap *, struct sockaddr_encap *);
 struct	tdb *gettdbbysrc(u_int, union sockaddr_union *, u_int8_t,
 		struct ipsec_ids *,
 		struct sockaddr_encap *, struct sockaddr_encap *);
-struct	tdb *gettdbbysrcdst(u_int, u_int32_t, union sockaddr_union *,
-		union sockaddr_union *, u_int8_t);
+struct	tdb *gettdbbysrcdst_dir(u_int, u_int32_t, union sockaddr_union *,
+		union sockaddr_union *, u_int8_t, int);
+#define gettdbbysrcdst(a,b,c,d,e) gettdbbysrcdst_dir((a),(b),(c),(d),(e),0)
+#define gettdbbysrcdst_rev(a,b,c,d,e) gettdbbysrcdst_dir((a),(b),(c),(d),(e),1)
 void	puttdb(struct tdb *);
 void	tdb_delete(struct tdb *);
 struct	tdb *tdb_alloc(u_int);
diff --git a/sys/netinet/ipsec_input.c b/sys/netinet/ipsec_input.c
index b6d0083af40..7303c3cb303 100644
--- a/sys/netinet/ipsec_input.c
+++ b/sys/netinet/ipsec_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipsec_input.c,v 1.169 2019/09/30 01:53:05 dlg Exp $	*/
+/*	$OpenBSD: ipsec_input.c,v 1.170 2020/04/23 19:38:08 tobhe Exp $	*/
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -299,7 +299,7 @@ ipsec_common_input(struct mbuf *m, int skip, int protoff, int af, int sproto,
 	}
 
 	if (sproto != IPPROTO_IPCOMP) {
-		if ((encif = enc_getif(tdbp->tdb_rdomain,
+		if ((encif = enc_getif(tdbp->tdb_rdomain_post,
 		    tdbp->tdb_tap)) == NULL) {
 			DPRINTF(("%s: no enc%u interface for SA %s/%08x/%u\n",
 			    __func__,
@@ -657,6 +657,8 @@ ipsec_common_input_cb(struct mbuf *m, struct tdb *tdbp, int skip, int protoff)
 	pf_tag_packet(m, tdbp->tdb_tag, -1);
 	pf_pkt_addr_changed(m);
 #endif
+	if (tdbp->tdb_rdomain != tdbp->tdb_rdomain_post)
+		m->m_pkthdr.ph_rtableid = tdbp->tdb_rdomain_post;
 
 	if (tdbp->tdb_flags & TDBF_TUNNELING)
 		m->m_flags |= M_TUNNEL;
@@ -665,7 +667,7 @@ ipsec_common_input_cb(struct mbuf *m, struct tdb *tdbp, int skip, int protoff)
 	tdbp->tdb_idecompbytes += m->m_pkthdr.len;
 
 #if NBPFILTER > 0
-	if ((encif = enc_getif(tdbp->tdb_rdomain, tdbp->tdb_tap)) != NULL) {
+	if ((encif = enc_getif(tdbp->tdb_rdomain_post, tdbp->tdb_tap)) != NULL) {
 		encif->if_ipackets++;
 		encif->if_ibytes += m->m_pkthdr.len;
 
@@ -966,7 +968,7 @@ ipsec_common_ctlinput(u_int rdomain, int cmd, struct sockaddr *sa,
 
 		memcpy(&spi, (caddr_t)ip + hlen, sizeof(u_int32_t));
 
-		tdbp = gettdb(rdomain, spi, (union sockaddr_union *)&dst,
+		tdbp = gettdb_rev(rdomain, spi, (union sockaddr_union *)&dst,
 		    proto);
 		if (tdbp == NULL || tdbp->tdb_flags & TDBF_INVALID)
 			return;
@@ -1025,7 +1027,8 @@ udpencap_ctlinput(int cmd, struct sockaddr *sa, u_int rdomain, void *v)
 	src.sin_addr.s_addr = ip->ip_src.s_addr;
 	su_src = (union sockaddr_union *)&src;
 
-	tdbp = gettdbbysrcdst(rdomain, 0, su_src, su_dst, IPPROTO_ESP);
+	tdbp = gettdbbysrcdst_rev(rdomain, 0, su_src, su_dst,
+	    IPPROTO_ESP);
 
 	for (; tdbp != NULL; tdbp = tdbp->tdb_snext) {
 		if (tdbp->tdb_sproto == IPPROTO_ESP &&
diff --git a/sys/netinet/ipsec_output.c b/sys/netinet/ipsec_output.c
index 28ff5b92781..c5a13312f67 100644
--- a/sys/netinet/ipsec_output.c
+++ b/sys/netinet/ipsec_output.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipsec_output.c,v 1.75 2018/09/14 23:40:10 mestre Exp $ */
+/*	$OpenBSD: ipsec_output.c,v 1.76 2020/04/23 19:38:09 tobhe Exp $ */
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -592,6 +592,8 @@ ipsp_process_done(struct mbuf *m, struct tdb *tdb)
 	pf_tag_packet(m, tdb->tdb_tag, -1);
 	pf_pkt_addr_changed(m);
 #endif
+	if (tdb->tdb_rdomain != tdb->tdb_rdomain_post)
+		m->m_pkthdr.ph_rtableid = tdb->tdb_rdomain_post;
 
 	/*
 	 * We're done with IPsec processing, transmit the packet using the