Introduce attributes to interface groups.

As a first user, move the global carp(4) demotion counter
into the interface group. Thus we have the possibility
to define which carp interfaces are demoted together.

Put the demotion counter into the reserved field of the carp header.
With this, we can have carp act smarter if multiple errors occur.
It now always takes over other carp peers, that are advertising
with a higher demote count.  As a side effect, we can also have
group failovers without the need of running in preempt mode.
The protocol change does not break compability with older
implementations.

Collaborative work with mcbride@

OK mcbride@, henning@

6 files changed, 167 insertions, 40 deletions

diff --git a/sys/net/if.c b/sys/net/if.c
index 4b97d064b0c..97b517fa1f9 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.146 2006/03/22 14:37:44 henning Exp $	*/
+/*	$OpenBSD: if.c,v 1.147 2006/06/02 19:53:12 mpf Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -137,6 +137,8 @@ void	if_detached_watchdog(struct ifnet *);
 
 int	if_getgroup(caddr_t, struct ifnet *);
 int	if_getgroupmembers(caddr_t);
+int	if_getgroupattribs(caddr_t);
+int	if_setgroupattribs(caddr_t);
 
 int	if_clone_list(struct if_clonereq *);
 struct if_clone	*if_clone_lookup(const char *, int *);
@@ -1167,6 +1169,12 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p)
 		return (if_clone_list((struct if_clonereq *)data));
 	case SIOCGIFGMEMB:
 		return (if_getgroupmembers(data));
+	case SIOCGIFGATTR:
+		return (if_getgroupattribs(data));
+	case SIOCSIFGATTR:
+		if ((error = suser(p, 0)) != 0)
+			return (error);
+		return (if_setgroupattribs(data));
 	}
 
 	ifp = ifunit(ifr->ifr_name);
@@ -1280,6 +1288,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p)
 	case SIOCAIFGROUP:
 		if ((error = suser(p, 0)))
 			return (error);
+		(*ifp->if_ioctl)(ifp, cmd, data); /* XXX error check */
 		ifgr = (struct ifgroupreq *)data;
 		if ((error = if_addgroup(ifp, ifgr->ifgr_group)))
 			return (error);
@@ -1293,6 +1302,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p)
 	case SIOCDIFGROUP:
 		if ((error = suser(p, 0)))
 			return (error);
+		(*ifp->if_ioctl)(ifp, cmd, data); /* XXX error check */
 		ifgr = (struct ifgroupreq *)data;
 		if ((error = if_delgroup(ifp, ifgr->ifgr_group)))
 			return (error);
@@ -1590,6 +1600,7 @@ if_addgroup(struct ifnet *ifp, const char *groupname)
 		}
 		strlcpy(ifg->ifg_group, groupname, sizeof(ifg->ifg_group));
 		ifg->ifg_refcnt = 0;
+		ifg->ifg_carp_demoted = 0;
 		TAILQ_INIT(&ifg->ifg_members);
 #if NPF > 0
 		pfi_attach_ifgroup(ifg);
@@ -1732,6 +1743,48 @@ if_getgroupmembers(caddr_t data)
 	return (0);
 }
 
+int
+if_getgroupattribs(caddr_t data)
+{
+	struct ifgroupreq	*ifgr = (struct ifgroupreq *)data;
+	struct ifg_group	*ifg;
+
+	TAILQ_FOREACH(ifg, &ifg_head, ifg_next)
+		if (!strcmp(ifg->ifg_group, ifgr->ifgr_name))
+			break;
+	if (ifg == NULL)
+		return (ENOENT);
+
+	ifgr->ifgr_attrib.ifg_carp_demoted = ifg->ifg_carp_demoted;
+
+	return (0);
+}
+
+int
+if_setgroupattribs(caddr_t data)
+{
+	struct ifgroupreq	*ifgr = (struct ifgroupreq *)data;
+	struct ifg_group	*ifg;
+	int			 demote;
+
+	TAILQ_FOREACH(ifg, &ifg_head, ifg_next)
+		if (!strcmp(ifg->ifg_group, ifgr->ifgr_name))
+			break;
+	if (ifg == NULL)
+		return (ENOENT);
+
+	demote = ifgr->ifgr_attrib.ifg_carp_demoted;
+	if (demote > 1 || demote < -1)
+		return (E2BIG);
+
+	if (demote + ifg->ifg_carp_demoted >= 0)
+		ifg->ifg_carp_demoted += demote;
+	else
+		ifg->ifg_carp_demoted = 0;
+
+	return (0);
+}
+
 void
 if_group_routechange(struct sockaddr *dst, struct sockaddr *mask)
 {
diff --git a/sys/net/if.h b/sys/net/if.h
index cef94421001..2e54ccd76ad 100644
--- a/sys/net/if.h
+++ b/sys/net/if.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.h,v 1.81 2006/05/27 10:03:15 brad Exp $	*/
+/*	$OpenBSD: if.h,v 1.82 2006/06/02 19:53:12 mpf Exp $	*/
 /*	$NetBSD: if.h,v 1.23 1996/05/07 02:40:27 thorpej Exp $	*/
 
 /*
@@ -433,6 +433,7 @@ struct ifg_group {
 	char				 ifg_group[IFNAMSIZ];
 	u_int				 ifg_refcnt;
 	caddr_t				 ifg_pf_kif;
+	int				 ifg_carp_demoted;
 	TAILQ_HEAD(, ifg_member)	 ifg_members;
 	TAILQ_ENTRY(ifg_group)		 ifg_next;
 };
@@ -456,6 +457,10 @@ struct ifg_req {
 #define	ifgrq_member	ifgrq_ifgrqu.ifgrqu_member
 };
 
+struct ifg_attrib {
+	int	ifg_carp_demoted;
+};
+
 /*
  * Used to lookup groups for an interface
  */
@@ -463,11 +468,13 @@ struct ifgroupreq {
 	char	ifgr_name[IFNAMSIZ];
 	u_int	ifgr_len;
 	union {
-		char	ifgru_group[IFNAMSIZ];
-		struct	ifg_req *ifgru_groups;
+		char			 ifgru_group[IFNAMSIZ];
+		struct	ifg_req		*ifgru_groups;
+		struct	ifg_attrib	 ifgru_attrib;
 	} ifgr_ifgru;
 #define ifgr_group	ifgr_ifgru.ifgru_group
 #define ifgr_groups	ifgr_ifgru.ifgru_groups
+#define ifgr_attrib	ifgr_ifgru.ifgru_attrib
 };
 
 /*
diff --git a/sys/net/if_pfsync.c b/sys/net/if_pfsync.c
index 9f43fc4be85..5d9604540e5 100644
--- a/sys/net/if_pfsync.c
+++ b/sys/net/if_pfsync.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_pfsync.c,v 1.65 2006/05/28 02:04:15 mcbride Exp $	*/
+/*	$OpenBSD: if_pfsync.c,v 1.66 2006/06/02 19:53:12 mpf Exp $	*/
 
 /*
  * Copyright (c) 2002 Michael Shalayeff
@@ -59,7 +59,7 @@
 
 #include "carp.h"
 #if NCARP > 0
-extern int carp_suppress_preempt;
+#include <netinet/ip_carp.h>
 #endif
 
 #include <net/pfvar.h>
@@ -143,6 +143,10 @@ pfsyncattach(int npfsync)
 	if_attach(ifp);
 	if_alloc_sadl(ifp);
 
+#if NCARP > 0
+	if_addgroup(ifp, "carp");
+#endif
+
 #if NBPFILTER > 0
 	bpfattach(&pfsyncif.sc_if.if_bpf, ifp, DLT_PFSYNC, PFSYNC_HDRLEN);
 #endif
@@ -757,7 +761,7 @@ pfsync_input(struct mbuf *m, ...)
 				timeout_del(&sc->sc_bulkfail_tmo);
 #if NCARP > 0
 				if (!pfsync_sync_ok)
-					carp_suppress_preempt--;
+					carp_group_demote_adj(&sc->sc_if, -1);
 #endif
 				pfsync_sync_ok = 1;
 				if (pf_status.debug >= PF_DEBUG_MISC)
@@ -922,7 +926,7 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			sc->sc_ureq_sent = time_uptime;
 #if NCARP > 0
 			if (pfsync_sync_ok)
-				carp_suppress_preempt++;
+				carp_group_demote_adj(&sc->sc_if, 1);
 #endif
 			pfsync_sync_ok = 0;
 			if (pf_status.debug >= PF_DEBUG_MISC)
@@ -1424,7 +1428,7 @@ pfsync_bulkfail(void *v)
 		sc->sc_bulk_tries = 0;
 #if NCARP > 0
 		if (!pfsync_sync_ok)
-			carp_suppress_preempt--;
+			carp_group_demote_adj(&sc->sc_if, -1);
 #endif
 		pfsync_sync_ok = 1;
 		if (pf_status.debug >= PF_DEBUG_MISC)
diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index 9d392445a89..91949c77db5 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_carp.c,v 1.125 2006/05/22 23:25:15 krw Exp $	*/
+/*	$OpenBSD: ip_carp.c,v 1.126 2006/06/02 19:53:12 mpf Exp $	*/
 
 /*
  * Copyright (c) 2002 Michael Shalayeff. All rights reserved.
@@ -145,7 +145,6 @@ struct carp_softc {
 	LIST_HEAD(__carp_mchead, carp_mc_entry)	carp_mc_listhead;
 };
 
-int carp_suppress_preempt = 0;
 int carp_opts[CARPCTL_MAXID] = { 0, 1, 0, 0, 0 };	/* XXX for now */
 struct carpstats carpstats;
 
@@ -183,6 +182,7 @@ void	carp_send_ad(void *);
 void	carp_send_arp(struct carp_softc *);
 void	carp_master_down(void *);
 int	carp_ioctl(struct ifnet *, u_long, caddr_t);
+void	carp_ifgroup_ioctl(struct ifnet *, u_long, caddr_t);
 void	carp_start(struct ifnet *);
 void	carp_setrun(struct carp_softc *, sa_family_t);
 void	carp_set_state(struct carp_softc *, int);
@@ -206,6 +206,7 @@ int	carp_clone_destroy(struct ifnet *);
 int	carp_ether_addmulti(struct carp_softc *, struct ifreq *);
 int	carp_ether_delmulti(struct carp_softc *, struct ifreq *);
 void	carp_ether_purgemulti(struct carp_softc *);
+int	carp_group_demote_count(struct carp_softc *);
 
 struct if_clone carp_cloner =
     IF_CLONE_INITIALIZER("carp", carp_clone_create, carp_clone_destroy);
@@ -664,7 +665,7 @@ carp_proto_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af)
 
 
 	sc_tv.tv_sec = sc->sc_advbase;
-	if (carp_suppress_preempt && sc->sc_advskew <  240)
+	if (carp_group_demote_count(sc) && sc->sc_advskew <  240)
 		sc_tv.tv_usec = 240 * 1000000 / 256;
 	else
 		sc_tv.tv_usec = sc->sc_advskew * 1000000 / 256;
@@ -680,7 +681,9 @@ carp_proto_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af)
 		 * be more frequent than us, go into BACKUP state.
 		 */
 		if (timercmp(&sc_tv, &ch_tv, >) ||
-		    timercmp(&sc_tv, &ch_tv, ==)) {
+		    (timercmp(&sc_tv, &ch_tv, ==) &&
+		    ch->carp_demote <=
+		    (carp_group_demote_count(sc) & 0xff))) {
 			timeout_del(&sc->sc_ad_tmo);
 			carp_set_state(sc, BACKUP);
 			carp_setrun(sc, 0);
@@ -698,6 +701,15 @@ carp_proto_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af)
 		}
 
 		/*
+		 * Take over masters advertising with a higher demote count,
+		 * regardless of CARPCTL_PREEMPT.
+		 */ 
+		if (ch->carp_demote > (carp_group_demote_count(sc) & 0xff)) {
+			carp_master_down(sc);
+			break;
+		}
+
+		/*
 		 *  If the master is going to advertise at such a low frequency
 		 *  that he's guaranteed to time out, we'd might as well just
 		 *  treat him as timed out now.
@@ -821,11 +833,11 @@ carpdetach(struct carp_softc *sc)
 	timeout_del(&sc->sc_md6_tmo);
 
 	if (sc->sc_suppress)
-		carp_suppress_preempt--;
+		carp_group_demote_adj(&sc->sc_if, -1);
 	sc->sc_suppress = 0;
 
 	if (sc->sc_sendad_errors >= CARP_SENDAD_MAX_ERRORS)
-		carp_suppress_preempt--;
+		carp_group_demote_adj(&sc->sc_if, -1);
 	sc->sc_sendad_errors = 0;
 
 	carp_set_state(sc, INIT);
@@ -929,7 +941,7 @@ carp_send_ad(void *v)
 		advskew = 255;
 	} else {
 		advbase = sc->sc_advbase;
-		if (!carp_suppress_preempt || sc->sc_advskew > 240)
+		if (!carp_group_demote_count(sc) || sc->sc_advskew > 240)
 			advskew = sc->sc_advskew;
 		else
 			advskew = 240;
@@ -940,10 +952,10 @@ carp_send_ad(void *v)
 	ch.carp_version = CARP_VERSION;
 	ch.carp_type = CARP_ADVERTISEMENT;
 	ch.carp_vhid = sc->sc_vhid;
+	ch.carp_demote = carp_group_demote_count(sc) & 0xff;
 	ch.carp_advbase = advbase;
 	ch.carp_advskew = advskew;
 	ch.carp_authlen = 7;	/* XXX DEFINE */
-	ch.carp_pad1 = 0;	/* must be zero */
 	ch.carp_cksum = 0;
 
 
@@ -1009,17 +1021,14 @@ carp_send_ad(void *v)
 			sc->sc_if.if_oerrors++;
 			if (sc->sc_sendad_errors < INT_MAX)
 				sc->sc_sendad_errors++;
-			if (sc->sc_sendad_errors == CARP_SENDAD_MAX_ERRORS) {
-				carp_suppress_preempt++;
-				if (carp_suppress_preempt == 1)
-					carp_send_ad_all();
-			}
+			if (sc->sc_sendad_errors == CARP_SENDAD_MAX_ERRORS)
+				carp_group_demote_adj(&sc->sc_if, 1);
 			sc->sc_sendad_success = 0;
 		} else {
 			if (sc->sc_sendad_errors >= CARP_SENDAD_MAX_ERRORS) {
 				if (++sc->sc_sendad_success >=
 				    CARP_SENDAD_MIN_SUCCESS) {
-					carp_suppress_preempt--;
+					carp_group_demote_adj(&sc->sc_if, -1);
 					sc->sc_sendad_errors = 0;
 				}
 			} else
@@ -1088,17 +1097,14 @@ carp_send_ad(void *v)
 			sc->sc_if.if_oerrors++;
 			if (sc->sc_sendad_errors < INT_MAX)
 				sc->sc_sendad_errors++;
-			if (sc->sc_sendad_errors == CARP_SENDAD_MAX_ERRORS) {
-				carp_suppress_preempt++;
-				if (carp_suppress_preempt == 1)
-					carp_send_ad_all();
-			}
+			if (sc->sc_sendad_errors == CARP_SENDAD_MAX_ERRORS)
+				carp_group_demote_adj(&sc->sc_if, 1);
 			sc->sc_sendad_success = 0;
 		} else {
 			if (sc->sc_sendad_errors >= CARP_SENDAD_MAX_ERRORS) {
 				if (++sc->sc_sendad_success >=
 				    CARP_SENDAD_MIN_SUCCESS) {
-					carp_suppress_preempt--;
+					carp_group_demote_adj(&sc->sc_if, -1);
 					sc->sc_sendad_errors = 0;
 				}
 			} else
@@ -2016,7 +2022,11 @@ carp_ioctl(struct ifnet *ifp, u_long cmd, caddr_t addr)
 	case SIOCDELMULTI:
 		error = carp_ether_delmulti(sc, ifr);
 		break;
-
+	case SIOCAIFGROUP:
+	case SIOCDIFGROUP:
+		if (sc->sc_suppress)
+			carp_ifgroup_ioctl(ifp, cmd, addr);
+		break;
 	default:
 		error = EINVAL;
 	}
@@ -2025,6 +2035,23 @@ carp_ioctl(struct ifnet *ifp, u_long cmd, caddr_t addr)
 	return (error);
 }
 
+void
+carp_ifgroup_ioctl(struct ifnet *ifp, u_long cmd, caddr_t addr)
+{
+	struct ifgroupreq *ifgr = (struct ifgroupreq *)addr;
+	struct ifg_list	*ifgl;
+
+	if (!strcmp(ifgr->ifgr_group, IFG_ALL))
+		return;
+	TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
+		if (!strcmp(ifgl->ifgl_group->ifg_group, ifgr->ifgr_group)) {
+			if (cmd == SIOCAIFGROUP)
+				ifgl->ifgl_group->ifg_carp_demoted++;
+			else if (cmd == SIOCDIFGROUP &&
+			    ifgl->ifgl_group->ifg_carp_demoted)
+				ifgl->ifgl_group->ifg_carp_demoted--;
+		}
+}
 
 /*
  * Start output on carp interface. This function should never be called.
@@ -2073,6 +2100,42 @@ carp_set_state(struct carp_softc *sc, int state)
 }
 
 void
+carp_group_demote_adj(struct ifnet *ifp, int adj)
+{
+	struct ifg_list	*ifgl;
+	int *dm;
+	struct carp_softc *nil = NULL;
+
+	TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) {
+		if (!strcmp(ifgl->ifgl_group->ifg_group, IFG_ALL))
+			continue;
+		dm = &ifgl->ifgl_group->ifg_carp_demoted;
+
+		if (*dm + adj >= 0)
+			*dm += adj;
+		else
+			*dm = 0;
+
+		if (adj > 0 && *dm == 1)
+			carp_send_ad_all();
+		CARP_LOG(nil, ("%s demoted group %s to %d", ifp->if_xname,
+		    ifgl->ifgl_group->ifg_group, *dm));
+	}
+}
+
+int
+carp_group_demote_count(struct carp_softc *sc)
+{
+	struct ifg_list	*ifgl;
+	int count = 0;
+
+	TAILQ_FOREACH(ifgl, &sc->sc_if.if_groups, ifgl_next)
+		count += ifgl->ifgl_group->ifg_carp_demoted;
+
+	return (count);
+}
+
+void
 carp_carpdev_state(void *v)
 {
 	struct carp_if *cif;
@@ -2096,17 +2159,14 @@ carp_carpdev_state(void *v)
 			carp_set_state(sc, INIT);
 			sc->sc_suppress = 1;
 			carp_setrun(sc, 0);
-			if (!suppressed) {
-				carp_suppress_preempt++;
-				if (carp_suppress_preempt == 1)
-					carp_send_ad_all();
-			}
+			if (!suppressed)
+				carp_group_demote_adj(&sc->sc_if, 1);
 		} else {
 			carp_set_state(sc, INIT);
 			sc->sc_suppress = 0;
 			carp_setrun(sc, 0);
 			if (suppressed)
-				carp_suppress_preempt--;
+				carp_group_demote_adj(&sc->sc_if, -1);
 		}
 	}
 }
diff --git a/sys/netinet/ip_carp.h b/sys/netinet/ip_carp.h
index 2ea0155941e..a2d966736bf 100644
--- a/sys/netinet/ip_carp.h
+++ b/sys/netinet/ip_carp.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_carp.h,v 1.20 2006/04/27 02:19:32 tedu Exp $	*/
+/*	$OpenBSD: ip_carp.h,v 1.21 2006/06/02 19:53:12 mpf Exp $	*/
 
 /*
  * Copyright (c) 2002 Michael Shalayeff. All rights reserved.
@@ -37,7 +37,7 @@
  *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *    |Version| Type  | VirtualHostID |    AdvSkew    |    Auth Len   |
  *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *    |   Reserved    |     AdvBase   |          Checksum             |
+ *    |    Demotion   |     AdvBase   |          Checksum             |
  *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *    |                         Counter (1)                           |
  *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -68,7 +68,7 @@ struct carp_header {
 	u_int8_t	carp_vhid;	/* virtual host id */
 	u_int8_t	carp_advskew;	/* advertisement skew */
 	u_int8_t	carp_authlen;   /* size of counter+md, 32bit chunks */
-	u_int8_t	carp_pad1;	/* reserved */
+	u_int8_t	carp_demote;	/* demotion indicator */
 	u_int8_t	carp_advbase;	/* advertisement interval */
 	u_int16_t	carp_cksum;
 	u_int32_t	carp_counter[2];
@@ -155,6 +155,7 @@ struct carpreq {
 void		 carp_ifdetach (struct ifnet *);
 void		 carp_proto_input (struct mbuf *, ...);
 void		 carp_carpdev_state(void *);
+void		 carp_group_demote_adj(struct ifnet *, int);
 int		 carp6_proto_input(struct mbuf **, int *, int);
 int		 carp_iamatch(struct in_ifaddr *, u_char *,
 		     u_int32_t *, u_int32_t);
diff --git a/sys/sys/sockio.h b/sys/sys/sockio.h
index 5c27502914e..2023b24a0dd 100644
--- a/sys/sys/sockio.h
+++ b/sys/sys/sockio.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sockio.h,v 1.36 2006/01/05 14:57:24 norby Exp $	*/
+/*	$OpenBSD: sockio.h,v 1.37 2006/06/02 19:53:12 mpf Exp $	*/
 /*	$NetBSD: sockio.h,v 1.5 1995/08/23 00:40:47 thorpej Exp $	*/
 
 /*-
@@ -145,6 +145,8 @@
 #define	SIOCGIFGROUP   _IOWR('i', 136, struct ifgroupreq) /* get ifgroups */
 #define	SIOCDIFGROUP    _IOW('i', 137, struct ifgroupreq) /* delete ifgroup */
 #define	SIOCGIFGMEMB   _IOWR('i', 138, struct ifgroupreq) /* get members */
+#define	SIOCGIFGATTR   _IOWR('i', 139, struct ifgroupreq) /* get ifgroup attribs */
+#define	SIOCSIFGATTR   _IOW('i', 140, struct ifgroupreq) /* set ifgroup attribs */
 
 #define	SIOCSIFDESCR	 _IOW('i', 128, struct ifreq)	/* set ifnet descr */
 #define	SIOCGIFDESCR	_IOWR('i', 129, struct ifreq)	/* get ifnet descr */