change the way relayd reports check results: instead of logging an

arbitrary string in debugging mode, it will store an error code
(HCE_*) for each host.  the error code can be translated to a string
(in log.c) for debugging but it will also be passed to relayctl via
the control socket.

from a user point of view, this will print a human-readable error
message in the "relayctl show hosts" output if a host is down because
the check failed.  the relayctl(8) manpage includes detailed
explanations of the error messages including mitigations for the
most-common problems.

ok jmc@ (manpages)
ok phessler@

11 files changed, 324 insertions, 68 deletions

diff --git a/usr.sbin/relayctl/relayctl.8 b/usr.sbin/relayctl/relayctl.8
index 3bffefae443..a57d471ba48 100644
--- a/usr.sbin/relayctl/relayctl.8
+++ b/usr.sbin/relayctl/relayctl.8
@@ -1,4 +1,4 @@
-.\" $OpenBSD: relayctl.8,v 1.21 2007/12/20 20:22:11 reyk Exp $
+.\" $OpenBSD: relayctl.8,v 1.22 2008/12/05 16:37:56 reyk Exp $
 .\"
 .\" Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
 .\"
@@ -14,7 +14,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: December 20 2007 $
+.Dd $Mdocdate: December 5 2008 $
 .Dt RELAYCTL 8
 .Os
 .Sh NAME
@@ -60,6 +60,10 @@ the backup table as well.
 Reload the configuration file.
 .It Cm show hosts
 Show detailed status of hosts and tables.
+It will also print the last error for failed host checks;
+see the
+.Sx ERRORS
+section below.
 .It Cm show redirects
 Show detailed status of redirections including the current and average
 access statistics.
@@ -94,6 +98,98 @@ again.
 Unix-domain socket used for communication with
 .Xr relayd 8 .
 .El
+.Sh ERRORS
+If a host is down and a previous check failed,
+.Nm
+will display the last error in the output of the
+.Cm show hosts
+command.
+This is especially useful for debugging server or configuration failures.
+The following errors will be reported:
+.Pp
+.Bl -tag -width Ds -compact
+.It Em none
+No specific error was reported by the check engine.
+.Pp
+.It Em aborted
+All checks were aborted by an external event, like a configuration reload.
+.Pp
+.It Em interval timeout
+The check did not finish in the configured time of an interval.
+This can happen if there are too many hosts that have to be checked by
+.Xr relayd 8
+and can be avoided by increasing the global
+.Ic interval
+option in
+.Xr relayd.conf 5 .
+.Pp
+.It Em icmp read timeout
+.It Em ssl read timeout
+.It Em tcp read timeout
+The check failed because the remote host did not send a reply within
+the configured timeout.
+.Pp
+.It Em icmp write timeout
+.It Em ssl write timeout
+.It Em tcp write timeout
+.It Em ssl connect timeout
+.It Em tcp connect timeout
+The check failed because
+.Xr relayd 8
+was not ready to send the request within the configured timeout.
+.Pp
+.It Em ssl connect error
+.It Em ssl read error
+.It Em ssl write error
+.It Em tcp connect error
+.It Em tcp read failed
+.It Em tcp write failed
+An I/O error occurred.
+This indicates that
+.Xr relayd 8
+was running low on resources,
+file descriptors, or was too busy to run the request.
+It can also indicate that an SSL/TCP protocol error occurred or that the
+connection was unexpectedly aborted.
+.Pp
+.It Em ssl connect failed
+.It Em tcp connect failed
+The check failed because the protocol handshake did not succeed in
+opening a stateful connection with the remote host.
+.Pp
+.It Em script failed
+The external script executed by the check did not return a valid return code.
+.Pp
+.It Em send/expect failed
+The payload data returned by the remote host did not match the
+expected pattern.
+.Pp
+.It Em http code malformed
+.It Em http digest malformed
+The remote host did not return a valid HTTP header or body.
+.Pp
+.It Em http code mismatch
+The remote host did not return a matching HTTP error code.
+This may indicate a real server problem (a server error, the page was
+not found, permission was denied) or a configuration error.
+For example, it is a very common mistake that
+.Xr relayd 8
+was configured to expect a
+HTTP 200 OK
+status but the host is returning a
+HTTP 302 Found
+redirection.
+See
+.Xr relayd.conf 5
+for more information on validating the HTTP return code.
+.Pp
+.It Em http digest mismatch
+The remote host did not return the expected content and the computed
+digest was different to the configured value.
+See
+.Xr relayd.conf 5
+for more information on validating the digest.
+.El
 .Sh SEE ALSO
 .Xr relayd 8
 .Sh HISTORY
diff --git a/usr.sbin/relayctl/relayctl.c b/usr.sbin/relayctl/relayctl.c
index c93c0062eb0..d7b8f42d38e 100644
--- a/usr.sbin/relayctl/relayctl.c
+++ b/usr.sbin/relayctl/relayctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayctl.c,v 1.34 2008/07/19 12:10:07 reyk Exp $	*/
+/*	$OpenBSD: relayctl.c,v 1.35 2008/12/05 16:37:56 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -351,6 +351,8 @@ show_summary_msg(struct imsg *imsg, int type)
 			    "", host->up_cnt, host->check_cnt);
 			if (host->retry_cnt)
 				printf(", %d retries", host->retry_cnt);
+			if (host->he && host->up == HOST_DOWN)
+				printf(", error: %s", host_error(host->he));
 			printf("\n");
 		}
 		break;
@@ -461,7 +463,7 @@ print_table_status(int up, int fl)
 	} else if (!up) {
 		snprintf(buf, sizeof(buf) - 1, "empty");
 	} else
-		snprintf(buf, sizeof(buf) - 1, "active (%d hosts up)", up);
+		snprintf(buf, sizeof(buf) - 1, "active (%d hosts)", up);
 	return (buf);
 }
 
diff --git a/usr.sbin/relayd/check_icmp.c b/usr.sbin/relayd/check_icmp.c
index f5518582f63..37321529dae 100644
--- a/usr.sbin/relayd/check_icmp.c
+++ b/usr.sbin/relayd/check_icmp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: check_icmp.c,v 1.25 2008/08/08 08:51:21 thib Exp $	*/
+/*	$OpenBSD: check_icmp.c,v 1.26 2008/12/05 16:37:55 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -43,7 +43,7 @@ void	icmp_setup(struct relayd *, struct ctl_icmp_event *, int);
 void	check_icmp_add(struct ctl_icmp_event *, int, struct timeval *,
 	    void (*)(int, short, void *));
 int	icmp_checks_done(struct ctl_icmp_event *);
-void	icmp_checks_timeout(struct ctl_icmp_event *, const char *);
+void	icmp_checks_timeout(struct ctl_icmp_event *, enum host_error);
 void	send_icmp(int, short, void *);
 void	recv_icmp(int, short, void *);
 int	in_cksum(u_short *, int);
@@ -135,7 +135,7 @@ icmp_checks_done(struct ctl_icmp_event *cie)
 }
 
 void
-icmp_checks_timeout(struct ctl_icmp_event *cie, const char *msg)
+icmp_checks_timeout(struct ctl_icmp_event *cie, enum host_error he)
 {
 	struct table	*table;
 	struct host	*host;
@@ -150,7 +150,7 @@ icmp_checks_timeout(struct ctl_icmp_event *cie, const char *msg)
 				continue;
 			if (!(host->flags & F_CHECK_DONE)) {
 				host->up = HOST_DOWN;
-				hce_notify_done(host, msg);
+				hce_notify_done(host, he);
 			}
 		}
 	}
@@ -171,7 +171,7 @@ send_icmp(int s, short event, void *arg)
 	int			 i = 0;
 
 	if (event == EV_TIMEOUT) {
-		icmp_checks_timeout(cie, "send_icmp: timeout");
+		icmp_checks_timeout(cie, HCE_ICMP_WRITE_TIMEOUT);
 		return;
 	}
 
@@ -255,7 +255,7 @@ recv_icmp(int s, short event, void *arg)
 	objid_t			 id;
 
 	if (event == EV_TIMEOUT) {
-		icmp_checks_timeout(cie, NULL);
+		icmp_checks_timeout(cie, HCE_ICMP_READ_TIMEOUT);
 		return;
 	}
 
@@ -293,7 +293,7 @@ recv_icmp(int s, short event, void *arg)
 
 	host->up = HOST_UP;
 	host->flags |= F_CHECK_DONE;
-	hce_notify_done(host, "recv_icmp: done");
+	hce_notify_done(host, HCE_ICMP_OK);
 
 	if (icmp_checks_done(cie))
 		return;
diff --git a/usr.sbin/relayd/check_script.c b/usr.sbin/relayd/check_script.c
index 16754b28c01..f11cb13f5eb 100644
--- a/usr.sbin/relayd/check_script.c
+++ b/usr.sbin/relayd/check_script.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: check_script.c,v 1.6 2008/02/13 11:32:59 reyk Exp $	*/
+/*	$OpenBSD: check_script.c,v 1.7 2008/12/05 16:37:55 reyk Exp $	*/
 
 /*
  * Copyright (c) 2007, 2008 Reyk Floeter <reyk@openbsd.org>
@@ -71,7 +71,7 @@ script_done(struct relayd *env, struct ctl_script *scr)
 	host->flags |= F_CHECK_DONE;
 
 	hce_notify_done(host, host->up == HOST_UP ?
-	    "script: done" : "script: failed");
+	    HCE_SCRIPT_OK : HCE_SCRIPT_FAIL);
 }
 
 void
diff --git a/usr.sbin/relayd/check_tcp.c b/usr.sbin/relayd/check_tcp.c
index 00245fc9d76..85f4e74de13 100644
--- a/usr.sbin/relayd/check_tcp.c
+++ b/usr.sbin/relayd/check_tcp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: check_tcp.c,v 1.32 2008/03/03 16:58:41 reyk Exp $	*/
+/*	$OpenBSD: check_tcp.c,v 1.33 2008/12/05 16:37:55 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -54,6 +54,7 @@ check_tcp(struct ctl_tcp_event *cte)
 	socklen_t		 len;
 	struct timeval		 tv;
 	struct linger		 lng;
+	int			 he = HCE_TCP_CONNECT_ERROR;
 
 	switch (cte->host->conf.ss.ss_family) {
 	case AF_INET:
@@ -84,8 +85,10 @@ check_tcp(struct ctl_tcp_event *cte)
 
 	bcopy(&cte->table->conf.timeout, &tv, sizeof(tv));
 	if (connect(s, (struct sockaddr *)&cte->host->conf.ss, len) == -1) {
-		if (errno != EINPROGRESS)
+		if (errno != EINPROGRESS) {
+			he = HCE_TCP_CONNECT_FAIL;
 			goto bad;
+		}
 	}
 
 	cte->host->up = HOST_UP;
@@ -96,7 +99,7 @@ check_tcp(struct ctl_tcp_event *cte)
 bad:
 	close(s);
 	cte->host->up = HOST_DOWN;
-	hce_notify_done(cte->host, "check_tcp: cannot connect");
+	hce_notify_done(cte->host, he);
 }
 
 void
@@ -109,7 +112,7 @@ tcp_write(int s, short event, void *arg)
 	if (event == EV_TIMEOUT) {
 		close(s);
 		cte->host->up = HOST_DOWN;
-		hce_notify_done(cte->host, "tcp_write: connect timed out");
+		hce_notify_done(cte->host, HCE_TCP_CONNECT_TIMEOUT);
 		return;
 	}
 
@@ -119,7 +122,7 @@ tcp_write(int s, short event, void *arg)
 	if (err != 0) {
 		close(s);
 		cte->host->up = HOST_DOWN;
-		hce_notify_done(cte->host, "tcp_write: connect failed");
+		hce_notify_done(cte->host, HCE_TCP_CONNECT_FAIL);
 		return;
 	}
 
@@ -137,7 +140,7 @@ tcp_host_up(int s, struct ctl_tcp_event *cte)
 		if (cte->table->conf.flags & F_SSL)
 			break;
 		close(s);
-		hce_notify_done(cte->host, "tcp_host_up: connect successful");
+		hce_notify_done(cte->host, HCE_TCP_CONNECT_OK);
 		return;
 	case CHECK_HTTP_CODE:
 		cte->validate_read = NULL;
@@ -181,7 +184,7 @@ tcp_send_req(int s, short event, void *arg)
 	if (event == EV_TIMEOUT) {
 		cte->host->up = HOST_DOWN;
 		close(cte->s);
-		hce_notify_done(cte->host, "tcp_send_req: timeout");
+		hce_notify_done(cte->host, HCE_TCP_WRITE_TIMEOUT);
 		return;
 	}
 	len = strlen(cte->req);
@@ -193,7 +196,7 @@ tcp_send_req(int s, short event, void *arg)
 			log_warnx("tcp_send_req: cannot send request");
 			cte->host->up = HOST_DOWN;
 			close(cte->s);
-			hce_notify_done(cte->host, "tcp_send_req: write");
+			hce_notify_done(cte->host, HCE_TCP_WRITE_FAIL);
 			return;
 		}
 		cte->req += bs;
@@ -222,7 +225,7 @@ tcp_read_buf(int s, short event, void *arg)
 		cte->host->up = HOST_DOWN;
 		buf_free(cte->buf);
 		close(s);
-		hce_notify_done(cte->host, "tcp_read_buf: timeout");
+		hce_notify_done(cte->host, HCE_TCP_READ_TIMEOUT);
 		return;
 	}
 
@@ -235,19 +238,14 @@ tcp_read_buf(int s, short event, void *arg)
 		cte->host->up = HOST_DOWN;
 		buf_free(cte->buf);
 		close(cte->s);
-		hce_notify_done(cte->host, "tcp_read_buf: read failed");
+		hce_notify_done(cte->host, HCE_TCP_READ_FAIL);
 		return;
 	case 0:
 		cte->host->up = HOST_DOWN;
 		(void)cte->validate_close(cte);
 		close(cte->s);
 		buf_free(cte->buf);
-		if (cte->host->up == HOST_UP)
-			hce_notify_done(cte->host,
-			    "tcp_read_buf: check succeeded");
-		else
-			hce_notify_done(cte->host,
-			    "tcp_read_buf: check failed");
+		hce_notify_done(cte->host, cte->host->he);
 		return;
 	default:
 		if (buf_add(cte->buf, rbuf, br) == -1)
@@ -258,12 +256,7 @@ tcp_read_buf(int s, short event, void *arg)
 
 			close(cte->s);
 			buf_free(cte->buf);
-			if (cte->host->up == HOST_UP)
-				hce_notify_done(cte->host,
-				    "tcp_read_buf: check succeeded");
-			else
-				hce_notify_done(cte->host,
-				    "tcp_read_buf: check failed");
+			hce_notify_done(cte->host, cte->host->he);
 			return;
 		}
 		break; /* retry */
@@ -286,9 +279,11 @@ check_send_expect(struct ctl_tcp_event *cte)
 		fatal("out of memory");
 	*b = '\0';
 	if (fnmatch(cte->table->conf.exbuf, cte->buf->buf, 0) == 0) {
+		cte->host->he = HCE_SEND_EXPECT_OK;
 		cte->host->up = HOST_UP;
 		return (0);
 	}
+	cte->host->he = HCE_SEND_EXPECT_FAIL;
 	cte->host->up = HOST_UNKNOWN;
 
 	/*
@@ -318,6 +313,8 @@ check_http_code(struct ctl_tcp_event *cte)
 
 	head = cte->buf->buf;
 	host = cte->host;
+	host->he = HCE_HTTP_CODE_ERROR;
+
 	if (strncmp(head, "HTTP/1.1 ", strlen("HTTP/1.1 ")) &&
 	    strncmp(head, "HTTP/1.0 ", strlen("HTTP/1.0 "))) {
 		log_debug("check_http_code: %s failed "
@@ -341,9 +338,12 @@ check_http_code(struct ctl_tcp_event *cte)
 	if (code != cte->table->conf.retcode) {
 		log_debug("check_http_code: %s failed "
 		    "(invalid HTTP code returned)", host->conf.name);
+		host->he = HCE_HTTP_CODE_FAIL;
 		host->up = HOST_DOWN;
-	} else
+	} else {
+		host->he = HCE_HTTP_CODE_OK;
 		host->up = HOST_UP;
+	}
 	return (!(host->up == HOST_UP));
 }
 
@@ -365,6 +365,8 @@ check_http_digest(struct ctl_tcp_event *cte)
 
 	head = cte->buf->buf;
 	host = cte->host;
+	host->he = HCE_HTTP_DIGEST_ERROR;
+
 	if ((head = strstr(head, "\r\n\r\n")) == NULL) {
 		log_debug("check_http_digest: %s failed "
 		    "(no end of headers)", host->conf.name);
@@ -378,8 +380,11 @@ check_http_digest(struct ctl_tcp_event *cte)
 	if (strcmp(cte->table->conf.digest, digest)) {
 		log_warnx("check_http_digest: %s failed "
 		    "(wrong digest)", host->conf.name);
+		host->he = HCE_HTTP_DIGEST_FAIL;
 		host->up = HOST_DOWN;
-	} else
+	} else {
+		host->he = HCE_HTTP_DIGEST_OK;
 		host->up = HOST_UP;
+	}
 	return (!(host->up == HOST_UP));
 }
diff --git a/usr.sbin/relayd/hce.c b/usr.sbin/relayd/hce.c
index a851c1caf4d..2f2dca574c6 100644
--- a/usr.sbin/relayd/hce.c
+++ b/usr.sbin/relayd/hce.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: hce.c,v 1.45 2008/09/29 15:12:22 reyk Exp $	*/
+/*	$OpenBSD: hce.c,v 1.46 2008/12/05 16:37:55 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -192,6 +192,7 @@ hce_disable_events(void)
 	evtimer_del(&env->sc_ev);
 	TAILQ_FOREACH(table, env->sc_tables, entry) {
 		TAILQ_FOREACH(host, &table->hosts, entry) {
+			host->he = HCE_ABORT;
 			event_del(&host->cte.ev);
 			close(host->cte.s);
 		}
@@ -219,6 +220,8 @@ hce_launch_checks(int fd, short event, void *arg)
 	imsg_compose(ibuf_pfe, IMSG_SYNC, 0, 0, -1, NULL, 0);
 	TAILQ_FOREACH(table, env->sc_tables, entry) {
 		TAILQ_FOREACH(host, &table->hosts, entry) {
+			if ((host->flags & F_CHECK_DONE) == 0)
+				host->he = HCE_INTERVAL_TIMEOUT;
 			host->flags &= ~(F_CHECK_SENT|F_CHECK_DONE);
 			event_del(&host->cte.ev);
 		}
@@ -269,7 +272,7 @@ hce_launch_checks(int fd, short event, void *arg)
 }
 
 void
-hce_notify_done(struct host *host, const char *msg)
+hce_notify_done(struct host *host, enum host_error he)
 {
 	struct table		*table;
 	struct ctl_status	 st;
@@ -278,8 +281,10 @@ hce_notify_done(struct host *host, const char *msg)
 	u_int			 logopt;
 	struct host		*h;
 	int			 hostup;
+	const char		*msg;
 
 	hostup = host->up;
+	host->he = he;
 
 	if (host->up == HOST_DOWN && host->retry_cnt) {
 		log_debug("hce_notify_done: host %s retry %d",
@@ -297,7 +302,9 @@ hce_notify_done(struct host *host, const char *msg)
 	st.up = host->up;
 	st.check_cnt = host->check_cnt;
 	st.retry_cnt = host->retry_cnt;
+	st.he = he;
 	host->flags |= (F_CHECK_SENT|F_CHECK_DONE);
+	msg = host_error(he);
 	if (msg)
 		log_debug("hce_notify_done: %s (%s)", host->conf.name, msg);
 
@@ -338,7 +345,7 @@ hce_notify_done(struct host *host, const char *msg)
 	/* Notify for all other hosts that inherit the state from this one */
 	SLIST_FOREACH(h, &host->children, child) {
 		h->up = hostup;
-		hce_notify_done(h, msg);
+		hce_notify_done(h, he);
 	}
 }
 
@@ -395,6 +402,7 @@ hce_dispatch_imsg(int fd, short event, void *ptr)
 			host->up = HOST_UNKNOWN;
 			host->check_cnt = 0;
 			host->up_cnt = 0;
+			host->he = HCE_NONE;
 			break;
 		case IMSG_HOST_ENABLE:
 			memcpy(&id, imsg.data, sizeof(id));
@@ -402,6 +410,7 @@ hce_dispatch_imsg(int fd, short event, void *ptr)
 				fatalx("hce_dispatch_imsg: desynchronized");
 			host->flags &= ~(F_DISABLE);
 			host->up = HOST_UNKNOWN;
+			host->he = HCE_NONE;
 			break;
 		case IMSG_TABLE_DISABLE:
 			memcpy(&id, imsg.data, sizeof(id));
diff --git a/usr.sbin/relayd/log.c b/usr.sbin/relayd/log.c
index b5cf7e80cfc..411d08c442d 100644
--- a/usr.sbin/relayd/log.c
+++ b/usr.sbin/relayd/log.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: log.c,v 1.13 2008/07/18 12:26:52 reyk Exp $	*/
+/*	$OpenBSD: log.c,v 1.14 2008/12/05 16:37:55 reyk Exp $	*/
 
 /*
  * Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
@@ -167,6 +167,111 @@ fatalx(const char *emsg)
 }
 
 const char *
+host_error(enum host_error he)
+{
+	switch (he) {
+	case HCE_NONE:
+		return ("none");
+		break;
+	case HCE_ABORT:
+		return ("aborted");
+		break;
+	case HCE_INTERVAL_TIMEOUT:
+		return ("interval timeout");
+		break;
+	case HCE_ICMP_OK:
+		return ("icmp ok");
+		break;
+	case HCE_ICMP_READ_TIMEOUT:
+		return ("icmp read timeout");
+		break;
+	case HCE_ICMP_WRITE_TIMEOUT:
+		return ("icmp write timeout");
+		break;
+	case HCE_TCP_CONNECT_ERROR:
+		return ("tcp connect error");
+		break;
+	case HCE_TCP_CONNECT_FAIL:
+		return ("tcp connect failed");
+		break;
+	case HCE_TCP_CONNECT_TIMEOUT:
+		return ("tcp connect timeout");
+		break;
+	case HCE_TCP_CONNECT_OK:
+		return ("tcp connect ok");
+		break;
+	case HCE_TCP_WRITE_TIMEOUT:
+		return ("tcp write timeout");
+		break;
+	case HCE_TCP_WRITE_FAIL:
+		return ("tcp write failed");
+		break;
+	case HCE_TCP_READ_TIMEOUT:
+		return ("tcp read timeout");
+		break;
+	case HCE_TCP_READ_FAIL:
+		return ("tcp read failed");
+		break;
+	case HCE_SCRIPT_OK:
+		return ("script ok");
+		break;
+	case HCE_SCRIPT_FAIL:
+		return ("script failed");
+		break;
+	case HCE_SSL_CONNECT_OK:
+		return ("ssl connect ok");
+		break;
+	case HCE_SSL_CONNECT_FAIL:
+		return ("ssl connect failed");
+		break;
+	case HCE_SSL_CONNECT_TIMEOUT:
+		return ("ssl connect timeout");
+		break;
+	case HCE_SSL_CONNECT_ERROR:
+		return ("ssl connect error");
+		break;
+	case HCE_SSL_READ_TIMEOUT:
+		return ("ssl read timeout");
+		break;
+	case HCE_SSL_WRITE_TIMEOUT:
+		return ("ssl write timeout");
+		break;
+	case HCE_SSL_READ_ERROR:
+		return ("ssl read error");
+		break;
+	case HCE_SSL_WRITE_ERROR:
+		return ("ssl write error");
+		break;
+	case HCE_SEND_EXPECT_FAIL:
+		return ("send/expect failed");
+		break;
+	case HCE_SEND_EXPECT_OK:
+		return ("send/expect ok");
+		break;
+	case HCE_HTTP_CODE_ERROR:
+		return ("http code malformed");
+		break;
+	case HCE_HTTP_CODE_FAIL:
+		return ("http code mismatch");
+		break;
+	case HCE_HTTP_CODE_OK:
+		return ("http code ok");
+		break;
+	case HCE_HTTP_DIGEST_ERROR:
+		return ("http digest malformed");
+		break;
+	case HCE_HTTP_DIGEST_FAIL:
+		return ("http digest mismatch");
+		break;
+	case HCE_HTTP_DIGEST_OK:
+		return ("http digest ok");
+		break;
+	}
+	/* NOTREACHED */
+	return ("invalid");
+}
+
+const char *
 host_status(enum host_status status)
 {
 	switch (status) {
diff --git a/usr.sbin/relayd/pfe.c b/usr.sbin/relayd/pfe.c
index 12acfbf05b0..a3ba407ba01 100644
--- a/usr.sbin/relayd/pfe.c
+++ b/usr.sbin/relayd/pfe.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pfe.c,v 1.52 2008/09/03 13:41:49 jsg Exp $	*/
+/*	$OpenBSD: pfe.c,v 1.53 2008/12/05 16:37:55 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -263,6 +263,7 @@ pfe_dispatch_imsg(int fd, short event, void *ptr)
 			memcpy(&st, imsg.data, sizeof(st));
 			if ((host = host_find(env, st.id)) == NULL)
 				fatalx("pfe_dispatch_imsg: invalid host id");
+			host->he = st.he;
 			if (host->flags & F_DISABLE)
 				break;
 			host->retry_cnt = st.retry_cnt;
diff --git a/usr.sbin/relayd/relayd.conf.5 b/usr.sbin/relayd/relayd.conf.5
index 2e9047edcc9..3b88ad485dc 100644
--- a/usr.sbin/relayd/relayd.conf.5
+++ b/usr.sbin/relayd/relayd.conf.5
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: relayd.conf.5,v 1.98 2008/11/09 12:34:47 tobias Exp $
+.\"	$OpenBSD: relayd.conf.5,v 1.99 2008/12/05 16:37:56 reyk Exp $
 .\"
 .\" Copyright (c) 2006, 2007 Reyk Floeter <reyk@openbsd.org>
 .\" Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: November 9 2008 $
+.Dd $Mdocdate: December 5 2008 $
 .Dt RELAYD.CONF 5
 .Os
 .Sh NAME
@@ -248,6 +248,16 @@ If
 is specified, it is used as the
 .Dq Host:
 header to query a specific hostname at the target host.
+To validate the HTTP return code, use this shell command:
+.Bd -literal -offset indent
+$ echo -n "HEAD <path> HTTP/1.0\er\en\er\en" | \e
+	nc <host> <port> | head -n1
+.Ed
+.Pp
+This prints the status header including the actual return code:
+.Bd -literal -offset indent
+HTTP/1.1 200 OK
+.Ed
 .It Xo
 .Ic check https Ar path
 .Op Ic host Ar hostname
@@ -280,8 +290,7 @@ To compute the digest, use this simple command:
 $ ftp -o - http://host[:port]/path | sha1
 .Ed
 .Pp
-This gives a digest
-that can be used as-is in a digest statement:
+This gives a digest that can be used as-is in a digest statement:
 .Bd -literal -offset indent
 a9993e36476816aba3e25717850c26c9cd0d89d
 .Ed
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index 8c381aed172..2eb76fc4e75 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.h,v 1.112 2008/09/29 14:53:36 reyk Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.113 2008/12/05 16:37:56 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -186,6 +186,7 @@ struct ctl_status {
 	int		 up;
 	int		 retry_cnt;
 	u_long		 check_cnt;
+	u_int16_t	 he;
 };
 
 struct ctl_id {
@@ -372,10 +373,46 @@ struct host {
 	u_long			 up_cnt;
 	int			 retry_cnt;
 	int			 idx;
+	u_int16_t		 he;
 	struct ctl_tcp_event	 cte;
 };
 TAILQ_HEAD(hostlist, host);
 
+enum host_error {
+	HCE_NONE		= 0,
+	HCE_ABORT,
+	HCE_INTERVAL_TIMEOUT,
+	HCE_ICMP_OK,
+	HCE_ICMP_READ_TIMEOUT,
+	HCE_ICMP_WRITE_TIMEOUT,
+	HCE_TCP_CONNECT_ERROR,
+	HCE_TCP_CONNECT_FAIL,
+	HCE_TCP_CONNECT_TIMEOUT,
+	HCE_TCP_CONNECT_OK,
+	HCE_TCP_WRITE_TIMEOUT,
+	HCE_TCP_WRITE_FAIL,
+	HCE_TCP_READ_TIMEOUT,
+	HCE_TCP_READ_FAIL,
+	HCE_SCRIPT_OK,
+	HCE_SCRIPT_FAIL,
+	HCE_SSL_CONNECT_ERROR,
+	HCE_SSL_CONNECT_FAIL,
+	HCE_SSL_CONNECT_OK,
+	HCE_SSL_CONNECT_TIMEOUT,
+	HCE_SSL_READ_TIMEOUT,
+	HCE_SSL_WRITE_TIMEOUT,
+	HCE_SSL_READ_ERROR,
+	HCE_SSL_WRITE_ERROR,
+	HCE_SEND_EXPECT_FAIL,
+	HCE_SEND_EXPECT_OK,
+	HCE_HTTP_CODE_ERROR,
+	HCE_HTTP_CODE_FAIL,
+	HCE_HTTP_CODE_OK,
+	HCE_HTTP_DIGEST_ERROR,
+	HCE_HTTP_DIGEST_FAIL,
+	HCE_HTTP_DIGEST_OK,
+};
+
 enum host_status {
 	HOST_DOWN	= -1,
 	HOST_UNKNOWN	= 0,
@@ -734,6 +771,7 @@ void	log_info(const char *, ...);
 void	log_debug(const char *, ...);
 __dead void fatal(const char *);
 __dead void fatalx(const char *);
+const char *host_error(enum host_error);
 const char *host_status(enum host_status);
 const char *table_check(enum table_check);
 const char *print_availability(u_long, u_long);
@@ -796,7 +834,7 @@ u_int64_t
 /* hce.c */
 pid_t	 hce(struct relayd *, int [2], int [2], int [RELAY_MAXPROC][2],
 	    int [2], int [RELAY_MAXPROC][2]);
-void	 hce_notify_done(struct host *, const char *);
+void	 hce_notify_done(struct host *, enum host_error);
 
 /* relay.c */
 pid_t	 relay(struct relayd *, int [2], int [2], int [RELAY_MAXPROC][2],
diff --git a/usr.sbin/relayd/ssl.c b/usr.sbin/relayd/ssl.c
index 93e9d1fbd7b..a8523b5f55b 100644
--- a/usr.sbin/relayd/ssl.c
+++ b/usr.sbin/relayd/ssl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ssl.c,v 1.13 2007/12/07 17:17:01 reyk Exp $	*/
+/*	$OpenBSD: ssl.c,v 1.14 2008/12/05 16:37:56 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -54,7 +54,7 @@ ssl_read(int s, short event, void *arg)
 	if (event == EV_TIMEOUT) {
 		cte->host->up = HOST_DOWN;
 		ssl_cleanup(cte);
-		hce_notify_done(cte->host, "ssl_read: timeout");
+		hce_notify_done(cte->host, HCE_SSL_READ_TIMEOUT);
 		return;
 	}
 
@@ -78,12 +78,7 @@ ssl_read(int s, short event, void *arg)
 				cte->host->up = HOST_DOWN;
 				(void)cte->validate_close(cte);
 				ssl_cleanup(cte);
-				if (cte->host->up == HOST_UP)
-					hce_notify_done(cte->host,
-					    "ssl_read: check succeeded");
-				else
-					hce_notify_done(cte->host,
-					    "ssl_read: check failed");
+				hce_notify_done(cte->host, cte->host->he);
 				return;
 			}
 			/* FALLTHROUGH */
@@ -91,7 +86,7 @@ ssl_read(int s, short event, void *arg)
 			cte->host->up = HOST_DOWN;
 			ssl_error(cte->host->conf.name, "cannot read");
 			ssl_cleanup(cte);
-			hce_notify_done(cte->host, "ssl_read: SSL error");
+			hce_notify_done(cte->host, HCE_SSL_READ_ERROR);
 			break;
 		}
 		return;
@@ -103,10 +98,7 @@ ssl_read(int s, short event, void *arg)
 			goto retry;
 
 		ssl_cleanup(cte);
-		if (cte->host->up == HOST_UP)
-			hce_notify_done(cte->host, "ssl_read: check succeeded");
-		else
-			hce_notify_done(cte->host, "ssl_read: check failed");
+		hce_notify_done(cte->host, cte->host->he);
 		return;
 	}
 
@@ -128,7 +120,7 @@ ssl_write(int s, short event, void *arg)
 	if (event == EV_TIMEOUT) {
 		cte->host->up = HOST_DOWN;
 		ssl_cleanup(cte);
-		hce_notify_done(cte->host, "ssl_write: timeout");
+		hce_notify_done(cte->host, HCE_SSL_WRITE_TIMEOUT);
 		return;
 	}
 
@@ -149,7 +141,7 @@ ssl_write(int s, short event, void *arg)
 			cte->host->up = HOST_DOWN;
 			ssl_error(cte->host->conf.name, "cannot write");
 			ssl_cleanup(cte);
-			hce_notify_done(cte->host, "ssl_write: SSL error");
+			hce_notify_done(cte->host, HCE_SSL_WRITE_ERROR);
 			return;
 		}
 	}
@@ -174,7 +166,7 @@ ssl_connect(int s, short event, void *arg)
 
 	if (event == EV_TIMEOUT) {
 		cte->host->up = HOST_DOWN;
-		hce_notify_done(cte->host, "ssl_connect: timeout");
+		hce_notify_done(cte->host, HCE_SSL_CONNECT_TIMEOUT);
 		ssl_cleanup(cte);
 		return;
 	}
@@ -194,7 +186,7 @@ ssl_connect(int s, short event, void *arg)
 		default:
 			cte->host->up = HOST_DOWN;
 			ssl_error(cte->host->conf.name, "cannot connect");
-			hce_notify_done(cte->host, "ssl_connect: SSL error");
+			hce_notify_done(cte->host, HCE_SSL_CONNECT_FAIL);
 			ssl_cleanup(cte);
 			return;
 		}
@@ -202,7 +194,7 @@ ssl_connect(int s, short event, void *arg)
 
 	if (cte->table->conf.check == CHECK_TCP) {
 		cte->host->up = HOST_UP;
-		hce_notify_done(cte->host, "ssl_connect: connect successful");
+		hce_notify_done(cte->host, HCE_SSL_CONNECT_OK);
 		ssl_cleanup(cte);
 		return;
 	}
@@ -272,8 +264,7 @@ ssl_transaction(struct ctl_tcp_event *cte)
 		cte->host->up = HOST_UNKNOWN;
 		ssl_error(cte->host->conf.name, "cannot set fd");
 		ssl_cleanup(cte);
-		hce_notify_done(cte->host,
-		    "ssl_transaction: cannot set SSL fd");
+		hce_notify_done(cte->host, HCE_SSL_CONNECT_ERROR);
 		return;
 	}
 	SSL_set_connect_state(cte->ssl);