When syslogd is writing over TLS, the error "SSL3_WRITE_PENDING:bad

write retry" may occur.  Unfortunately libtls tls_write() has
inherited the strange semantics regarding partial writes and buffer
movement from SSL_write().  This will be investigated after unlock,
the goal is to have the behavior of write(2) in libtls.

For now add a workaround in syslogd.  If tls_write() indicates that
it needs a read or write again, stop modifying the output buffer.
Instead drop and count the syslog messages.  After writing over TLS
was successful, continue to queue the messages.  This solution has
minimum inpact and will be improved after 5.7 release.

discussed with tedu@ reyk@ jsing@;  OK tedu@

3 files changed, 23 insertions, 4 deletions

diff --git a/usr.sbin/syslogd/evbuffer_tls.c b/usr.sbin/syslogd/evbuffer_tls.c
index 09913c71d10..31b3f4399de 100644
--- a/usr.sbin/syslogd/evbuffer_tls.c
+++ b/usr.sbin/syslogd/evbuffer_tls.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: evbuffer_tls.c,v 1.2 2015/01/30 14:00:55 bluhm Exp $ */
+/*	$OpenBSD: evbuffer_tls.c,v 1.3 2015/02/20 00:56:32 bluhm Exp $ */
 
 /*
  * Copyright (c) 2002-2004 Niels Provos <provos@citi.umich.edu>
@@ -186,6 +186,7 @@ buffertls_writecb(int fd, short event, void *arg)
 		if (res <= 0)
 			goto error;
 	}
+	buftls->bt_flags &= ~BT_WRITE_AGAIN;
 
 	event_set(&bufev->ev_write, fd, EV_WRITE, buffertls_writecb, buftls);
 	if (EVBUFFER_LENGTH(bufev->output) != 0)
@@ -202,6 +203,7 @@ buffertls_writecb(int fd, short event, void *arg)
 	return;
 
  reschedule:
+	buftls->bt_flags |= BT_WRITE_AGAIN;
 	if (EVBUFFER_LENGTH(bufev->output) != 0)
 		bufferevent_add(&bufev->ev_write, bufev->timeout_write);
 	return;
@@ -276,6 +278,7 @@ buffertls_set(struct buffertls *buftls, struct bufferevent *bufev,
 	event_set(&bufev->ev_write, fd, EV_WRITE, buffertls_writecb, buftls);
 	buftls->bt_bufev = bufev;
 	buftls->bt_ctx = ctx;
+	buftls->bt_flags = 0;
 }
 
 void
diff --git a/usr.sbin/syslogd/evbuffer_tls.h b/usr.sbin/syslogd/evbuffer_tls.h
index ba81f80c322..e28ed53cc6b 100644
--- a/usr.sbin/syslogd/evbuffer_tls.h
+++ b/usr.sbin/syslogd/evbuffer_tls.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: evbuffer_tls.h,v 1.1 2015/01/18 19:37:59 bluhm Exp $ */
+/*	$OpenBSD: evbuffer_tls.h,v 1.2 2015/02/20 00:56:32 bluhm Exp $ */
 
 /*
  * Copyright (c) 2014-2015 Alexander Bluhm <bluhm@openbsd.org>
@@ -28,6 +28,8 @@ struct buffertls {
 	struct bufferevent	*bt_bufev;
 	struct tls		*bt_ctx;
 	const char		*bt_hostname;
+	int			 bt_flags;
+#define BT_WRITE_AGAIN		0x1
 };
 
 void	buffertls_set(struct buffertls *, struct bufferevent *, struct tls *,
diff --git a/usr.sbin/syslogd/syslogd.c b/usr.sbin/syslogd/syslogd.c
index 32f4d1a3006..d381c1402bd 100644
--- a/usr.sbin/syslogd/syslogd.c
+++ b/usr.sbin/syslogd/syslogd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syslogd.c,v 1.156 2015/02/14 09:02:15 bluhm Exp $	*/
+/*	$OpenBSD: syslogd.c,v 1.157 2015/02/20 00:56:32 bluhm Exp $	*/
 
 /*
  * Copyright (c) 1983, 1988, 1993, 1994
@@ -1265,8 +1265,22 @@ fprintlog(struct filed *f, int flags, char *msg)
 		}
 		break;
 
-	case F_FORWTCP:
 	case F_FORWTLS:
+		if (f->f_un.f_forw.f_buftls.bt_flags & BT_WRITE_AGAIN) {
+			/*
+			 * After an OpenSSL SSL_ERROR_WANT_WRITE you must not
+			 * modify the buffer pointer or length until the next
+			 * successful write.  Otherwise there will be an
+			 * error SSL3_WRITE_PENDING:bad write retry.
+			 * XXX This should be handled in the buffertls layer.
+			 */
+			dprintf(" %s (dropped tls write again)\n",
+			    f->f_un.f_forw.f_loghost);
+			f->f_un.f_forw.f_dropped++;
+			break;
+		}
+		/* FALLTHROUGH */
+	case F_FORWTCP:
 		dprintf(" %s", f->f_un.f_forw.f_loghost);
 		if (EVBUFFER_LENGTH(f->f_un.f_forw.f_bufev->output) >=
 		    MAX_TCPBUF) {