Implement flushing for TLSv1.3 handshakes.

When we finish sending a flight of records, flush the record layer output.
This effectively means calling BIO_flush() on the wbio.

Some things (such as apache2) have custom BIOs that perform buffering and
do not actually send on BIO_write(). Without BIO_flush() the server thinks
it has sent data and starts receiving records, however the client never
sends records since it never received those that the server should have
sent.

Joint work with tb@

ok tb@

5 files changed, 69 insertions, 7 deletions

diff --git a/lib/libssl/tls13_handshake.c b/lib/libssl/tls13_handshake.c
index 310a2116b87..cca8560fc27 100644
--- a/lib/libssl/tls13_handshake.c
+++ b/lib/libssl/tls13_handshake.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tls13_handshake.c,v 1.69 2021/07/01 17:53:39 jsing Exp $	*/
+/*	$OpenBSD: tls13_handshake.c,v 1.70 2021/09/16 19:25:30 jsing Exp $	*/
 /*
  * Copyright (c) 2018-2021 Theo Buehler <tb@openbsd.org>
  * Copyright (c) 2019 Joel Sing <jsing@openbsd.org>
@@ -331,6 +331,18 @@ tls13_handshake_advance_state_machine(struct tls13_ctx *ctx)
 	return 1;
 }
 
+static int
+tls13_handshake_end_of_flight(struct tls13_ctx *ctx,
+    const struct tls13_handshake_action *previous)
+{
+	const struct tls13_handshake_action *current;
+
+	if ((current = tls13_handshake_active_action(ctx)) == NULL)
+		return 1;
+
+	return current->sender != previous->sender;
+}
+
 int
 tls13_handshake_msg_record(struct tls13_ctx *ctx)
 {
@@ -344,6 +356,7 @@ int
 tls13_handshake_perform(struct tls13_ctx *ctx)
 {
 	const struct tls13_handshake_action *action;
+	int sending;
 	int ret;
 
 	if (!ctx->handshake_started) {
@@ -367,6 +380,13 @@ tls13_handshake_perform(struct tls13_ctx *ctx)
 		if ((action = tls13_handshake_active_action(ctx)) == NULL)
 			return TLS13_IO_FAILURE;
 
+		if (ctx->need_flush) {
+			if ((ret = tls13_record_layer_flush(ctx->rl)) !=
+			    TLS13_IO_SUCCESS)
+				return ret;
+			ctx->need_flush = 0;
+		}
+
 		if (action->handshake_complete) {
 			ctx->handshake_completed = 1;
 			tls13_record_layer_handshake_completed(ctx->rl);
@@ -379,14 +399,16 @@ tls13_handshake_perform(struct tls13_ctx *ctx)
 			return TLS13_IO_SUCCESS;
 		}
 
+		sending = action->sender == ctx->mode;
+
 		DEBUGF("%s %s %s\n", tls13_handshake_mode_name(ctx->mode),
-		    (action->sender == ctx->mode) ? "sending" : "receiving",
+		    sending ? "sending" : "receiving",
 		    tls13_handshake_message_name(action->handshake_type));
 
 		if (ctx->alert != 0)
 			return tls13_send_alert(ctx->rl, ctx->alert);
 
-		if (action->sender == ctx->mode)
+		if (sending)
 			ret = tls13_handshake_send_action(ctx, action);
 		else
 			ret = tls13_handshake_recv_action(ctx, action);
@@ -408,6 +430,10 @@ tls13_handshake_perform(struct tls13_ctx *ctx)
 		if (!tls13_handshake_advance_state_machine(ctx))
 			return TLS13_IO_FAILURE;
 
+		if (sending)
+			ctx->need_flush = tls13_handshake_end_of_flight(ctx,
+			    action);
+
 		if (!tls13_handshake_set_legacy_state(ctx))
 			return TLS13_IO_FAILURE;
 	}
diff --git a/lib/libssl/tls13_internal.h b/lib/libssl/tls13_internal.h
index cb59634edc4..20cb52ebdd1 100644
--- a/lib/libssl/tls13_internal.h
+++ b/lib/libssl/tls13_internal.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls13_internal.h,v 1.93 2021/09/14 14:35:09 tb Exp $ */
+/* $OpenBSD: tls13_internal.h,v 1.94 2021/09/16 19:25:30 jsing Exp $ */
 /*
  * Copyright (c) 2018 Bob Beck <beck@openbsd.org>
  * Copyright (c) 2018 Theo Buehler <tb@openbsd.org>
@@ -92,6 +92,7 @@ typedef void (*tls13_phh_sent_cb)(void *_cb_arg);
 typedef ssize_t (*tls13_read_cb)(void *_buf, size_t _buflen, void *_cb_arg);
 typedef ssize_t (*tls13_write_cb)(const void *_buf, size_t _buflen,
     void *_cb_arg);
+typedef ssize_t (*tls13_flush_cb)(void *_cb_arg);
 typedef void (*tls13_handshake_message_cb)(void *_cb_arg);
 typedef void (*tls13_info_cb)(void *_cb_arg, int _state, int _ret);
 typedef int (*tls13_ocsp_status_cb)(void *_cb_arg);
@@ -200,6 +201,7 @@ struct tls13_record_layer;
 struct tls13_record_layer_callbacks {
 	tls13_read_cb wire_read;
 	tls13_write_cb wire_write;
+	tls13_flush_cb wire_flush;
 	tls13_alert_cb alert_recv;
 	tls13_alert_cb alert_sent;
 	tls13_phh_recv_cb phh_recv;
@@ -226,6 +228,7 @@ int tls13_record_layer_set_write_traffic_key(struct tls13_record_layer *rl,
     struct tls13_secret *write_key);
 ssize_t tls13_record_layer_send_pending(struct tls13_record_layer *rl);
 ssize_t tls13_record_layer_phh(struct tls13_record_layer *rl, CBS *cbs);
+ssize_t tls13_record_layer_flush(struct tls13_record_layer *rl);
 
 ssize_t tls13_read_handshake_data(struct tls13_record_layer *rl, uint8_t *buf, size_t n);
 ssize_t tls13_write_handshake_data(struct tls13_record_layer *rl, const uint8_t *buf,
@@ -283,6 +286,7 @@ struct tls13_ctx {
 	struct tls13_handshake_stage handshake_stage;
 	int handshake_started;
 	int handshake_completed;
+	int need_flush;
 	int middlebox_compat;
 	int send_dummy_ccs;
 	int send_dummy_ccs_after;
@@ -328,6 +332,7 @@ int tls13_legacy_connect(SSL *ssl);
 int tls13_legacy_return_code(SSL *ssl, ssize_t ret);
 ssize_t tls13_legacy_wire_read_cb(void *buf, size_t n, void *arg);
 ssize_t tls13_legacy_wire_write_cb(const void *buf, size_t n, void *arg);
+ssize_t tls13_legacy_wire_flush_cb(void *arg);
 int tls13_legacy_pending(const SSL *ssl);
 int tls13_legacy_read_bytes(SSL *ssl, int type, unsigned char *buf, int len,
     int peek);
diff --git a/lib/libssl/tls13_legacy.c b/lib/libssl/tls13_legacy.c
index 3368600c60f..f668dd4ea36 100644
--- a/lib/libssl/tls13_legacy.c
+++ b/lib/libssl/tls13_legacy.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tls13_legacy.c,v 1.30 2021/09/14 14:31:21 tb Exp $ */
+/*	$OpenBSD: tls13_legacy.c,v 1.31 2021/09/16 19:25:30 jsing Exp $ */
 /*
  * Copyright (c) 2018, 2019 Joel Sing <jsing@openbsd.org>
  *
@@ -96,6 +96,30 @@ tls13_legacy_wire_write_cb(const void *buf, size_t n, void *arg)
 	return tls13_legacy_wire_write(ctx->ssl, buf, n);
 }
 
+static ssize_t
+tls13_legacy_wire_flush(SSL *ssl)
+{
+	if (BIO_flush(ssl->wbio) <= 0) {
+		if (BIO_should_write(ssl->wbio))
+			return TLS13_IO_WANT_POLLOUT;
+
+		if (ERR_peek_error() == 0 && errno != 0)
+			SYSerror(errno);
+
+		return TLS13_IO_FAILURE;
+	}
+
+	return TLS13_IO_SUCCESS;
+}
+
+ssize_t
+tls13_legacy_wire_flush_cb(void *arg)
+{
+	struct tls13_ctx *ctx = arg;
+
+	return tls13_legacy_wire_flush(ctx->ssl);
+}
+
 static void
 tls13_legacy_error(SSL *ssl)
 {
diff --git a/lib/libssl/tls13_lib.c b/lib/libssl/tls13_lib.c
index 6615efc4d2b..1a9596adca8 100644
--- a/lib/libssl/tls13_lib.c
+++ b/lib/libssl/tls13_lib.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tls13_lib.c,v 1.61 2021/09/02 11:58:30 beck Exp $ */
+/*	$OpenBSD: tls13_lib.c,v 1.62 2021/09/16 19:25:30 jsing Exp $ */
 /*
  * Copyright (c) 2018, 2019 Joel Sing <jsing@openbsd.org>
  * Copyright (c) 2019 Bob Beck <beck@openbsd.org>
@@ -374,6 +374,7 @@ tls13_phh_received_cb(void *cb_arg, CBS *cbs)
 static const struct tls13_record_layer_callbacks rl_callbacks = {
 	.wire_read = tls13_legacy_wire_read_cb,
 	.wire_write = tls13_legacy_wire_write_cb,
+	.wire_flush = tls13_legacy_wire_flush_cb,
 	.alert_recv = tls13_alert_received_cb,
 	.alert_sent = tls13_alert_sent_cb,
 	.phh_recv = tls13_phh_received_cb,
diff --git a/lib/libssl/tls13_record_layer.c b/lib/libssl/tls13_record_layer.c
index 2e32cb8a37c..6b9f5d14191 100644
--- a/lib/libssl/tls13_record_layer.c
+++ b/lib/libssl/tls13_record_layer.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls13_record_layer.c,v 1.63 2021/09/04 16:26:12 jsing Exp $ */
+/* $OpenBSD: tls13_record_layer.c,v 1.64 2021/09/16 19:25:30 jsing Exp $ */
 /*
  * Copyright (c) 2018, 2019 Joel Sing <jsing@openbsd.org>
  *
@@ -1096,6 +1096,12 @@ tls13_record_layer_write(struct tls13_record_layer *rl, uint8_t content_type,
 	return ret;
 }
 
+ssize_t
+tls13_record_layer_flush(struct tls13_record_layer *rl)
+{
+	return rl->cb.wire_flush(rl->cb_arg);
+}
+
 static const uint8_t tls13_dummy_ccs[] = { 0x01 };
 
 ssize_t