Provide record layer overhead for DTLS.

Rather than manually calculating the maximum record layer overhead in the
DTLS code, have the record layer provide this information. This also makes
it work correctly with AEAD ciphersuites.

ok inoguchi@ tb@

3 files changed, 38 insertions, 17 deletions

diff --git a/lib/libssl/d1_both.c b/lib/libssl/d1_both.c
index ae5ebfacb46..d6bf6dfd1b8 100644
--- a/lib/libssl/d1_both.c
+++ b/lib/libssl/d1_both.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: d1_both.c,v 1.63 2020/12/05 19:34:57 tb Exp $ */
+/* $OpenBSD: d1_both.c,v 1.64 2021/01/19 18:51:08 jsing Exp $ */
 /*
  * DTLS implementation written by Nagendra Modadugu
  * (nagendra@cs.stanford.edu) for the OpenSSL project 2005.
@@ -218,7 +218,8 @@ dtls1_do_write(SSL *s, int type)
 {
 	int ret;
 	int curr_mtu;
-	unsigned int len, frag_off, mac_size, blocksize;
+	unsigned int len, frag_off;
+	size_t overhead;
 
 	/* AHA!  Figure out the MTU, and stick to the right size */
 	if (D1I(s)->mtu < dtls1_min_mtu() &&
@@ -246,21 +247,13 @@ dtls1_do_write(SSL *s, int type)
 		OPENSSL_assert(s->internal->init_num ==
 		    (int)D1I(s)->w_msg_hdr.msg_len + DTLS1_HM_HEADER_LENGTH);
 
-	if (s->internal->write_hash)
-		mac_size = EVP_MD_CTX_size(s->internal->write_hash);
-	else
-		mac_size = 0;
-
-	if (s->internal->enc_write_ctx &&
-	    (EVP_CIPHER_mode( s->internal->enc_write_ctx->cipher) & EVP_CIPH_CBC_MODE))
-		blocksize = 2 * EVP_CIPHER_block_size(s->internal->enc_write_ctx->cipher);
-	else
-		blocksize = 0;
+	if (!tls12_record_layer_write_overhead(s->internal->rl, &overhead))
+		return -1;
 
 	frag_off = 0;
 	while (s->internal->init_num) {
 		curr_mtu = D1I(s)->mtu - BIO_wpending(SSL_get_wbio(s)) -
-		    DTLS1_RT_HEADER_LENGTH - mac_size - blocksize;
+		    DTLS1_RT_HEADER_LENGTH - overhead;
 
 		if (curr_mtu <= DTLS1_HM_HEADER_LENGTH) {
 			/* grr.. we could get an error if MTU picked was wrong */
@@ -268,7 +261,7 @@ dtls1_do_write(SSL *s, int type)
 			if (ret <= 0)
 				return ret;
 			curr_mtu = D1I(s)->mtu - DTLS1_RT_HEADER_LENGTH -
-			    mac_size - blocksize;
+			    overhead;
 		}
 
 		if (s->internal->init_num > curr_mtu)
@@ -276,7 +269,6 @@ dtls1_do_write(SSL *s, int type)
 		else
 			len = s->internal->init_num;
 
-
 		/* XDTLS: this function is too long.  split out the CCS part */
 		if (type == SSL3_RT_HANDSHAKE) {
 			if (s->internal->init_off != 0) {
diff --git a/lib/libssl/ssl_locl.h b/lib/libssl/ssl_locl.h
index 560fcdc1a4a..e09f6681219 100644
--- a/lib/libssl/ssl_locl.h
+++ b/lib/libssl/ssl_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_locl.h,v 1.312 2021/01/13 18:20:54 jsing Exp $ */
+/* $OpenBSD: ssl_locl.h,v 1.313 2021/01/19 18:51:08 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -477,6 +477,8 @@ struct tls12_record_layer *tls12_record_layer_new(void);
 void tls12_record_layer_free(struct tls12_record_layer *rl);
 void tls12_record_layer_alert(struct tls12_record_layer *rl,
     uint8_t *alert_desc);
+int tls12_record_layer_write_overhead(struct tls12_record_layer *rl,
+    size_t *overhead);
 void tls12_record_layer_set_version(struct tls12_record_layer *rl,
     uint16_t version);
 void tls12_record_layer_set_write_epoch(struct tls12_record_layer *rl,
diff --git a/lib/libssl/tls12_record_layer.c b/lib/libssl/tls12_record_layer.c
index 04699f9a834..7fa31707d3b 100644
--- a/lib/libssl/tls12_record_layer.c
+++ b/lib/libssl/tls12_record_layer.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls12_record_layer.c,v 1.10 2021/01/19 18:34:02 jsing Exp $ */
+/* $OpenBSD: tls12_record_layer.c,v 1.11 2021/01/19 18:51:08 jsing Exp $ */
 /*
  * Copyright (c) 2020 Joel Sing <jsing@openbsd.org>
  *
@@ -168,6 +168,33 @@ tls12_record_layer_alert(struct tls12_record_layer *rl, uint8_t *alert_desc)
 	*alert_desc = rl->alert_desc;
 }
 
+int
+tls12_record_layer_write_overhead(struct tls12_record_layer *rl,
+    size_t *overhead)
+{
+	size_t block_size, eiv_len, mac_len;
+
+	*overhead = 0;
+
+	if (rl->write->aead_ctx != NULL) {
+		*overhead = rl->write->aead_ctx->tag_len;
+	} else if (rl->write->cipher_ctx != NULL) {
+		eiv_len = 0;
+		if (rl->version != TLS1_VERSION) {
+			if (!tls12_record_protection_eiv_len(rl->write, &eiv_len))
+				return 0;
+		}
+		if (!tls12_record_protection_block_size(rl->write, &block_size))
+			return 0;
+		if (!tls12_record_protection_mac_len(rl->write, &mac_len))
+			return 0;
+
+		*overhead = eiv_len + block_size + mac_len;
+	}
+
+	return 1;
+}
+
 void
 tls12_record_layer_set_version(struct tls12_record_layer *rl, uint16_t version)
 {