Fix two bugs in IPsec/HMAC-SHA2:

(1) use correct (message) block size of 128 byte (instead of 64
    bytes) for HMAC-SHA512/384 (RFC4634).
(2) RFC4868 specifies that HMAC-SHA-{256,384,512} is truncated to
    nnn/2 bits, while we still use 96 bits. 96 bits have been
    specified in draft-ietf-ipsec-ciph-sha-256-00 while
    draft-ietf-ipsec-ciph-sha-256-01 changed it to 128 bits.

WARNING: this change makes IPsec with SHA-256 (the default)
incompatible with older OpenBSD versions and other IPsec-implementations
that share this bug.

ok+tests naddy, fries; requested by reyk/deraadt

2 files changed, 14 insertions, 14 deletions

diff --git a/sys/arch/i386/i386/via.c b/sys/arch/i386/i386/via.c
index df4d40547d5..16d3d5388c8 100644
--- a/sys/arch/i386/i386/via.c
+++ b/sys/arch/i386/i386/via.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: via.c,v 1.19 2008/06/09 07:07:15 djm Exp $	*/
+/*	$OpenBSD: via.c,v 1.20 2010/01/10 12:43:07 markus Exp $	*/
 /*	$NetBSD: machdep.c,v 1.214 1996/11/10 03:16:17 thorpej Exp $	*/
 
 /*-
@@ -89,8 +89,8 @@ struct viac3_softc {
 static struct viac3_softc *vc3_sc;
 extern int i386_has_xcrypt;
 
-extern const u_int8_t hmac_ipad_buffer[64];
-extern const u_int8_t hmac_opad_buffer[64];
+extern const u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
+extern const u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
 
 void viac3_crypto_setup(void);
 int viac3_crypto_newsession(u_int32_t *, struct cryptoini *);
@@ -220,13 +220,13 @@ viac3_crypto_newsession(u_int32_t *sidp, struct cryptoini *cri)
 			axf = &auth_hash_hmac_ripemd_160_96;
 			goto authcommon;
 		case CRYPTO_SHA2_256_HMAC:
-			axf = &auth_hash_hmac_sha2_256_96;
+			axf = &auth_hash_hmac_sha2_256_128;
 			goto authcommon;
 		case CRYPTO_SHA2_384_HMAC:
-			axf = &auth_hash_hmac_sha2_384_96;
+			axf = &auth_hash_hmac_sha2_384_192;
 			goto authcommon;
 		case CRYPTO_SHA2_512_HMAC:
-			axf = &auth_hash_hmac_sha2_512_96;
+			axf = &auth_hash_hmac_sha2_512_256;
 		authcommon:
 			swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA,
 			    M_NOWAIT|M_ZERO);
@@ -256,7 +256,7 @@ viac3_crypto_newsession(u_int32_t *sidp, struct cryptoini *cri)
 			axf->Init(swd->sw_ictx);
 			axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8);
 			axf->Update(swd->sw_ictx, hmac_ipad_buffer,
-			    HMAC_BLOCK_LEN - (c->cri_klen / 8));
+			    axf->blocksize - (c->cri_klen / 8));
 
 			for (i = 0; i < c->cri_klen / 8; i++)
 				c->cri_key[i] ^= (HMAC_IPAD_VAL ^
@@ -265,7 +265,7 @@ viac3_crypto_newsession(u_int32_t *sidp, struct cryptoini *cri)
 			axf->Init(swd->sw_octx);
 			axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8);
 			axf->Update(swd->sw_octx, hmac_opad_buffer,
-			    HMAC_BLOCK_LEN - (c->cri_klen / 8));
+			    axf->blocksize - (c->cri_klen / 8));
 
 			for (i = 0; i < c->cri_klen / 8; i++)
 				c->cri_key[i] ^= HMAC_OPAD_VAL;
diff --git a/sys/arch/i386/pci/glxsb.c b/sys/arch/i386/pci/glxsb.c
index 4465fd73971..094483f1fbf 100644
--- a/sys/arch/i386/pci/glxsb.c
+++ b/sys/arch/i386/pci/glxsb.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: glxsb.c,v 1.17 2009/10/30 18:18:09 deraadt Exp $	*/
+/*	$OpenBSD: glxsb.c,v 1.18 2010/01/10 12:43:07 markus Exp $	*/
 
 /*
  * Copyright (c) 2006 Tom Cosgrove <tom@openbsd.org>
@@ -411,13 +411,13 @@ glxsb_crypto_newsession(uint32_t *sidp, struct cryptoini *cri)
 			axf = &auth_hash_hmac_ripemd_160_96;
 			goto authcommon;
 		case CRYPTO_SHA2_256_HMAC:
-			axf = &auth_hash_hmac_sha2_256_96;
+			axf = &auth_hash_hmac_sha2_256_128;
 			goto authcommon;
 		case CRYPTO_SHA2_384_HMAC:
-			axf = &auth_hash_hmac_sha2_384_96;
+			axf = &auth_hash_hmac_sha2_384_192;
 			goto authcommon;
 		case CRYPTO_SHA2_512_HMAC:
-			axf = &auth_hash_hmac_sha2_512_96;
+			axf = &auth_hash_hmac_sha2_512_256;
 		authcommon:
 			swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA,
 			    M_NOWAIT|M_ZERO);
@@ -447,7 +447,7 @@ glxsb_crypto_newsession(uint32_t *sidp, struct cryptoini *cri)
 			axf->Init(swd->sw_ictx);
 			axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8);
 			axf->Update(swd->sw_ictx, hmac_ipad_buffer,
-			    HMAC_BLOCK_LEN - (c->cri_klen / 8));
+			    axf->blocksize - (c->cri_klen / 8));
 
 			for (i = 0; i < c->cri_klen / 8; i++)
 				c->cri_key[i] ^= (HMAC_IPAD_VAL ^
@@ -456,7 +456,7 @@ glxsb_crypto_newsession(uint32_t *sidp, struct cryptoini *cri)
 			axf->Init(swd->sw_octx);
 			axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8);
 			axf->Update(swd->sw_octx, hmac_opad_buffer,
-			    HMAC_BLOCK_LEN - (c->cri_klen / 8));
+			    axf->blocksize - (c->cri_klen / 8));
 
 			for (i = 0; i < c->cri_klen / 8; i++)
 				c->cri_key[i] ^= HMAC_OPAD_VAL;