Allow the setkey function of a transform to fail, eg. when an insufficient

number of key bits is supplied.

Only AES and DES/3DES might fail.

ok and help markus@

5 files changed, 80 insertions, 35 deletions

diff --git a/sys/crypto/cryptosoft.c b/sys/crypto/cryptosoft.c
index f61cc6248c7..dfb2329ccc6 100644
--- a/sys/crypto/cryptosoft.c
+++ b/sys/crypto/cryptosoft.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cryptosoft.c,v 1.41 2004/05/07 14:42:26 millert Exp $	*/
+/*	$OpenBSD: cryptosoft.c,v 1.42 2004/12/20 20:31:18 hshoexer Exp $	*/
 
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
@@ -619,8 +619,11 @@ swcr_newsession(u_int32_t *sid, struct cryptoini *cri)
 			txf = &enc_xform_null;
 			goto enccommon;
 		enccommon:
-			txf->setkey(&((*swd)->sw_kschedule), cri->cri_key,
-			    cri->cri_klen / 8);
+			if (txf->setkey(&((*swd)->sw_kschedule), cri->cri_key,
+			    cri->cri_klen / 8) < 0) {
+				swcr_freesession(i);
+				return EINVAL;
+			}
 			(*swd)->sw_exf = txf;
 			break;
 
diff --git a/sys/crypto/rijndael.c b/sys/crypto/rijndael.c
index 47cbe4331af..e1dcb0dae64 100644
--- a/sys/crypto/rijndael.c
+++ b/sys/crypto/rijndael.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rijndael.c,v 1.16 2004/12/14 17:01:08 hshoexer Exp $ */
+/*	$OpenBSD: rijndael.c,v 1.17 2004/12/20 20:31:18 hshoexer Exp $ */
 
 /**
  * rijndael-alg-fst.c
@@ -1220,20 +1220,37 @@ rijndaelDecrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 ct[16],
 }
 
 /* setup key context for encryption only */
-void
+int
 rijndael_set_key_enc_only(rijndael_ctx *ctx, u_char *key, int bits)
 {
-	ctx->Nr = rijndaelKeySetupEnc(ctx->ek, key, bits);
+	int rounds;
+
+	rounds = rijndaelKeySetupEnc(ctx->ek, key, bits);
+	if (rounds == 0)
+		return -1;
+
+	ctx->Nr = rounds;
 	ctx->enc_only = 1;
+
+	return 0;
 }
 
 /* setup key context for both encryption and decryption */
-void
+int
 rijndael_set_key(rijndael_ctx *ctx, u_char *key, int bits)
 {
-	ctx->Nr = rijndaelKeySetupEnc(ctx->ek, key, bits);
-	rijndaelKeySetupDec(ctx->dk, key, bits);
+	int rounds;
+
+	rounds = rijndaelKeySetupEnc(ctx->ek, key, bits);
+	if (rounds == 0)
+		return -1;
+	if (rijndaelKeySetupDec(ctx->dk, key, bits) != rounds)
+		return -1;
+
+	ctx->Nr = rounds;
 	ctx->enc_only = 0;
+
+	return 0;
 }
 
 void
diff --git a/sys/crypto/rijndael.h b/sys/crypto/rijndael.h
index d1f3f4b57f4..6b509576c6a 100644
--- a/sys/crypto/rijndael.h
+++ b/sys/crypto/rijndael.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rijndael.h,v 1.9 2004/02/05 18:55:20 deraadt Exp $ */
+/*	$OpenBSD: rijndael.h,v 1.10 2004/12/20 20:31:18 hshoexer Exp $ */
 
 /**
  * rijndael-alg-fst.h
@@ -44,8 +44,8 @@ typedef struct {
 	u32	dk[4*(MAXNR + 1)];	/* decrypt key schedule */
 } rijndael_ctx;
 
-void	 rijndael_set_key(rijndael_ctx *, u_char *, int);
-void	 rijndael_set_key_enc_only(rijndael_ctx *, u_char *, int);
+int	 rijndael_set_key(rijndael_ctx *, u_char *, int);
+int	 rijndael_set_key_enc_only(rijndael_ctx *, u_char *, int);
 void	 rijndael_decrypt(rijndael_ctx *, u_char *, u_char *);
 void	 rijndael_encrypt(rijndael_ctx *, u_char *, u_char *);
 
diff --git a/sys/crypto/xform.c b/sys/crypto/xform.c
index 87f0f6a8adb..c9f42a648e4 100644
--- a/sys/crypto/xform.c
+++ b/sys/crypto/xform.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xform.c,v 1.27 2004/12/15 17:49:14 hshoexer Exp $	*/
+/*	$OpenBSD: xform.c,v 1.28 2004/12/20 20:31:18 hshoexer Exp $	*/
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -60,13 +60,13 @@
 extern void des_ecb3_encrypt(caddr_t, caddr_t, caddr_t, caddr_t, caddr_t, int);
 extern void des_ecb_encrypt(caddr_t, caddr_t, caddr_t, int);
 
-void des_set_key(caddr_t, caddr_t);
-void des1_setkey(u_int8_t **, u_int8_t *, int);
-void des3_setkey(u_int8_t **, u_int8_t *, int);
-void blf_setkey(u_int8_t **, u_int8_t *, int);
-void cast5_setkey(u_int8_t **, u_int8_t *, int);
-void skipjack_setkey(u_int8_t **, u_int8_t *, int);
-void rijndael128_setkey(u_int8_t **, u_int8_t *, int);
+int  des_set_key(caddr_t, caddr_t);
+int  des1_setkey(u_int8_t **, u_int8_t *, int);
+int  des3_setkey(u_int8_t **, u_int8_t *, int);
+int  blf_setkey(u_int8_t **, u_int8_t *, int);
+int  cast5_setkey(u_int8_t **, u_int8_t *, int);
+int  skipjack_setkey(u_int8_t **, u_int8_t *, int);
+int  rijndael128_setkey(u_int8_t **, u_int8_t *, int);
 void des1_encrypt(caddr_t, u_int8_t *);
 void des3_encrypt(caddr_t, u_int8_t *);
 void blf_encrypt(caddr_t, u_int8_t *);
@@ -87,7 +87,7 @@ void skipjack_zerokey(u_int8_t **);
 void rijndael128_zerokey(u_int8_t **);
 void null_encrypt(caddr_t, u_int8_t *);
 void null_zerokey(u_int8_t **);
-void null_setkey(u_int8_t **, u_int8_t *, int);
+int  null_setkey(u_int8_t **, u_int8_t *, int);
 void null_decrypt(caddr_t, u_int8_t *);
 
 int MD5Update_int(void *, u_int8_t *, u_int16_t);
@@ -273,12 +273,18 @@ des1_decrypt(caddr_t key, u_int8_t *blk)
 	des_ecb_encrypt(blk, blk, key, 0);
 }
 
-void
+int
 des1_setkey(u_int8_t **sched, u_int8_t *key, int len)
 {
 	MALLOC(*sched, u_int8_t *, 128, M_CRYPTO_DATA, M_WAITOK);
 	bzero(*sched, 128);
-	des_set_key(key, *sched);
+
+	if (des_set_key(key, *sched) < 0) {
+		des1_zerokey(sched);
+		return -1;
+	}
+
+	return 0;
 }
 
 void
@@ -301,14 +307,19 @@ des3_decrypt(caddr_t key, u_int8_t *blk)
 	des_ecb3_encrypt(blk, blk, key + 256, key + 128, key, 0);
 }
 
-void
+int
 des3_setkey(u_int8_t **sched, u_int8_t *key, int len)
 {
 	MALLOC(*sched, u_int8_t *, 384, M_CRYPTO_DATA, M_WAITOK);
 	bzero(*sched, 384);
-	des_set_key(key, *sched);
-	des_set_key(key + 8, *sched + 128);
-	des_set_key(key + 16, *sched + 256);
+
+	if (des_set_key(key, *sched) < 0 || des_set_key(key + 8, *sched + 128)
+	    < 0 || des_set_key(key + 16, *sched + 256) < 0) {
+		des3_zerokey(sched);
+		return -1;
+	}
+
+	return 0;
 }
 
 void
@@ -331,12 +342,14 @@ blf_decrypt(caddr_t key, u_int8_t *blk)
 	blf_ecb_decrypt((blf_ctx *) key, blk, 8);
 }
 
-void
+int
 blf_setkey(u_int8_t **sched, u_int8_t *key, int len)
 {
 	MALLOC(*sched, u_int8_t *, sizeof(blf_ctx), M_CRYPTO_DATA, M_WAITOK);
 	bzero(*sched, sizeof(blf_ctx));
 	blf_key((blf_ctx *)*sched, key, len);
+
+	return 0;
 }
 
 void
@@ -347,9 +360,10 @@ blf_zerokey(u_int8_t **sched)
 	*sched = NULL;
 }
 
-void
+int
 null_setkey(u_int8_t **sched, u_int8_t *key, int len)
 {
+	return 0;
 }
 
 void
@@ -379,12 +393,14 @@ cast5_decrypt(caddr_t key, u_int8_t *blk)
 	cast_decrypt((cast_key *) key, blk, blk);
 }
 
-void
+int
 cast5_setkey(u_int8_t **sched, u_int8_t *key, int len)
 {
 	MALLOC(*sched, u_int8_t *, sizeof(cast_key), M_CRYPTO_DATA, M_WAITOK);
 	bzero(*sched, sizeof(cast_key));
 	cast_setkey((cast_key *)*sched, key, len);
+
+	return 0;
 }
 
 void
@@ -407,13 +423,15 @@ skipjack_decrypt(caddr_t key, u_int8_t *blk)
 	skipjack_backwards(blk, blk, (u_int8_t **) key);
 }
 
-void
+int
 skipjack_setkey(u_int8_t **sched, u_int8_t *key, int len)
 {
 	MALLOC(*sched, u_int8_t *, 10 * sizeof(u_int8_t *), M_CRYPTO_DATA,
 	    M_WAITOK);
 	bzero(*sched, 10 * sizeof(u_int8_t *));
 	subkey_table_gen(key, (u_int8_t **) *sched);
+
+	return 0;
 }
 
 void
@@ -444,13 +462,20 @@ rijndael128_decrypt(caddr_t key, u_int8_t *blk)
 	rijndael_decrypt((rijndael_ctx *) key, (u_char *) blk, (u_char *) blk);
 }
 
-void
+int
 rijndael128_setkey(u_int8_t **sched, u_int8_t *key, int len)
 {
 	MALLOC(*sched, u_int8_t *, sizeof(rijndael_ctx), M_CRYPTO_DATA,
 	    M_WAITOK);
 	bzero(*sched, sizeof(rijndael_ctx));
-	rijndael_set_key((rijndael_ctx *) *sched, (u_char *) key, len * 8);
+
+	if (rijndael_set_key((rijndael_ctx *)*sched, (u_char *)key, len * 8)
+	    < 0) {
+		rijndael128_zerokey(sched);
+		return -1;
+	}
+
+	return 0;
 }
 
 void
diff --git a/sys/crypto/xform.h b/sys/crypto/xform.h
index bc9be17fcd8..fa7838665c2 100644
--- a/sys/crypto/xform.h
+++ b/sys/crypto/xform.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xform.h,v 1.14 2004/05/07 14:42:26 millert Exp $	*/
+/*	$OpenBSD: xform.h,v 1.15 2004/12/20 20:31:18 hshoexer Exp $	*/
 
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
@@ -49,7 +49,7 @@ struct enc_xform {
 	u_int16_t minkey, maxkey;
 	void (*encrypt) (caddr_t, u_int8_t *);
 	void (*decrypt) (caddr_t, u_int8_t *);
-	void (*setkey) (u_int8_t **, u_int8_t *, int len);
+	int  (*setkey) (u_int8_t **, u_int8_t *, int len);
 	void (*zerokey) (u_int8_t **);
 };