split auth_rsa() for better readability and privsep; ok provos@

3 files changed, 121 insertions, 89 deletions

diff --git a/usr.bin/ssh/auth-rh-rsa.c b/usr.bin/ssh/auth-rh-rsa.c
index d1b7ae996d5..397b0a55c26 100644
--- a/usr.bin/ssh/auth-rh-rsa.c
+++ b/usr.bin/ssh/auth-rh-rsa.c
@@ -13,7 +13,7 @@
  */
 
 #include "includes.h"
-RCSID("$OpenBSD: auth-rh-rsa.c,v 1.29 2002/03/04 12:43:06 markus Exp $");
+RCSID("$OpenBSD: auth-rh-rsa.c,v 1.30 2002/03/14 16:56:33 markus Exp $");
 
 #include "packet.h"
 #include "uidswap.h"
@@ -63,7 +63,7 @@ auth_rhosts_rsa(struct passwd *pw, const char *client_user, Key *client_host_key
 	/* A matching host key was found and is known. */
 
 	/* Perform the challenge-response dialog with the client for the host key. */
-	if (!auth_rsa_challenge_dialog(client_host_key->rsa)) {
+	if (!auth_rsa_challenge_dialog(client_host_key)) {
 		log("Client on %.800s failed to respond correctly to host authentication.",
 		    canonical_hostname);
 		return 0;
diff --git a/usr.bin/ssh/auth-rsa.c b/usr.bin/ssh/auth-rsa.c
index f7ae03cfdaf..ff9bf3b6420 100644
--- a/usr.bin/ssh/auth-rsa.c
+++ b/usr.bin/ssh/auth-rsa.c
@@ -14,7 +14,7 @@
  */
 
 #include "includes.h"
-RCSID("$OpenBSD: auth-rsa.c,v 1.50 2001/12/28 14:50:54 markus Exp $");
+RCSID("$OpenBSD: auth-rsa.c,v 1.51 2002/03/14 16:56:33 markus Exp $");
 
 #include <openssl/rsa.h>
 #include <openssl/md5.h>
@@ -52,6 +52,51 @@ extern u_char session_id[16];
  * description of the options.
  */
 
+static BIGNUM *
+auth_rsa_generate_challenge(Key *key)
+{
+	BIGNUM *challenge;
+	BN_CTX *ctx;
+
+	if ((challenge = BN_new()) == NULL)
+		fatal("auth_rsa_generate_challenge: BN_new() failed");
+	/* Generate a random challenge. */
+	BN_rand(challenge, 256, 0, 0);
+	if ((ctx = BN_CTX_new()) == NULL)
+		fatal("auth_rsa_generate_challenge: BN_CTX_new() failed");
+	BN_mod(challenge, challenge, key->rsa->n, ctx);
+	BN_CTX_free(ctx);
+
+	return challenge;
+}
+
+static int
+auth_rsa_verify_response(Key *key, BIGNUM *challenge, u_char response[16])
+{
+	u_char buf[32], mdbuf[16];
+	MD5_CTX md;
+	int len;
+
+	/* The response is MD5 of decrypted challenge plus session id. */
+	len = BN_num_bytes(challenge);
+	if (len <= 0 || len > 32)
+		fatal("auth_rsa_verify_response: bad challenge length %d", len);
+	memset(buf, 0, 32);
+	BN_bn2bin(challenge, buf + 32 - len);
+	MD5_Init(&md);
+	MD5_Update(&md, buf, 32);
+	MD5_Update(&md, session_id, 16);
+	MD5_Final(mdbuf, &md);
+
+	/* Verify that the response is the original challenge. */
+	if (memcmp(response, mdbuf, 16) != 0) {
+		/* Wrong answer. */
+		return (0);
+	}
+	/* Correct answer. */
+	return (1);
+}
+
 /*
  * Performs the RSA authentication challenge-response dialog with the client,
  * and returns true (non-zero) if the client gave the correct answer to
@@ -59,29 +104,19 @@ extern u_char session_id[16];
  */
 
 int
-auth_rsa_challenge_dialog(RSA *pk)
+auth_rsa_challenge_dialog(Key *key)
 {
 	BIGNUM *challenge, *encrypted_challenge;
-	BN_CTX *ctx;
-	u_char buf[32], mdbuf[16], response[16];
-	MD5_CTX md;
-	u_int i;
-	int len;
+	u_char response[16];
+	int i, success;
 
 	if ((encrypted_challenge = BN_new()) == NULL)
 		fatal("auth_rsa_challenge_dialog: BN_new() failed");
-	if ((challenge = BN_new()) == NULL)
-		fatal("auth_rsa_challenge_dialog: BN_new() failed");
 
-	/* Generate a random challenge. */
-	BN_rand(challenge, 256, 0, 0);
-	if ((ctx = BN_CTX_new()) == NULL)
-		fatal("auth_rsa_challenge_dialog: BN_CTX_new() failed");
-	BN_mod(challenge, challenge, pk->n, ctx);
-	BN_CTX_free(ctx);
+	challenge = auth_rsa_generate_challenge(key);
 
 	/* Encrypt the challenge with the public key. */
-	rsa_public_encrypt(encrypted_challenge, challenge, pk);
+	rsa_public_encrypt(encrypted_challenge, challenge, key->rsa);
 
 	/* Send the encrypted challenge to the client. */
 	packet_start(SSH_SMSG_AUTH_RSA_CHALLENGE);
@@ -96,48 +131,26 @@ auth_rsa_challenge_dialog(RSA *pk)
 		response[i] = packet_get_char();
 	packet_check_eom();
 
-	/* The response is MD5 of decrypted challenge plus session id. */
-	len = BN_num_bytes(challenge);
-	if (len <= 0 || len > 32)
-		fatal("auth_rsa_challenge_dialog: bad challenge length %d", len);
-	memset(buf, 0, 32);
-	BN_bn2bin(challenge, buf + 32 - len);
-	MD5_Init(&md);
-	MD5_Update(&md, buf, 32);
-	MD5_Update(&md, session_id, 16);
-	MD5_Final(mdbuf, &md);
+	success = auth_rsa_verify_response(key, challenge, response);
 	BN_clear_free(challenge);
-
-	/* Verify that the response is the original challenge. */
-	if (memcmp(response, mdbuf, 16) != 0) {
-		/* Wrong answer. */
-		return 0;
-	}
-	/* Correct answer. */
-	return 1;
+	return (success);
 }
 
 /*
- * Performs the RSA authentication dialog with the client.  This returns
- * 0 if the client could not be authenticated, and 1 if authentication was
- * successful.  This may exit if there is a serious protocol violation.
+ * check if there's user key matching client_n,
+ * return key if login is allowed, NULL otherwise
  */
 
-int
-auth_rsa(struct passwd *pw, BIGNUM *client_n)
+static int
+auth_rsa_key_allowed(struct passwd *pw, BIGNUM *client_n, Key **rkey)
 {
 	char line[8192], *file;
-	int authenticated;
+	int allowed;
 	u_int bits;
 	FILE *f;
 	u_long linenum = 0;
 	struct stat st;
 	Key *key;
-	char *fp;
-
-	/* no user given */
-	if (pw == NULL)
-		return 0;
 
 	/* Temporarily use the user's uid. */
 	temporarily_use_uid(pw);
@@ -151,29 +164,27 @@ auth_rsa(struct passwd *pw, BIGNUM *client_n)
 		/* Restore the privileged uid. */
 		restore_uid();
 		xfree(file);
-		return 0;
+		return (NULL);
 	}
 	/* Open the file containing the authorized keys. */
 	f = fopen(file, "r");
 	if (!f) {
 		/* Restore the privileged uid. */
 		restore_uid();
-		packet_send_debug("Could not open %.900s for reading.", file);
-		packet_send_debug("If your home is on an NFS volume, it may need to be world-readable.");
 		xfree(file);
-		return 0;
+		return (NULL);
 	}
 	if (options.strict_modes &&
 	    secure_filename(f, file, pw, line, sizeof(line)) != 0) {
 		xfree(file);
 		fclose(f);
 		log("Authentication refused: %s", line);
-		packet_send_debug("Authentication refused: %s", line);
 		restore_uid();
-		return 0;
+		return (NULL);
 	}
-	/* Flag indicating whether authentication has succeeded. */
-	authenticated = 0;
+
+	/* Flag indicating whether the key is allowed. */
+	allowed = 0;
 
 	key = key_new(KEY_RSA1);
 
@@ -238,32 +249,8 @@ auth_rsa(struct passwd *pw, BIGNUM *client_n)
 		if (!auth_parse_options(pw, options, file, linenum))
 			continue;
 
-		/* Perform the challenge-response dialog for this key. */
-		if (!auth_rsa_challenge_dialog(key->rsa)) {
-			/* Wrong response. */
-			verbose("Wrong response to RSA authentication challenge.");
-			packet_send_debug("Wrong response to RSA authentication challenge.");
-			/*
-			 * Break out of the loop. Otherwise we might send
-			 * another challenge and break the protocol.
-			 */
-			break;
-		}
-		/*
-		 * Correct response.  The client has been successfully
-		 * authenticated. Note that we have not yet processed the
-		 * options; this will be reset if the options cause the
-		 * authentication to be rejected.
-		 * Break out of the loop if authentication was successful;
-		 * otherwise continue searching.
-		 */
-		authenticated = 1;
-
-	 	fp = key_fingerprint(key, SSH_FP_MD5, SSH_FP_HEX);
-		verbose("Found matching %s key: %s",
-		    key_type(key), fp);
-		xfree(fp);
-
+		/* break out, this key is allowed */
+		allowed = 1;
 		break;
 	}
 
@@ -274,13 +261,58 @@ auth_rsa(struct passwd *pw, BIGNUM *client_n)
 	xfree(file);
 	fclose(f);
 
-	key_free(key);
-
-	if (authenticated)
-		packet_send_debug("RSA authentication accepted.");
+	/* return key if allowed */
+	if (allowed && rkey != NULL)
+		*rkey = key;
 	else
+		key_free(key);
+	return (allowed);
+}
+
+/*
+ * Performs the RSA authentication dialog with the client.  This returns
+ * 0 if the client could not be authenticated, and 1 if authentication was
+ * successful.  This may exit if there is a serious protocol violation.
+ */
+int
+auth_rsa(struct passwd *pw, BIGNUM *client_n)
+{
+	Key *key;
+	char *fp;
+
+	/* no user given */
+	if (pw == NULL)
+		return 0;
+
+	if (auth_rsa_key_allowed(pw, client_n, &key) == 0) {
 		auth_clear_options();
+		return (0);
+	}
+
+	/* Perform the challenge-response dialog for this key. */
+	if (!auth_rsa_challenge_dialog(key)) {
+		/* Wrong response. */
+		verbose("Wrong response to RSA authentication challenge.");
+		packet_send_debug("Wrong response to RSA authentication challenge.");
+		/*
+		 * Break out of the loop. Otherwise we might send
+		 * another challenge and break the protocol.
+		 */
+		key_free(key);
+		return (0);
+	}
+	/*
+	 * Correct response.  The client has been successfully
+	 * authenticated. Note that we have not yet processed the
+	 * options; this will be reset if the options cause the
+	 * authentication to be rejected.
+	 */
+	fp = key_fingerprint(key, SSH_FP_MD5, SSH_FP_HEX);
+	verbose("Found matching %s key: %s",
+	    key_type(key), fp);
+	xfree(fp);
+	key_free(key);
 
-	/* Return authentication result. */
-	return authenticated;
+	packet_send_debug("RSA authentication accepted.");
+	return (1);
 }
diff --git a/usr.bin/ssh/auth.h b/usr.bin/ssh/auth.h
index 56ef2de0357..37884198451 100644
--- a/usr.bin/ssh/auth.h
+++ b/usr.bin/ssh/auth.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: auth.h,v 1.29 2002/03/04 17:27:39 stevesk Exp $	*/
+/*	$OpenBSD: auth.h,v 1.30 2002/03/14 16:56:33 markus Exp $	*/
 
 /*
  * Copyright (c) 2000 Markus Friedl.  All rights reserved.
@@ -95,7 +95,7 @@ auth_rhosts2(struct passwd *, const char *, const char *, const char *);
 int	 auth_rhosts_rsa(struct passwd *, const char *, Key *);
 int      auth_password(Authctxt *, const char *);
 int      auth_rsa(struct passwd *, BIGNUM *);
-int      auth_rsa_challenge_dialog(RSA *);
+int      auth_rsa_challenge_dialog(Key *);
 
 #ifdef KRB4
 #include <krb.h>