Add TLS_ERROR_INVALID_ARGUMENT error code to libtls

This is an initial pass, defining the error code and using it for
"too long"/length-related errors.

ok beck jsing

3 files changed, 15 insertions, 12 deletions

diff --git a/lib/libtls/tls.c b/lib/libtls/tls.c
index a8b03f0d4a1..c2f7f3722c4 100644
--- a/lib/libtls/tls.c
+++ b/lib/libtls/tls.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls.c,v 1.102 2024/03/26 08:54:48 joshua Exp $ */
+/* $OpenBSD: tls.c,v 1.103 2024/03/27 07:35:30 joshua Exp $ */
 /*
  * Copyright (c) 2014 Joel Sing <jsing@openbsd.org>
  *
@@ -359,9 +359,9 @@ tls_keypair_to_pkey(struct tls *ctx, struct tls_keypair *keypair, EVP_PKEY **pke
 		return (0);
 
 	if (len > INT_MAX) {
-		tls_set_errorx(ctx, TLS_ERROR_UNKNOWN,
+		tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
 		    ctx->config->use_fake_private_key ?
-		    "cert too long" : "key too long");
+		    "certificate too long" : "key too long");
 		goto err;
 	}
 
@@ -491,7 +491,7 @@ tls_configure_ssl_keypair(struct tls *ctx, SSL_CTX *ssl_ctx,
 
 	if (keypair->cert_mem != NULL) {
 		if (keypair->cert_len > INT_MAX) {
-			tls_set_errorx(ctx, TLS_ERROR_UNKNOWN,
+			tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
 			    "certificate too long");
 			goto err;
 		}
@@ -647,7 +647,8 @@ tls_configure_ssl_verify(struct tls *ctx, SSL_CTX *ssl_ctx, int verify)
 
 	if (ca_mem != NULL) {
 		if (ca_len > INT_MAX) {
-			tls_set_errorx(ctx, TLS_ERROR_UNKNOWN, "ca too long");
+			tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
+			    "ca too long");
 			goto err;
 		}
 		if (SSL_CTX_load_verify_mem(ssl_ctx, ca_mem, ca_len) != 1) {
@@ -664,7 +665,8 @@ tls_configure_ssl_verify(struct tls *ctx, SSL_CTX *ssl_ctx, int verify)
 
 	if (crl_mem != NULL) {
 		if (crl_len > INT_MAX) {
-			tls_set_errorx(ctx, TLS_ERROR_UNKNOWN, "crl too long");
+			tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
+			    "crl too long");
 			goto err;
 		}
 		if ((bio = BIO_new_mem_buf(crl_mem, crl_len)) == NULL) {
@@ -865,7 +867,7 @@ tls_read(struct tls *ctx, void *buf, size_t buflen)
 	}
 
 	if (buflen > INT_MAX) {
-		tls_set_errorx(ctx, TLS_ERROR_UNKNOWN,
+		tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
 		    "buflen too long");
 		goto out;
 	}
@@ -897,7 +899,7 @@ tls_write(struct tls *ctx, const void *buf, size_t buflen)
 	}
 
 	if (buflen > INT_MAX) {
-		tls_set_errorx(ctx, TLS_ERROR_UNKNOWN,
+		tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
 		    "buflen too long");
 		goto out;
 	}
diff --git a/lib/libtls/tls.h b/lib/libtls/tls.h
index b69c4af58c0..67804d7cd83 100644
--- a/lib/libtls/tls.h
+++ b/lib/libtls/tls.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls.h,v 1.65 2024/03/26 08:54:48 joshua Exp $ */
+/* $OpenBSD: tls.h,v 1.66 2024/03/27 07:35:30 joshua Exp $ */
 /*
  * Copyright (c) 2014 Joel Sing <jsing@openbsd.org>
  *
@@ -81,6 +81,7 @@ extern "C" {
 #define TLS_ERROR_UNKNOWN			0x0000
 #define TLS_ERROR_OUT_OF_MEMORY			0x1000
 #define TLS_ERROR_INVALID_CONTEXT		0x2000
+#define TLS_ERROR_INVALID_ARGUMENT		0x2001
 #endif
 
 struct tls;
diff --git a/lib/libtls/tls_config.c b/lib/libtls/tls_config.c
index 449071641bc..645562e838b 100644
--- a/lib/libtls/tls_config.c
+++ b/lib/libtls/tls_config.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls_config.c,v 1.68 2024/03/26 06:24:52 joshua Exp $ */
+/* $OpenBSD: tls_config.c,v 1.69 2024/03/27 07:35:30 joshua Exp $ */
 /*
  * Copyright (c) 2014 Joel Sing <jsing@openbsd.org>
  *
@@ -321,12 +321,12 @@ tls_config_parse_alpn(struct tls_config *config, const char *alpn,
 	q = s;
 	while ((p = strsep(&q, ",")) != NULL) {
 		if ((len = strlen(p)) == 0) {
-			tls_config_set_errorx(config, TLS_ERROR_UNKNOWN,
+			tls_config_set_errorx(config, TLS_ERROR_INVALID_ARGUMENT,
 			    "alpn protocol with zero length");
 			goto err;
 		}
 		if (len > 255) {
-			tls_config_set_errorx(config, TLS_ERROR_UNKNOWN,
+			tls_config_set_errorx(config, TLS_ERROR_INVALID_ARGUMENT,
 			    "alpn protocol too long");
 			goto err;
 		}