Limit ASN.1 constructed types recursive definition depth

Fixes for CVE-2018-0739.
Copied from commit below, and modified for adaption to our code.
https://github.com/openssl/openssl/commit/9310d45087ae546e27e61ddf8f6367f29848220d

ok bcook@ beck@ jsing@

3 files changed, 45 insertions, 22 deletions

diff --git a/lib/libcrypto/asn1/asn1.h b/lib/libcrypto/asn1/asn1.h
index 6fc4cd75270..3a2da6463ef 100644
--- a/lib/libcrypto/asn1/asn1.h
+++ b/lib/libcrypto/asn1/asn1.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: asn1.h,v 1.44 2018/02/14 16:46:04 jsing Exp $ */
+/* $OpenBSD: asn1.h,v 1.45 2018/03/29 02:29:24 inoguchi Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1405,6 +1405,7 @@ void ERR_load_ASN1_strings(void);
 #define ASN1_R_MSTRING_NOT_UNIVERSAL			 139
 #define ASN1_R_MSTRING_WRONG_TAG			 140
 #define ASN1_R_NESTED_ASN1_STRING			 197
+#define ASN1_R_NESTED_TOO_DEEP				 219
 #define ASN1_R_NON_HEX_CHARACTERS			 141
 #define ASN1_R_NOT_ASCII_FORMAT				 190
 #define ASN1_R_NOT_ENOUGH_DATA				 142
diff --git a/lib/libcrypto/asn1/asn1_err.c b/lib/libcrypto/asn1/asn1_err.c
index 0c827a92365..5cc355084f0 100644
--- a/lib/libcrypto/asn1/asn1_err.c
+++ b/lib/libcrypto/asn1/asn1_err.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: asn1_err.c,v 1.20 2017/01/29 17:49:22 beck Exp $ */
+/* $OpenBSD: asn1_err.c,v 1.21 2018/03/29 02:29:24 inoguchi Exp $ */
 /* ====================================================================
  * Copyright (c) 1999-2011 The OpenSSL Project.  All rights reserved.
  *
@@ -150,6 +150,7 @@ static ERR_STRING_DATA ASN1_str_reasons[] = {
 	{ERR_REASON(ASN1_R_MSTRING_NOT_UNIVERSAL), "mstring not universal"},
 	{ERR_REASON(ASN1_R_MSTRING_WRONG_TAG)    , "mstring wrong tag"},
 	{ERR_REASON(ASN1_R_NESTED_ASN1_STRING)   , "nested asn1 string"},
+	{ERR_REASON(ASN1_R_NESTED_TOO_DEEP)      , "nested too deep"},
 	{ERR_REASON(ASN1_R_NON_HEX_CHARACTERS)   , "non hex characters"},
 	{ERR_REASON(ASN1_R_NOT_ASCII_FORMAT)     , "not ascii format"},
 	{ERR_REASON(ASN1_R_NOT_ENOUGH_DATA)      , "not enough data"},
diff --git a/lib/libcrypto/asn1/tasn_dec.c b/lib/libcrypto/asn1/tasn_dec.c
index 3f680c60fde..bf536cb667f 100644
--- a/lib/libcrypto/asn1/tasn_dec.c
+++ b/lib/libcrypto/asn1/tasn_dec.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tasn_dec.c,v 1.34 2017/01/29 17:49:22 beck Exp $ */
+/* $OpenBSD: tasn_dec.c,v 1.35 2018/03/29 02:29:24 inoguchi Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -65,6 +65,12 @@
 #include <openssl/buffer.h>
 #include <openssl/err.h>
 
+/* Constructed types with a recursive definition (such as can be found in PKCS7)
+ * could eventually exceed the stack given malicious input with excessive
+ * recursion. Therefore we limit the stack depth.
+ */
+#define ASN1_MAX_CONSTRUCTED_NEST 30
+
 static int asn1_check_eoc(const unsigned char **in, long len);
 static int asn1_find_end(const unsigned char **in, long len, char inf);
 
@@ -78,9 +84,9 @@ static int asn1_check_tlen(long *olen, int *otag, unsigned char *oclass,
     int expclass, char opt, ASN1_TLC *ctx);
 
 static int asn1_template_ex_d2i(ASN1_VALUE **pval, const unsigned char **in,
-    long len, const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx);
+    long len, const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx, int depth);
 static int asn1_template_noexp_d2i(ASN1_VALUE **val, const unsigned char **in,
-    long len, const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx);
+    long len, const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx, int depth);
 static int asn1_d2i_ex_primitive(ASN1_VALUE **pval, const unsigned char **in,
     long len, const ASN1_ITEM *it, int tag, int aclass, char opt,
     ASN1_TLC *ctx);
@@ -142,7 +148,7 @@ ASN1_template_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
 	ASN1_TLC c;
 
 	asn1_tlc_clear_nc(&c);
-	return asn1_template_ex_d2i(pval, in, len, tt, 0, &c);
+	return asn1_template_ex_d2i(pval, in, len, tt, 0, &c, 0);
 }
 
 
@@ -150,9 +156,10 @@ ASN1_template_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
  * If 'opt' set and tag mismatch return -1 to handle OPTIONAL
  */
 
-int
-ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
-    const ASN1_ITEM *it, int tag, int aclass, char opt, ASN1_TLC *ctx)
+static int
+asn1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
+    const ASN1_ITEM *it, int tag, int aclass, char opt, ASN1_TLC *ctx,
+    int depth)
 {
 	const ASN1_TEMPLATE *tt, *errtt = NULL;
 	const ASN1_EXTERN_FUNCS *ef;
@@ -179,6 +186,11 @@ ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
 	else
 		asn1_cb = 0;
 
+	if (++depth > ASN1_MAX_CONSTRUCTED_NEST) {
+		ASN1error(ASN1_R_NESTED_TOO_DEEP);
+		goto err;
+	}
+
 	switch (it->itype) {
 	case ASN1_ITYPE_PRIMITIVE:
 		if (it->templates) {
@@ -193,7 +205,7 @@ ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
 				goto err;
 			}
 			return asn1_template_ex_d2i(pval, in, len,
-			    it->templates, opt, ctx);
+			    it->templates, opt, ctx, depth);
 		}
 		return asn1_d2i_ex_primitive(pval, in, len, it,
 		    tag, aclass, opt, ctx);
@@ -258,7 +270,8 @@ ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
 			/* We mark field as OPTIONAL so its absence
 			 * can be recognised.
 			 */
-			ret = asn1_template_ex_d2i(pchptr, &p, len, tt, 1, ctx);
+			ret = asn1_template_ex_d2i(pchptr, &p, len, tt, 1, ctx,
+				depth);
 			/* If field not present, try the next one */
 			if (ret == -1)
 				continue;
@@ -376,7 +389,7 @@ ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
 			 * OPTIONAL */
 
 			ret = asn1_template_ex_d2i(pseqval, &p, len,
-			    seqtt, isopt, ctx);
+			    seqtt, isopt, ctx, depth);
 			if (!ret) {
 				errtt = seqtt;
 				goto err;
@@ -448,13 +461,20 @@ err:
 	return 0;
 }
 
+int
+ASN1_item_ex_d2i(ASN1_VALUE **pval, const unsigned char **in, long len,
+    const ASN1_ITEM *it, int tag, int aclass, char opt, ASN1_TLC *ctx)
+{
+	return asn1_item_ex_d2i(pval, in, len, it, tag, aclass, opt, ctx, 0);
+}
+
 /* Templates are handled with two separate functions.
  * One handles any EXPLICIT tag and the other handles the rest.
  */
 
 static int
 asn1_template_ex_d2i(ASN1_VALUE **val, const unsigned char **in, long inlen,
-    const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx)
+    const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx, int depth)
 {
 	int flags, aclass;
 	int ret;
@@ -489,7 +509,7 @@ asn1_template_ex_d2i(ASN1_VALUE **val, const unsigned char **in, long inlen,
 			return 0;
 		}
 		/* We've found the field so it can't be OPTIONAL now */
-		ret = asn1_template_noexp_d2i(val, &p, len, tt, 0, ctx);
+		ret = asn1_template_noexp_d2i(val, &p, len, tt, 0, ctx, depth);
 		if (!ret) {
 			ASN1error(ERR_R_NESTED_ASN1_ERROR);
 			return 0;
@@ -511,7 +531,8 @@ asn1_template_ex_d2i(ASN1_VALUE **val, const unsigned char **in, long inlen,
 			}
 		}
 	} else
-		return asn1_template_noexp_d2i(val, in, inlen, tt, opt, ctx);
+		return asn1_template_noexp_d2i(val, in, inlen, tt, opt, ctx,
+			depth);
 
 	*in = p;
 	return 1;
@@ -523,7 +544,7 @@ err:
 
 static int
 asn1_template_noexp_d2i(ASN1_VALUE **val, const unsigned char **in, long len,
-    const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx)
+    const ASN1_TEMPLATE *tt, char opt, ASN1_TLC *ctx, int depth)
 {
 	int flags, aclass;
 	int ret;
@@ -594,8 +615,8 @@ asn1_template_noexp_d2i(ASN1_VALUE **val, const unsigned char **in, long len,
 				break;
 			}
 			skfield = NULL;
-			if (!ASN1_item_ex_d2i(&skfield, &p, len,
-			    tt->item, -1, 0, 0, ctx)) {
+			if (!asn1_item_ex_d2i(&skfield, &p, len,
+			    tt->item, -1, 0, 0, ctx, depth)) {
 				ASN1error(ERR_R_NESTED_ASN1_ERROR);
 				goto err;
 			}
@@ -612,8 +633,8 @@ asn1_template_noexp_d2i(ASN1_VALUE **val, const unsigned char **in, long len,
 		}
 	} else if (flags & ASN1_TFLG_IMPTAG) {
 		/* IMPLICIT tagging */
-		ret = ASN1_item_ex_d2i(val, &p, len,
-		    tt->item, tt->tag, aclass, opt, ctx);
+		ret = asn1_item_ex_d2i(val, &p, len,
+		    tt->item, tt->tag, aclass, opt, ctx, depth);
 		if (!ret) {
 			ASN1error(ERR_R_NESTED_ASN1_ERROR);
 			goto err;
@@ -621,8 +642,8 @@ asn1_template_noexp_d2i(ASN1_VALUE **val, const unsigned char **in, long len,
 			return -1;
 	} else {
 		/* Nothing special */
-		ret = ASN1_item_ex_d2i(val, &p, len, tt->item,
-		    -1, tt->flags & ASN1_TFLG_COMBINE, opt, ctx);
+		ret = asn1_item_ex_d2i(val, &p, len, tt->item,
+		    -1, tt->flags & ASN1_TFLG_COMBINE, opt, ctx, depth);
 		if (!ret) {
 			ASN1error(ERR_R_NESTED_ASN1_ERROR);
 			goto err;