Extend fread() and fwrite() to check for integer overflow, in which case

errno EOVERFLOW is returned and error is set on the FILE.
ok kettenis miod beck

3 files changed, 40 insertions, 4 deletions

diff --git a/lib/libc/stdio/fread.3 b/lib/libc/stdio/fread.3
index 3dd624e7284..04bd6a4f3ca 100644
--- a/lib/libc/stdio/fread.3
+++ b/lib/libc/stdio/fread.3
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: fread.3,v 1.7 2013/07/17 05:42:11 schwarze Exp $
+.\"	$OpenBSD: fread.3,v 1.8 2014/05/01 16:40:36 deraadt Exp $
 .\"
 .\" Copyright (c) 1990, 1991, 1993
 .\"	The Regents of the University of California.  All rights reserved.
@@ -31,7 +31,7 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd $Mdocdate: July 17 2013 $
+.Dd $Mdocdate: May 1 2014 $
 .Dt FREAD 3
 .Os
 .Sh NAME
@@ -84,6 +84,13 @@ and
 .Fn fwrite
 return 0 with no change made to the
 .Fa stream .
+If the product of
+.Fa size
+and
+.Fa nemb
+results in integer overflow, 0 is returned and errno
+is set to
+.Er EOVERFLOW .
 If an error occurs, or the end-of-file is reached,
 the return value is a short object count (or zero).
 .Pp
diff --git a/lib/libc/stdio/fread.c b/lib/libc/stdio/fread.c
index 430865d022f..8a592f6d3f1 100644
--- a/lib/libc/stdio/fread.c
+++ b/lib/libc/stdio/fread.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: fread.c,v 1.11 2009/11/21 09:53:44 guenther Exp $ */
+/*	$OpenBSD: fread.c,v 1.12 2014/05/01 16:40:36 deraadt Exp $ */
 /*-
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -33,8 +33,12 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <stdint.h>
+#include <errno.h>
 #include "local.h"
 
+#define MUL_NO_OVERFLOW	(1UL << (sizeof(size_t) * 4))
+
 size_t
 fread(void *buf, size_t size, size_t count, FILE *fp)
 {
@@ -44,6 +48,16 @@ fread(void *buf, size_t size, size_t count, FILE *fp)
 	size_t total;
 
 	/*
+	 * Extension:  Catch integer overflow
+	 */
+	if ((size >= MUL_NO_OVERFLOW || count >= MUL_NO_OVERFLOW) &&
+	    size > 0 && SIZE_MAX / size < count) {
+		errno = EOVERFLOW;
+		fp->_flags |= __SERR;
+		return (0);
+	}
+
+	/*
 	 * ANSI and SUSv2 require a return value of 0 if size or count are 0.
 	 */
 	if ((resid = count * size) == 0)
diff --git a/lib/libc/stdio/fwrite.c b/lib/libc/stdio/fwrite.c
index 41784f9312a..f0a17bfb9a8 100644
--- a/lib/libc/stdio/fwrite.c
+++ b/lib/libc/stdio/fwrite.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: fwrite.c,v 1.10 2009/11/21 09:53:44 guenther Exp $ */
+/*	$OpenBSD: fwrite.c,v 1.11 2014/05/01 16:40:36 deraadt Exp $ */
 /*-
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -32,9 +32,14 @@
  */
 
 #include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <errno.h>
 #include "local.h"
 #include "fvwrite.h"
 
+#define MUL_NO_OVERFLOW	(1UL << (sizeof(size_t) * 4))
+
 /*
  * Write `count' objects (each size `size') from memory to the given file.
  * Return the number of whole objects written.
@@ -48,6 +53,16 @@ fwrite(const void *buf, size_t size, size_t count, FILE *fp)
 	int ret;
 
 	/*
+	 * Extension:  Catch integer overflow
+	 */
+	if ((size >= MUL_NO_OVERFLOW || count >= MUL_NO_OVERFLOW) &&
+	    size > 0 && SIZE_MAX / size < count) {
+		errno = EOVERFLOW;
+		fp->_flags |= __SERR;
+		return (0);
+	}
+
+	/*
 	 * ANSI and SUSv2 require a return value of 0 if size or count are 0.
 	 */
 	if ((n = count * size) == 0)