Fix mmap() error checking to be correct 64-bit addresses. Consistently

use _dl_mmap_error() to check for mmap() errors. Adjust datatypes of
some local vars for 64-bit safety.

okay millert@ drahn@

15 files changed, 62 insertions, 61 deletions

diff --git a/libexec/ld.so/alpha/syscall.h b/libexec/ld.so/alpha/syscall.h
index f1161d6c76d..ab63c4b4c3a 100644
--- a/libexec/ld.so/alpha/syscall.h
+++ b/libexec/ld.so/alpha/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.15 2006/05/03 16:10:51 drahn Exp $ */
+/*	$OpenBSD: syscall.h,v 1.16 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 2001 Niklas Hallqvist
@@ -33,10 +33,10 @@
 #include <sys/signal.h>
 
 #ifndef _dl_MAX_ERRNO
-#define _dl_MAX_ERRNO 4096
+#define _dl_MAX_ERRNO 512L
 #endif
-#define _dl_check_error(__res) \
-    ((int) __res < 0 && (int) __res >= -_dl_MAX_ERRNO)
+#define _dl_mmap_error(__res) \
+    ((long) __res < 0 && (long) __res >= -_dl_MAX_ERRNO)
 
 int	_dl_close(int);
 int	_dl_exit(int);
diff --git a/libexec/ld.so/amd64/syscall.h b/libexec/ld.so/amd64/syscall.h
index 0f6a73181b0..406b4bfe4a2 100644
--- a/libexec/ld.so/amd64/syscall.h
+++ b/libexec/ld.so/amd64/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.3 2006/05/03 16:10:52 drahn Exp $	*/
+/*	$OpenBSD: syscall.h,v 1.4 2008/10/02 20:12:08 kurt Exp $	*/
 
 /*
  * Copyright (c) 2001 Niklas Hallqvist
@@ -34,10 +34,10 @@
 #include <sys/signal.h>
 
 #ifndef _dl_MAX_ERRNO
-#define _dl_MAX_ERRNO 4096
+#define _dl_MAX_ERRNO 512L
 #endif
-#define _dl_check_error(__res) \
-    ((int) __res < 0 && (int) __res >= -_dl_MAX_ERRNO)
+#define _dl_mmap_error(__res) \
+    ((long)__res < 0 && (long)__res >= -_dl_MAX_ERRNO)
 
 int	_dl_close(int);
 int	_dl_exit(int);
diff --git a/libexec/ld.so/arm/syscall.h b/libexec/ld.so/arm/syscall.h
index 74cd62d828f..465e0301c70 100644
--- a/libexec/ld.so/arm/syscall.h
+++ b/libexec/ld.so/arm/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.3 2006/05/03 16:10:52 drahn Exp $ */
+/*	$OpenBSD: syscall.h,v 1.4 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 2001 Niklas Hallqvist
@@ -34,10 +34,10 @@
 #include <sys/signal.h>
 
 #ifndef _dl_MAX_ERRNO
-#define _dl_MAX_ERRNO 4096
+#define _dl_MAX_ERRNO 512L
 #endif
-#define _dl_check_error(__res) \
-    ((int) __res < 0 && (int) __res >= -_dl_MAX_ERRNO)
+#define _dl_mmap_error(__res) \
+    ((long)__res < 0 && (long)__res >= -_dl_MAX_ERRNO)
 
 int	_dl_close(int);
 int	_dl_exit(int);
diff --git a/libexec/ld.so/hppa/syscall.h b/libexec/ld.so/hppa/syscall.h
index 0f6a73181b0..406b4bfe4a2 100644
--- a/libexec/ld.so/hppa/syscall.h
+++ b/libexec/ld.so/hppa/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.3 2006/05/03 16:10:52 drahn Exp $	*/
+/*	$OpenBSD: syscall.h,v 1.4 2008/10/02 20:12:08 kurt Exp $	*/
 
 /*
  * Copyright (c) 2001 Niklas Hallqvist
@@ -34,10 +34,10 @@
 #include <sys/signal.h>
 
 #ifndef _dl_MAX_ERRNO
-#define _dl_MAX_ERRNO 4096
+#define _dl_MAX_ERRNO 512L
 #endif
-#define _dl_check_error(__res) \
-    ((int) __res < 0 && (int) __res >= -_dl_MAX_ERRNO)
+#define _dl_mmap_error(__res) \
+    ((long)__res < 0 && (long)__res >= -_dl_MAX_ERRNO)
 
 int	_dl_close(int);
 int	_dl_exit(int);
diff --git a/libexec/ld.so/i386/syscall.h b/libexec/ld.so/i386/syscall.h
index 2d2f36b7b9d..a409e17015e 100644
--- a/libexec/ld.so/i386/syscall.h
+++ b/libexec/ld.so/i386/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.7 2006/05/03 16:10:52 drahn Exp $ */
+/*	$OpenBSD: syscall.h,v 1.8 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 2001 Niklas Hallqvist
@@ -34,10 +34,10 @@
 #include <sys/signal.h>
 
 #ifndef _dl_MAX_ERRNO
-#define _dl_MAX_ERRNO 4096
+#define _dl_MAX_ERRNO 512L
 #endif
-#define _dl_check_error(__res) \
-    ((int) __res < 0 && (int) __res >= -_dl_MAX_ERRNO)
+#define _dl_mmap_error(__res) \
+    ((long)__res < 0 && (long)__res >= -_dl_MAX_ERRNO)
 
 int	_dl_close(int);
 int	_dl_exit(int);
diff --git a/libexec/ld.so/library.c b/libexec/ld.so/library.c
index 2851b2928cf..0ddbb48293e 100644
--- a/libexec/ld.so/library.c
+++ b/libexec/ld.so/library.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: library.c,v 1.57 2008/05/05 02:29:02 kurt Exp $ */
+/*	$OpenBSD: library.c,v 1.58 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 2002 Dale Rahn
@@ -77,10 +77,10 @@ _dl_unload_shlib(elf_object_t *object)
 elf_object_t *
 _dl_tryload_shlib(const char *libname, int type, int flags)
 {
-	int	libfile, i, align = _dl_pagesz - 1;
+	int	libfile, i;
 	struct load_list *next_load, *load_list = NULL;
 	Elf_Addr maxva = 0, minva = ELFDEFNNAME(NO_ADDR);
-	Elf_Addr libaddr, loff;
+	Elf_Addr libaddr, loff, align = _dl_pagesz - 1;
 	elf_object_t *object;
 	char	hbuf[4096];
 	Elf_Dyn *dynp = 0;
@@ -174,7 +174,7 @@ _dl_tryload_shlib(const char *libname, int type, int flags)
 	 */
 	libaddr = (Elf_Addr)_dl_mmap(0, maxva - minva, PROT_NONE,
 	    MAP_PRIVATE|MAP_FILE, libfile, 0);
-	if (_dl_check_error(libaddr)) {
+	if (_dl_mmap_error(libaddr)) {
 		_dl_printf("%s: rtld mmap failed mapping %s.\n",
 		    _dl_progname, libname);
 		_dl_close(libfile);
@@ -188,8 +188,8 @@ _dl_tryload_shlib(const char *libname, int type, int flags)
 	for (i = 0; i < ehdr->e_phnum; i++, phdp++) {
 		if (phdp->p_type == PT_LOAD) {
 			char *start = (char *)(TRUNC_PG(phdp->p_vaddr)) + loff;
-			int off = (phdp->p_vaddr & align);
-			int size = off + phdp->p_filesz;
+			Elf_Addr off = (phdp->p_vaddr & align);
+			Elf_Addr size = off + phdp->p_filesz;
 			void *res;
 
 			res = _dl_mmap(start, ROUND_PG(size),
@@ -202,7 +202,7 @@ _dl_tryload_shlib(const char *libname, int type, int flags)
 			next_load->start = start;
 			next_load->size = size;
 			next_load->prot = PFLAGS(phdp->p_flags);
-			if (_dl_check_error((long)res)) {
+			if (_dl_mmap_error(res)) {
 				_dl_printf("%s: rtld mmap failed mapping %s.\n",
 				    _dl_progname, libname);
 				_dl_close(libfile);
@@ -225,7 +225,7 @@ _dl_tryload_shlib(const char *libname, int type, int flags)
 				res = _dl_mmap(start, size,
 				    PFLAGS(phdp->p_flags),
 				    MAP_FIXED|MAP_PRIVATE|MAP_ANON, -1, 0);
-				if (_dl_check_error((long)res)) {
+				if (_dl_mmap_error(res)) {
 					_dl_printf("%s: rtld mmap failed mapping %s.\n",
 					    _dl_progname, libname);
 					_dl_close(libfile);
diff --git a/libexec/ld.so/library_mquery.c b/libexec/ld.so/library_mquery.c
index b9d40a365a8..cd0ea376426 100644
--- a/libexec/ld.so/library_mquery.c
+++ b/libexec/ld.so/library_mquery.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: library_mquery.c,v 1.35 2008/05/05 02:29:02 kurt Exp $ */
+/*	$OpenBSD: library_mquery.c,v 1.36 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 2002 Dale Rahn
@@ -47,7 +47,7 @@ void
 _dl_load_list_free(struct load_list *load_list)
 {
 	struct load_list *next;
-	int align = _dl_pagesz - 1;
+	Elf_Addr align = _dl_pagesz - 1;
 
 	while (load_list != NULL) {
 		if (load_list->start != NULL)
@@ -83,13 +83,14 @@ _dl_unload_shlib(elf_object_t *object)
 elf_object_t *
 _dl_tryload_shlib(const char *libname, int type, int flags)
 {
-	int libfile, i, align = _dl_pagesz - 1, off, size;
+	int libfile, i;
 	struct load_list *ld, *lowld = NULL;
 	elf_object_t *object;
 	Elf_Dyn *dynp = 0;
 	Elf_Ehdr *ehdr;
 	Elf_Phdr *phdp;
 	Elf_Addr load_end = 0;
+	Elf_Addr align = _dl_pagesz - 1, off, size;
 	struct stat sb;
 	void *prebind_data;
 	char hbuf[4096];
@@ -232,11 +233,11 @@ retry:
 		 */
 		ld->start = _dl_mquery(ld->start, ROUND_PG(ld->size), ld->prot,
 		    flags, fd, foff);
-		if (_dl_check_error(ld->start)) {
+		if (_dl_mmap_error(ld->start)) {
 			ld->start = (void *)(LOFF + ld->moff);
 			ld->start = _dl_mquery(ld->start, ROUND_PG(ld->size),
 			    ld->prot, flags & ~MAP_FIXED, fd, foff);
-			if (_dl_check_error(ld->start))
+			if (_dl_mmap_error(ld->start))
 				goto fail;
 		}
 
@@ -266,7 +267,7 @@ retry:
 		}
 		res = _dl_mmap(ld->start, ROUND_PG(ld->size), ld->prot, flags,
 		    fd, foff);
-		if (_dl_check_error((long)res))
+		if (_dl_mmap_error(res))
 			goto fail;
 		/* Zero out everything past the EOF */
 		if ((ld->prot & PROT_WRITE) != 0 && (ld->size & align) != 0)
diff --git a/libexec/ld.so/loader.c b/libexec/ld.so/loader.c
index 5182cb83d66..e1e29649a81 100644
--- a/libexec/ld.so/loader.c
+++ b/libexec/ld.so/loader.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: loader.c,v 1.115 2008/06/13 23:52:42 kurt Exp $ */
+/*	$OpenBSD: loader.c,v 1.116 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 1998 Per Fogelstrom, Opsycon AB
@@ -816,7 +816,7 @@ _dl_rtld(elf_object_t *object)
 
 		_dl_symcache = (void *)_dl_mmap(0, sz, PROT_READ|PROT_WRITE,
 		    MAP_PRIVATE|MAP_ANON, -1, 0);
-		if (_dl_symcache == (void *)MAP_FAILED) {
+		if (_dl_mmap_error(_dl_symcache)) {
 			sz = 0;
 			_dl_symcache = NULL;
 		}
diff --git a/libexec/ld.so/mips64/syscall.h b/libexec/ld.so/mips64/syscall.h
index 9be5ed1c2ef..b45fd1eb512 100644
--- a/libexec/ld.so/mips64/syscall.h
+++ b/libexec/ld.so/mips64/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.4 2006/05/03 16:10:52 drahn Exp $ */
+/*	$OpenBSD: syscall.h,v 1.5 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 1998-2002 Opsycon AB, Sweden.
@@ -35,10 +35,10 @@
 extern long _dl__syscall(quad_t val, ...);
 
 #ifndef _dl_MAX_ERRNO
-#define _dl_MAX_ERRNO 4096
+#define _dl_MAX_ERRNO 512L
 #endif
-#define _dl_check_error(__res)	\
-	((int) __res < 0 && (int) __res >= -_dl_MAX_ERRNO)
+#define _dl_mmap_error(__res) \
+    ((long)__res < 0 && (long)__res >= -_dl_MAX_ERRNO)
 
 /*
  *  Inlined system call functions that can be used before
diff --git a/libexec/ld.so/powerpc/syscall.h b/libexec/ld.so/powerpc/syscall.h
index 29878f3a988..b1212f6bde0 100644
--- a/libexec/ld.so/powerpc/syscall.h
+++ b/libexec/ld.so/powerpc/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.20 2006/05/03 16:10:52 drahn Exp $ */
+/*	$OpenBSD: syscall.h,v 1.21 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 1998 Per Fogelstrom, Opsycon AB
@@ -38,10 +38,10 @@
 static off_t	_dl_lseek(int, off_t, int);
 
 #ifndef _dl_MAX_ERRNO
-#define _dl_MAX_ERRNO 4096
+#define _dl_MAX_ERRNO 512L
 #endif
-#define _dl_check_error(__res) \
-	((int) __res < 0 && (int) __res >= -_dl_MAX_ERRNO)
+#define _dl_mmap_error(__res) \
+    ((long)__res < 0 && (long)__res >= -_dl_MAX_ERRNO)
 
 /*
  *  Inlined system call functions that can be used before
diff --git a/libexec/ld.so/sh/syscall.h b/libexec/ld.so/sh/syscall.h
index be119fc778a..85c1857022d 100644
--- a/libexec/ld.so/sh/syscall.h
+++ b/libexec/ld.so/sh/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.1 2006/11/10 07:44:58 drahn Exp $ */
+/*	$OpenBSD: syscall.h,v 1.2 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 2001 Niklas Hallqvist
@@ -34,10 +34,10 @@
 #include <sys/signal.h>
 
 #ifndef _dl_MAX_ERRNO
-#define _dl_MAX_ERRNO 4096
+#define _dl_MAX_ERRNO 512L
 #endif
-#define _dl_check_error(__res) \
-    ((int) __res < 0 && (int) __res >= -_dl_MAX_ERRNO)
+#define _dl_mmap_error(__res) \
+    ((long)__res < 0 && (long)__res >= -_dl_MAX_ERRNO)
 
 int	_dl_close(int);
 int	_dl_exit(int);
diff --git a/libexec/ld.so/sod.c b/libexec/ld.so/sod.c
index d93bc62d23b..7b421b24c54 100644
--- a/libexec/ld.so/sod.c
+++ b/libexec/ld.so/sod.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sod.c,v 1.22 2004/10/17 03:56:49 drahn Exp $	*/
+/*	$OpenBSD: sod.c,v 1.23 2008/10/02 20:12:08 kurt Exp $	*/
 
 /*
  * Copyright (c) 1993 Paul Kranenburg
@@ -156,7 +156,7 @@ _dl_maphints(void)
 
 	hsize = (long)sb.st_size;
 	addr = (void *)_dl_mmap(0, hsize, PROT_READ, MAP_PRIVATE, hfd, 0);
-	if (addr == MAP_FAILED)
+	if (_dl_mmap_error(addr))
 		goto bad_hints;
 
 	hheader = (struct hints_header *)addr;
@@ -178,7 +178,7 @@ _dl_maphints(void)
 	return;
 
 bad_hints:
-	if (addr != MAP_FAILED)
+	if (!_dl_mmap_error(addr))
 		_dl_munmap(addr, hsize);
 	if (hfd != -1)
 		_dl_close(hfd);
diff --git a/libexec/ld.so/sparc/syscall.h b/libexec/ld.so/sparc/syscall.h
index bb07f56cab0..093221ea451 100644
--- a/libexec/ld.so/sparc/syscall.h
+++ b/libexec/ld.so/sparc/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.8 2006/05/03 16:10:52 drahn Exp $ */
+/*	$OpenBSD: syscall.h,v 1.9 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 2001 Niklas Hallqvist
@@ -34,10 +34,10 @@
 #include <sys/signal.h>
 
 #ifndef _dl_MAX_ERRNO
-#define _dl_MAX_ERRNO 4096
+#define _dl_MAX_ERRNO 512L
 #endif
-#define _dl_check_error(__res) \
-    ((int) __res < 0 && (int) __res >= -_dl_MAX_ERRNO)
+#define _dl_mmap_error(__res) \
+    ((long)__res < 0 && (long)__res >= -_dl_MAX_ERRNO)
 
 int	_dl_close(int);
 int	_dl_exit(int);
diff --git a/libexec/ld.so/sparc64/syscall.h b/libexec/ld.so/sparc64/syscall.h
index 45c05b0dcbc..d01861364ef 100644
--- a/libexec/ld.so/sparc64/syscall.h
+++ b/libexec/ld.so/sparc64/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.15 2006/05/03 16:10:52 drahn Exp $ */
+/*	$OpenBSD: syscall.h,v 1.16 2008/10/02 20:12:08 kurt Exp $ */
 
 /*
  * Copyright (c) 2001 Niklas Hallqvist
@@ -34,10 +34,10 @@
 #include <sys/signal.h>
 
 #ifndef _dl_MAX_ERRNO
-#define _dl_MAX_ERRNO 4096
+#define _dl_MAX_ERRNO 512L
 #endif
-#define _dl_check_error(__res) \
-    ((int) __res < 0 && (int) __res >= -_dl_MAX_ERRNO)
+#define _dl_mmap_error(__res) \
+    ((long)__res < 0 && (long)__res >= -_dl_MAX_ERRNO)
 
 int	_dl_close(int);
 int	_dl_exit(int);
diff --git a/libexec/ld.so/util.c b/libexec/ld.so/util.c
index a5e8c75c59f..00982811297 100644
--- a/libexec/ld.so/util.c
+++ b/libexec/ld.so/util.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: util.c,v 1.19 2007/02/09 14:51:13 drahn Exp $	*/
+/*	$OpenBSD: util.c,v 1.20 2008/10/02 20:12:08 kurt Exp $	*/
 
 /*
  * Copyright (c) 1998 Per Fogelstrom, Opsycon AB
@@ -104,7 +104,7 @@ _dl_malloc(size_t need)
 		_dl_malloc_pool = (void *)_dl_mmap((void *)0,
 		    _dl_round_page(need), PROT_READ|PROT_WRITE,
 		    MAP_ANON|MAP_PRIVATE, -1, 0);
-		if (_dl_malloc_pool == 0 || _dl_malloc_pool == MAP_FAILED ) {
+		if (_dl_malloc_pool == 0 || _dl_mmap_error(_dl_malloc_pool)) {
 			_dl_printf("Dynamic loader failure: malloc.\n");
 			_dl_exit(7);
 		}