Add a new mmap(2) flag __MAP_NOREMAP for use with MAP_FIXED to

indicate that the kernel should fail with MAP_FAILED if the specified
address is not currently available instead of unmapping it.

Change ld.so on i386 to make use of __MAP_NOREMAP to improve
reliability.

__MAP_NOREMAP diff by guenther based on an earlier diff by Ariane;
ld.so bits by guenther and me
bulk build stress testing of earlier diffs by sthen
ok deraadt; committing now for further testing

1 files changed, 47 insertions, 54 deletions

diff --git a/libexec/ld.so/library_mquery.c b/libexec/ld.so/library_mquery.c
index 5b4c9c0a2c4..aedfd091f8b 100644
--- a/libexec/ld.so/library_mquery.c
+++ b/libexec/ld.so/library_mquery.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: library_mquery.c,v 1.42 2012/06/12 20:32:17 matthew Exp $ */
+/*	$OpenBSD: library_mquery.c,v 1.43 2012/07/21 06:46:58 matthew Exp $ */
 
 /*
  * Copyright (c) 2002 Dale Rahn
@@ -201,71 +201,64 @@ retry:
 	for (ld = lowld; ld != NULL; ld = ld->next) {
 		off_t foff;
 		int fd, flags;
+		void *res;
 
-		/*
-		 * We don't want to provide the fd/off hint for anything
-		 * but the first mapping, all other might have
-		 * cache-incoherent aliases and will cause this code to
-		 * loop forever.
-		 */
-		if (ld == lowld) {
-			fd = libfile;
-			foff = ld->foff;
-			flags = 0;
-		} else {
+		flags = MAP_PRIVATE;
+		if (LOFF + ld->moff != 0)
+			flags |= MAP_FIXED | __MAP_NOREPLACE;
+
+		if (ld->foff < 0) {
 			fd = -1;
 			foff = 0;
-			flags = MAP_FIXED;
+			flags |= MAP_ANON;
+		} else {
+			fd = libfile;
+			foff = ld->foff;
 		}
 
-		ld->start = (void *)(LOFF + ld->moff);
-
-		/*
-		 * Magic here.
-		 * The first mquery is done with MAP_FIXED to see if
-		 * the mapping we want is free. If it's not, we redo the
-		 * mquery without MAP_FIXED to get the next free mapping,
-		 * adjust the base mapping address to match this free mapping
-		 * and restart the process again.
-		 */
-		ld->start = _dl_mquery(ld->start, ROUND_PG(ld->size), ld->prot,
-		    flags, fd, foff);
-		if (_dl_mmap_error(ld->start)) {
-			ld->start = (void *)(LOFF + ld->moff);
-			ld->start = _dl_mquery(ld->start, ROUND_PG(ld->size),
-			    ld->prot, flags & ~MAP_FIXED, fd, foff);
-			if (_dl_mmap_error(ld->start))
+		res = _dl_mmap((void *)(LOFF + ld->moff), ROUND_PG(ld->size),
+		    ld->prot, flags, fd, foff);
+		if (_dl_mmap_error(res)) {
+			/*
+			 * The mapping we wanted isn't free, so we do an
+			 * mquery without MAP_FIXED to get the next free
+			 * mapping, adjust the base mapping address to match
+			 * this free mapping and restart the process again.
+			 *
+			 * XXX - we need some kind of boundary condition
+			 * here, or fix mquery to not run into the stack
+			 */
+			res = _dl_mquery((void *)(LOFF + ld->moff),
+			    ROUND_PG(ld->size), ld->prot,
+			    flags & ~(MAP_FIXED | __MAP_NOREPLACE), fd, foff);
+
+			/*
+			 * If ld == lowld, then ld->start is just a hint and
+			 * thus shouldn't be unmapped.
+			 */
+			ld->start = NULL;
+
+			/* Unmap any mappings that we did get in. */
+			for (ld = lowld; ld != NULL; ld = ld->next) {
+				if (ld->start == NULL)
+					break;
+				_dl_munmap(ld->start, ROUND_PG(ld->size));
+				ld->start = NULL;
+			}
+
+			/* if the mquery failed, give up */
+			if (_dl_mmap_error(res))
 				goto fail;
-		}
 
-		if (ld->start != (void *)(LOFF + ld->moff)) {
-			lowld->start = ld->start - ld->moff + lowld->moff;
+			/* otherwise, reset the start of the base mapping */
+			lowld->start = res - ld->moff + lowld->moff;
 			goto retry;
 		}
-		/*
-		 * XXX - we need some kind of boundary condition here,
-		 * or fix mquery to not run into the stack
-		 */
+
+		ld->start = res;
 	}
 
 	for (ld = lowld; ld != NULL; ld = ld->next) {
-		int fd, flags;
-		off_t foff;
-		void *res;
-
-		if (ld->foff < 0) {
-			fd = -1;
-			foff = 0;
-			flags = MAP_FIXED|MAP_PRIVATE|MAP_ANON;
-		} else {
-			fd = libfile;
-			foff = ld->foff;
-			flags = MAP_FIXED|MAP_PRIVATE;
-		}
-		res = _dl_mmap(ld->start, ROUND_PG(ld->size), ld->prot, flags,
-		    fd, foff);
-		if (_dl_mmap_error(res))
-			goto fail;
 		/* Zero out everything past the EOF */
 		if ((ld->prot & PROT_WRITE) != 0 && (ld->size & align) != 0)
 			_dl_memset((char *)ld->start + ld->size, 0,