The next step for mimmutable(). ld.so figures out what regions of memory

of startup shared library mappings can be made immutable, and also does
this for dlope() RTLD_NODELETE and subsidiary libraries.  Complexity in this
diff is due to the GNU_RELRO and OPENBSD_MUTABLE sections.
Tested in snaps for about 3 weeks, with some bootstrap related pain felt in ports
ok kettenis, much help from others.

4 files changed, 161 insertions, 8 deletions

diff --git a/libexec/ld.so/library.c b/libexec/ld.so/library.c
index eb641d99c53..f22b2db306b 100644
--- a/libexec/ld.so/library.c
+++ b/libexec/ld.so/library.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: library.c,v 1.88 2022/11/07 10:35:26 deraadt Exp $ */
+/*	$OpenBSD: library.c,v 1.89 2022/12/04 15:42:07 deraadt Exp $ */
 
 /*
  * Copyright (c) 2002 Dale Rahn
@@ -98,6 +98,7 @@ unload:
 elf_object_t *
 _dl_tryload_shlib(const char *libname, int type, int flags, int nodelete)
 {
+	struct mutate imut[MAXMUT], mut[MAXMUT];
 	int	libfile, i;
 	struct load_list *next_load, *load_list = NULL;
 	Elf_Addr maxva = 0, minva = ELF_NO_ADDR;
@@ -150,6 +151,9 @@ _dl_tryload_shlib(const char *libname, int type, int flags, int nodelete)
 		return(0);
 	}
 
+	_dl_memset(&mut, 0, sizeof mut);
+	_dl_memset(&imut, 0, sizeof imut);
+
 	/*
 	 *  Alright, we might have a winner!
 	 *  Figure out how much VM space we need.
@@ -211,6 +215,9 @@ _dl_tryload_shlib(const char *libname, int type, int flags, int nodelete)
 	loff = libaddr - minva;
 	phdp = (Elf_Phdr *)(hbuf + ehdr->e_phoff);
 
+	/* Entire mapping can become immutable, minus exceptions chosen later */
+	_dl_defer_immut(imut, loff, maxva - minva);
+
 	for (i = 0; i < ehdr->e_phnum; i++, phdp++) {
 		switch (phdp->p_type) {
 		case PT_LOAD: {
@@ -293,6 +300,11 @@ _dl_tryload_shlib(const char *libname, int type, int flags, int nodelete)
 		case PT_GNU_RELRO:
 			relro_addr = phdp->p_vaddr + loff;
 			relro_size = phdp->p_memsz;
+			_dl_defer_mut(mut, phdp->p_vaddr + loff, phdp->p_memsz);
+			break;
+
+		case PT_OPENBSD_MUTABLE:
+			_dl_defer_mut(mut, phdp->p_vaddr + loff, phdp->p_memsz);
 			break;
 
 		default:
@@ -329,6 +341,8 @@ _dl_tryload_shlib(const char *libname, int type, int flags, int nodelete)
 				_dl_printf("msyscall %lx %lx error\n",
 				    exec_start, exec_size);
 		}
+		_dl_bcopy(mut, object->mut, sizeof mut);
+		_dl_bcopy(imut, object->imut, sizeof imut);
 	} else {
 		_dl_munmap((void *)libaddr, maxva - minva);
 		_dl_load_list_free(load_list);
diff --git a/libexec/ld.so/library_mquery.c b/libexec/ld.so/library_mquery.c
index 6903263b7f9..143c44c526b 100644
--- a/libexec/ld.so/library_mquery.c
+++ b/libexec/ld.so/library_mquery.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: library_mquery.c,v 1.68 2022/11/07 10:35:26 deraadt Exp $ */
+/*	$OpenBSD: library_mquery.c,v 1.69 2022/12/04 15:42:07 deraadt Exp $ */
 
 /*
  * Copyright (c) 2002 Dale Rahn
@@ -103,6 +103,7 @@ unload:
 elf_object_t *
 _dl_tryload_shlib(const char *libname, int type, int flags, int nodelete)
 {
+	struct mutate imut[MAXMUT], mut[MAXMUT];
 	int libfile, i;
 	struct load_list *ld, *lowld = NULL;
 	elf_object_t *object;
@@ -231,6 +232,8 @@ _dl_tryload_shlib(const char *libname, int type, int flags, int nodelete)
 #define LOFF ((Elf_Addr)lowld->start - lowld->moff)
 
 retry:
+	_dl_memset(&mut, 0, sizeof mut);
+	_dl_memset(&imut, 0, sizeof imut);
 	exec_start = NULL;
 	exec_size = 0;
 	for (ld = lowld; ld != NULL; ld = ld->next) {
@@ -285,6 +288,9 @@ retry:
 			exec_size = ROUND_PG(ld->size);
 		}
 
+		/* Entire mapping can become immutable, minus exceptions chosen later */
+		_dl_defer_immut(imut, LOFF + ld->moff, ROUND_PG(ld->size));
+
 		ld->start = res;
 	}
 
@@ -298,12 +304,19 @@ retry:
 
 	phdp = (Elf_Phdr *)(hbuf + ehdr->e_phoff);
 	for (i = 0; i < ehdr->e_phnum; i++, phdp++) {
-		if (phdp->p_type == PT_OPENBSD_RANDOMIZE)
+		switch (phdp->p_type) {
+		case PT_OPENBSD_RANDOMIZE:
 			_dl_arc4randombuf((char *)(phdp->p_vaddr + LOFF),
 			    phdp->p_memsz);
-		else if (phdp->p_type == PT_GNU_RELRO) {
+			break;
+		case PT_GNU_RELRO:
 			relro_addr = phdp->p_vaddr + LOFF;
 			relro_size = phdp->p_memsz;
+			_dl_defer_mut(mut, phdp->p_vaddr + LOFF, phdp->p_memsz);
+			break;
+		case PT_OPENBSD_MUTABLE:
+			_dl_defer_mut(mut, phdp->p_vaddr + LOFF, phdp->p_memsz);
+			break;
 		}
 	}
 
@@ -337,6 +350,8 @@ retry:
 				_dl_printf("msyscall %lx %lx error\n",
 				    exec_start, exec_size);
 		}
+		_dl_bcopy(mut, object->mut, sizeof mut);
+		_dl_bcopy(imut, object->imut, sizeof imut);
 	} else {
 		_dl_load_list_free(lowld);
 	}
diff --git a/libexec/ld.so/loader.c b/libexec/ld.so/loader.c
index c1922d06ee5..0cd15cdb524 100644
--- a/libexec/ld.so/loader.c
+++ b/libexec/ld.so/loader.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: loader.c,v 1.204 2022/11/09 19:50:25 deraadt Exp $ */
+/*	$OpenBSD: loader.c,v 1.205 2022/12/04 15:42:07 deraadt Exp $ */
 
 /*
  * Copyright (c) 1998 Per Fogelstrom, Opsycon AB
@@ -840,8 +840,10 @@ _dl_call_init_recurse(elf_object_t *object, int initfirst)
 	if (initfirst && (object->obj_flags & DF_1_INITFIRST) == 0)
 		return;
 
-	if (!initfirst)
+	if (!initfirst) {
 		_dl_relro(object);
+		_dl_apply_immutable(object);
+	}
 
 	if (object->dyn.init) {
 		DL_DEB(("doing ctors obj %p @%p: [%s]\n",
@@ -860,8 +862,10 @@ _dl_call_init_recurse(elf_object_t *object, int initfirst)
 			    environ, &_dl_cb_cb);
 	}
 
-	if (initfirst)
+	if (initfirst) {
 		_dl_relro(object);
+		_dl_apply_immutable(object);
+	}
 
 	object->status |= STAT_INIT_DONE;
 }
@@ -1008,3 +1012,114 @@ _dl_rreloc(elf_object_t *object)
 	}
 }
 
+void
+_dl_defer_immut(struct mutate *m, vaddr_t start, vsize_t len)
+{
+	int i;
+
+	for (i = 0; i < MAXMUT; i++) {
+		if (m[i].valid == 0) {
+//			_dl_printf("%dimut\t%lx-%lx (len %x)\n",
+//			    i, start, start + len, len);
+			m[i].start = start;
+			m[i].end = start + len;
+			m[i].valid = 1;
+			return;
+		}
+	}
+	if (i == MAXMUT)
+		_dl_die("too many _dl_defer_immut");
+}
+
+void
+_dl_defer_mut(struct mutate *m, vaddr_t start, size_t len)
+{
+	int i;
+
+	for (i = 0; i < MAXMUT; i++) {
+		if (m[i].valid == 0) {
+//			_dl_printf("%dmut\t%lx-%lx (len %x)\n",
+//			    i, start, start + len, len);
+			m[i].start = start;
+			m[i].end = start + len;
+			m[i].valid = 1;
+			return;
+		}
+	}
+	if (i == MAXMUT)
+		_dl_die("too many _dl_defer_mut");
+}
+
+void
+_dl_apply_immutable(elf_object_t *object)
+{
+	struct mutate *m, *im, *imtail;
+	int mut, imut;
+	
+	if (object->obj_type != OBJTYPE_LIB)
+		return;
+
+	imtail = &object->imut[MAXMUT - 1];
+
+//	_dl_printf("library %s %lx:\n", object->load_name);
+	for (imut = 0; imut < MAXMUT; imut++) {
+		im = &object->imut[imut];
+		if (im->valid == 0)
+			continue;
+
+		for (mut = 0; mut < MAXMUT; mut++) {
+			m = &object->mut[mut];
+			if (m->valid == 0)
+				continue;
+//			_dl_printf("- mut%d %lx-%lx (%x) from imut%d %lx-%lx (%x): ",
+//			    mut, m->start, m->end, m->end - m->start,
+//			    imut, im->start, im->end, im->end - im->start);
+			if (m->start <= im->start) {
+				if (m->end < im->start) {
+//					_dl_printf("before ignored");
+					;
+				} else if (m->end >= im->end) {
+					im->start = im->end = im->valid = 0;
+//					_dl_printf("whole: %lx-%lx", im->start, im->end);
+				} else {
+					im->start = m->end;
+//					_dl_printf("early: %lx-%lx", im->start, im->end);
+				}
+			} else if (m->start > im->start) {
+				if (m->end > im->end) {
+//					_dl_printf("after ignored");
+					;
+				} else if (m->end == im->end) {
+					im->end = m->start;
+//					_dl_printf("end: %lx-%lx", im->start, im->end);
+				} else if (m->end < im->end) {
+					imtail->start = im->start;
+					imtail->end = m->start;
+					imtail->valid = 1;
+					imtail--;
+					imtail->start = m->end;
+					imtail->end = im->end;
+					imtail->valid = 1;
+					imtail--;
+					im->start = im->end = im->valid = 0;
+//					_dl_printf("split %lx-%lx %lx-%lx",
+//					    imtail[1].start, imtail[1].end,
+//					    imtail[2].start, imtail[2].end);
+				}
+			}
+//			_dl_printf("\n");
+		}
+	}
+
+	/* and now, install immutability for objects */
+	for (imut = 0; imut < MAXMUT; imut++) {
+		im = &object->imut[imut];
+		if (im->valid == 0)
+			continue;
+//		_dl_printf("IMUT %s %lx-%lx (len %x) (%lx,%lx) [%lx,%lx]\n",
+//		    object->load_name, im->start, im->end, im->end - im->start,
+//		    (void *)im->start, (void *)im->end,
+//		    (void *)im->start, im->end - im->start);
+		_dl_mimmutable((void *)im->start, im->end - im->start);
+	}
+}
diff --git a/libexec/ld.so/resolve.h b/libexec/ld.so/resolve.h
index 941ab891110..550fd7d5102 100644
--- a/libexec/ld.so/resolve.h
+++ b/libexec/ld.so/resolve.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: resolve.h,v 1.102 2022/11/07 10:35:26 deraadt Exp $ */
+/*	$OpenBSD: resolve.h,v 1.103 2022/12/04 15:42:07 deraadt Exp $ */
 
 /*
  * Copyright (c) 1998 Per Fogelstrom, Opsycon AB
@@ -238,6 +238,10 @@ struct elf_object {
 
 	/* nonzero if trace enabled for this object */
 	int traced;
+
+#define MAXMUT 40
+	struct mutate imut[MAXMUT];
+	struct mutate mut[MAXMUT];
 };
 
 struct dep_node {
@@ -325,6 +329,11 @@ int	_dl_match_file(struct sod *sodp, const char *name, int namelen);
 char	*_dl_find_shlib(struct sod *sodp, char **searchpath, int nohints);
 void	_dl_load_list_free(struct load_list *load_list);
 
+void _dl_find_immutables(int type, elf_object_t *object, Elf_Ehdr *);
+void _dl_defer_mut(struct mutate *m, vaddr_t start, vsize_t len);
+void _dl_defer_immut(struct mutate *m, vaddr_t start, vsize_t len);
+void _dl_apply_immutable(elf_object_t *object);
+
 typedef void lock_cb(int);
 void	_dl_thread_kern_go(lock_cb *);
 lock_cb	*_dl_thread_kern_stop(void);