Do a better job of running destructors in the right order.

7 files changed, 236 insertions, 41 deletions

diff --git a/libexec/ld.so/dlfcn.c b/libexec/ld.so/dlfcn.c
index fff632ee745..2826cdf8cf4 100644
--- a/libexec/ld.so/dlfcn.c
+++ b/libexec/ld.so/dlfcn.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: dlfcn.c,v 1.44 2005/03/08 20:01:59 drahn Exp $ */
+/*	$OpenBSD: dlfcn.c,v 1.45 2005/04/05 19:29:09 drahn Exp $ */
 
 /*
  * Copyright (c) 1998 Per Fogelstrom, Opsycon AB
@@ -40,12 +40,10 @@
 
 int _dl_errno;
 
-static int _dl_real_close(void *handle);
-static void _dl_unload_deps(elf_object_t *object);
-static void _dl_thread_kern_stop(void);
-static void _dl_thread_kern_go(void);
-static void (*_dl_thread_fnc)(int) = NULL;
-static elf_object_t *obj_from_addr(const void *addr);
+int _dl_real_close(void *handle);
+void _dl_unload_deps(elf_object_t *object);
+void (*_dl_thread_fnc)(int) = NULL;
+elf_object_t *obj_from_addr(const void *addr);
 
 void *
 dlopen(const char *libname, int flags)
@@ -113,6 +111,9 @@ dlopen(const char *libname, int flags)
 	_dl_rtld(object);
 	_dl_call_init(object);
 
+	_dl_link_dlopen(object);
+
+
 	if (_dl_debug_map->r_brk) {
 		_dl_debug_map->r_state = RT_ADD;
 		(*((void (*)(void))_dl_debug_map->r_brk))();
@@ -229,7 +230,7 @@ dlclose(void *handle)
 	return (retval);
 }
 
-static int
+int
 _dl_real_close(void *handle)
 {
 	elf_object_t	*object;
@@ -250,6 +251,7 @@ _dl_real_close(void *handle)
 			_dl_unload_deps(dynobj);
 	}
 
+	_dl_unlink_dlopen(object);
 	_dl_unload_shlib(object);
 	return (0);
 }
@@ -258,7 +260,7 @@ _dl_real_close(void *handle)
  * Scan through the shadow dep list and 'unload' every library
  * we depend upon. Shadow objects are removed when removing ourself.
  */
-static void
+void
 _dl_unload_deps(elf_object_t *object)
 {
 	elf_object_t *depobj;
@@ -379,14 +381,14 @@ _dl_show_objects(void)
 		    _dl_symcachestat_lookups));
 }
 
-static void
+void
 _dl_thread_kern_stop(void)
 {
 	if (_dl_thread_fnc != NULL)
 		(*_dl_thread_fnc)(0);
 }
 
-static void
+void
 _dl_thread_kern_go(void)
 {
 	if (_dl_thread_fnc != NULL)
diff --git a/libexec/ld.so/library.c b/libexec/ld.so/library.c
index a3c87d77e33..d144978b9c9 100644
--- a/libexec/ld.so/library.c
+++ b/libexec/ld.so/library.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: library.c,v 1.36 2005/03/23 19:48:05 drahn Exp $ */
+/*	$OpenBSD: library.c,v 1.37 2005/04/05 19:29:09 drahn Exp $ */
 
 /*
  * Copyright (c) 2002 Dale Rahn
@@ -55,10 +55,16 @@ _dl_load_list_free(struct load_list *load_list)
 }
 
 void
-_dl_unload_shlib(elf_object_t *object)
+_dl_notify_unload_shlib(elf_object_t *object)
 {
 	if (--object->refcount == 0) {
 		_dl_run_dtors(object);
+	}
+}
+void
+_dl_unload_shlib(elf_object_t *object)
+{
+	if (object->refcount == 0) {
 		_dl_load_list_free(object->load_list);
 		_dl_munmap((void *)object->load_addr, object->load_size);
 		_dl_remove_object(object);
diff --git a/libexec/ld.so/library_mquery.c b/libexec/ld.so/library_mquery.c
index 2fa1165d627..61807f55fa0 100644
--- a/libexec/ld.so/library_mquery.c
+++ b/libexec/ld.so/library_mquery.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: library_mquery.c,v 1.16 2005/03/23 19:48:05 drahn Exp $ */
+/*	$OpenBSD: library_mquery.c,v 1.17 2005/04/05 19:29:09 drahn Exp $ */
 
 /*
  * Copyright (c) 2002 Dale Rahn
@@ -59,12 +59,27 @@ _dl_load_list_free(struct load_list *load_list)
 }
 
 void
+_dl_notify_unload_shlib(elf_object_t *object)
+{
+	struct dep_node *n;
+
+	if (--object->refcount == 0)
+		for (n = object->first_child; n; n = n->next_sibling)
+			_dl_notify_unload_shlib(n->data);
+}
+
+void
 _dl_unload_shlib(elf_object_t *object)
 {
-	if (--object->refcount == 0) {
-		_dl_run_dtors(object);
+	struct dep_node *n;
+
+	if (object->refcount == 0) {
 		_dl_load_list_free(object->load_list);
 		_dl_remove_object(object);
+
+		for (n = object->first_child; n; n = n->next_sibling) {
+			_dl_unload_shlib(n->data);
+		}
 	}
 }
 
diff --git a/libexec/ld.so/library_subr.c b/libexec/ld.so/library_subr.c
index b50afda6063..bd1b2bb85d3 100644
--- a/libexec/ld.so/library_subr.c
+++ b/libexec/ld.so/library_subr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: library_subr.c,v 1.1 2005/03/23 19:48:05 drahn Exp $ */
+/*	$OpenBSD: library_subr.c,v 1.2 2005/04/05 19:29:09 drahn Exp $ */
 
 /*
  * Copyright (c) 2002 Dale Rahn
@@ -41,6 +41,12 @@
 
 #define DEFAULT_PATH "/usr/lib"
 
+static void _dl_unload_dlopen_recurse(struct dep_node *node);
+
+/* STATIC DATA */
+static struct dep_node *_dlopened_first_child;
+static struct dep_node *_dlopened_last_child;
+
 /*
  * _dl_match_file()
  *
@@ -329,6 +335,83 @@ again:
 
 
 void
+_dl_link_dlopen(elf_object_t *dep)
+{
+	struct dep_node *n;
+
+	n = _dl_malloc(sizeof *n);
+	if (n == NULL)
+		_dl_exit(5);
+
+	n->data = dep;
+	n->next_sibling = NULL;
+	if (_dlopened_first_child) {
+		_dlopened_last_child->next_sibling = n;
+		_dlopened_last_child = n;
+	} else
+		_dlopened_first_child = _dlopened_last_child = n;
+
+	DL_DEB(("linking %s as dlopen()ed\n", dep->load_name));
+}
+
+void
+_dl_unlink_dlopen(elf_object_t *dep)
+{
+	struct dep_node **dnode;
+	struct dep_node *pnode;
+	struct dep_node *next;
+
+	dnode = &_dlopened_first_child;
+
+	if (_dlopened_first_child == NULL)
+		return;
+
+	if (_dlopened_first_child->data == dep) {
+		next = _dlopened_first_child->next_sibling;
+		_dl_free(_dlopened_first_child);
+		_dlopened_first_child = next;
+		return;
+	}
+	pnode = _dlopened_first_child;
+
+	while (pnode->next_sibling != NULL) {
+		if (pnode->next_sibling->data == dep) {
+			next = pnode->next_sibling->next_sibling;
+			if (pnode->next_sibling == _dlopened_last_child)
+				_dlopened_last_child = pnode;
+			_dl_free(pnode->next_sibling);
+			pnode->next_sibling = next;
+			break;
+		}
+		pnode = pnode->next_sibling;
+	}
+}
+
+void
+_dl_unload_dlopen(void)
+{
+	if (_dlopened_first_child != NULL)
+		_dl_unload_dlopen_recurse(_dlopened_first_child);
+}
+
+/*
+ * is recursion here a good thing? 
+ */
+void
+_dl_unload_dlopen_recurse(struct dep_node *node)
+{
+	if (node->next_sibling != NULL) {
+		_dl_unload_dlopen_recurse(node->next_sibling);
+	}
+	_dl_notify_unload_shlib(node->data);
+	_dl_run_all_dtors();
+	if (_dl_exiting == 0)
+		_dl_unload_shlib(node->data);
+	_dl_free(node);
+}
+
+
+void
 _dl_link_sub(elf_object_t *dep, elf_object_t *p)
 {
 	struct dep_node *n;
diff --git a/libexec/ld.so/loader.c b/libexec/ld.so/loader.c
index 35743a17160..88552a04de0 100644
--- a/libexec/ld.so/loader.c
+++ b/libexec/ld.so/loader.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: loader.c,v 1.83 2004/08/11 17:13:10 pefo Exp $ */
+/*	$OpenBSD: loader.c,v 1.84 2005/04/05 19:29:09 drahn Exp $ */
 
 /*
  * Copyright (c) 1998 Per Fogelstrom, Opsycon AB
@@ -71,6 +71,8 @@ struct r_debug *_dl_debug_map;
 
 void _dl_dopreload(char *paths);
 
+int _dl_exiting;
+
 void
 _dl_debug_state(void)
 {
@@ -78,27 +80,102 @@ _dl_debug_state(void)
 }
 
 /*
- * Routine to walk through all of the objects except the first
- * (main executable).
+ * Run dtors for the current object, then notify all of the DT_NEEDED
+ * libraries that it can be unloaded (or ref count lowered).
  */
+
+void
+_dl_run_all_dtors()
+{
+	elf_object_t *node;
+	int fini_complete;
+	struct dep_node *dnode;
+
+	fini_complete = 0;
+
+	while (fini_complete == 0) {
+		fini_complete = 1;
+		for (node = _dl_objects->next;
+		    node != NULL;
+		    node = node->next) {
+			if ((node->dyn.fini) &&
+			    (node->refcount == 0) &&
+			    (node->status & STAT_INIT_DONE) &&
+			    ((node->status & STAT_FINI_DONE) == 0)) {
+				node->status |= STAT_FINI_READY;
+			    }
+		}
+		for (node = _dl_objects->next;
+		    node != NULL;
+		    node = node->next ) {
+			if ((node->dyn.fini) &&
+			    (node->refcount == 0) &&
+			    (node->status & STAT_INIT_DONE) &&
+			    ((node->status & STAT_FINI_DONE) == 0))
+				for (dnode = node->first_child;
+				    dnode != NULL;
+				    dnode = dnode->next_sibling)
+					dnode->data->status &= ~STAT_FINI_READY;
+		}
+
+
+		for (node = _dl_objects->next;
+		    node != NULL;
+		    node = node->next ) {
+			if (node->status & STAT_FINI_READY) {
+				DL_DEB(("doing dtors obj %p @%p: [%s]\n",
+				    node, node->dyn.fini,
+				    node->load_name));
+
+				fini_complete = 0;
+				node->status |= STAT_FINI_DONE;
+				node->status &= ~STAT_FINI_READY;
+				(*node->dyn.fini)();
+			}
+		}
+	}
+}
+
 void
 _dl_run_dtors(elf_object_t *object)
 {
-	if (object->dyn.fini) {
-		DL_DEB(("doing dtors @%p: [%s]\n",
-		    object->dyn.fini, object->load_name));
-		(*object->dyn.fini)();
+	struct dep_node *n;
+
+	for (n = object->first_child; n; n = n->next_sibling) {
+		_dl_notify_unload_shlib(n->data);
 	}
-	if (object->next)
-		_dl_run_dtors(object->next);
+
+	_dl_run_all_dtors();
+
+
+	if (_dl_exiting == 0)
+		for (n = object->first_child; n; n = n->next_sibling)
+			_dl_unload_shlib(n->data);
 }
 
+/*
+ * Routine to walk through all of the objects except the first
+ * (main executable).
+ *
+ * Big question, should dlopen()ed objects be unloaded before or after
+ * the destructor for the main application runs?
+ */
 void
 _dl_dtors(void)
 {
+	_dl_thread_kern_stop();
+	_dl_exiting = 1;
+
+	/* ORDER? */
+	_dl_unload_dlopen();
+
 	DL_DEB(("doing dtors\n"));
-	if (_dl_objects->next)
-		_dl_run_dtors(_dl_objects->next);
+
+	/* main program runs its dtors itself
+	 * but we want to run dtors on all it's children);
+	 */
+	_dl_objects->status |= STAT_FINI_DONE;
+	_dl_run_dtors(_dl_objects);
 }
 
 void
@@ -676,8 +753,8 @@ _dl_call_init(elf_object_t *object)
 		return;
 
 	if (object->dyn.init) {
-		DL_DEB(("doing ctors @%p: [%s]\n",
-		    object->dyn.init, object->load_name));
+		DL_DEB(("doing ctors obj %p @%p: [%s]\n",
+		    object, object->dyn.init, object->load_name));
 		(*object->dyn.init)();
 	}
 
diff --git a/libexec/ld.so/resolve.c b/libexec/ld.so/resolve.c
index 2c6155b3e9d..8dfcbac48b0 100644
--- a/libexec/ld.so/resolve.c
+++ b/libexec/ld.so/resolve.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: resolve.c,v 1.24 2004/07/05 00:47:40 kjell Exp $ */
+/*	$OpenBSD: resolve.c,v 1.25 2005/04/05 19:29:09 drahn Exp $ */
 
 /*
  * Copyright (c) 1998 Per Fogelstrom, Opsycon AB
@@ -187,8 +187,9 @@ _dl_lookup_object(const char *name)
 	return(0);
 }
 
-int find_symbol_obj(elf_object_t *object, const char *name, unsigned long hash,
-    int flags, const Elf_Sym **ref, const Elf_Sym **weak_sym,
+int _dl_find_symbol_obj(elf_object_t *object, const char *name,
+    unsigned long hash, int flags, const Elf_Sym **ref,
+    const Elf_Sym **weak_sym,
     elf_object_t **weak_object);
 
 sym_cache *_dl_symcache;
@@ -261,7 +262,7 @@ _dl_find_symbol(const char *name, elf_object_t *startlook,
 	}
 
 	if (req_obj->dyn.symbolic)
-		if (find_symbol_obj(req_obj, name, h, flags, ref, &weak_sym,
+		if (_dl_find_symbol_obj(req_obj, name, h, flags, ref, &weak_sym,
 		    &weak_object)) {
 			object = req_obj;
 			found = 1;
@@ -278,7 +279,7 @@ retry_nonglobal_dlo:
 		    (object != req_obj))
 			continue;
 
-		if (find_symbol_obj(object, name, h, flags, ref, &weak_sym,
+		if (_dl_find_symbol_obj(object, name, h, flags, ref, &weak_sym,
 		    &weak_object)) {
 			found = 1;
 			break;
@@ -318,7 +319,7 @@ found:
 }
 
 int
-find_symbol_obj(elf_object_t *object, const char *name, unsigned long hash,
+_dl_find_symbol_obj(elf_object_t *object, const char *name, unsigned long hash,
     int flags, const Elf_Sym **ref, const Elf_Sym **weak_sym,
     elf_object_t **weak_object)
 {
diff --git a/libexec/ld.so/resolve.h b/libexec/ld.so/resolve.h
index 1f583fe23ad..af303fae8d9 100644
--- a/libexec/ld.so/resolve.h
+++ b/libexec/ld.so/resolve.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: resolve.h,v 1.33 2005/03/23 19:48:05 drahn Exp $ */
+/*	$OpenBSD: resolve.h,v 1.34 2005/04/05 19:29:09 drahn Exp $ */
 
 /*
  * Copyright (c) 1998 Per Fogelstrom, Opsycon AB
@@ -99,9 +99,11 @@ typedef struct elf_object {
 	struct elf_object *dep_next;	/* Shadow objects for resolve search */
 
 	int		status;
-#define	STAT_RELOC_DONE	1
-#define	STAT_GOT_DONE	2
-#define	STAT_INIT_DONE	4
+#define	STAT_RELOC_DONE	0x01
+#define	STAT_GOT_DONE	0x02
+#define	STAT_INIT_DONE	0x04
+#define	STAT_FINI_DONE	0x08
+#define	STAT_FINI_READY	0x10
 
 	Elf_Phdr	*phdrp;
 	int		phdrc;
@@ -118,7 +120,7 @@ typedef struct elf_object {
 	u_int32_t	nbuckets;
 	Elf_Word	*chains;
 	u_int32_t	nchains;
-	Elf_Dyn	*dynamic;
+	Elf_Dyn		*dynamic;
 
 	struct dep_node *first_child;
 	struct dep_node *last_child;
@@ -180,8 +182,13 @@ Elf_Addr _dl_find_symbol_bysym(elf_object_t *req_obj, unsigned int symidx,
 void _dl_rtld(elf_object_t *object);
 void _dl_call_init(elf_object_t *object);
 void _dl_link_sub(elf_object_t *dep, elf_object_t *p);
+void _dl_link_dlopen(elf_object_t *dep);
+void _dl_unlink_dlopen(elf_object_t *dep);
+void _dl_notify_unload_shlib(elf_object_t *object);
+void _dl_unload_dlopen(void);
 
 void _dl_run_dtors(elf_object_t *object);
+void _dl_run_all_dtors(void);
 
 Elf_Addr _dl_bind(elf_object_t *object, int index);
 
@@ -189,6 +196,9 @@ int	_dl_match_file(struct sod *sodp, char *name, int namelen);
 char	*_dl_find_shlib(struct sod *sodp, const char *searchpath, int nohints);
 void	_dl_load_list_free(struct load_list *load_list);
 
+void	_dl_thread_kern_go(void);
+void	_dl_thread_kern_stop(void);
+
 extern elf_object_t *_dl_objects;
 extern elf_object_t *_dl_last_object;
 
@@ -197,6 +207,7 @@ extern struct r_debug *_dl_debug_map;
 
 extern int  _dl_pagesz;
 extern int  _dl_errno;
+extern int  _dl_exiting;
 
 extern char *_dl_libpath;
 extern char *_dl_preload;