btrace(8) support to symbolize utrace addresses.

This only works for a single static binary where everything was compiled
with -fno-omit-frame-pointer since the stack unwinder requires the
frame-pointer.

A possible btrace script to capture performace of a single process is:
profile:hz:100 / pid == $1 / {
        @[ustack] = count();
}
Then using btrace -p program uprofile.bt `pgrep program` will collect
the information for program.

This is far from perfect but should allow other people to play with this
and hopefully improve work.
OK mpi@

3 files changed, 126 insertions, 64 deletions

diff --git a/usr.sbin/btrace/btrace.c b/usr.sbin/btrace/btrace.c
index ae583a5e15f..21c91884a64 100644
--- a/usr.sbin/btrace/btrace.c
+++ b/usr.sbin/btrace/btrace.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: btrace.c,v 1.69 2023/03/10 23:02:30 bluhm Exp $ */
+/*	$OpenBSD: btrace.c,v 1.70 2023/05/12 14:14:16 claudio Exp $ */
 
 /*
  * Copyright (c) 2019 - 2021 Martin Pieuchot <mpi@openbsd.org>
@@ -119,9 +119,12 @@ struct dtioc_arg_info  **dt_args;	/* array of probe arguments */
 struct dt_evt		 bt_devt;	/* fake event for BEGIN/END */
 uint64_t		 bt_filtered;	/* # of events filtered out */
 
+struct syms		*kelf, *uelf;
+
 char			**vargs;
 int			 nargs = 0;
 int			 verbose = 0;
+int			 dtfd;
 volatile sig_atomic_t	 quit_pending;
 
 static void
@@ -153,6 +156,9 @@ main(int argc, char *argv[])
 		case 'n':
 			noaction = 1;
 			break;
+		case 'p':
+			uelf = kelf_open(optarg);
+			break;
 		case 'v':
 			verbose++;
 			break;
@@ -204,6 +210,7 @@ main(int argc, char *argv[])
 		fd = open(__PATH_DEVDT, O_RDONLY);
 		if (fd == -1)
 			err(1, "could not open %s", __PATH_DEVDT);
+		dtfd = fd;
 	}
 
 	if (showprobes) {
@@ -516,7 +523,7 @@ rules_setup(int fd)
 	}
 
 	if (dokstack)
-		kelf_open();
+		kelf = kelf_open(_PATH_KSYMS);
 
 	/* Initialize "fake" event for BEGIN/END */
 	bt_devt.dtev_pbn = -1;
@@ -593,8 +600,10 @@ rules_teardown(int fd)
 		}
 	}
 
-	if (dokstack)
-		kelf_close();
+	kelf_close(kelf);
+	kelf = NULL;
+	kelf_close(uelf);
+	uelf = NULL;
 
 	/* Update "fake" event for BEGIN/END */
 	clock_gettime(CLOCK_REALTIME, &bt_devt.dtev_tsp);
@@ -703,17 +712,21 @@ builtin_nsecs(struct dt_evt *dtev)
 }
 
 const char *
-builtin_stack(struct dt_evt *dtev, int kernel)
+builtin_stack(struct dt_evt *dtev, int kernel, unsigned long offset)
 {
 	struct stacktrace *st = &dtev->dtev_kstack;
-	static char buf[4096], *bp;
+	static char buf[4096];
+	const char *last = "\nkernel\n";
+	char *bp;
 	size_t i;
 	int sz;
 
-	if (!kernel)
-		return "";
-	if (st->st_count == 0)
+	if (!kernel) {
+		st = &dtev->dtev_ustack;
+		last = "\nuserland\n";
+	} else if (st->st_count == 0) {
 		return "\nuserland\n";
+	}
 
 	buf[0] = '\0';
 	bp = buf;
@@ -721,7 +734,12 @@ builtin_stack(struct dt_evt *dtev, int kernel)
 	for (i = 0; i < st->st_count; i++) {
 		int l;
 
-		l = kelf_snprintsym(bp, sz - 1, st->st_pc[i]);
+		if (!kernel)
+			l = kelf_snprintsym(uelf, bp, sz - 1, st->st_pc[i],
+			    offset);
+		else
+			l = kelf_snprintsym(kelf, bp, sz - 1, st->st_pc[i],
+			    offset);
 		if (l < 0)
 			break;
 		if (l >= sz - 1) {
@@ -732,7 +750,7 @@ builtin_stack(struct dt_evt *dtev, int kernel)
 		bp += l;
 		sz -= l;
 	}
-	snprintf(bp, sz, "\nkernel\n");
+	snprintf(bp, sz, "%s", last);
 
 	return buf;
 }
@@ -1525,10 +1543,10 @@ ba2str(struct bt_arg *ba, struct dt_evt *dtev)
 		str = "";
 		break;
 	case B_AT_BI_KSTACK:
-		str = builtin_stack(dtev, 1);
+		str = builtin_stack(dtev, 1, 0);
 		break;
 	case B_AT_BI_USTACK:
-		str = builtin_stack(dtev, 0);
+		str = builtin_stack(dtev, 0, dt_get_offset(dtev->dtev_pid));
 		break;
 	case B_AT_BI_COMM:
 		str = dtev->dtev_comm;
@@ -1796,3 +1814,30 @@ debug_probe_name(struct bt_probe *bp)
 
 	return buf;
 }
+
+unsigned long
+dt_get_offset(pid_t pid)
+{
+	static struct dtioc_getaux	cache[32];
+	static int			next;
+	struct dtioc_getaux		*aux = NULL;
+	int				 i;
+
+	for (i = 0; i < 32; i++) {
+		if (cache[i].dtga_pid != pid)
+			continue;
+		aux = cache + i;
+		break;
+	}
+
+	if (aux == NULL) {
+		aux = &cache[next++];
+		next %= 32;
+
+		aux->dtga_pid = pid;
+		if (ioctl(dtfd, DTIOCGETAUXBASE, aux))
+			aux->dtga_auxbase = 0;
+	}
+
+	return aux->dtga_auxbase;
+}
diff --git a/usr.sbin/btrace/btrace.h b/usr.sbin/btrace/btrace.h
index d763ae5a5fc..78d4618d416 100644
--- a/usr.sbin/btrace/btrace.h
+++ b/usr.sbin/btrace/btrace.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: btrace.h,v 1.11 2021/12/07 22:17:03 guenther Exp $ */
+/*	$OpenBSD: btrace.h,v 1.12 2023/05/12 14:14:16 claudio Exp $ */
 
 /*
  * Copyright (c) 2019 - 2020 Martin Pieuchot <mpi@openbsd.org>
@@ -33,11 +33,15 @@ const char *		 ba_name(struct bt_arg *);
 long			 ba2long(struct bt_arg *, struct dt_evt *);
 const char		*ba2str(struct bt_arg *, struct dt_evt *);
 long			 bacmp(struct bt_arg *, struct bt_arg *);
+unsigned long		 dt_get_offset(pid_t);
 
 /* ksyms.c */
-int			 kelf_open(void);
-void			 kelf_close(void);
-int			 kelf_snprintsym(char *, size_t, unsigned long);
+struct syms;
+struct syms		*kelf_open(const char *);
+void			 kelf_offset(struct syms *, unsigned long);
+void			 kelf_close(struct syms *);
+int			 kelf_snprintsym(struct syms *, char *, size_t,
+			    unsigned long, unsigned long);
 
 /* map.c */
 struct map;
@@ -52,7 +56,7 @@ void			 map_zero(struct map *);
 struct hist		*hist_increment(struct hist *, const char *, long);
 void			 hist_print(struct hist *, const char *);
 
-#define KLEN	512	/* # of characters in map key, contain a stack trace */
+#define KLEN	1024	/* # of characters in map key, contain a stack trace */
 #define STRLEN	64	/* maximum # of bytes to output via str() function */
 
 /* printf.c */
diff --git a/usr.sbin/btrace/ksyms.c b/usr.sbin/btrace/ksyms.c
index 09fbd81cc25..7e0ca0ad157 100644
--- a/usr.sbin/btrace/ksyms.c
+++ b/usr.sbin/btrace/ksyms.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ksyms.c,v 1.4 2021/02/10 00:34:57 deraadt Exp $ */
+/*	$OpenBSD: ksyms.c,v 1.5 2023/05/12 14:14:16 claudio Exp $ */
 
 /*
  * Copyright (c) 2016 Martin Pieuchot <mpi@openbsd.org>
@@ -16,73 +16,82 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
+#define _DYN_LOADER	/* needed for AuxInfo */
+
+#include <sys/types.h>
+
 #include <assert.h>
 #include <err.h>
 #include <fcntl.h>
 #include <gelf.h>
 #include <paths.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
 #include "btrace.h"
 
-int		 kfd = -1;
-Elf		*kelf;
-Elf_Scn		*ksymtab;
-size_t		 kstrtabndx, knsymb;
+struct syms {
+	int		 fd;
+	Elf		*elf;
+	Elf_Scn		*symtab;
+	size_t		 strtabndx, nsymb;
+};
 
-int		 kelf_parse(void);
+int		 kelf_parse(struct syms *);
 
-int
-kelf_open(void)
+struct syms *
+kelf_open(const char *path)
 {
+	struct syms *syms;
 	int error;
 
-	assert(kfd == -1);
-
 	if (elf_version(EV_CURRENT) == EV_NONE)
 		errx(1, "elf_version: %s", elf_errmsg(-1));
 
-	kfd = open(_PATH_KSYMS, O_RDONLY);
-	if (kfd == -1) {
-		warn("open");
-		return 1;
+	if ((syms = calloc(1, sizeof(*syms))) == NULL)
+		err(1, NULL);
+
+	syms->fd = open(path, O_RDONLY);
+	if (syms->fd == -1) {
+		warn("open: %s", path);
+		free(syms);
+		return NULL;
 	}
 
-	if ((kelf = elf_begin(kfd, ELF_C_READ, NULL)) == NULL) {
+	if ((syms->elf = elf_begin(syms->fd, ELF_C_READ, NULL)) == NULL) {
 		warnx("elf_begin: %s", elf_errmsg(-1));
-		error = 1;
 		goto bad;
 	}
 
-	if (elf_kind(kelf) != ELF_K_ELF) {
-		error = 1;
+	if (elf_kind(syms->elf) != ELF_K_ELF)
 		goto bad;
-	}
 
-	error = kelf_parse();
+	error = kelf_parse(syms);
 	if (error)
 		goto bad;
 
-	return 0;
+	return syms;
 
 bad:
-	kelf_close();
-	return error;
+	kelf_close(syms);
+	return NULL;
 }
 
 void
-kelf_close(void)
+kelf_close(struct syms *syms)
 {
-	elf_end(kelf);
-	kelf = NULL;
-	close(kfd);
-	kfd = -1;
+	if (syms == NULL)
+		return;
+	elf_end(syms->elf);
+	close(syms->fd);
+	free(syms);
 }
 
 int
-kelf_snprintsym(char *str, size_t size, unsigned long pc)
+kelf_snprintsym(struct syms *syms, char *str, size_t size, unsigned long pc,
+    unsigned long off)
 {
 	GElf_Sym	 sym;
 	Elf_Data	*data = NULL;
@@ -91,32 +100,35 @@ kelf_snprintsym(char *str, size_t size, unsigned long pc)
 	char		*name;
 	int		 cnt;
 
-	data = elf_rawdata(ksymtab, data);
+	if (syms == NULL)
+		goto fallback;
+
+	data = elf_rawdata(syms->symtab, data);
 	if (data == NULL)
 		goto fallback;
 
-	for (i = 0; i < knsymb; i++) {
+	for (i = 0; i < syms->nsymb; i++) {
 		if (gelf_getsym(data, i, &sym) == NULL)
 			continue;
 		if (GELF_ST_TYPE(sym.st_info) != STT_FUNC)
 			continue;
-		if (pc >= sym.st_value) {
-			if (pc < (sym.st_value + sym.st_size))
+		if (pc >= sym.st_value + off) {
+			if (pc < (sym.st_value + off + sym.st_size))
 				break;
 			/* Workaround for symbols w/o size, usually asm ones. */
-			if (sym.st_size == 0 && sym.st_value > bestoff) {
+			if (sym.st_size == 0 && sym.st_value + off > bestoff) {
 				bestidx = i;
-				bestoff = sym.st_value;
+				bestoff = sym.st_value + off;
 			}
 		}
 	}
 
-	if (i == knsymb) {
+	if (i == syms->nsymb) {
 		if (bestidx == 0 || gelf_getsym(data, bestidx, &sym) == NULL)
 			goto fallback;
 	}
 
-	name = elf_strptr(kelf, kstrtabndx, sym.st_name);
+	name = elf_strptr(syms->elf, syms->strtabndx, sym.st_name);
 	if (name != NULL)
 		cnt = snprintf(str, size, "\n%s", name);
 	else
@@ -124,7 +136,7 @@ kelf_snprintsym(char *str, size_t size, unsigned long pc)
 	if (cnt < 0)
 		return cnt;
 
-	offset = pc - sym.st_value;
+	offset = pc - (sym.st_value + off);
 	if (offset != 0) {
 		int l;
 
@@ -142,43 +154,44 @@ fallback:
 }
 
 int
-kelf_parse(void)
+kelf_parse(struct syms *syms)
 {
 	GElf_Shdr	 shdr;
 	Elf_Scn		*scn, *scnctf;
 	char		*name;
 	size_t		 shstrndx;
 
-	if (elf_getshdrstrndx(kelf, &shstrndx) != 0) {
+	if (elf_getshdrstrndx(syms->elf, &shstrndx) != 0) {
 		warnx("elf_getshdrstrndx: %s", elf_errmsg(-1));
 		return 1;
 	}
 
 	scn = scnctf = NULL;
-	while ((scn = elf_nextscn(kelf, scn)) != NULL) {
+	while ((scn = elf_nextscn(syms->elf, scn)) != NULL) {
 		if (gelf_getshdr(scn, &shdr) != &shdr) {
 			warnx("elf_getshdr: %s", elf_errmsg(-1));
 			return 1;
 		}
 
-		if ((name = elf_strptr(kelf, shstrndx, shdr.sh_name)) == NULL) {
+		if ((name = elf_strptr(syms->elf, shstrndx,
+		    shdr.sh_name)) == NULL) {
 			warnx("elf_strptr: %s", elf_errmsg(-1));
 			return 1;
 		}
 
 		if (strcmp(name, ELF_SYMTAB) == 0 &&
 		    shdr.sh_type == SHT_SYMTAB && shdr.sh_entsize != 0) {
-			ksymtab = scn;
-			knsymb = shdr.sh_size / shdr.sh_entsize;
+			syms->symtab = scn;
+			syms->nsymb = shdr.sh_size / shdr.sh_entsize;
 		}
 
 		if (strcmp(name, ELF_STRTAB) == 0 &&
 		    shdr.sh_type == SHT_STRTAB) {
-			kstrtabndx = elf_ndxscn(scn);
+			syms->strtabndx = elf_ndxscn(scn);
 		}
 	}
 
-	if (ksymtab == NULL)
+	if (syms->symtab == NULL)
 		warnx("symbol table not found");
 
 	return 0;