Fixes endianess problems with old a.out tools. This makes building

some cross toolchains possible.

This also changes some utilities's behavior slightly:
- nm, strip, size now handle any a.out format they know about.
- ranlib complains if it detects mixed archives (several object
formats for different boxes).

In the presence of mixed objects, you still have file, ranlib or ld
to prevent you from getting too confused...

8 files changed, 194 insertions, 57 deletions

diff --git a/usr.bin/nm/byte.c b/usr.bin/nm/byte.c
new file mode 100644
index 00000000000..7aa2408f9e4
--- /dev/null
+++ b/usr.bin/nm/byte.c
@@ -0,0 +1,110 @@
+/* $OpenBSD: byte.c,v 1.1 1999/05/10 16:14:07 espie Exp $ */
+/*
+ * Copyright (c) 1999  
+ *	Marc Espie.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ */
+
+/* a set of routines to read object files and compensate for host
+ * endianness
+ */
+
+
+static int byte_sex(mid)
+	int mid;
+{
+	switch(mid) {
+	case MID_I386:
+	case MID_VAX:
+	case MID_ALPHA:
+	case MID_PMAX:
+		return LITTLE_ENDIAN;
+	case MID_M68K:
+	case MID_M68K4K:
+	case MID_M88K:
+	case MID_SUN010:
+	case MID_SUN020:
+	case MID_HP200:
+	case MID_HP300:
+	case MID_HPUX800:
+	case MID_HPUX:
+	case MID_SPARC:
+	case MID_MIPS:
+		return BIG_ENDIAN;
+	default:	/* we don't know what this is, so we don't want to process it */
+		return 0;
+	}
+}
+
+#define BAD_OBJECT(h)  (N_BADMAG(h) || !byte_sex(N_GETMID(h)))
+
+/* handles endianess swaps */
+static void swap_u32s(h, n)
+	u_int32_t *h;
+	size_t n;
+{
+	size_t i;
+
+	for (i = 0; i < n; i++)
+		h[i] = swap32(h[i]);
+}
+
+static void fix_header_order(h)
+	struct exec *h;
+{
+	if (byte_sex(N_GETMID(*h)) != BYTE_ORDER)
+		swap_u32s( ((u_int32_t *)(h))+1, sizeof *h/sizeof(u_int32_t) - 1);
+}
+
+static long fix_long_order(l, mid)
+	long l;
+	int mid;
+{
+	if (byte_sex(mid) != BYTE_ORDER)
+		return swap32(l);
+	else
+		return l;
+}
+
+static void swap_nlist(p)
+	struct nlist *p;
+{
+	p->n_un.n_strx = swap32(p->n_un.n_strx);
+	p->n_desc = swap16(p->n_desc);
+	p->n_value = swap32(p->n_value);
+}
+
+static void fix_nlist_order(p, mid)
+	struct nlist *p;
+	int mid;
+{
+	if (byte_sex(mid) != BYTE_ORDER)
+		swap_nlist(p);
+}
+
+static void fix_nlists_order(p, n, mid)
+	struct nlist *p;
+	size_t n;
+	int mid;
+{
+	int i;
+
+	if (byte_sex(mid) != BYTE_ORDER)
+		for (i = 0; i < n; i++)
+			swap_nlist(p+i);
+}
+
+static fix_ranlib_order(r, mid)
+	struct ranlib *r;
+	int mid;
+{
+	if (byte_sex(mid) != BYTE_ORDER) {
+		r->ran_un.ran_strx = swap32(r->ran_un.ran_strx);
+		r->ran_off = swap32(r->ran_off);
+	}
+}
diff --git a/usr.bin/nm/nm.c b/usr.bin/nm/nm.c
index b3d6433608c..1cbfb3a4f91 100644
--- a/usr.bin/nm/nm.c
+++ b/usr.bin/nm/nm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nm.c,v 1.7 1998/05/11 20:34:58 niklas Exp $	*/
+/*	$OpenBSD: nm.c,v 1.8 1999/05/10 16:14:07 espie Exp $	*/
 /*	$NetBSD: nm.c,v 1.7 1996/01/14 23:04:03 pk Exp $	*/
 
 /*
@@ -47,7 +47,7 @@ static char copyright[] =
 #if 0
 static char sccsid[] = "@(#)nm.c	8.1 (Berkeley) 6/6/93";
 #endif
-static char rcsid[] = "$OpenBSD: nm.c,v 1.7 1998/05/11 20:34:58 niklas Exp $";
+static char rcsid[] = "$OpenBSD: nm.c,v 1.8 1999/05/10 16:14:07 espie Exp $";
 #endif /* not lint */
 
 #include <sys/param.h>
@@ -62,11 +62,9 @@ static char rcsid[] = "$OpenBSD: nm.c,v 1.7 1998/05/11 20:34:58 niklas Exp $";
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+/* XXX get shared code to handle byte-order swaps */
+#include "byte.c"
 
-#ifdef MID_MACHINE_OVERRIDE
-#undef MID_MACHINE
-#define MID_MACHINE MID_MACHINE_OVERRIDE
-#endif
 
 int ignore_bad_archive_entries = 1;
 int print_only_external_symbols;
@@ -79,6 +77,7 @@ int rev;
 int fname(), rname(), value();
 int (*sfunc)() = fname;
 
+
 /* some macros for symbol type (nlist.n_type) handling */
 #define	IS_DEBUGGER_SYMBOL(x)	((x) & N_STAB)
 #define	IS_EXTERNAL(x)		((x) & N_EXT)
@@ -178,13 +177,8 @@ process_file(fname)
 	}
 	rewind(fp);
 
+	if (BAD_OBJECT(exec_head)) {
 	/* this could be an archive */
-#if (MID_MACHINE == MID_M68K)
-	if (N_BADMAG(exec_head) || ((N_GETMID(exec_head) != MID_MACHINE) &&
-	    (N_GETMID(exec_head) != MID_M68K4K))) {
-#else
-	if (N_BADMAG(exec_head) || N_GETMID(exec_head) != MID_MACHINE) {
-#endif
 		if (fread(magic, sizeof(magic), (size_t)1, fp) != 1 ||
 		    strncmp(magic, ARMAG, SARMAG)) {
 			warnx("%s: not object file or archive", fname);
@@ -281,7 +275,7 @@ show_archive(fname, fp)
 			return(1);
 		}
 
-		if (N_BADMAG(exec_head)) {
+		if (BAD_OBJECT(exec_head)) {
 			if (!ignore_bad_archive_entries) {
 				 warnx("%s: bad format", name);
 				rval = 1;
@@ -340,12 +334,16 @@ show_objfile(objname, fp)
 		return(1);
 	}
 
-	/* stop if this is no valid object file */
-	if (N_BADMAG(head)) {
+	/* stop if this is no valid object file, or a format we don't dare
+	 * playing with
+	 */
+	if (BAD_OBJECT(head)) {
 		warnx("%s: bad format", objname);
 		return(1);
 	}
 
+	fix_header_order(&head);
+
 	/* stop if the object file contains no symbol table */
 	if (!head.a_syms) {
 		warnx("%s: no name list", objname);
@@ -365,6 +363,7 @@ show_objfile(objname, fp)
 		(void)free((char *)names);
 		return(1);
 	}
+	fix_nlists_order(names, nrawnames, N_GETMID(head));
 
 	/*
 	 * Following the symbol table comes the string table.  The first
@@ -376,6 +375,7 @@ show_objfile(objname, fp)
 		(void)free((char *)names);
 		return(1);
 	}
+	stabsize = fix_long_order(stabsize, N_GETMID(head));
 	stab = emalloc((size_t)stabsize);
 
 	/*
diff --git a/usr.bin/ranlib/Makefile b/usr.bin/ranlib/Makefile
index 2ea6c6d0108..e040bec3bb7 100644
--- a/usr.bin/ranlib/Makefile
+++ b/usr.bin/ranlib/Makefile
@@ -1,8 +1,8 @@
-#	$OpenBSD: Makefile,v 1.2 1996/06/26 05:38:03 deraadt Exp $
+#	$OpenBSD: Makefile,v 1.3 1999/05/10 16:14:07 espie Exp $
 
 PROG=	ranlib
 SRCS=	archive.c build.c misc.c ranlib.c touch.c
-CFLAGS+=-I${.CURDIR} -I${.CURDIR}/../ar
+CFLAGS+=-I${.CURDIR} -I${.CURDIR}/../ar -I${.CURDIR}/../nm
 MAN=	ranlib.1 ranlib.5
 VPATH=	${.CURDIR}/../ar
 
diff --git a/usr.bin/ranlib/build.c b/usr.bin/ranlib/build.c
index 5625f307927..c953492a694 100644
--- a/usr.bin/ranlib/build.c
+++ b/usr.bin/ranlib/build.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: build.c,v 1.5 1997/11/07 03:42:47 deraadt Exp $	*/
+/*	$OpenBSD: build.c,v 1.6 1999/05/10 16:14:07 espie Exp $	*/
 
 /*-
  * Copyright (c) 1990 The Regents of the University of California.
@@ -38,7 +38,7 @@
 
 #ifndef lint
 /*static char sccsid[] = "from: @(#)build.c	5.3 (Berkeley) 3/12/91";*/
-static char rcsid[] = "$OpenBSD: build.c,v 1.5 1997/11/07 03:42:47 deraadt Exp $";
+static char rcsid[] = "$OpenBSD: build.c,v 1.6 1999/05/10 16:14:07 espie Exp $";
 #endif /* not lint */
 
 #include <sys/types.h>
@@ -54,6 +54,8 @@ static char rcsid[] = "$OpenBSD: build.c,v 1.5 1997/11/07 03:42:47 deraadt Exp $
 #include <stdio.h>
 #include <string.h>
 #include <archive.h>
+#include "byte.c"
+
 
 extern CHDR chdr;			/* converted header */
 extern char *archive;			/* archive name */
@@ -71,15 +73,19 @@ static FILE	*fp;
 static long	symcnt;			/* symbol count */
 static long	tsymlen;		/* total string length */
 
-static void	rexec(), symobj();
+static int rexec();
+static void symobj();
 extern void	*emalloc();
 
 build()
 {
 	CF cf;
 	int afd, tfd;
+	int check_mid;
+	int current_mid;
 	off_t size;
 
+	check_mid = 0;
 	afd = open_archive(O_RDWR);
 	fp = fdopen(afd, "r+");
 	tfd = tmp();
@@ -90,17 +96,24 @@ build()
 	symcnt = tsymlen = 0;
 	pnext = &rhead;
 	while(get_arobj(afd)) {
+		int new_mid;
+
 		if (!strcmp(chdr.name, RANLIBMAG)) {
 			skip_arobj(afd);
 			continue;
 		}
-		rexec(afd, tfd);
+		new_mid = rexec(afd, tfd);
+		if (check_mid && new_mid != current_mid)
+			errx(1, "Mixed object format archive: %d / %d", 
+				new_mid, current_mid);
+		current_mid = new_mid;
+		check_mid = 1;
 		put_arobj(&cf, (struct stat *)NULL);
 	}
 	*pnext = NULL;
 
-	/* Create the symbol table. */
-	symobj();
+	/* Create the symbol table.  Endianess the same as last mid seen */
+	symobj(current_mid);
 
 	/* Copy the saved objects into the archive. */
 	size = lseek(tfd, (off_t)0, SEEK_CUR);
@@ -119,9 +132,9 @@ build()
 /*
  * rexec
  *	Read the exec structure; ignore any files that don't look
- *	exactly right.
+ *	exactly right. Return MID.
  */
-static void
+static int
 rexec(rfd, wfd)
 	register int rfd;
 	int wfd;
@@ -145,8 +158,9 @@ rexec(rfd, wfd)
 		goto badread;
 
 	/* Check magic number and symbol count. */
-	if (N_BADMAG(ebuf) || ebuf.a_syms == 0)
+	if (BAD_OBJECT(ebuf) || ebuf.a_syms == 0)
 		goto bad1;
+	fix_header_order(&ebuf);
 
 	/* Seek to string table. */
 	if (lseek(rfd, N_STROFF(ebuf) + r_off, SEEK_SET) == (off_t)-1)
@@ -157,6 +171,7 @@ rexec(rfd, wfd)
 	if (nr != sizeof(strsize))
 		goto badread;
 
+	strsize = fix_long_order(strsize, N_GETMID(ebuf));
 	/* Read in the string table. */
 	strsize -= sizeof(strsize);
 	strtab = (char *)emalloc(strsize);
@@ -179,6 +194,7 @@ badread:	if (nr < 0)
 				badfmt();
 			error(archive);
 		}
+	fix_nlist_order(&nl, N_GETMID(ebuf));
 
 		/* Ignore if no name or local. */
 		if (!nl.n_un.n_strx || !(nl.n_type & N_EXT))
@@ -211,15 +227,17 @@ badread:	if (nr < 0)
 
 bad2:	free(strtab);
 bad1:	(void)lseek(rfd, (off_t)r_off, SEEK_SET);
+	return N_GETMID(ebuf);
 }
 
 /*
  * symobj --
  *	Write the symbol table into the archive, computing offsets as
- *	writing.
+ *	writing.  Use the right format depending on mid.
  */
 static void
-symobj()
+symobj(mid)
+	int mid;
 {
 	register RLIB *rp, *rnext;
 	struct ranlib rn;
@@ -260,7 +278,7 @@ symobj()
 		error(tname);
 
 	/* First long is the size of the ranlib structure section. */
-	size = symcnt * sizeof(struct ranlib);
+	size = fix_long_order(symcnt * sizeof(struct ranlib), mid);
 	if (!fwrite((char *)&size, sizeof(size), 1, fp))
 		error(tname);
 
@@ -276,12 +294,15 @@ symobj()
 		rn.ran_un.ran_strx = stroff;
 		stroff += rp->symlen;
 		rn.ran_off = size + rp->pos;
+		fix_ranlib_order(&rn, mid);
 		if (!fwrite((char *)&rn, sizeof(struct ranlib), 1, fp))
 			error(archive);
 	}
 
 	/* Second long is the size of the string table. */
-	if (!fwrite((char *)&tsymlen, sizeof(tsymlen), 1, fp))
+
+	size = fix_long_order(tsymlen, mid);
+	if (!fwrite((char *)&size, sizeof(size), 1, fp))
 		error(tname);
 
 	/* Write out the string table. */
diff --git a/usr.bin/size/Makefile b/usr.bin/size/Makefile
index 20467f578d7..273c3c12b76 100644
--- a/usr.bin/size/Makefile
+++ b/usr.bin/size/Makefile
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.4 1998/05/11 07:40:30 niklas Exp $
+#	$OpenBSD: Makefile,v 1.5 1999/05/10 16:14:07 espie Exp $
 
 TARGET_MACHINE_ARCH?=	${MACHINE_ARCH}
 
@@ -17,6 +17,8 @@ CFLAGS+=	-DMID_MACHINE_OVERRIDE=MID_VAX
 .endif
 .endif
 
+CFLAGS+= -I${.CURDIR}/../nm
+
 PROG=	size
 
 .include <bsd.prog.mk>
diff --git a/usr.bin/size/size.c b/usr.bin/size/size.c
index 41342ed1071..0c02ec60c67 100644
--- a/usr.bin/size/size.c
+++ b/usr.bin/size/size.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: size.c,v 1.10 1998/05/11 20:20:55 niklas Exp $	*/
+/*	$OpenBSD: size.c,v 1.11 1999/05/10 16:14:07 espie Exp $	*/
 /*	$NetBSD: size.c,v 1.7 1996/01/14 23:07:12 pk Exp $	*/
 
 /*
@@ -44,7 +44,7 @@ static char copyright[] =
 #if 0
 static char sccsid[] = "@(#)size.c	8.2 (Berkeley) 12/9/93";
 #endif
-static char rcsid[] = "$OpenBSD: size.c,v 1.10 1998/05/11 20:20:55 niklas Exp $";
+static char rcsid[] = "$OpenBSD: size.c,v 1.11 1999/05/10 16:14:07 espie Exp $";
 #endif /* not lint */
 
 #include <sys/param.h>
@@ -58,6 +58,7 @@ static char rcsid[] = "$OpenBSD: size.c,v 1.10 1998/05/11 20:20:55 niklas Exp $"
 #include <string.h>
 #include <ctype.h>
 #include <err.h>
+#include "byte.c"
 
 #ifdef MID_MACHINE_OVERRIDE
 #undef MID_MACHINE
@@ -245,16 +246,11 @@ show_archive(count, fname, fp)
 			return(1);
 		}
 
-		if (N_BADMAG(exec_head)) {
+		if (BAD_OBJECT(exec_head)) {
 			if (!ignore_bad_archive_entries) {
 				warnx("%s: bad format", name);
 				rval = 1;
 			}
-		} else if (N_GETMID(exec_head) != MID_MACHINE) {
-			if (!ignore_bad_archive_entries) {
-				warnx("%s: wrong architecture", name);
-				rval = 1;
-			}
 		} else {
 			(void)fseek(fp, (long)-sizeof(exec_head),
 			    SEEK_CUR);
@@ -292,15 +288,12 @@ show_objfile(count, name, fp)
 		return(1);
 	}
 
-	if (N_BADMAG(head)) {
+	if (BAD_OBJECT(head)) {
 		warnx("%s: bad format", name);
 		return(1);
 	}
 
-	if (N_GETMID(head) && N_GETMID(head) != MID_MACHINE) {
-		warnx("%s: wrong architecture", name);
-		return(1);
-	}
+	fix_header_order(&head);
 
 	if (first) {
 		first = 0;
diff --git a/usr.bin/strip/Makefile b/usr.bin/strip/Makefile
index 6443a4dea34..8adf1fa3205 100644
--- a/usr.bin/strip/Makefile
+++ b/usr.bin/strip/Makefile
@@ -1,9 +1,10 @@
-#	$OpenBSD: Makefile,v 1.7 1998/05/11 07:41:23 niklas Exp $
+#	$OpenBSD: Makefile,v 1.8 1999/05/10 16:14:07 espie Exp $
 
 TARGET_MACHINE_ARCH?=	${MACHINE_ARCH}
 
 PROG=	strip
 
+CFLAGS+=-I${.CURDIR}/../nm
 .if ${TARGET_MACHINE_ARCH} != ${MACHINE_ARCH}
 # XXX should make this automatic
 .if ${TARGET_MACHINE_ARCH} == "i386"
diff --git a/usr.bin/strip/strip.c b/usr.bin/strip/strip.c
index 44aca452a37..29a18f332ce 100644
--- a/usr.bin/strip/strip.c
+++ b/usr.bin/strip/strip.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: strip.c,v 1.10 1998/05/11 07:41:25 niklas Exp $	*/
+/*	$OpenBSD: strip.c,v 1.11 1999/05/10 16:14:07 espie Exp $	*/
 
 /*
  * Copyright (c) 1988 Regents of the University of California.
@@ -41,7 +41,7 @@ char copyright[] =
 
 #ifndef lint
 /*static char sccsid[] = "from: @(#)strip.c	5.8 (Berkeley) 11/6/91";*/
-static char rcsid[] = "$OpenBSD: strip.c,v 1.10 1998/05/11 07:41:25 niklas Exp $";
+static char rcsid[] = "$OpenBSD: strip.c,v 1.11 1999/05/10 16:14:07 espie Exp $";
 #endif /* not lint */
 
 #include <sys/param.h>
@@ -56,6 +56,8 @@ static char rcsid[] = "$OpenBSD: strip.c,v 1.10 1998/05/11 07:41:25 niklas Exp $
 #include <stdlib.h>
 #include <string.h>
 #include <err.h>
+#include <ranlib.h>
+#include "byte.c"
 
 #ifdef MID_MACHINE_OVERRIDE
 #undef MID_MACHINE
@@ -120,17 +122,17 @@ main(argc, argv)
 			(void)close(fd);
 			ERROR(errno);
 		}
-#if (MID_MACHINE == MID_M68K)
-		if (N_BADMAG(*ep) || ((N_GETMID(*ep) != MID_MACHINE) &&
-		    (N_GETMID(*ep) != MID_M68K4K))) {
-#else
-		if (N_BADMAG(*ep) || N_GETMID(*ep) != MID_MACHINE) {
-#endif
+		if (BAD_OBJECT(*ep)) {
 			munmap((caddr_t)ep, sb.st_size);
 			(void)close(fd);
 			ERROR(EFTYPE);
 		}
+		/* since we're dealing with an mmap there, we have to convert once
+		   for dealing with data in memory, and a second time for out
+		 */
+		fix_header_order(ep);
 		errors |= sfcn(fn, fd, ep, &sb);
+		fix_header_order(ep);
 		munmap((caddr_t)ep, sb.st_size);
 		if (close(fd)) {
 			ERROR(errno);
@@ -216,6 +218,8 @@ s_stab(fn, fd, ep, sp)
 	register int cnt, len, nsymcnt;
 	register char *nstr, *nstrbase, *p, *strbase;
 	register NLIST *sym, *nsym;
+	u_long allocsize;
+	int mid;
 	NLIST *symbase;
 
 	/* Quit if no symbols. */
@@ -227,6 +231,8 @@ s_stab(fn, fd, ep, sp)
 		return 1;
 	}
 
+	mid = N_GETMID(*ep);
+
 	/*
 	 * Initialize old and new symbol pointers.  They both point to the
 	 * beginning of the symbol table in memory, since we're deleting
@@ -240,7 +246,8 @@ s_stab(fn, fd, ep, sp)
 	 * of the string table.
 	 */
 	strbase = (char *)ep + N_STROFF(*ep);
-	if ((nstrbase = malloc((u_int)*(u_long *)strbase)) == NULL) {
+	allocsize = fix_long_order(*(u_long *)strbase, mid);
+	if ((nstrbase = malloc((u_int) allocsize)) == NULL) {
 		warnx("%s", strerror(ENOMEM));
 		return 1;
 	}
@@ -251,7 +258,8 @@ s_stab(fn, fd, ep, sp)
 	 * copy it and save its string in the new string table.  Keep
 	 * track of the number of symbols.
 	 */
-	for (cnt = ep->a_syms / sizeof(NLIST); cnt--; ++sym)
+	for (cnt = ep->a_syms / sizeof(NLIST); cnt--; ++sym) {
+		fix_nlist_order(sym, mid);
 		if (!(sym->n_type & N_STAB) && sym->strx) {
 			*nsym = *sym;
 			nsym->strx = nstr - nstrbase;
@@ -267,14 +275,16 @@ s_stab(fn, fd, ep, sp)
 			len = strlen(p) + 1;
 			bcopy(p, nstr, len);
 			nstr += len;
-			++nsym;
+			fix_nlist_order(nsym++, mid);
 		}
+	}
 
 	/* Fill in new symbol table size. */
 	ep->a_syms = (nsym - symbase) * sizeof(NLIST);
 
 	/* Fill in the new size of the string table. */
-	*(u_long *)nstrbase = len = nstr - nstrbase;
+	len = nstr - nstrbase;
+	*(u_long *)nstrbase = fix_long_order(len, mid);
 
 	/*
 	 * Copy the new string table into place.  Nsym should be pointing