refresh the test infrastructure a bit.

3 files changed, 93 insertions, 90 deletions

diff --git a/regress/lib/libc/asr/Makefile b/regress/lib/libc/asr/Makefile
index 76891ec176b..12e59536e0d 100644
--- a/regress/lib/libc/asr/Makefile
+++ b/regress/lib/libc/asr/Makefile
@@ -1,53 +1,52 @@
-#	$OpenBSD: Makefile,v 1.3 2013/04/01 10:18:28 eric Exp $
+#	$OpenBSD: Makefile,v 1.4 2017/03/10 17:23:48 eric Exp $
 
 #
-# Note on building and running the regress tests:
+# Notes on building and running the regression tests
 #
-# The regress suite builds two sets of static executables: one linked
-# against the current libc, and one linked against a libc with the resolver
-# replaced. The idea is to compare the output of all programs in both cases.
+# The regress suite builds two sets of static executables: the former linked
+# against the system libc, and the latter against the libc found in /usr/obj
+# The idea is to compare the output of all programs with internal changes in
+# the libc (more specifically the resolver).  They will be run in a chrooted
+# environment to test various /etc configurations without touching the local
+# machine config files.
 #
-# So before building the regression test programs, you need to have two
-# static libc: the system libc.a in /usr/lib/ which is assumed to be built
-# with asr, and a pre-asr libc.a in the "./lib/" directory (you can build
-# one by commenting out the ".include "${LIBCSRCDIR}/asr/Makefile.inc"
-# line from the /usr/src/lib/libc/Makefile.in and rebuild the libc).
-# 
-# When done, run "make && make install". This builds and installs the two sets
-# of programs in REGRESSDIR. They will run chrooted there to allow testing with
-# various /etc environment without messing up the local machine config files.
+# First, build a libc that needs testing, then:
 #
-# When you are sure everything is in place, run "make regress". This will
-# create two files: $REGRESSDIR/output.log with the output of all tests,
-# and $REGRESSDIR/regress.log with the diffs for those that "failed".
+# $ make
+# $ doas make install
+# $ doas make regress
 #
-# Note that the "install" and "regress" targets need to be run as root.
+# Tests output goes into a $REGRESSDIR/tmp.XXXXXXXXXX directory, and a digest
+# is displayed at the end.
 #
 
-LIBCDIRSTD?=	${.CURDIR}/lib
-LIBCDIRASR?=	/usr/lib
-
-REGRESSDIR?=	/tmp/regress
+REGRESSDIR?=	/tmp/regress-asr
 REGRESS?=	regress.sh
 
+A?=	.a
+B?=	.b
+
+DIRA?=	/usr/lib
+DIRB?=	/usr/obj/lib/libc
+
+
 all: build
 
 build:
-	cd bin && EXT=.asr LDFLAGS=-L${LIBCDIRASR} make
-	cd bin && EXT=.std LDFLAGS=-L${LIBCDIRSTD} make
+	cd bin && EXT=${A} LDFLAGS=-L${DIRA} make
+	cd bin && EXT=${B} LDFLAGS=-L${DIRB} make
 
 clean:
-	cd bin && EXT=.std make clean
-	cd bin && EXT=.asr make clean
+	cd bin && EXT=${A} make clean
+	cd bin && EXT=${B} make clean
 
 install:
-	mkdir -p ${REGRESSDIR}/etc
 	mkdir -p ${REGRESSDIR}/bin
-	cd bin && BINDIR=${REGRESSDIR}/bin EXT=.std make install
-	cd bin && BINDIR=${REGRESSDIR}/bin EXT=.asr make install
+	cd bin && BINDIR=${REGRESSDIR}/bin EXT=${A} make install
+	cd bin && BINDIR=${REGRESSDIR}/bin EXT=${B} make install
 
 uninstall:
 	rm -rf ${REGRESSDIR}
 
 regress:
-	RUNDIR=${REGRESSDIR} sh ${.CURDIR}/${REGRESS}
+	RUNDIR=${REGRESSDIR} A=${A} B=${B} sh ${.CURDIR}/${REGRESS}
diff --git a/regress/lib/libc/asr/regress.sh b/regress/lib/libc/asr/regress.sh
index 3388f0dd2cd..a0950cefde1 100644
--- a/regress/lib/libc/asr/regress.sh
+++ b/regress/lib/libc/asr/regress.sh
@@ -1,9 +1,7 @@
-#	$OpenBSD: regress.sh,v 1.6 2013/04/01 10:17:45 eric Exp $
+#	$OpenBSD: regress.sh,v 1.7 2017/03/10 17:23:48 eric Exp $
 
 . regress.subr
 
-EFLAG=
-
 test_res_mkquery()
 {
 	for i in $@; do
diff --git a/regress/lib/libc/asr/regress.subr b/regress/lib/libc/asr/regress.subr
index 3e8cc0ebd5a..43b7280158d 100644
--- a/regress/lib/libc/asr/regress.subr
+++ b/regress/lib/libc/asr/regress.subr
@@ -1,13 +1,5 @@
 #!/bin/sh
-#	$OpenBSD: regress.subr,v 1.2 2013/03/28 09:36:03 eric Exp $
-
-TOTAL=0
-FAIL=0
-OK=0
-
-EXT0=.std
-EXT1=.asr
-EFLAG=-ee
+#	$OpenBSD: regress.subr,v 1.3 2017/03/10 17:23:48 eric Exp $
 
 set -e
 
@@ -19,70 +11,75 @@ fail()
 
 regress()
 {
-	local out;
-	local _cmd=$1;
-	local _bin0=/bin/$_cmd$EXT0
-	local _bin1=/bin/$_cmd$EXT1
-	shift;
-
-	TOTAL=$((TOTAL+1))
-
-	# XXX with user "bin"
-	test -x $_RUNDIR$_bin0 || fail $_RUNDIR$_bin0 not executable
-	test -x $_RUNDIR$_bin1 || fail $_RUNDIR$_bin1 not executable
-
-	out=/tmp/asr_regress
-
-	echo -n $_cmd $EFLAG $@ "."
+	local cmd="$1"
+	local bin0="/bin/${cmd}${A}"
+	local bin1="/bin/${cmd}${B}"
+	local out="${OUTDIR}/test-${COUNT}.out"
+	local out0="${OUTDIR}/test-${COUNT}${A}.out"
+	local out1="${OUTDIR}/test-${COUNT}${B}.out"
+	local err0="${OUTDIR}/test-${COUNT}${A}.err"
+	local err1="${OUTDIR}/test-${COUNT}${B}.err"
+	local outdiff="${OUTDIR}/test-${COUNT}${A}${B}.diff"
+	shift
+
+	test -x "${ROOTDIR}${bin0}" || fail "${ROOTDIR}${bin0}" not executable
+	test -x "${ROOTDIR}${bin1}" || fail "${ROOTDIR}${bin1}" not executable
+
+	echo -n "${cmd} $@ ."
 
 	set +e
-	chroot -u bin "$_RUNDIR" $_bin0 $EFLAG $@ > $out.0
+	chroot -u bin "${ROOTDIR}" ${bin0} $@ > ${out0}
 	echo -n .
-	chroot -u bin "$_RUNDIR" $_bin1 $EFLAG $@ > $out.1
-	echo -n ". "
+	chroot -u bin "${ROOTDIR}" ${bin1} $@ > ${out1}
+	echo -n "."
 
-	diff -u $out.0 $out.1 > $out.diff
+	diff -u ${out0} ${out1} > ${outdiff}
 	set -e
-	if test -s $out.diff; then
+	if test -s ${outdiff}; then
 		FAIL=$((FAIL+1))
 		echo fail
-		echo "*** FAIL (env=$REGRESSENV)" $_cmd $EFLAG $@ >> $REG
-		tail -n +3 $out.diff >> $REG
-		echo >> $REG
+		echo "### FAIL [#${COUNT} env=${REGRESSENV}] ${cmd} $@" >> $ERR
+		cat ${outdiff} >> $ERR
+		echo >> $ERR
 	else
+		rm ${outdiff}
+		rm ${out1}
+		mv ${out0} ${out}
 		OK=$((OK+1))
 		echo ok
-		echo "OK (env=$REGRESSENV)" $_cmd $EFLAG $@ >> $OUT
-		cat $out.0 >> $OUT
+		echo "### OK [#${COUNT} env=$REGRESSENV] ${cmd} $@" >> $OUT
+		cat ${out} >> $OUT
 		echo >> $OUT
 	fi
-	rm $out.diff $out.0 $out.1
+
+	COUNT=$((COUNT+1))
 }
 
 regress_setenv()
 {
 	local _name="$1"
 
-	echo "===> using env $_name"
+	echo "==> using env $_name"
 
-	cp /etc/hosts $_RUNDIR/etc/
-	cp /etc/resolv.conf $_RUNDIR/etc/
-	cp /etc/protocols $_RUNDIR/etc/
-	cp /etc/networks $_RUNDIR/etc/
+	mkdir -p "${ROOTDIR}/etc/"
+	cp /etc/hosts "${ROOTDIR}/etc/"
+	cp /etc/resolv.conf "${ROOTDIR}/etc/"
+	cp /etc/protocols "${ROOTDIR}/etc/"
+	cp /etc/networks "${ROOTDIR}/etc/"
 
 	case $_name in
 	empty)
-		rm -f $_RUNDIR/etc/*
+		rm -f "${ROOTDIR}/etc/*"
 		;;
 	local)
 		;;
 	file)
-		grep -v lookup /etc/resolv.conf > $_RUNDIR/etc/resolv.conf
-		echo "lookup file" >> $_RUNDIR/etc/resolv.conf
+		grep -v lookup /etc/resolv.conf > "${ROOTDIR}/etc/resolv.conf"
+		echo "lookup file" >> "${ROOTDIR}/etc/resolv.conf"
 		;;
 	bind)
-		grep -v lookup /etc/resolv.conf > $_RUNDIR/etc/resolv.conf
-		echo "lookup bind" >> $_RUNDIR/etc/resolv.conf
+		grep -v lookup /etc/resolv.conf > "${ROOTDIR}/etc/resolv.conf"
+		echo "lookup bind" >> "${ROOTDIR}/etc/resolv.conf"
 		;;
 	*)
 		fail unknown env $_name
@@ -93,23 +90,32 @@ regress_setenv()
 
 regress_digest()
 {
-	echo
-	cat $REG
-	echo "===>" run=$TOTAL fail=$FAIL
-}
+	echo "===> done"
+ 	test -f $ERR && (echo; cat $ERR; echo "===> errfile=$ERR")
+	echo "===> logfile=$OUT"
+	echo "===> run=$COUNT ok=$OK fail=$FAIL error=$ERROR"
 
+}
 
 # needed for chroot
 test "$(id -u)" -ne 0 && fail need root privileges to run this script
 
 # we really really want to avoid erasing /etc later
-test "$RUNDIR" || fail RUNDIR is not set
-_RUNDIR=$(readlink -fn ${RUNDIR})
-test "$_RUNDIR" == / && fail RUNDIR is root dir: $RUNDIR
+test "${RUNDIR}" || fail RUNDIR is not set
+
+ROOTDIR=$(readlink -fn ${RUNDIR})
+test "${ROOTDIR}" == "/" && fail RUNDIR is root dir: ${RUNDIR}
+
+COUNT=0
+OK=0
+FAIL=0
+ERROR=0
+A=${A:=.a}
+B=${B:=.b}
 
-OUT=$_RUNDIR/output.log
-REG=$_RUNDIR/regress.log
-ETC=$_RUNDIR/etc
+OUTDIR=$(mktemp -d -p $ROOTDIR)
+chmod a+rx ${OUTDIR}
+echo "===> ${OUTDIR}"
 
-echo -n > $REG
-echo -n > $OUT
+OUT=$OUTDIR/regress.log
+ERR=$OUTDIR/regress.err