- Precede internal bsd.regress.mk variables with a _

- Rename old variables to be easier to read, and deprecate old
variables.  The old variables will still work, but emit warnings.
- REGRESS_SKIP_SLOW only needs to be non-empty now, not explicitly
set to 'yes'
- REGRESS_LOG can contain more than one file to append results to
- ERRORS variable can be set in Makefile to emit warnings or indicate
fatal errors.
- Add REGRESS_MAIL variable which can be set to an email address to
send results to.  Currently this only works for a full regression
run, since art doesnt want partial run results
- sync bsd.regress.mk(5) with these changes

ok art@ , also looked over by miod@

3 files changed, 124 insertions, 48 deletions

diff --git a/regress/Makefile b/regress/Makefile
index f88df9c48e4..7690c3940ca 100644
--- a/regress/Makefile
+++ b/regress/Makefile
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.12 2002/02/23 01:25:10 art Exp $
+#	$OpenBSD: Makefile,v 1.13 2002/09/02 19:56:55 avsm Exp $
 
 #
 # To run full regression tests you just need two steps:
@@ -9,12 +9,53 @@
 # as "regress" (possibly with more bugs). This allows us to include
 # compilation as a part of the regression test.
 #
-# To get a log of test results set the REGRESSLOG make variable to
+# To get a log of test results set the REGRESS_LOG make variable to
 # a log file, use an absolute path.
-# To skip the really slow tests, set the REGRESSSKIPSLOW variable.
+# To mail the test results set the REGRESS_MAIL make variable to an
+# email address, preferably 'regress@openbsd.org'.
+# To skip the really slow tests, set the REGRESS_SKIP_SLOW variable.
 
 SUBDIR+= etc bin include lib libexec sbin sys share usr.bin
 
 install:
 
+REGRESS_MAIL?=
+
+.if !empty(REGRESS_MAIL)
+_REGRESS_TMP!=/usr/bin/mktemp
+.endif
+
+.if empty(REGRESS_MAIL) || empty(_REGRESS_TMP)
+_REGRESS_TMP=
+_REGRESS_CLEAN=exit 0
+.else
+_REGRESS_CLEAN=rm -f ${_REGRESS_TMP}
+.endif
+.MAKEFLAGS+= _REGRESS_TMP=${_REGRESS_TMP}
+
+.INTERRUPT:
+	${_REGRESS_CLEAN}
+
+.END: .SILENT
+.if ${.TARGETS:Mall} || ${.TARGETS:Mregress} 
+	echo ====================================================================
+.if empty(REGRESS_MAIL) || empty(_REGRESS_TMP)
+	echo Please mail a copy of these regression tests to regress@openbsd.org
+	echo to help with future development.
+	echo
+	echo You can do this automatically by setting the REGRESS_MAIL variable
+	echo $$ make REGRESS_MAIL=regress@openbsd.org regress
+	echo
+	echo You can also set REGRESS_LOG to point to a file and mail it manually.
+	echo $$ make REGRESS_LOG=/var/log/regress-tests regress
+.else
+	echo Mailing regress results to ${REGRESS_MAIL} ...
+	echo If you want them to be also logged locally, you can set the
+	echo REGRESS_LOG make variable to the full path of a log file.
+	(sysctl -n kern.version && cat ${_REGRESS_TMP}) | /usr/bin/mail -s "Regress:  `uname -a`" ${REGRESS_MAIL}
+.endif
+	echo ====================================================================
+	${_REGRESS_CLEAN}
+.endif
+
 .include <bsd.subdir.mk>
diff --git a/share/man/man5/bsd.regress.mk.5 b/share/man/man5/bsd.regress.mk.5
index c4410b91bd0..6a370f8ffe6 100644
--- a/share/man/man5/bsd.regress.mk.5
+++ b/share/man/man5/bsd.regress.mk.5
@@ -1,4 +1,4 @@
-.\" $OpenBSD: bsd.regress.mk.5,v 1.2 2002/08/30 11:59:39 mpech Exp $
+.\" $OpenBSD: bsd.regress.mk.5,v 1.3 2002/09/02 19:56:55 avsm Exp $
 .\"
 .\" Copyright (c) 2002 Anil Madhavapeddy
 .\" Copyright (c) 2000 Marc Espie
@@ -58,33 +58,33 @@ If the exit status of the program indicates an error or timeout,
 then a failure is logged, otherwise the test is marked as a success.
 .El 
 .Sh VARIABLES
-.Bl -tag -width REGRESSLOG
-.It Ev REGRESSLOG
+.Bl -tag -width REGRESS_LOG
+.It Ev REGRESS_LOG
 Points to the fully-qualified path of a file to which regression
 results are appended.
 Defaults to
 .Pa /dev/null .
-.It Ev REGRESSMAXTIME
+.It Ev REGRESS_MAXTIME
 Maximum limit of CPU seconds to spend on the regression test.
 Exceeding this time will result in a failure being logged.
-.It Ev REGRESSROOTTARGETS
+.It Ev REGRESS_ROOT_TARGETS
 Targets for which root access is required to run the test.
 The
 .Ev SUDO
 variable is invoked for these targets.
 See also
 .Ev SUDO .
-.It Ev REGRESSSKIPSLOW
-Skip over all the regression tests which have been marked as
-being 'slow' using the
-.Ev REGRESSSLOWTARGETS
+.It Ev REGRESS_SKIP_SLOW
+If this variable is not empty, skip over all the regression 
+tests which have been marked as being 'slow' using the
+.Ev REGRESS_SLOW_TARGETS
 variable.
-.It Ev REGRESSSLOWTARGETS
+.It Ev REGRESS_SLOW_TARGETS
 Targets which are defined as 'slow'.
 All of these tests can be skipped by setting the 
-.Ev REGRESSSKIPSLOW 
+.Ev REGRESS_SKIP_SLOW 
 variable.
-.It Ev REGRESSTARGETS
+.It Ev REGRESS_TARGETS
 Targets which are invoked to run the set of regression tests
 for this Makefile.
 Defaults to 
@@ -95,7 +95,7 @@ test targets which require it.
 .El
 .Sh BUGS AND LIMITATIONS
 The build system is unable to distinguish between timeouts due to 
-.Ev REGRESSMAXTIME
+.Ev REGRESS_MAXTIME
 being exceeded, or a genuine failure occurring.
 .Sh HISTORY
 The regression system originally came from
diff --git a/share/mk/bsd.regress.mk b/share/mk/bsd.regress.mk
index ff70d71a5ce..71776683f7e 100644
--- a/share/mk/bsd.regress.mk
+++ b/share/mk/bsd.regress.mk
@@ -1,12 +1,14 @@
-#	$OpenBSD: bsd.regress.mk,v 1.9 2002/02/17 01:10:15 marc Exp $
+# $OpenBSD: bsd.regress.mk,v 1.10 2002/09/02 19:56:55 avsm Exp $
+# Documented in bsd.regress.mk(5)
+
 # No man pages for regression tests.
 NOMAN=
 
 # No installation.
 install:
 
-# If REGRESSTARGETS is defined and PROG is not defined, set NOPROG
-.if defined(REGRESSTARGETS) && !defined(PROG)
+# If REGRESS_TARGETS is defined and PROG is not defined, set NOPROG
+.if defined(REGRESS_TARGETS) && !defined(PROG)
 NOPROG=
 .endif
 
@@ -15,63 +17,96 @@ NOPROG=
 .MAIN: all
 all: regress
 
-# XXX - Need full path to REGRESSLOG, otherwise there will be much pain.
+# Check for deprecated REGRESS* variables and assign them to the
+# new versions if the new version is not already defined. 
+_REGRESS_DEPRECATED=LOG:LOG SKIPTARGETS:SKIP_TARGETS SKIPSLOW:SKIP_SLOW \
+	SKIP:SKIP TARGETS:TARGETS MAXTIME:MAXTIME ROOTTARGETS:ROOT_TARGETS
+
+.for _I in ${_REGRESS_DEPRECATED}
+_REGRESS_OLD=REGRESS${_I:C/\:.*//}
+_REGRESS_NEW=REGRESS_${_I:C/.*\://}
+.  if defined(${_REGRESS_OLD})
+ERRORS:= ${ERRORS} "Warning: ${_REGRESS_OLD} is deprecated, use ${_REGRESS_NEW} instead."
+.    if !defined(${_REGRESS_NEW})
+${_REGRESS_NEW}:=${${_REGRESS_OLD}}
+.    endif
+.  endif
+.endfor
+
+# XXX - Need full path to REGRESS_LOG, otherwise there will be much pain.
+REGRESS_LOG?=/dev/null
+REGRESS_SKIP_TARGETS?=
+REGRESS_SKIP_SLOW?=no
 
-REGRESSLOG?=/dev/null
-REGRESSNAME=${.CURDIR:S/${BSDSRCDIR}\/regress\///}
+_REGRESS_NAME=${.CURDIR:S/${BSDSRCDIR}\/regress\///}
+_REGRESS_TMP?=/dev/null
+_REGRESS_OUT= | tee -a ${REGRESS_LOG} ${_REGRESS_TMP} 2>&1 > /dev/null
 
 .if defined(PROG) && !empty(PROG)
 run-regress-${PROG}: ${PROG}
 	./${PROG}
 .endif
 
-.if !defined(REGRESSTARGETS)
-REGRESSTARGETS=run-regress-${PROG}
-.  if defined(REGRESSSKIP)
-REGRESSSKIPTARGETS=run-regress-${PROG}
+.if !defined(REGRESS_TARGETS)
+REGRESS_TARGETS=run-regress-${PROG}
+.  if defined(REGRESS_SKIP)
+REGRESS_SKIP_TARGETS=run-regress-${PROG}
 .  endif
 .endif
 
-REGRESSSKIPSLOW?=no
-
-.if (${REGRESSSKIPSLOW:L} == "yes") && defined(REGRESSSLOWTARGETS)
-REGRESSSKIPTARGETS+=${REGRESSSLOWTARGETS}
+.if defined(REGRESS_SLOW_TARGETS) && !empty(REGRESS_SKIP_SLOW)
+REGRESS_SKIP_TARGETS+=${REGRESS_SLOW_TARGETS}
 .endif
 
-.if defined(REGRESSROOTTARGETS)
-ROOTUSER!=id -g
+.if defined(REGRESS_ROOT_TARGETS)
+_ROOTUSER!=id -g
 SUDO?=
-. if (${ROOTUSER} != 0) && empty(SUDO)
-REGRESSSKIPTARGETS+=${REGRESSROOTTARGETS}
-. endif
+.  if (${_ROOTUSER} != 0) && empty(SUDO)
+REGRESS_SKIP_TARGETS+=${REGRESS_ROOT_TARGETS}
+.  endif
 .endif
 
-REGRESSSKIPTARGETS?=
+.if defined(ERRORS)
+.BEGIN:
+.  for _m in ${ERRORS}
+	@echo 1>&2 ${_m}
+.  endfor
+.  if !empty(ERRORS:M"Fatal\:*") || !empty(ERRORS:M'Fatal\:*')
+	@exit 1
+.  endif
+.endif 
 
-regress:
-.for RT in ${REGRESSTARGETS} 
-.  if ${REGRESSSKIPTARGETS:M${RT}}
-	@echo -n "SKIP " >> ${REGRESSLOG}
+regress: .SILENT
+.if ! ${REGRESS_LOG:M/*}
+	echo =========================================================
+	echo REGRESS_LOG must contain an absolute path to the log-file.
+	echo It currently points to: ${REGRESS_LOG}
+	echo =========================================================
+	exit 1
+.endif
+.for RT in ${REGRESS_TARGETS} 
+.  if ${REGRESS_SKIP_TARGETS:M${RT}}
+	@echo -n "SKIP " ${_REGRESS_OUT}
 .  else
 # XXX - we need a better method to see if a test fails due to timeout or just
 #       normal failure.
-.   if !defined(REGRESSMAXTIME)
-	@if cd ${.CURDIR} && ${MAKE} ${RT}; then \
-	    echo -n "SUCCESS " >> ${REGRESSLOG} ; \
+.   if !defined(REGRESS_MAXTIME)
+	-if cd ${.CURDIR} && ${MAKE} ${RT}; then \
+	    echo -n "SUCCESS " ${_REGRESS_OUT} ; \
 	else \
-	    echo -n "FAIL " >> ${REGRESSLOG} ; \
+	    echo -n "FAIL " ${_REGRESS_OUT} ; \
 	    echo FAILED ; \
 	fi
 .   else
-	@if cd ${.CURDIR} && (ulimit -t ${REGRESSMAXTIME} ; ${MAKE} ${RT}); then \
-	    echo -n "SUCCESS " >> ${REGRESSLOG} ; \
+	-if cd ${.CURDIR} && (ulimit -t ${REGRESS_MAXTIME} ; ${MAKE} ${RT}); then \
+	    echo -n "SUCCESS " ${_REGRESS_OUT} ; \
 	else \
-	    echo -n "FAIL (possible timeout) " >> ${REGRESSLOG} ; \
+	    echo -n "FAIL (possible timeout) " ${_REGRESS_OUT} ; \
 	    echo FAILED ; \
 	fi
 .   endif
 .  endif
-	@echo ${REGRESSNAME}/${RT:S/^run-regress-//} >> ${REGRESSLOG}
+	@echo ${_REGRESS_NAME}/${RT:S/^run-regress-//} ${_REGRESS_OUT}
 .endfor
 
 .PHONY: regress