This commit adds a new feature to sysmerge(8), usable in '-a' mode.

From now on, checksums of reference files (sets and/or src) will be
stored under /var/db/sysmerge/{etcsum,xetcsum,srcsum} . This allows for
"remembering last choice" several people have requested.
This deprecates the need for -X and -S which have been removed (these
switches may come back at some point for the auto-patch feature which
halex@ proposed).

It will only compare files which reference sources have changed since
the last run and will attempt to automatically upgrade them to the
newest version provided that they have no local changes.
i.e. first run should dislay differences, second run should not display
anything except if something changed between old and new sets/src. This
way sysmerge can also warn you from files which may have been obsoleted.

General idea taken from Net/FreeBSD but we're doing things differently.

Intensively discussed with oga@, sthen@ and halex@
ok oga@ sthen@

2 files changed, 80 insertions, 147 deletions

diff --git a/usr.sbin/sysmerge/sysmerge.8 b/usr.sbin/sysmerge/sysmerge.8
index 46bd6fc210c..ca655384060 100644
--- a/usr.sbin/sysmerge/sysmerge.8
+++ b/usr.sbin/sysmerge/sysmerge.8
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: sysmerge.8,v 1.21 2009/04/12 07:43:17 ajacoutot Exp $
+.\"	$OpenBSD: sysmerge.8,v 1.22 2009/05/11 20:25:55 ajacoutot Exp $
 .\"
 .\" Copyright (c) 2008 Antoine Jacoutot <ajacoutot@openbsd.org>
 .\"
@@ -14,7 +14,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: April 12 2009 $
+.Dd $Mdocdate: May 11 2009 $
 .Dt SYSMERGE 8
 .Os
 .Sh NAME
@@ -24,9 +24,7 @@
 .Nm
 .Bk -words
 .Op Fl ab
-.Op Fl S Ar etcXX.tgz
 .Op Fl s Ar src \*(Ba etcXX.tgz
-.Op Fl X Ar xetcXX.tgz
 .Op Fl x Ar xetcXX.tgz
 .Ek
 .Sh DESCRIPTION
@@ -103,7 +101,10 @@ The options are as follows:
 Automatic mode.
 If this option is specified,
 .Nm
-will automatically install missing files
+will only compare files which reference sources have changed since
+the last run and will attempt to automatically upgrade them to the
+newest version provided that they have no local changes.
+It will automatically install missing files
 and files differing only by CVS Id,
 and will disable strict file comparison when possible (using CVS Id).
 .Pa /etc/fbtab ,
@@ -122,38 +123,12 @@ If this option is specified,
 .Nm
 will run unattended (non-interactively), saving differing files for
 later manual processing.
-.It Fl S Ar etcXX.tgz
-Specify a path to an
-etcXX.tgz tarball,
-where XX represents the version to be upgraded from.
-.Nm
-will attempt to automatically upgrade files from this version
-to the one specified by
-.Fl s ,
-provided that the file has no local changes.
-Files that have not been modified between the supplied tarballs will
-not be offered for comparison.
-A tarball path specified as an FTP or HTTP URL will be passed
-to ${FETCH_CMD}.
 .It Fl s Ar src \*(Ba etcXX.tgz
 Specify a path to an
 .Ox
 top src directory or an etcXX.tgz tarball.
 A tarball path specified as an FTP or HTTP URL will be passed
 to ${FETCH_CMD}.
-.It Fl X Ar xetcXX.tgz
-Specify a path to an
-xetcXX.tgz tarball,
-where XX represents the version to be upgraded from.
-.Nm
-will attempt to automatically upgrade files from this version
-to the one specified by
-.Fl x ,
-provided that the file has no local changes.
-Files that have not been modified between the supplied tarballs will
-not be offered for comparison.
-A tarball path specified as an FTP or HTTP URL will be passed
-to ${FETCH_CMD}.
 .It Fl x Ar xetcXX.tgz
 Specify a path to an
 xetcXX.tgz tarball.
@@ -185,6 +160,10 @@ A special test was added to handle this
 and they are offered for comparison only if they really differ.
 .Sh ENVIRONMENT
 .Bl -tag -width "DESTDIRXXX"
+.It Ev DBDIR
+Directory in which checksum files are stored.
+If unset, this defaults to
+.Pa /var/db/sysmerge .
 .It Ev DESTDIR
 Directory in which to merge and install files.
 If unset, this defaults to
diff --git a/usr.sbin/sysmerge/sysmerge.sh b/usr.sbin/sysmerge/sysmerge.sh
index 7964a60ee54..d9bd10d4705 100644
--- a/usr.sbin/sysmerge/sysmerge.sh
+++ b/usr.sbin/sysmerge/sysmerge.sh
@@ -1,6 +1,6 @@
 #!/bin/sh -
 #
-# $OpenBSD: sysmerge.sh,v 1.39 2009/04/28 08:25:27 ajacoutot Exp $
+# $OpenBSD: sysmerge.sh,v 1.40 2009/05/11 20:25:55 ajacoutot Exp $
 #
 # This script is based on the FreeBSD mergemaster script, written by
 # Douglas Barton <DougB@FreeBSD.org>
@@ -30,6 +30,7 @@ WRKDIR=`mktemp -d -p /var/tmp sysmerge.XXXXX` || exit 1
 SWIDTH=`stty size | awk '{w=$2} END {if (w==0) {w=80} print w}'`
 MERGE_CMD="${MERGE_CMD:=sdiff -as -w ${SWIDTH} -o}"
 REPORT="${REPORT:=${WRKDIR}/sysmerge.log}"
+DBDIR="${DBDIR:=/var/db/sysmerge}"
 
 EDITOR="${EDITOR:=/usr/bin/vi}"
 PAGER="${PAGER:=/usr/bin/more}"
@@ -41,6 +42,14 @@ clean_src() {
 	fi
 }
 
+# restore files from backups
+restore_bak() {
+	for i in ${DBDIR}/.*.bak; do
+		_i=`basename ${i} .bak`
+		mv ${i} ${DBDIR}/${_i#.}
+	done
+}
+
 # remove newly created work directory and exit with status 1
 error_rm_wrkdir() {
 	rmdir ${WRKDIR} 2> /dev/null
@@ -51,7 +60,7 @@ usage() {
 	echo "usage: ${0##*/} [-ab] [-S etcXX.tgz] [-s src | etcXX.tgz] [-X xetcXX.tgz] [-x xetcXX.tgz]" >&2
 }
 
-trap "clean_src; rm -rf ${WRKDIR}; exit 1" 1 2 3 13 15
+trap "restore_bak; clean_src; rm -rf ${WRKDIR}; exit 1" 1 2 3 13 15
 
 if [ "`id -u`" -ne 0 ]; then
 	echo " *** Error: need root privileges to run this script"
@@ -77,36 +86,6 @@ do_pre() {
 		fi
 	fi
 
-	if [ "${OTGZ}" ]; then
-		TGZV=`echo ${TGZ} | sed -e 's,^.*/,,' -e 's,etc,,' -e 's,.tgz,,'`
-		OTGZV=`echo ${OTGZ} | sed -e 's,^.*/,,' -e 's,etc,,' -e 's,.tgz,,'`
-		if [ -z "${TGZ}" ]; then
-			echo " *** Error: please specify a valid path to the new etcXX.tgz"
-			error_rm_wrkdir
-		elif cmp -s ${OTGZ} ${TGZ}; then
-			echo " *** Error: old and new etcXX.tgz are identical"
-			error_rm_wrkdir
-		elif [ "${OTGZV}" -gt "${TGZV}" ]; then
-			echo " *** Error: old etc${OTGZV}.tgz version is higher than new etc${TGZV}.tgz"
-			error_rm_wrkdir
-		fi
-	fi
-	
-	if [ "${OXTGZ}" ]; then
-		XTGZV=`echo ${XTGZ} | sed -e 's,^.*/,,' -e 's,etc,,' -e 's,.tgz,,'`
-		OXTGZV=`echo ${OXTGZ} | sed -e 's,^.*/,,' -e 's,etc,,' -e 's,.tgz,,'`
-		if [ -z "${XTGZ}" ]; then
-			echo " *** Error: please specify a valid path to the new xetcXX.tgz"
-			error_rm_wrkdir
-		elif cmp -s ${OXTGZ} ${XTGZ}; then
-			echo " *** Error: old and new xetcXX.tgz are identical"
-			error_rm_wrkdir
-		elif [ "${OXTGZV}" -gt "${XTGZV}" ]; then
-			echo " *** Error: old xetc${OXTGZV}.tgz version is higher than new xetc${XTGZV}.tgz"
-			error_rm_wrkdir
-		fi
-	fi
-
 	TEMPROOT="${WRKDIR}/temproot"
 	BKPDIR="${WRKDIR}/backups"
 
@@ -120,25 +99,12 @@ do_pre() {
 		elif [ "${SRCDIR}" ]; then
 			echo " etc source:          ${SRCDIR}"
 		fi
-		if [ "${OTGZURL}" ]; then
-			echo " old etc source:      ${OTGZURL}"
-			echo "                      (fetched in ${OTGZ})"
-		else
-			[ "${OTGZ}" ] && echo " old etc source:      ${OTGZ}"
-		fi
-		if [ "${OTGZ}" -o "${OXTGZ}" ]; then echo ""; fi
 		if [ "${XTGZURL}" ]; then
 			echo " xetc source:         ${XTGZURL}"
 			echo "                      (fetched in ${XTGZ})"
 		else
 			[ "${XTGZ}" ] && echo " xetc source:         ${XTGZ}"
 		fi
-		if [ "${OXTGZURL}" ]; then
-			echo " old xetc source:     ${OXTGZURL}"
-			echo "                      (fetched in ${OXTGZ})"
-		else
-			[ "${OXTGZ}" ] && echo " old xetc source:     ${OXTGZ}"
-		fi
 		echo ""
 		echo " base work directory: ${WRKDIR}"
 		echo " temp root directory: ${TEMPROOT}"
@@ -159,33 +125,67 @@ do_pre() {
 
 
 do_populate() {
+	mkdir -p ${DBDIR} || error_rm_wrkdir
 	echo "===> Creating and populating temporary root under"
 	echo "     ${TEMPROOT}"
 	mkdir -p ${TEMPROOT}
 	if [ "${SRCDIR}" ]; then
+		local SRCSUM=srcsum
 		cd ${SRCDIR}/etc
 		make DESTDIR=${TEMPROOT} distribution-etc-root-var > /dev/null 2>&1
+		(cd ${TEMPROOT} && find . -type f | xargs cksum >> ${WRKDIR}/${SRCSUM})
 	fi
 
 	if [ "${TGZ}" -o "${XTGZ}" ]; then
 		for i in ${TGZ} ${XTGZ}; do
 			tar -xzphf ${i} -C ${TEMPROOT};
 		done
+		if [ "${TGZ}" ]; then
+			local ETCSUM=etcsum
+			_E=$(cd `dirname ${TGZ}` && pwd)/`basename ${TGZ}`
+			(cd ${TEMPROOT} && tar -tzf ${_E} | xargs cksum >> ${WRKDIR}/${ETCSUM})
+		fi
+		if [ "${XTGZ}" ]; then
+			local XETCSUM=xetcsum
+			_X=$(cd `dirname ${XTGZ}` && pwd)/`basename ${XTGZ}`
+			(cd ${TEMPROOT} && tar -tzf ${_X} | xargs cksum >> ${WRKDIR}/${XETCSUM})
+		fi
 	fi
 
-	if [ "${OTGZ}" -o "${OXTGZ}" ]; then
-		OTEMPROOT="${WRKDIR}/.otemproot"
-		mkdir -p ${OTEMPROOT}
-		for i in ${OTGZ} ${OXTGZ}; do
-			tar -xzphf ${i} -C ${OTEMPROOT};
-			tar -tzf ${i} >> ${WRKDIR}/olist;
-		done
-		for i in ${TGZ} ${XTGZ}; do
-			tar -tzf ${i} >> ${WRKDIR}/nlist;
-		done
-		OBSOLETE_FILES=`diff -C 0 ${WRKDIR}/olist ${WRKDIR}/nlist | grep -E '^- .' | sed -e 's,^- .,,g'`
-		rm -f ${WRKDIR}/olist ${WRKDIR}/nlist
-	fi
+	for i in ${SRCSUM} ${ETCSUM} ${XETCSUM}; do
+		if [ -f ${DBDIR}/${i} ]; then
+			# delete file in temproot if it has not changed since last release
+			# and is present in current installation
+			if [ "${AUTOMODE}" ]; then
+				_R=$(cd ${TEMPROOT} && cksum -c ${DBDIR}/${i} 2> /dev/null | grep OK | awk '{ print $2 }' | sed 's/[:]//')
+				for _r in ${_R}; do
+					if [ -f ${DESTDIR}/${_r} -a -f ${TEMPROOT}/${_r} ]; then
+						rm -f ${TEMPROOT}/${_r}
+					fi
+				done
+			fi
+
+			# set auto-upgradable files
+			_D=`diff -u ${WRKDIR}/${i} ${DBDIR}/${i} | grep -E '^\+' | sed '1d' | awk '{print $3}'`
+			for _d in ${_D}; do
+				CURSUM=$(cd ${DESTDIR:=/} && cksum ${_d} 2> /dev/null)
+				if [ -n "`grep "${CURSUM}" ${DBDIR}/${i}`" -a -z "`grep "${CURSUM}" ${WRKDIR}/${i}`" ]; then
+					set -A AUTO_UPG -- ${_d}
+				fi
+			done
+
+			# check for obsolete files
+			awk '{ print $3 }' ${DBDIR}/${i} > ${WRKDIR}/new
+			awk '{ print $3 }' ${WRKDIR}/${i} > ${WRKDIR}/old
+			if [ -n "`diff -q ${WRKDIR}/old ${WRKDIR}/new`" ]; then
+				OBSOLETE_FILES="${OBSOLETE_FILES} `diff -C 0 ${WRKDIR}/new ${WRKDIR}/old | grep -E '^- .' | sed -e 's,^- .,,g'`"
+			fi
+			rm ${WRKDIR}/new ${WRKDIR}/old
+			
+			mv ${DBDIR}/${i} ${DBDIR}/.${i}.bak
+		fi
+		mv ${WRKDIR}/${i} ${DBDIR}/${i}
+	done
 
 	# files we don't want/need to deal with
 	IGNORE_FILES="/etc/*.db /etc/mail/*.db /etc/passwd /etc/motd /etc/myname /var/mail/root"
@@ -369,29 +369,22 @@ diff_loop() {
 
 	unset NO_INSTALLED
 	unset CAN_INSTALL
+	unset FORCE_UPG
 
 	while [ "${HANDLE_COMPFILE}" = "v" -o "${HANDLE_COMPFILE}" = "todo" ]; do
 		if [ "${HANDLE_COMPFILE}" = "v" ]; then
 			echo "\n========================================================================\n"
 		fi
 		if [ -f "${DESTDIR}${COMPFILE#.}" -a -f "${COMPFILE}" ]; then
-			# automatically install files which differ only by CVS Id
 			if [ "${AUTOMODE}" ]; then
-				if diff -q -I'[$]OpenBSD:.*$' "${DESTDIR}${COMPFILE#.}" "${COMPFILE}" > /dev/null 2>&1; then
-					if mm_install "${COMPFILE}"; then
-						echo "===> ${COMPFILE} installed successfully"
-						AUTO_INSTALLED_FILES="${AUTO_INSTALLED_FILES}${DESTDIR}${COMPFILE#.}\n"
-					else
-						echo " *** Warning: problem installing ${COMPFILE}, it will remain to merge by hand"
+				# automatically install files if current != new and current = old
+				for i in "${AUTO_UPG[@]}"; do
+					if [ "${i}" = "${COMPFILE}" ]; then
+						FORCE_UPG=1
 					fi
-					return
-				fi
-			fi
-			if [ "${OTGZ}" -o "${OXTGZ}" ]; then
-				# if current != new and current = old, auto-install new
-				if diff -q "${DESTDIR}${COMPFILE#.}" "${OTEMPROOT}${COMPFILE#.}" > /dev/null 2>&1; then
-					echo "===> ${COMPFILE} has not been modified since previous release/snapshot,"
-					echo "     automatically installing new version"
+				done
+				# automatically install files which differ only by CVS Id
+				if [ -z "`diff -q -I'[$]OpenBSD:.*$' "${DESTDIR}${COMPFILE#.}" "${COMPFILE}"`" -o -n "${FORCE_UPG}" ]; then
 					if mm_install "${COMPFILE}"; then
 						echo "===> ${COMPFILE} installed successfully"
 						AUTO_INSTALLED_FILES="${AUTO_INSTALLED_FILES}${DESTDIR}${COMPFILE#.}\n"
@@ -521,9 +514,6 @@ do_compare() {
 			# make sure files are different; if not, delete the one in temproot
 			if diff -q "${DESTDIR}${COMPFILE#.}" "${COMPFILE}" > /dev/null 2>&1; then
 				rm "${COMPFILE}"
-			# delete file in temproot if it has not changed since last release
-			elif [ "${OTGZ}" -o "${OXTGZ}" ] && diff -q "${COMPFILE}" "${OTEMPROOT}${COMPFILE#.}" > /dev/null 2>&1; then
-				rm "${COMPFILE}"
 			# xetcXX.tgz contains binary files; set IS_BINFILE to disable sdiff
 			elif diff -q "${DESTDIR}${COMPFILE#.}" "${COMPFILE}" | grep "Binary" > /dev/null 2>&1; then
 				IS_BINFILE=1
@@ -540,9 +530,6 @@ do_compare() {
 
 
 do_post() {
-	clean_src
-	rm -rf ${OTEMPROOT}
-
 	echo "===> Making sure your directory hierarchy has correct perms, running mtree"
 	mtree -qdef ${DESTDIR}/etc/mtree/4.4BSD.dist -p ${DESTDIR:=/} -U > /dev/null
 
@@ -564,7 +551,7 @@ do_post() {
 		echo "${BKPDIR}\n" >> ${REPORT}
 	fi
 	if [ "${OBSOLETE_FILES}" ]; then
-		echo "===> File(s) removed from previous set (maybe obsolete)" >> ${REPORT}
+		echo "===> File(s) removed from previous source (maybe obsolete)" >> ${REPORT}
 		echo "${OBSOLETE_FILES}" >> ${REPORT}
 	fi
 	if [ "${FILES_IN_TEMPROOT}" ]; then
@@ -583,6 +570,9 @@ do_post() {
 		echo "===> Removing ${WRKDIR}"
 		rm -rf "${WRKDIR}"
 	fi
+
+	clean_src
+	rm -f ${DBDIR}/.*.bak
 }
 
 
@@ -632,42 +622,6 @@ while getopts abs:x:S:X: arg; do
 			error_rm_wrkdir
 		fi
 		;;
-	S)
-		if [ -f "${OPTARG}" ] && echo -n ${OPTARG} | \
-		    awk -F/ '{print $NF}' | \
-		    grep '^etc[0-9][0-9]\.tgz$' > /dev/null 2>&1 ; then
-			OTGZ=${OPTARG}
-		elif echo ${OPTARG} | \
-		    grep -qE '^(http|ftp)://.*/etc[0-9][0-9]\.tgz$'; then
-			OTGZ=${WRKDIR}/etc.tgz
-			OTGZURL=${OPTARG}
-			if ! ${FETCH_CMD} -o ${OTGZ} ${OTGZURL}; then
-				echo " *** Error: could not retrieve ${OTGZURL}"
-				error_rm_wrkdir
-			fi
-		else
-			echo " *** Error: ${OPTARG} is not a path to etcXX.tgz"
-			error_rm_wrkdir
-		fi
-		;;
-	X)
-		if [ -f "${OPTARG}" ] && echo -n ${OPTARG} | \
-		    awk -F/ '{print $NF}' | \
-		    grep '^xetc[0-9][0-9]\.tgz$' > /dev/null 2>&1 ; then
-			OXTGZ=${OPTARG}
-		elif echo ${OPTARG} | \
-		    grep -qE '^(http|ftp)://.*/xetc[0-9][0-9]\.tgz$'; then
-			OXTGZ=${WRKDIR}/xetc.tgz
-			OXTGZURL=${OPTARG}
-			if ! ${FETCH_CMD} -o ${OXTGZ} ${OXTGZURL}; then
-				echo " *** Error: could not retrieve ${OXTGZURL}"
-				error_rm_wrkdir
-			fi
-		else
-			echo " *** Error: ${OPTARG} is not a path to xetcXX.tgz"
-			error_rm_wrkdir
-		fi
-		;;
 	*)
 		usage
 		error_rm_wrkdir