in an UpdateSet, match new files with old files based on their sha256.

avoid extracting again identical files, leading to less file system churn.
put in a few failsafes (size check) for people with fubar'd file
systems (hey oga@).

The more often you update, the more you gain. Over 6 months, about one
out of five files doesn't change.

apparently, improves things a lot on slow fs, as said landry@ and sthen@.
tested by quite a few people.

5 files changed, 70 insertions, 9 deletions

diff --git a/usr.sbin/pkg_add/OpenBSD/Add.pm b/usr.sbin/pkg_add/OpenBSD/Add.pm
index 60cfc7f25b9..2e20ebbe9a7 100644
--- a/usr.sbin/pkg_add/OpenBSD/Add.pm
+++ b/usr.sbin/pkg_add/OpenBSD/Add.pm
@@ -1,5 +1,5 @@
 # ex:ts=8 sw=4:
-# $OpenBSD: Add.pm,v 1.122 2011/01/25 11:46:57 espie Exp $
+# $OpenBSD: Add.pm,v 1.123 2011/06/20 09:46:22 espie Exp $
 #
 # Copyright (c) 2003-2007 Marc Espie <espie@openbsd.org>
 #
@@ -358,7 +358,8 @@ sub prepare_for_addition
 		$state->{problems}++;
 		return;
 	}
-	my $s = $state->vstat->add($fname, $self->{size}, $pkgname);
+	my $s = $state->vstat->add($fname, $self->{tieto} ? 0 : $self->{size}, 
+	    $pkgname);
 	return unless defined $s;
 	if ($s->ro) {
 		$s->report_ro($state, $fname);
diff --git a/usr.sbin/pkg_add/OpenBSD/Delete.pm b/usr.sbin/pkg_add/OpenBSD/Delete.pm
index 98b24d5480f..f5c23a14faa 100644
--- a/usr.sbin/pkg_add/OpenBSD/Delete.pm
+++ b/usr.sbin/pkg_add/OpenBSD/Delete.pm
@@ -1,5 +1,5 @@
 # ex:ts=8 sw=4:
-# $OpenBSD: Delete.pm,v 1.113 2011/01/23 06:56:53 espie Exp $
+# $OpenBSD: Delete.pm,v 1.114 2011/06/20 09:46:23 espie Exp $
 #
 # Copyright (c) 2003-2007 Marc Espie <espie@openbsd.org>
 #
@@ -378,10 +378,11 @@ sub prepare_for_deletion
 
 	my $fname = $state->{destdir}.$self->fullname;
 	my $s;
+	my $size = $self->{tied} ? 0 : $self->{size};
 	if ($state->{delete_first}) {
-		$s = $state->vstat->remove_first($fname, $self->{size});
+		$s = $state->vstat->remove_first($fname, $size);
 	} else {
-		$s = $state->vstat->remove($fname, $self->{size});
+		$s = $state->vstat->remove($fname, $size);
 	}
 	return unless defined $s;
 	if ($s->ro) {
diff --git a/usr.sbin/pkg_add/OpenBSD/PkgAdd.pm b/usr.sbin/pkg_add/OpenBSD/PkgAdd.pm
index dc03e8c8a48..6a2345a5459 100644
--- a/usr.sbin/pkg_add/OpenBSD/PkgAdd.pm
+++ b/usr.sbin/pkg_add/OpenBSD/PkgAdd.pm
@@ -1,7 +1,7 @@
 #! /usr/bin/perl
 
 # ex:ts=8 sw=4:
-# $OpenBSD: PkgAdd.pm,v 1.21 2011/01/03 19:01:04 espie Exp $
+# $OpenBSD: PkgAdd.pm,v 1.22 2011/06/20 09:46:23 espie Exp $
 #
 # Copyright (c) 2003-2010 Marc Espie <espie@openbsd.org>
 #
@@ -62,6 +62,41 @@ sub has_different_sig
 	return $plist->{different_sig};
 }
 
+package OpenBSD::PackingElement;
+sub hash_files
+{
+}
+sub tie_files
+{
+}
+
+package OpenBSD::PackingElement::FileBase;
+sub hash_files
+{
+	my ($self, $sha, $state) = @_;
+	return if $self->{link} or $self->{symlink} or $self->{nochecksum};
+	if (defined $self->{d}) {
+		$sha->{${$self->{d}}} = $self;
+	}
+}
+
+sub tie_files
+{
+	my ($self, $sha, $state) = @_;
+	return if $self->{link} or $self->{symlink} or $self->{nochecksum};
+	if (defined $sha->{${$self->{d}}}) {
+		my $tied = $sha->{${$self->{d}}};
+		# don't tie if there's a problem with the file
+		return unless -f $tied->realname($state);
+		# and do a sanity check that this file wasn't altered
+		return unless (stat _)[7] == $self->{size};
+		$self->{tieto} = $tied;
+		$tied->{tied} = 1;
+		$state->say("Tieing #1 to #2", $self->stringize, 
+		    $tied->stringize) if $state->verbose >= 3;
+	}
+}
+
 package OpenBSD::PkgAdd::State;
 our @ISA = qw(OpenBSD::AddDelete::State);
 
@@ -938,6 +973,16 @@ sub install_set
 			return ();
 		}
 	}
+	if ($set->newer > 0 && $set->older_to_do > 0 && !$state->defines('donttie')) {
+		$set->{sha} = {};
+
+		for my $o ($set->older_to_do) {
+			$o->{plist}->hash_files($set->{sha}, $state);
+		}
+		for my $n ($set->newer) {
+			$n->{plist}->tie_files($set->{sha}, $state);
+		}
+	}
 	if ($set->newer > 0 || $set->older_to_do > 0) {
 		for my $h ($set->newer) {
 			$h->plist->set_infodir($h->location->info);
diff --git a/usr.sbin/pkg_add/OpenBSD/Replace.pm b/usr.sbin/pkg_add/OpenBSD/Replace.pm
index 4a6fd65274f..3b8d27a2a87 100644
--- a/usr.sbin/pkg_add/OpenBSD/Replace.pm
+++ b/usr.sbin/pkg_add/OpenBSD/Replace.pm
@@ -1,5 +1,5 @@
 # ex:ts=8 sw=4:
-# $OpenBSD: Replace.pm,v 1.80 2010/12/24 09:04:14 espie Exp $
+# $OpenBSD: Replace.pm,v 1.81 2011/06/20 09:46:23 espie Exp $
 #
 # Copyright (c) 2004-2010 Marc Espie <espie@openbsd.org>
 #
@@ -74,9 +74,22 @@ sub extract
 			File::Path::mkpath($d);
 		}
 		my ($fh, $tempname) = OpenBSD::Temp::permanent_file($d, "pkg");
+		$self->{tempname} = $tempname;
+		if ($self->{tieto}) {
+			my $src = $self->{tieto}->realname($state);
+			unlink($tempname);
+			$state->say("linking #1 to #2", $src, $tempname) 
+			    if $state->verbose >= 3;
+			if (link($src, $tempname) ||
+			    $state->copy_file($src, $tempname)) {
+				$state->{archive}->skip;
+				return;
+			}
+			# okay, it didn't work. recreate tempname.
+			open $fh, ">", $tempname;
+		}
 
 		$state->say("extracting #1", $tempname) if $state->verbose >= 3;
-		$self->{tempname} = $tempname;
 
 		# XXX don't apply destdir twice
 		$file->{destdir} = '';
diff --git a/usr.sbin/pkg_add/OpenBSD/UpdateSet.pm b/usr.sbin/pkg_add/OpenBSD/UpdateSet.pm
index cdcc068a600..188a0c37bf5 100644
--- a/usr.sbin/pkg_add/OpenBSD/UpdateSet.pm
+++ b/usr.sbin/pkg_add/OpenBSD/UpdateSet.pm
@@ -1,5 +1,5 @@
 # ex:ts=8 sw=4:
-# $OpenBSD: UpdateSet.pm,v 1.64 2010/12/24 09:04:14 espie Exp $
+# $OpenBSD: UpdateSet.pm,v 1.65 2011/06/20 09:46:23 espie Exp $
 #
 # Copyright (c) 2007-2010 Marc Espie <espie@openbsd.org>
 #
@@ -122,6 +122,7 @@ sub cleanup
 	$self->{errorinfo} //= $errorinfo;
 	delete $self->{solver};
 	delete $self->{conflict_cache};
+	delete $self->{sha};
 	$self->{finished} = 1;
 }