X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FImport.pm;h=b2aae9a7a800c8dbc0d738a8ef63eeea6e2f104d;hb=c34a83286234ea1e876ebdf92a33744272bb6f4e;hp=6824faca21dc3205ecd12be9899cb1e14f0e3834;hpb=f6285ab9d73a4eae490dda325096e61eadc415cd;p=public-inbox.git

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 6824faca..b2aae9a7 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -49,7 +49,14 @@ sub gfi_start {
 	$self->lock_acquire;
 
 	local $/ = "\n";
-	chomp($self->{tip} = $git->qx(qw(rev-parse --revs-only), $self->{ref}));
+	my $ref = $self->{ref};
+	chomp($self->{tip} = $git->qx(qw(rev-parse --revs-only), $ref));
+	if ($self->{path_type} ne '2/38' && $self->{tip}) {
+		local $/ = "\0";
+		my @tree = $git->qx(qw(ls-tree -r -z --name-only), $ref);
+		chomp @tree;
+		$self->{-tree} = { map { $_ => 1 } @tree };
+	}
 
 	my $git_dir = $git->{git_dir};
 	my @cmd = ('git', "--git-dir=$git_dir", qw(fast-import
@@ -238,7 +245,8 @@ sub remove {
 	if (defined $path) {
 		print $w "D $path\n\n" or wfail;
 	} else {
-		print $w "M 100644 :$blob _/D\n\n" or wfail;
+		clean_tree_v2($self, $w, 'd');
+		print $w "M 100644 :$blob d\n\n" or wfail;
 	}
 	$self->{nchg}++;
 	(($self->{tip} = ":$commit"), $cur);
@@ -297,12 +305,12 @@ sub drop_unwanted_headers ($) {
 }
 
 # used by V2Writable, too
-sub prepend_mid ($$) {
+sub append_mid ($$) {
 	my ($hdr, $mid0) = @_;
 	# @cur is likely empty if we need to call this sub, but it could
 	# have random unparseable crap which we'll preserve, too.
-	my @cur = $hdr->header_raw('Message-Id');
-	$hdr->header_set('Message-Id', "<$mid0>", @cur);
+	my @cur = $hdr->header_raw('Message-ID');
+	$hdr->header_set('Message-ID', @cur, "<$mid0>");
 }
 
 sub v1_mid0 ($) {
@@ -312,11 +320,20 @@ sub v1_mid0 ($) {
 
 	if (!scalar(@$mids)) { # spam often has no Message-Id
 		my $mid0 = digest2mid(content_digest($mime));
-		prepend_mid($hdr, $mid0);
+		append_mid($hdr, $mid0);
 		return $mid0;
 	}
 	$mids->[0];
 }
+sub clean_tree_v2 ($$$) {
+	my ($self, $w, $keep) = @_;
+	my $tree = $self->{-tree} or return; #v2 only
+	delete $tree->{$keep};
+	foreach (keys %$tree) {
+		print $w "D $_\n" or wfail;
+	}
+	%$tree = ($keep => 1);
+}
 
 # returns undef on duplicate
 # returns the :MARK of the most recent commit
@@ -382,14 +399,15 @@ sub add {
 	if ($tip ne '') {
 		print $w 'from ', ($parent ? $parent : $tip), "\n" or wfail;
 	}
+	clean_tree_v2($self, $w, $path);
 	print $w "M 100644 :$blob $path\n\n" or wfail;
 	$self->{nchg}++;
 	$self->{tip} = ":$commit";
 }
 
-sub run_die ($;$) {
-	my ($cmd, $env) = @_;
-	my $pid = spawn($cmd, $env, undef);
+sub run_die ($;$$) {
+	my ($cmd, $env, $rdr) = @_;
+	my $pid = spawn($cmd, $env, $rdr);
 	defined $pid or die "spawning ".join(' ', @$cmd)." failed: $!";
 	waitpid($pid, 0) == $pid or die join(' ', @$cmd) .' did not finish';
 	$? == 0 or die join(' ', @$cmd) . " failed: $?\n";
@@ -430,6 +448,101 @@ sub digest2mid ($) {
 	$b64 . '@localhost';
 }
 
+sub clean_purge_buffer {
+	my ($oids, $buf) = @_;
+	my $cmt_msg = 'purged '.join(' ',@$oids)."\n";
+	@$oids = ();
+
+	foreach my $i (0..$#$buf) {
+		my $l = $buf->[$i];
+		if ($l =~ /^author .* (\d+ [\+-]?\d+)$/) {
+			$buf->[$i] = "author <> $1\n";
+		} elsif ($l =~ /^data (\d+)/) {
+			$buf->[$i++] = "data " . length($cmt_msg) . "\n";
+			$buf->[$i] = $cmt_msg;
+			last;
+		}
+	}
+}
+
+sub purge_oids {
+	my ($self, $purge) = @_;
+	my $tmp = "refs/heads/purge-".((keys %$purge)[0]);
+	my $old = $self->{'ref'};
+	my $git = $self->{git};
+	my @export = (qw(fast-export --no-data --use-done-feature), $old);
+	my ($rd, $pid) = $git->popen(@export);
+	my ($r, $w) = $self->gfi_start;
+	my @buf;
+	my $npurge = 0;
+	my @oids;
+	my $tree = $self->{-tree};
+	while (<$rd>) {
+		if (/^reset (?:.+)/) {
+			push @buf, "reset $tmp\n";
+		} elsif (/^commit (?:.+)/) {
+			if (@buf) {
+				$w->print(@buf) or wfail;
+				@buf = ();
+			}
+			push @buf, "commit $tmp\n";
+		} elsif (/^data (\d+)/) {
+			# only commit message, so $len is small:
+			my $len = $1; # + 1 for trailing "\n"
+			push @buf, $_;
+			my $n = read($rd, my $buf, $len) or die "read: $!";
+			$len == $n or die "short read ($n < $len)";
+			push @buf, $buf;
+		} elsif (/^M 100644 ([a-f0-9]+) (\w+)/) {
+			my ($oid, $path) = ($1, $2);
+			if ($purge->{$oid}) {
+				push @oids, $oid;
+				delete $tree->{$path};
+			} else {
+				$tree->{$path} = 1;
+				push @buf, $_;
+			}
+		} elsif (/^D (\w+)/) {
+			my $path = $1;
+			push @buf, $_ if $tree->{$path};
+		} elsif ($_ eq "\n") {
+			my $out = join('', @buf);
+			$out =~ s/^/# /sgm;
+			warn "purge rewriting\n", $out, "\n";
+			clean_purge_buffer(\@oids, \@buf);
+			$out = join('', @buf);
+			$w->print(@buf, "\n") or wfail;
+			@buf = ();
+			$npurge++;
+		} else {
+			push @buf, $_;
+		}
+	}
+	if (@buf) {
+		$w->print(@buf) or wfail;
+	}
+	$w = $r = undef;
+	$self->done;
+	my @git = ('git', "--git-dir=$git->{git_dir}");
+
+	run_die([@git, qw(update-ref), $old, $tmp]) if $npurge;
+
+	run_die([@git, qw(update-ref -d), $tmp]);
+
+	return if $npurge == 0;
+
+	run_die([@git, qw(-c gc.reflogExpire=now gc --prune=all)]);
+	my $err = 0;
+	foreach my $oid (keys %$purge) {
+		my @info = $git->check($oid);
+		if (@info) {
+			warn "$oid not purged\n";
+			$err++;
+		}
+	}
+	die "Failed to purge $err object(s)\n" if $err;
+}
+
 1;
 __END__
 =pod