X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=blobdiff_plain;f=lib%2FPublicInbox%2FV2Writable.pm;h=34dd139b8ab82495296ddb9abd76847dee54b5fc;hp=fc2f33f9dd2cd09f48b89cc5d2629ae2f1e6f3b7;hb=1218a4126807951a0f47286338dc04d7f197bb78;hpb=95bdac7f09c69036efed537a4d03d5bdd2ae4eb6

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index fc2f33f9..34dd139b 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -150,9 +150,15 @@ sub add {
 # indexes a message, returns true if checkpointing is needed
 sub do_idx ($$$$$$$) {
 	my ($self, $msgref, $mime, $len, $num, $oid, $mid0) = @_;
-	$self->{over}->add_overview($mime, $len, $num, $oid, $mid0);
+	my $smsg = bless {
+		bytes => $len,
+		num => $num,
+		blob => $oid,
+		mid => $mid0,
+	}, 'PublicInbox::Smsg';
+	$self->{over}->add_overview($mime, $smsg, $self);
 	my $idx = idx_shard($self, $num % $self->{shards});
-	$idx->index_raw($len, $msgref, $num, $oid, $mid0, $mime);
+	$idx->index_raw($len, $msgref, $num, $oid, $mid0, $mime, $self);
 	my $n = $self->{transact_bytes} += $len;
 	$n >= (PublicInbox::SearchIdx::BATCH_BYTES * $self->{shards});
 }
@@ -176,7 +182,7 @@ sub _add {
 	defined $num or return; # duplicate
 	defined $mid0 or die "BUG: $mid0 undefined\n";
 	my $im = $self->importer;
-	my $cmt = $im->add($mime);
+	my $cmt = $im->add($mime, undef, $self); # sets $self->{(au|co)time}
 	$cmt = $im->get_mark($cmt);
 	$self->{last_commit}->[$self->{epoch_max}] = $cmt;
 
@@ -201,11 +207,10 @@ sub v2_num_for {
 		# crap, Message-ID is already known, hope somebody just resent:
 		foreach my $m (@$mids) {
 			# read-only lookup now safe to do after above barrier
-			my $existing = lookup_content($self, $mime, $m);
 			# easy, don't store duplicates
 			# note: do not add more diagnostic info here since
 			# it gets noisy on public-inbox-watch restarts
-			return () if $existing;
+			return () if content_exists($self, $mime, $m);
 		}
 
 		# AltId may pre-populate article numbers (e.g. X-Mail-Count
@@ -381,7 +386,7 @@ sub rewrite_internal ($$;$$$) {
 	}
 	my $over = $self->{over};
 	my $cids = content_ids($old_mime);
-	my $removed;
+	my @removed;
 	my $mids = mids($old_mime->header_obj);
 
 	# We avoid introducing new blobs into git since the raw content
@@ -391,7 +396,7 @@ sub rewrite_internal ($$;$$$) {
 	my $mark;
 
 	foreach my $mid (@$mids) {
-		my %gone; # num => [ smsg, raw ]
+		my %gone; # num => [ smsg, $mime, raw ]
 		my ($id, $prev);
 		while (my $smsg = $over->next_by_mid($mid, \$id, \$prev)) {
 			my $msg = get_blob($self, $smsg);
@@ -402,8 +407,7 @@ sub rewrite_internal ($$;$$$) {
 			my $orig = $$msg;
 			my $cur = PublicInbox::MIME->new($msg);
 			if (content_matches($cids, $cur)) {
-				$smsg->{mime} = $cur;
-				$gone{$smsg->{num}} = [ $smsg, \$orig ];
+				$gone{$smsg->{num}} = [ $smsg, $cur, \$orig ];
 			}
 		}
 		my $n = scalar keys %gone;
@@ -413,15 +417,16 @@ sub rewrite_internal ($$;$$$) {
 				join(',', sort keys %gone), "\n";
 		}
 		foreach my $num (keys %gone) {
-			my ($smsg, $orig) = @{$gone{$num}};
-			# $removed should only be set once assuming
+			my ($smsg, $mime, $orig) = @{$gone{$num}};
+			# @removed should only be set once assuming
 			# no bugs in our deduplication code:
-			$removed = $smsg;
+			@removed = (undef, $mime, $smsg);
 			my $oid = $smsg->{blob};
 			if ($replace_map) {
 				$replace_map->{$oid} = $sref;
 			} else {
 				($mark, undef) = $im->remove($orig, $cmt_msg);
+				$removed[0] = $mark;
 			}
 			$orig = undef;
 			if ($need_reindex) { # ->replace
@@ -441,15 +446,18 @@ sub rewrite_internal ($$;$$$) {
 		my $rewrites = _replace_oids($self, $new_mime, $replace_map);
 		return { rewrites => $rewrites, need_reindex => $need_reindex };
 	}
-	$removed;
+	defined($mark) ? @removed : undef;
 }
 
-# public
+# public (see PublicInbox::Import->remove), but note the 3rd element
+# (retval[2]) is not part of the stable API shared with Import->remove
 sub remove {
 	my ($self, $mime, $cmt_msg) = @_;
+	my @ret;
 	$self->{-inbox}->with_umask(sub {
-		rewrite_internal($self, $mime, $cmt_msg);
+		@ret = rewrite_internal($self, $mime, $cmt_msg);
 	});
+	defined($ret[0]) ? @ret : undef;
 }
 
 sub _replace ($$;$$) {
@@ -821,7 +829,7 @@ sub get_blob ($$) {
 	$ibx->msg_by_smsg($smsg);
 }
 
-sub lookup_content ($$$) {
+sub content_exists ($$$) {
 	my ($self, $mime, $mid) = @_;
 	my $over = $self->{over};
 	my $cids = content_ids($mime);
@@ -833,11 +841,7 @@ sub lookup_content ($$$) {
 			next;
 		}
 		my $cur = PublicInbox::MIME->new($msg);
-		if (content_matches($cids, $cur)) {
-			$smsg->{mime} = $cur;
-			return $smsg;
-		}
-
+		return 1 if content_matches($cids, $cur);
 
 		# XXX DEBUG_DIFF is experimental and may be removed
 		diff($mid, $cur, $mime) if $ENV{DEBUG_DIFF};
@@ -1268,15 +1272,17 @@ sub index_epoch ($$$) {
 		$pr->("$i.git indexing $range\n");
 	}
 
-	my @cmd = qw(log --raw -r --pretty=tformat:%H
+	my @cmd = qw(log --raw -r --pretty=tformat:%H.%at.%ct
 			--no-notes --no-color --no-abbrev --no-renames);
 	my $fh = $self->{reindex_pipe} = $git->popen(@cmd, $range);
 	my $cmt;
 	while (<$fh>) {
 		chomp;
 		$self->{current_info} = "$i.git $_";
-		if (/\A$x40$/o && !defined($cmt)) {
-			$cmt = $_;
+		if (/\A($x40)\.([0-9]+)\.([0-9]+)$/o) {
+			$cmt //= $1;
+			$self->{autime} = $2;
+			$self->{cotime} = $3;
 		} elsif (/\A:\d{6} 100644 $x40 ($x40) [AM]\tm$/o) {
 			reindex_oid($self, $sync, $git, $1);
 		} elsif (/\A:\d{6} 100644 $x40 ($x40) [AM]\td$/o) {