v2writable: remove "resent" message for duplicate Message-IDs

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 261f9d91a0c7e7849eb21d22b6d48c6ffaf5b4ed..dc96b87a7239f4da40078fd002f8e5787ec4d2ee 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -56,6 +56,7 @@ sub new {
                xap_rw => undef, # PublicInbox::V2SearchIdx
                xap_ro => undef,
                partitions => $nparts,
+               parallel => 1,
                transact_bytes => 0,
                lock_path => "$dir/inbox.lock",
                # limit each repo to 1GB or so
@@ -93,7 +94,7 @@ sub add {
        my $nparts = $self->{partitions};
        my $part = $num % $nparts;
        my $idx = $self->idx_part($part);
-       $idx->index_raw($len, $msgref, $num, $oid, $mid0);
+       $idx->index_raw($len, $msgref, $num, $oid, $mid0, $mime);
        my $n = $self->{transact_bytes} += $len;
        if ($n > (PublicInbox::SearchIdx::BATCH_BYTES * $nparts)) {
                $self->checkpoint;
@@ -118,10 +119,10 @@ sub num_for {
                foreach my $m (@$mids) {
                        # read-only lookup now safe to do after above barrier
                        my $existing = $self->lookup_content($mime, $m);
-                       if ($existing) {
-                               warn "<$m> resent\n";
-                               return; # easy, don't store duplicates
-                       }
+                       # easy, don't store duplicates
+                       # note: do not add more diagnostic info here since
+                       # it gets noisy on public-inbox-watch restarts
+                       return if $existing;
                }
 
                # very unlikely:
@@ -405,6 +406,26 @@ sub import_init {
        $self->{im} = $im;
 }
 
+# XXX experimental
+sub diff ($$$) {
+       my ($mid, $cur, $new) = @_;
+       use File::Temp qw(tempfile);
+       use PublicInbox::Spawn qw(spawn);
+
+       my ($ah, $an) = tempfile('email-cur-XXXXXXXX');
+       print $ah $cur->as_string or die "print: $!";
+       close $ah or die "close: $!";
+       my ($bh, $bn) = tempfile('email-new-XXXXXXXX');
+       print $bh $new->as_string or die "print: $!";
+       close $bh or die "close: $!";
+       my $cmd = [ qw(diff -u), $an, $bn ];
+       print STDERR "# MID conflict <$mid>\n";
+       my $pid = spawn($cmd, undef, { 1 => 2 });
+       defined $pid or die "diff failed to spawn $!";
+       waitpid($pid, 0) == $pid or die "diff did not finish";
+       unlink($an, $bn);
+}
+
 sub lookup_content {
        my ($self, $mime, $mid) = @_;
        my $ibx = $self->{-inbox};
@@ -426,6 +447,10 @@ sub lookup_content {
                        $found = $smsg;
                        return 0; # break out of loop
                }
+
+               # XXX DEBUG_DIFF is experimental and may be removed
+               diff($mid, $cur, $mime) if $ENV{DEBUG_DIFF};
+
                1; # continue
        });
        $found;