import: force Message-ID generation for v1 here

[public-inbox.git] / lib / PublicInbox / V2Writable.pm

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index b6c46a21a704b7a1a961eaa3007ad3c5acf723c4..a305842e311c7e1772d7856f0bd6efd2c842619f 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -146,19 +146,19 @@ sub num_for_harder {
 
        my $hdr = $mime->header_obj;
        my $dig = content_digest($mime);
-       $$mid0 = $dig->clone->hexdigest . '@localhost';
+       $$mid0 = PublicInbox::Import::digest2mid($dig);
        my $num = $self->{skel}->{mm}->mid_insert($$mid0);
        unless (defined $num) {
                # it's hard to spoof the last Received: header
                my @recvd = $hdr->header_raw('Received');
                $dig->add("Received: $_") foreach (@recvd);
-               $$mid0 = $dig->clone->hexdigest . '@localhost';
+               $$mid0 = PublicInbox::Import::digest2mid($dig);
                $num = $self->{skel}->{mm}->mid_insert($$mid0);
 
                # fall back to a random Message-ID and give up determinism:
                until (defined($num)) {
                        $dig->add(rand);
-                       $$mid0 = $dig->clone->hexdigest . '@localhost';
+                       $$mid0 = PublicInbox::Import::digest2mid($dig);
                        warn "using random Message-ID <$$mid0> as fallback\n";
                        $num = $self->{skel}->{mm}->mid_insert($$mid0);
                }
@@ -199,18 +199,49 @@ sub idx_init {
 }
 
 sub remove {
-       my ($self, $mime, $msg) = @_;
-       my $existing = $self->lookup_content($mime) or return;
-
-       # don't touch ghosts or already junked messages
-       return unless $existing->type eq 'mail';
-
-       # always write removals to the current (latest) git repo since
-       # we process chronologically
+       my ($self, $mime, $cmt_msg) = @_;
+       $self->barrier;
+       $self->idx_init;
        my $im = $self->importer;
-       my ($cmt, undef) = $im->remove($mime, $msg);
-       $cmt = $im->get_mark($cmt);
-       $self->unindex_msg($existing, $cmt);
+       my $ibx = $self->{-inbox};
+       my $srch = $ibx->search;
+       my $cid = content_id($mime);
+       my $skel = $self->{skel};
+       my $parts = $self->{idx_parts};
+       my $mm = $skel->{mm};
+       my $removed;
+       my $mids = mids($mime->header_obj);
+       foreach my $mid (@$mids) {
+               $srch->reopen->each_smsg_by_mid($mid, sub {
+                       my ($smsg) = @_;
+                       $smsg->load_expand;
+                       my $msg = $ibx->msg_by_smsg($smsg);
+                       if (!defined($msg)) {
+                               warn "broken smsg for $mid\n";
+                               return 1; # continue
+                       }
+                       my $orig = $$msg;
+                       my $cur = PublicInbox::MIME->new($msg);
+                       if (content_id($cur) eq $cid) {
+                               $mm->num_delete($smsg->num);
+                               # $removed should only be set once assuming
+                               # no bugs in our deduplication code:
+                               $removed = $smsg;
+                               $removed->{mime} = $cur;
+                               $im->remove(\$orig, $cmt_msg);
+                               $orig = undef;
+                               $removed->num; # memoize this for callers
+
+                               my $oid = $smsg->{blob};
+                               foreach my $idx (@$parts, $skel) {
+                                       $idx->remote_remove($oid, $mid);
+                               }
+                       }
+                       1; # continue
+               });
+               $self->barrier;
+       }
+       $removed;
 }
 
 sub done {
@@ -232,13 +263,8 @@ sub checkpoint {
 sub barrier {
        my ($self) = @_;
 
-       # For safety, we ensure git checkpoint is complete before because
-       # the data in git is still more important than what is in Xapian.
-       # Performance may be gained by delaying ->progress call but we
-       # lose safety
        if (my $im = $self->{im}) {
-               $im->checkpoint;
-               $im->progress('checkpoint');
+               $im->barrier;
        }
        my $skel = $self->{skel};
        my $parts = $self->{idx_parts};
@@ -378,8 +404,7 @@ sub lookup_content {
        my ($self, $mime, $mid) = @_;
        my $ibx = $self->{-inbox};
 
-       my $srch = $ibx->search;
-       $srch->reopen;
+       my $srch = $ibx->search->reopen;
        my $cid = content_id($mime);
        my $found;
        $srch->each_smsg_by_mid($mid, sub {