v2writable: unindex deleted messages after incremental fetch

The normal behavior is to prevent the deleted messages from
being indexed in the first place.  However, when fetching
incrementally via git; public-inbox-index needs to account for
deleted files which were created outside of the most recent
fetch/reindexing window.

Reported-by: Eric W. Biederman <ebiederm@xmission.com>

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 412eb6a9a4cc0ff3aa9b8ec98c348a042bab6c18..934640eb672ccfa3d57b5caa84c3628df8bbcfb3 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -653,7 +653,7 @@ sub mark_deleted {
        my $mids = mids($mime->header_obj);
        my $cid = content_id($mime);
        foreach my $mid (@$mids) {
-               $D->{"$mid\0$cid"} = 1;
+               $D->{"$mid\0$cid"} = $oid;
        }
 }
 
@@ -671,7 +671,7 @@ sub reindex_oid {
        my $num = -1;
        my $del = 0;
        foreach my $mid (@$mids) {
-               $del += (delete $D->{"$mid\0$cid"} || 0);
+               $del += delete($D->{"$mid\0$cid"}) ? 1 : 0;
                my $n = $mm_tmp->num_for($mid);
                if (defined $n && $n > $num) {
                        $mid0 = $mid;
@@ -882,7 +882,7 @@ sub index_sync {
        my ($min, $max) = $mm_tmp->minmax;
        my $regen = $self->index_prepare($opts, $epoch_max, $ranges);
        $$regen += $max if $max;
-       my $D = {};
+       my $D = {}; # "$mid\0$cid" => $oid
        my @cmd = qw(log --raw -r --pretty=tformat:%H
                        --no-notes --no-color --no-abbrev --no-renames);
 
@@ -912,13 +912,13 @@ sub index_sync {
                delete $self->{reindex_pipe};
                $self->update_last_commit($git, $i, $cmt) if defined $cmt;
        }
-       my @d = sort keys %$D;
-       if (@d) {
-               warn "BUG: ", scalar(@d)," unseen deleted messages marked\n";
-               foreach (@d) {
-                       my ($mid, undef) = split(/\0/, $_, 2);
-                       warn "<$mid>\n";
-               }
+
+       # unindex is required for leftovers if "deletes" affect messages
+       # in a previous fetch+index window:
+       if (scalar keys %$D) {
+               my $git = $self->{-inbox}->git;
+               $self->unindex_oid($git, $_) for values %$D;
+               $git->cleanup;
        }
        $self->done;
 }

diff --git a/t/v2mirror.t b/t/v2mirror.t
index c0c329ca41defab379b3bd955816f6046638388a..f95ad0f54b5e527261757ebdc35b6b8129faba03 100644 (file)
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -182,7 +182,33 @@ is($mibx->git->check($to_purge), undef, 'unindex+prune successful in mirror');
        is_deeply(\@warn, [], 'no warnings from index_sync after purge');
 }
 
-$v2w->done;
+# deletes happen in a different fetch window
+{
+       $mset = $mibx->search->reopen->query('m:1@example.com', {mset => 1});
+       is(scalar($mset->items), 1, '1@example.com visible in mirror');
+       $mime->header_set('Message-ID', '<1@example.com>');
+       $mime->header_set('Subject', 'subject = 1');
+       ok($v2w->remove($mime), 'removed <1@example.com> from source');
+       $v2w->done;
+       fetch_each_epoch();
+
+       open my $err, '+>', "$tmpdir/index-err" or die "open: $!";
+       my $ipid = fork;
+       if ($ipid == 0) {
+               dup2(fileno($err), 2) or die "dup2 failed: $!";
+               exec("$script-index", "$tmpdir/m");
+               die "exec fail: $!";
+       }
+       ok($ipid, 'running index');
+       is(waitpid($ipid, 0), $ipid, 'index done');
+       is($?, 0, 'no error from index');
+       ok(seek($err, 0, 0), 'rewound stderr');
+       $err = eval { local $/; <$err> };
+       is($err, '', 'no errors reported by index');
+       $mset = $mibx->search->reopen->query('m:1@example.com', {mset => 1});
+       is(scalar($mset->items), 0, '1@example.com no longer visible in mirror');
+}
+
 ok(kill('TERM', $pid), 'killed httpd');
 $pid = undef;
 waitpid(-1, 0);