extindex: prune invalid alternate entries on --gc

[public-inbox.git] / lib / PublicInbox / ExtSearchIdx.pm

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index c2ab0447e17624544a319d7fb8b4ddb28f0f4ace..a08a9451558244f39818c1f9a7170c799d083909 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -18,6 +18,7 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::ExtSearch PublicInbox::Lock);
 use Carp qw(croak carp);
+use Scalar::Util qw(blessed);
 use Sys::Hostname qw(hostname);
 use POSIX qw(strftime);
 use File::Glob qw(bsd_glob GLOB_NOSORT);
@@ -121,7 +122,7 @@ sub apply_boost ($$) {
                $a->[1] <=> $b->[1] # break ties with {xnum}
        } @$xr3;
        my $new_smsg = $req->{new_smsg};
-       return if $xr3->[0]->[2] ne pack('H*', $new_smsg->{blob}); # loser
+       return if $xr3->[0]->[2] ne $new_smsg->oidbin; # loser
 
        # replace the old smsg with the more boosted one
        $new_smsg->{num} = $smsg->{num};
@@ -143,6 +144,14 @@ sub _unref_doc ($$$$$;$) {
                $smsg = $docid;
                $docid = $smsg->{num};
        }
+       if (defined($oidbin) && defined($xnum) && blessed($ibx) && $ibx->over) {
+               my $smsg = $ibx->over->get_art($xnum);
+               if ($smsg && $smsg->oidbin eq $oidbin) {
+                       carp("BUG: (non-fatal) ".$ibx->eidx_key.
+                               " #$xnum $smsg->{blob} still valid");
+                       return;
+               }
+       }
        my $s = 'DELETE FROM xref3 WHERE oidbin = ?';
        $s .= ' AND ibx_id = ?' if defined($ibx);
        $s .= ' AND xnum = ?' if defined($xnum);
@@ -193,6 +202,7 @@ sub do_xpost ($$) {
                $idx->ipc_do('add_eidx_info', $docid, $eidx_key, $eml);
                apply_boost($req, $smsg) if $req->{boost_in_use};
        } else { # 'd' no {xnum}
+               $self->git->async_wait_all;
                $oid = pack('H*', $oid);
                _unref_doc($req, $docid, $xibx, undef, $oid, $eml);
        }
@@ -260,8 +270,8 @@ sub _blob_missing ($$) { # called when a known $smsg->{blob} is gone
        my ($req, $smsg) = @_;
        # xnum and ibx are unknown, we only call this when an entry from
        # /ei*/over.sqlite3 is bad, not on entries from xap*/over.sqlite3
-       my $oidbin = pack('H*', $smsg->{blob});
-       _unref_doc($req, $smsg, undef, undef, $oidbin);
+       $req->{self}->git->async_wait_all;
+       _unref_doc($req, $smsg, undef, undef, $smsg->oidbin);
 }
 
 sub ck_existing { # git->cat_async callback
@@ -552,12 +562,13 @@ sub _reindex_finalize ($$$) {
        }
        return if $nr == 1; # likely, all good
 
+       $self->git->async_wait_all;
        warn "W: #$docid split into $nr due to deduplication change\n";
        my @todo;
        for my $ary (values %$by_chash) {
                for my $x (reverse @$ary) {
                        warn "removing #$docid xref3 $x->{blob}\n";
-                       my $bin = pack('H*', $x->{blob});
+                       my $bin = $x->oidbin;
                        my $n = _unref_doc($sync, $docid, undef, undef, $bin);
                        die "BUG: $x->{blob} invalidated #$docid" if $n == 0;
                }
@@ -592,11 +603,11 @@ sub _reindex_oid { # git->cat_async callback
                my $oidbin = pack('H*', $expect_oid);
                my $remain = _unref_doc($sync, $docid, undef, undef, $oidbin);
                if ($remain == 0) {
-                       warn "W: #$docid gone or corrupted\n";
+                       warn "W: #$docid ($oid) gone or corrupt\n";
                } elsif (my $next_oid = $req->{xr3r}->[++$req->{ix}]->[2]) {
                        $self->git->cat_async($next_oid, \&_reindex_oid, $req);
                } else {
-                       warn "BUG: #$docid gone (UNEXPECTED)\n";
+                       warn "BUG: #$docid ($oid) gone (UNEXPECTED)\n";
                }
                return;
        }
@@ -751,9 +762,8 @@ sub prep_id2pos ($) {
 
 sub eidxq_process ($$) { # for reindexing
        my ($self, $sync) = @_;
-       return unless $self->{cfg};
-
-       return unless eidxq_lock_acquire($self);
+       local $self->{current_info} = 'eidxq process';
+       return unless ($self->{cfg} && eidxq_lock_acquire($self));
        my $dbh = $self->{oidx}->dbh;
        my $tot = $dbh->selectrow_array('SELECT COUNT(*) FROM eidxq') or return;
        ${$sync->{nr}} = 0;
@@ -896,6 +906,7 @@ ibx_id = ? AND xnum >= ? AND xnum <= ?
                }
                return if $sync->{quit};
                next unless scalar keys %x3m;
+               $self->git->async_wait_all; # wait for reindex_unseen
 
                # eliminate stale/mismatched entries
                my %mismatch = map { $_->{num} => $_->{blob} } @$msgs;
@@ -975,7 +986,7 @@ sub dd_smsg { # git->cat_async callback
                my $oidx = $self->{oidx};
                for my $smsg (@$ary) {
                        my $gone = $smsg->{num};
-                       $oidx->merge_xref3($keep->{num}, $gone, $smsg->{blob});
+                       $oidx->merge_xref3($keep->{num}, $gone, $smsg->oidbin);
                        remove_doc($self, $gone);
                }
        }
@@ -1181,7 +1192,7 @@ sub idx_init { # similar to V2Writable
        $self->git->cleanup;
        my $mode = 0644;
        my $ALL = $self->git->{git_dir}; # topdir/ALL.git
-       my ($has_new, $alt, $seen);
+       my ($has_new, $alt, $seen, $prune, $prune_nr);
        if ($opt->{-private}) { # LeiStore
                my $local = "$self->{topdir}/local"; # lei/store
                $self->{mg} //= PublicInbox::MultiGit->new($self->{topdir},
@@ -1197,8 +1208,8 @@ sub idx_init { # similar to V2Writable
        } else { # extindex has no epochs
                $self->{mg} //= PublicInbox::MultiGit->new($self->{topdir},
                                                        'ALL.git');
-               ($alt, $seen) = $self->{mg}->read_alternates(\$mode,
-                                                       $opt->{-idx_gc});
+               $prune = $opt->{-idx_gc} ? \$prune_nr : undef;
+               ($alt, $seen) = $self->{mg}->read_alternates(\$mode, $prune);
                PublicInbox::Import::init_bare($ALL);
        }
 
@@ -1232,7 +1243,7 @@ sub idx_init { # similar to V2Writable
                }
                $new .= "$d\n";
        }
-       ($has_new || $new ne '') and
+       ($has_new || $prune_nr || $new ne '') and
                $self->{mg}->write_alternates($mode, $alt, $new);
        $git_midx and $self->with_umask(sub {
                my @cmd = ('multi-pack-index');