extindex: retry sync_inbox before reindex

[public-inbox.git] / lib / PublicInbox / ExtSearchIdx.pm

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index ba7588d53254c092f735994f309f722874ff3618..daff656d1ac54749090a137a85ea8a8542564a68 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -719,11 +719,12 @@ sub eidxq_lock_acquire ($) {
                return $locked if $locked eq $cur;
        }
        my ($pid, $time, $euid, $ident) = split(/-/, $cur, 4);
-       my $t = strftime('%Y-%m-%d %k:%M:%S', gmtime($time));
+       my $t = strftime('%Y-%m-%d %k:%M %z', localtime($time));
+       local $self->{current_info} = 'eidxq';
        if ($euid == $> && $ident eq host_ident) {
                if (kill(0, $pid)) {
                        warn <<EOM; return;
-I: PID:$pid (re)indexing Xapian since $t, it will continue our work
+I: PID:$pid (re)indexing since $t, it will continue our work
 EOM
                }
                if ($!{ESRCH}) {
@@ -839,7 +840,7 @@ sub _unref_stale_range ($$$) {
                $r = $sync->{self}->{oidx}->dbh->selectall_arrayref(
                        <<EOS, undef, $ibx->{-ibx_id});
 SELECT docid,xnum,oidbin FROM xref3
-WHERE ibx_id = ? AND xnum $lt_or_gt LIMIT $lim
+WHERE ibx_id = ? AND $lt_or_gt LIMIT $lim
 EOS
                return if $sync->{quit};
                for (@$r) { # hopefully rare, not worth optimizing:
@@ -858,18 +859,24 @@ sub _reindex_check_ibx ($$$) {
        my $slice = 10000;
        my $opt = { limit => $slice };
        my ($beg, $end) = (1, $slice);
-       my $err = sync_inbox($self, $sync, $ibx) and return;
-       my $max = $ibx->over->max;
+       my $ekey = $ibx->eidx_key;
+       my ($max, $max0);
+       do {
+               $max0 = $ibx->mm->num_highwater;
+               sync_inbox($self, $sync, $ibx) and return; # warned
+               $max = $ibx->mm->num_highwater;
+               return if $sync->{quit};
+       } while ($max > $max0 &&
+               warn("# $ekey moved $max0..$max, resyncing..\n"));
        $end = $max if $end > $max;
 
        # first, check if we missed any messages in target $ibx
        my $msgs;
        my $pr = $sync->{-opt}->{-progress};
-       my $ekey = $ibx->eidx_key;
        local $sync->{-regen_fmt} = "$ekey checking %u/$max\n";
        ${$sync->{nr}} = 0;
        my $fast = $sync->{-opt}->{fast};
-       my $dsu; # _unref_stale_range (< $lo) called
+       my $usr; # _unref_stale_range (< $lo) called
        my ($lo, $hi);
        while (scalar(@{$msgs = $ibx->over->query_xover($beg, $end, $opt)})) {
                ${$sync->{nr}} = $beg;
@@ -880,7 +887,7 @@ sub _reindex_check_ibx ($$$) {
                        reindex_checkpoint($self, $sync); # release lock
                }
                ($lo, $hi) = ($msgs->[0]->{num}, $msgs->[-1]->{num});
-               $dsu //= _unref_stale_range($sync, $ibx, "< $lo");
+               $usr //= _unref_stale_range($sync, $ibx, "xnum < $lo");
                my $x3a = $self->{oidx}->dbh->selectall_arrayref(
                        <<"", undef, $ibx_id, $lo, $hi);
 SELECT xnum,oidbin,docid FROM xref3 WHERE
@@ -921,7 +928,8 @@ ibx_id = ? AND xnum >= ? AND xnum <= ?
                        }
                }
        }
-       _unref_stale_range($sync, $ibx, "> $hi") if defined($hi);
+       defined($hi) and ($hi < $max) and
+               _unref_stale_range($sync, $ibx, "xnum > $hi AND xnum <= $max");
 }
 
 sub _reindex_inbox ($$$) {
@@ -1192,7 +1200,7 @@ sub idx_init { # similar to V2Writable
        $self->git->cleanup;
        my $mode = 0644;
        my $ALL = $self->git->{git_dir}; # topdir/ALL.git
-       my ($has_new, $alt, $seen);
+       my ($has_new, $alt, $seen, $prune, $prune_nr);
        if ($opt->{-private}) { # LeiStore
                my $local = "$self->{topdir}/local"; # lei/store
                $self->{mg} //= PublicInbox::MultiGit->new($self->{topdir},
@@ -1208,8 +1216,8 @@ sub idx_init { # similar to V2Writable
        } else { # extindex has no epochs
                $self->{mg} //= PublicInbox::MultiGit->new($self->{topdir},
                                                        'ALL.git');
-               ($alt, $seen) = $self->{mg}->read_alternates(\$mode,
-                                                       $opt->{-idx_gc});
+               $prune = $opt->{-idx_gc} ? \$prune_nr : undef;
+               ($alt, $seen) = $self->{mg}->read_alternates(\$mode, $prune);
                PublicInbox::Import::init_bare($ALL);
        }
 
@@ -1243,7 +1251,7 @@ sub idx_init { # similar to V2Writable
                }
                $new .= "$d\n";
        }
-       ($has_new || $new ne '') and
+       ($has_new || $prune_nr || $new ne '') and
                $self->{mg}->write_alternates($mode, $alt, $new);
        $git_midx and $self->with_umask(sub {
                my @cmd = ('multi-pack-index');