extindex: avoid triggering a buggy unref

[public-inbox.git] / lib / PublicInbox / ExtSearchIdx.pm

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 750ced5c0f453bc8e0241a1ea2e6603a64c00104..69d048fb73426b7e0656269f944807829f36b773 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -18,6 +18,7 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::ExtSearch PublicInbox::Lock);
 use Carp qw(croak carp);
+use Scalar::Util qw(blessed);
 use Sys::Hostname qw(hostname);
 use POSIX qw(strftime);
 use File::Glob qw(bsd_glob GLOB_NOSORT);
@@ -121,7 +122,7 @@ sub apply_boost ($$) {
                $a->[1] <=> $b->[1] # break ties with {xnum}
        } @$xr3;
        my $new_smsg = $req->{new_smsg};
-       return if $xr3->[0]->[2] ne pack('H*', $new_smsg->{blob}); # loser
+       return if $xr3->[0]->[2] ne $new_smsg->oidbin; # loser
 
        # replace the old smsg with the more boosted one
        $new_smsg->{num} = $smsg->{num};
@@ -143,6 +144,14 @@ sub _unref_doc ($$$$$;$) {
                $smsg = $docid;
                $docid = $smsg->{num};
        }
+       if (defined($oidbin) && defined($xnum) && blessed($ibx) && $ibx->over) {
+               my $smsg = $ibx->over->get_art($xnum);
+               if ($smsg && $smsg->oidbin eq $oidbin) {
+                       carp("BUG: (non-fatal) ".$ibx->eidx_key.
+                               " #$xnum $smsg->{blob} still valid");
+                       return;
+               }
+       }
        my $s = 'DELETE FROM xref3 WHERE oidbin = ?';
        $s .= ' AND ibx_id = ?' if defined($ibx);
        $s .= ' AND xnum = ?' if defined($xnum);
@@ -261,9 +270,8 @@ sub _blob_missing ($$) { # called when a known $smsg->{blob} is gone
        my ($req, $smsg) = @_;
        # xnum and ibx are unknown, we only call this when an entry from
        # /ei*/over.sqlite3 is bad, not on entries from xap*/over.sqlite3
-       my $oidbin = pack('H*', $smsg->{blob});
        $req->{self}->git->async_wait_all;
-       _unref_doc($req, $smsg, undef, undef, $oidbin);
+       _unref_doc($req, $smsg, undef, undef, $smsg->oidbin);
 }
 
 sub ck_existing { # git->cat_async callback
@@ -560,7 +568,7 @@ sub _reindex_finalize ($$$) {
        for my $ary (values %$by_chash) {
                for my $x (reverse @$ary) {
                        warn "removing #$docid xref3 $x->{blob}\n";
-                       my $bin = pack('H*', $x->{blob});
+                       my $bin = $x->oidbin;
                        my $n = _unref_doc($sync, $docid, undef, undef, $bin);
                        die "BUG: $x->{blob} invalidated #$docid" if $n == 0;
                }
@@ -831,7 +839,7 @@ sub _unref_stale_range ($$$) {
                $r = $sync->{self}->{oidx}->dbh->selectall_arrayref(
                        <<EOS, undef, $ibx->{-ibx_id});
 SELECT docid,xnum,oidbin FROM xref3
-WHERE ibx_id = ? AND xnum $lt_or_gt LIMIT $lim
+WHERE ibx_id = ? AND $lt_or_gt LIMIT $lim
 EOS
                return if $sync->{quit};
                for (@$r) { # hopefully rare, not worth optimizing:
@@ -851,7 +859,7 @@ sub _reindex_check_ibx ($$$) {
        my $opt = { limit => $slice };
        my ($beg, $end) = (1, $slice);
        my $err = sync_inbox($self, $sync, $ibx) and return;
-       my $max = $ibx->over->max;
+       my $max = $ibx->mm->num_highwater;
        $end = $max if $end > $max;
 
        # first, check if we missed any messages in target $ibx
@@ -861,7 +869,7 @@ sub _reindex_check_ibx ($$$) {
        local $sync->{-regen_fmt} = "$ekey checking %u/$max\n";
        ${$sync->{nr}} = 0;
        my $fast = $sync->{-opt}->{fast};
-       my $dsu; # _unref_stale_range (< $lo) called
+       my $usr; # _unref_stale_range (< $lo) called
        my ($lo, $hi);
        while (scalar(@{$msgs = $ibx->over->query_xover($beg, $end, $opt)})) {
                ${$sync->{nr}} = $beg;
@@ -872,7 +880,7 @@ sub _reindex_check_ibx ($$$) {
                        reindex_checkpoint($self, $sync); # release lock
                }
                ($lo, $hi) = ($msgs->[0]->{num}, $msgs->[-1]->{num});
-               $dsu //= _unref_stale_range($sync, $ibx, "< $lo");
+               $usr //= _unref_stale_range($sync, $ibx, "xnum < $lo");
                my $x3a = $self->{oidx}->dbh->selectall_arrayref(
                        <<"", undef, $ibx_id, $lo, $hi);
 SELECT xnum,oidbin,docid FROM xref3 WHERE
@@ -913,7 +921,8 @@ ibx_id = ? AND xnum >= ? AND xnum <= ?
                        }
                }
        }
-       _unref_stale_range($sync, $ibx, "> $hi") if defined($hi);
+       defined($hi) and ($hi < $max) and
+               _unref_stale_range($sync, $ibx, "xnum > $hi AND xnum <= $max");
 }
 
 sub _reindex_inbox ($$$) {
@@ -978,7 +987,7 @@ sub dd_smsg { # git->cat_async callback
                my $oidx = $self->{oidx};
                for my $smsg (@$ary) {
                        my $gone = $smsg->{num};
-                       $oidx->merge_xref3($keep->{num}, $gone, $smsg->{blob});
+                       $oidx->merge_xref3($keep->{num}, $gone, $smsg->oidbin);
                        remove_doc($self, $gone);
                }
        }
@@ -1184,7 +1193,7 @@ sub idx_init { # similar to V2Writable
        $self->git->cleanup;
        my $mode = 0644;
        my $ALL = $self->git->{git_dir}; # topdir/ALL.git
-       my ($has_new, $alt, $seen);
+       my ($has_new, $alt, $seen, $prune, $prune_nr);
        if ($opt->{-private}) { # LeiStore
                my $local = "$self->{topdir}/local"; # lei/store
                $self->{mg} //= PublicInbox::MultiGit->new($self->{topdir},
@@ -1200,8 +1209,8 @@ sub idx_init { # similar to V2Writable
        } else { # extindex has no epochs
                $self->{mg} //= PublicInbox::MultiGit->new($self->{topdir},
                                                        'ALL.git');
-               ($alt, $seen) = $self->{mg}->read_alternates(\$mode,
-                                                       $opt->{-idx_gc});
+               $prune = $opt->{-idx_gc} ? \$prune_nr : undef;
+               ($alt, $seen) = $self->{mg}->read_alternates(\$mode, $prune);
                PublicInbox::Import::init_bare($ALL);
        }
 
@@ -1235,7 +1244,7 @@ sub idx_init { # similar to V2Writable
                }
                $new .= "$d\n";
        }
-       ($has_new || $new ne '') and
+       ($has_new || $prune_nr || $new ne '') and
                $self->{mg}->write_alternates($mode, $alt, $new);
        $git_midx and $self->with_umask(sub {
                my @cmd = ('multi-pack-index');