X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FExtSearchIdx.pm;h=7c44a1a406308604c1319b91eaa7130a0be645d2;hb=3eec2f7792040f75f3988c520f308e2445baf645;hp=cb5256a2c5625826b90d6aea26e75bf5863aeef9;hpb=9406144c54215626e79ee65fc3c89ff350506317;p=public-inbox.git
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index cb5256a2..7c44a1a4 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors
+# Copyright (C) all contributors
# License: AGPL-3.0+
# Detached/external index cross inbox search indexing support
@@ -59,6 +59,7 @@ sub new {
nproc_shards({ nproc => $opt->{jobs} });
my $oidx = PublicInbox::OverIdx->new("$self->{xpfx}/over.sqlite3");
$self->{-no_fsync} = $oidx->{-no_fsync} = 1 if !$opt->{fsync};
+ $self->{-dangerous} = 1 if $opt->{dangerous};
$self->{oidx} = $oidx;
$self
}
@@ -292,8 +293,8 @@ sub ck_existing { # git->cat_async callback
# is the messages visible in the inbox currently being indexed?
# return the number if so
-sub cur_ibx_xnum ($$) {
- my ($req, $bref) = @_;
+sub cur_ibx_xnum ($$;$) {
+ my ($req, $bref, $mismatch) = @_;
my $ibx = $req->{ibx} or die 'BUG: current {ibx} missing';
$req->{eml} = PublicInbox::Eml->new($bref);
@@ -303,6 +304,7 @@ sub cur_ibx_xnum ($$) {
my ($id, $prev);
while (my $x = $ibx->over->next_by_mid($mid, \$id, \$prev)) {
return $x->{num} if $x->{blob} eq $req->{oid};
+ push @$mismatch, $x if $mismatch;
}
}
undef;
@@ -317,8 +319,15 @@ sub index_oid { # git->cat_async callback for 'm'
blob => $oid,
}, 'PublicInbox::Smsg';
$new_smsg->set_bytes($$bref, $size);
- defined($req->{xnum} = cur_ibx_xnum($req, $bref)) or return;
++${$req->{nr}};
+ my $mismatch = [];
+ $req->{xnum} = cur_ibx_xnum($req, $bref, $mismatch) // do {
+ warn "# deleted\n";
+ warn "# mismatch $_->{blob}\n" for @$mismatch;
+ ${$req->{latest_cmt}} = $req->{cur_cmt} //
+ die "BUG: {cur_cmt} unset ($oid)\n";
+ return;
+ };
do_step($req);
}
@@ -908,10 +917,9 @@ ibx_id = ? AND xnum >= ? AND xnum <= ?
for my $num (@$docids) {
$self->{oidx}->eidxq_add($num);
}
- return if $sync->{quit};
}
+ return if $sync->{quit};
}
- return if $sync->{quit};
next unless scalar keys %x3m;
$self->git->async_wait_all; # wait for reindex_unseen
@@ -936,6 +944,7 @@ BUG: (non-fatal) $ekey #$xnum $smsg->{blob} still matches (old exp: $exp)
for my $i (@$docids) {
_unref_doc($sync, $i, $ibx, $xnum, $bin);
}
+ return if $sync->{quit};
}
}
defined($hi) and ($hi < $max) and