X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FExtSearchIdx.pm;h=401b18d00754176a69eb1c2a90d380bc926a84b3;hb=HEAD;hp=cb5256a2c5625826b90d6aea26e75bf5863aeef9;hpb=9406144c54215626e79ee65fc3c89ff350506317;p=public-inbox.git

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index cb5256a2..401b18d0 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Detached/external index cross inbox search indexing support
@@ -59,6 +59,7 @@ sub new {
 		nproc_shards({ nproc => $opt->{jobs} });
 	my $oidx = PublicInbox::OverIdx->new("$self->{xpfx}/over.sqlite3");
 	$self->{-no_fsync} = $oidx->{-no_fsync} = 1 if !$opt->{fsync};
+	$self->{-dangerous} = 1 if $opt->{dangerous};
 	$self->{oidx} = $oidx;
 	$self
 }
@@ -292,8 +293,8 @@ sub ck_existing { # git->cat_async callback
 
 # is the messages visible in the inbox currently being indexed?
 # return the number if so
-sub cur_ibx_xnum ($$) {
-	my ($req, $bref) = @_;
+sub cur_ibx_xnum ($$;$) {
+	my ($req, $bref, $mismatch) = @_;
 	my $ibx = $req->{ibx} or die 'BUG: current {ibx} missing';
 
 	$req->{eml} = PublicInbox::Eml->new($bref);
@@ -303,6 +304,7 @@ sub cur_ibx_xnum ($$) {
 		my ($id, $prev);
 		while (my $x = $ibx->over->next_by_mid($mid, \$id, \$prev)) {
 			return $x->{num} if $x->{blob} eq $req->{oid};
+			push @$mismatch, $x if $mismatch;
 		}
 	}
 	undef;
@@ -317,8 +319,15 @@ sub index_oid { # git->cat_async callback for 'm'
 		blob => $oid,
 	}, 'PublicInbox::Smsg';
 	$new_smsg->set_bytes($$bref, $size);
-	defined($req->{xnum} = cur_ibx_xnum($req, $bref)) or return;
 	++${$req->{nr}};
+	my $mismatch = [];
+	$req->{xnum} = cur_ibx_xnum($req, $bref, $mismatch) // do {
+		warn "# deleted\n";
+		warn "# mismatch $_->{blob}\n" for @$mismatch;
+		${$req->{latest_cmt}} = $req->{cur_cmt} //
+			die "BUG: {cur_cmt} unset ($oid)\n";
+		return;
+	};
 	do_step($req);
 }
 
@@ -397,14 +406,14 @@ EOM
 	while (my ($ibx_id, $eidx_key) = $ibx_ck->fetchrow_array) {
 		next if $self->{ibx_map}->{$eidx_key};
 		$self->{midx}->remove_eidx_key($eidx_key);
-		warn "I: deleting messages for $eidx_key...\n";
+		warn "# deleting messages for $eidx_key...\n";
 		$x3_doc->execute($ibx_id);
 		my $ibx = { -ibx_id => $ibx_id, -gc_eidx_key => $eidx_key };
 		while (my ($docid, $xnum, $oid) = $x3_doc->fetchrow_array) {
 			my $r = _unref_doc($sync, $docid, $ibx, $xnum, $oid);
 			$oid = unpack('H*', $oid);
 			$r = $r ? 'unref' : 'remove';
-			warn "I: $r #$docid $eidx_key $oid\n";
+			warn "# $r #$docid $eidx_key $oid\n";
 			if (checkpoint_due($sync)) {
 				$x3_doc = $ibx_ck = undef;
 				reindex_checkpoint($self, $sync);
@@ -424,12 +433,12 @@ SELECT key FROM eidx_meta WHERE key LIKE ? ESCAPE ?
 		$lc_i->execute("lc-%:$pat//%", '\\');
 		while (my ($key) = $lc_i->fetchrow_array) {
 			next if $key !~ m!\Alc-v[1-9]+:\Q$eidx_key\E//!;
-			warn "I: removing $key\n";
+			warn "# removing $key\n";
 			$self->{oidx}->dbh->do(<<'', undef, $key);
 DELETE FROM eidx_meta WHERE key = ?
 
 		}
-		warn "I: $eidx_key removed\n";
+		warn "# $eidx_key removed\n";
 	}
 }
 
@@ -438,20 +447,20 @@ sub eidx_gc_scan_shards ($$) { # TODO: use for lei/store
 	my $nr = $self->{oidx}->dbh->do(<<'');
 DELETE FROM xref3 WHERE docid NOT IN (SELECT num FROM over)
 
-	warn "I: eliminated $nr stale xref3 entries\n" if $nr != 0;
+	warn "# eliminated $nr stale xref3 entries\n" if $nr != 0;
 	reindex_checkpoint($self, $sync) if checkpoint_due($sync);
 
 	# fixup from old bugs:
 	$nr = $self->{oidx}->dbh->do(<<'');
 DELETE FROM over WHERE num > 0 AND num NOT IN (SELECT docid FROM xref3)
 
-	warn "I: eliminated $nr stale over entries\n" if $nr != 0;
+	warn "# eliminated $nr stale over entries\n" if $nr != 0;
 	reindex_checkpoint($self, $sync) if checkpoint_due($sync);
 
 	$nr = $self->{oidx}->dbh->do(<<'');
 DELETE FROM eidxq WHERE docid NOT IN (SELECT num FROM over)
 
-	warn "I: eliminated $nr stale reindex queue entries\n" if $nr != 0;
+	warn "# eliminated $nr stale reindex queue entries\n" if $nr != 0;
 	reindex_checkpoint($self, $sync) if checkpoint_due($sync);
 
 	my ($cur) = $self->{oidx}->dbh->selectrow_array(<<EOM);
@@ -481,7 +490,7 @@ SELECT num FROM over WHERE num >= ? ORDER BY num ASC LIMIT 10000
 			reindex_checkpoint($self, $sync);
 		}
 	}
-	warn "I: eliminated $nr stale Xapian documents\n" if $nr != 0;
+	warn "# eliminated $nr stale Xapian documents\n" if $nr != 0;
 }
 
 sub eidx_gc {
@@ -722,13 +731,11 @@ sub eidxq_lock_acquire ($) {
 	my $t = strftime('%Y-%m-%d %k:%M %z', localtime($time));
 	local $self->{current_info} = 'eidxq';
 	if ($euid == $> && $ident eq host_ident) {
-		if (kill(0, $pid)) {
-			warn <<EOM; return;
-I: PID:$pid (re)indexing since $t, it will continue our work
+		kill(0, $pid) and warn <<EOM and return;
+# PID:$pid (re)indexing since $t, it will continue our work
 EOM
-		}
 		if ($!{ESRCH}) {
-			warn "I: eidxq_lock is stale ($cur), clobbering\n";
+			warn "# eidxq_lock is stale ($cur), clobbering\n";
 			return _eidxq_take($self);
 		}
 		warn "E: kill(0, $pid) failed: $!\n"; # fall-through:
@@ -828,7 +835,7 @@ sub reindex_unseen ($$$$) {
 		xnum => $xsmsg->{num},
 		# {mids} and {chash} will be filled in at _reindex_unseen
 	};
-	warn "I: reindex_unseen ${\$ibx->eidx_key}:$req->{xnum}:$req->{oid}\n";
+	warn "# reindex_unseen ${\$ibx->eidx_key}:$req->{xnum}:$req->{oid}\n";
 	$self->git->cat_async($xsmsg->{blob}, \&_reindex_unseen, $req);
 }
 
@@ -908,10 +915,9 @@ ibx_id = ? AND xnum >= ? AND xnum <= ?
 				for my $num (@$docids) {
 					$self->{oidx}->eidxq_add($num);
 				}
-				return if $sync->{quit};
 			}
+			return if $sync->{quit};
 		}
-		return if $sync->{quit};
 		next unless scalar keys %x3m;
 		$self->git->async_wait_all; # wait for reindex_unseen
 
@@ -936,6 +942,7 @@ BUG: (non-fatal) $ekey #$xnum $smsg->{blob} still matches (old exp: $exp)
 			for my $i (@$docids) {
 				_unref_doc($sync, $i, $ibx, $xnum, $bin);
 			}
+			return if $sync->{quit};
 		}
 	}
 	defined($hi) and ($hi < $max) and