Documentation/public-inbox-extindex.pod | 22 ++++++++++++++++++++++ lib/PublicInbox/ExtSearchIdx.pm | 6 +++--- script/public-inbox-extindex | 9 ++++++--- t/extsearch.t | 5 +++++ diff --git a/Documentation/public-inbox-extindex.pod b/Documentation/public-inbox-extindex.pod index 2e2e6383b79b35c31f7be30ed1daa7996a1674dd..a0fca83c6255b1bcfa5a89270501dba164004cdb 100644 --- a/Documentation/public-inbox-extindex.pod +++ b/Documentation/public-inbox-extindex.pod @@ -40,6 +40,28 @@ (C or C) directories removed. For multiple public-inboxes where cross-posting is common, this allows significant space savings on Xapian indices. +=item --gc + +Perform garbage collection instead of indexing. Use this if +inboxes are removed from the extindex, or if messages are +purged or removed from some inboxes. + +=item --reindex + +Forces a re-index of all messages in the extindex. This can be +used for in-place upgrades and bugfixes while read-only server +processes are utilizing the index. Keep in mind this roughly +doubles the size of the already-large Xapian database. + +The extindex locks will be released roughly every 10s to +allow L and L +processes to write to the extindex. + +=item --fast + +Used with C<--reindex>, it will only look for new and stale +entries and not touch already-indexed messages. + =back =head1 FILES diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm index 7cc8dd952559d17b3c5c600a7ac8155547ad34ce..20c4cf7807eae9cb152b9760c9ede22bfdc95595 100644 --- a/lib/PublicInbox/ExtSearchIdx.pm +++ b/lib/PublicInbox/ExtSearchIdx.pm @@ -811,7 +811,7 @@ my $ekey = $ibx->eidx_key; local $sync->{-regen_fmt} = "$ekey checking unseen %u/".$ibx->over->max."\n"; ${$sync->{nr}} = 0; - + my $fast = $sync->{-opt}->{fast}; while (scalar(@{$msgs = $ibx->over->query_xover($beg, $end)})) { ${$sync->{nr}} = $beg; $beg = $msgs->[-1]->{num} + 1; @@ -835,7 +835,7 @@ # index messages which were totally missed # the first time around ASAP: if (scalar(@$docids) == 0) { reindex_unseen($self, $sync, $ibx, $xsmsg); - } else { # already seen, reindex later + } elsif (!$fast) { # already seen, reindex later for my $r (@$docids) { $self->{oidx}->eidxq_add($r->[0]); } @@ -853,7 +853,7 @@ my $pr = $sync->{-opt}->{-progress}; my $fetching; my $ekey = $ibx->eidx_key; local $sync->{-regen_fmt} = - "$ekey check stale/missing %u/".$ibx->over->max."\n"; + "$ekey checking stale/missing %u/".$ibx->over->max."\n"; ${$sync->{nr}} = 0; do { if (checkpoint_due($sync)) { diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex index 1572a1d23d82dad663a81956c3b95ea955ee13f0..c63f5dc26fd29527319987ce160807f5e9f2879d 100755 --- a/script/public-inbox-extindex +++ b/script/public-inbox-extindex @@ -18,6 +18,8 @@ --batch-size=BYTES flush changes to OS after a given number of bytes --max-size=BYTES do not index messages larger than the given size --gc perform garbage collection instead of indexing --dedupe[=MSGID] fix prior deduplication errors (may be repeated) + --reindex index previously indexed inboxes + --fast only reindex unseen/stale messages --verbose | -v increase verbosity (may be repeated) --dry-run | -n dry-run on --dedupe @@ -26,7 +28,7 @@ See public-inbox-extindex(1) man page for full documentation. EOF my $opt = { quiet => -1, compact => 0, fsync => 1, scan => 1 }; GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i - fsync|sync! + fsync|sync! fast indexlevel|index-level|L=s max_size|max-size=s batch_size|batch-size=s dedupe:s@ gc commit-interval=i watch scan! dry-run|n @@ -59,9 +61,10 @@ } } else { @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt, $cfg); } -if ($opt->{'dry-run'} && !$opt->{dedupe}) { +$opt->{'dry-run'} && !$opt->{dedupe} and die "E: --dry-run only affects --dedupe\n"; -} +$opt->{fast} && !$opt->{reindex} and + die "E: --fast only affects --reindex\n"; PublicInbox::Admin::require_or_die(qw(-search)); PublicInbox::Config::json() or die "Cpanel::JSON::XS or similar missing\n"; diff --git a/t/extsearch.t b/t/extsearch.t index ca586f61c29f5ecab581d56a1dc7a1bdb71e51c5..896e270414bd8eff0e909fe2fbdacf291bec33cf 100644 --- a/t/extsearch.t +++ b/t/extsearch.t @@ -336,6 +336,11 @@ 'stale message has no xref3'); $es->{xdb}->reopen; $mset = $es->mset("mid:$new->{mid}"); is($mset->size, 0, 'stale mid gone Xapian'); + + ok(run_script([qw(-extindex --reindex --all --fast), "$home/extindex"], + undef, $opt), '--reindex w/ --fast'); + ok(!run_script([qw(-extindex --all --fast), "$home/extindex"], + undef, $opt), '--fast alone makes no sense'); } if ('reindex catches content bifurcation') {