--- /dev/null
+=head1 NAME
+
+lei-reindex - reindex messages already in lei/store
+
+=head1 SYNOPSIS
+
+lei reindex [OPTIONS]
+
+=head1 DESCRIPTION
+
+Forces a re-index of all messages previously-indexed by L<lei-import(1)>
+or L<lei-index(1)>. This can be used for in-place upgrades and bugfixes
+while other processes are querying the store. Keep in mind this roughly
+doubles the size of the already-large Xapian database.
+
+It does not re-index messages in externals, using the C<--reindex>
+switch of L<public-inbox-index(1)> or L<public-inbox-extindex(1)> is
+needed for that.
+
+=head1 OPTIONS
+
+=over
+
+=item -q
+
+=item --quiet
+
+Suppress feedback messages.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-index(1)>, L<lei-import(1)>
Documentation/lei-q.pod
Documentation/lei-rediff.pod
Documentation/lei-refresh-mail-sync.pod
+Documentation/lei-reindex.pod
Documentation/lei-rm-watch.pod
Documentation/lei-rm.pod
Documentation/lei-security.pod
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRefreshMailSync.pm
+lib/PublicInbox/LeiReindex.pm
lib/PublicInbox/LeiRemote.pm
lib/PublicInbox/LeiRm.pm
lib/PublicInbox/LeiRmWatch.pm
'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch',
qw(prune), @c_opt ],
+'reindex' => [ '', 'reindex all locally-indexed messages', @c_opt ],
+
'index' => [ 'LOCATION...', 'one-time index from URL or filesystem',
qw(in-format|F=s kw! offset=i recursive|r exclude=s include|I=s
verbose|v+ incremental!), @net_opt, # mainly for --proxy=
--- /dev/null
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei reindex" command to reindex everything in lei/store
+package PublicInbox::LeiReindex;
+use v5.12;
+use parent qw(PublicInbox::IPC);
+
+sub reindex_full {
+ my ($lei) = @_;
+ my $sto = $lei->{sto};
+ my $max = $sto->search->over(1)->max;
+ $lei->qerr("# reindexing 1..$max");
+ $sto->wq_do('reindex_art', $_) for (1..$max);
+}
+
+sub reindex_store { # via wq_do
+ my ($self) = @_;
+ my ($lei, $argv) = delete @$self{qw(lei argv)};
+ if (!@$argv) {
+ reindex_full($lei);
+ }
+}
+
+sub lei_reindex {
+ my ($lei, @argv) = @_;
+ my $sto = $lei->_lei_store or return $lei->fail('nothing indexed');
+ $sto->write_prepare($lei);
+ my $self = bless { lei => $lei, argv => \@argv }, __PACKAGE__;
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
+ $lei->{wq1} = $self;
+ $lei->wait_wq_events($op_c, $ops);
+ $self->wq_do('reindex_store');
+ $self->wq_close;
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+ my ($lei) = @_;
+ $lei->{sto}->wq_do('reindex_done');
+ $lei->wq_eof;
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ $self->{lei}->_lei_atfork_child;
+ $self->SUPER::ipc_atfork_child;
+}
+
+1;
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# Local storage (cache/memo) for lei(1), suitable for personal/private
($docids, [ sort keys %$kw ]);
}
+sub _reindex_1 { # git->cat_async callback
+ my ($bref, $hex, $type, $size, $smsg) = @_;
+ my ($self, $eidx, $tl) = delete @$smsg{qw(-self -eidx -tl)};
+ $bref //= _lms_rw($self)->local_blob($hex, 1);
+ if ($bref) {
+ my $eml = PublicInbox::Eml->new($bref);
+ $smsg->{-merge_vmd} = 1; # preserve existing keywords
+ $eidx->idx_shard($smsg->{num})->index_eml($eml, $smsg);
+ } else {
+ warn("E: $type $hex\n");
+ }
+}
+
+sub reindex_art {
+ my ($self, $art) = @_;
+ my ($eidx, $tl) = eidx_init($self);
+ my $smsg = $eidx->{oidx}->get_art($art) // return;
+ return if $smsg->{bytes} == 0; # external-only message
+ @$smsg{qw(-self -eidx -tl)} = ($self, $eidx, $tl);
+ $eidx->git->cat_async($smsg->{blob} // die("no blob (#$art)"),
+ \&_reindex_1, $smsg);
+}
+
+sub reindex_done {
+ my ($self) = @_;
+ my ($eidx, $tl) = eidx_init($self);
+ $eidx->git->async_wait_all;
+ # ->done to be called via sto_done_request
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch