X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiQuery.pm;h=358574eac255e8b3f218c11ee4bc0a39e4457ea6;hb=refs%2Fheads%2Fmaster;hp=cb5ac8fb84a72a06907075c7a65c2a7d94c3aec6;hpb=c1cfe2c2ff17bca3cafb9dea4d7a7697eeab9683;p=public-inbox.git
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index cb5ac8fb..358574ea 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors
+# Copyright (C) all contributors
# License: AGPL-3.0+
# handles "lei q" command and provides internals for
@@ -6,6 +6,7 @@
package PublicInbox::LeiQuery;
use strict;
use v5.10.1;
+use PublicInbox::OverIdx;
sub prep_ext { # externals_each callback
my ($lxs, $exclude, $loc) = @_;
@@ -17,26 +18,31 @@ sub _start_query { # used by "lei q" and "lei up"
require PublicInbox::LeiOverview;
PublicInbox::LeiOverview->new($self) or return;
my $opt = $self->{opt};
+ PublicInbox::OverIdx::fork_ok($opt);
my ($xj, $mj) = split(/,/, $opt->{jobs} // '');
- if (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) {
- return $self->fail("`$xj' search jobs must be >= 1");
- }
+ (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) and
+ die "`$xj' search jobs must be >= 1\n";
my $lxs = $self->{lxs};
$xj ||= $lxs->concurrency($opt); # allow: "--jobs ,$WRITER_ONLY"
my $nproc = $lxs->detect_nproc || 1; # don't memoize, schedtool(1) exists
$xj = $nproc if $xj > $nproc;
$lxs->{-wq_nr_workers} = $xj;
- if (defined($mj) && $mj !~ /\A[1-9][0-9]*\z/) {
- return $self->fail("`$mj' writer jobs must be >= 1");
- }
+ (defined($mj) && $mj !~ /\A[1-9][0-9]*\z/) and
+ die "`$mj' writer jobs must be >= 1\n";
my $l2m = $self->{l2m};
# we use \1 (a ref) to distinguish between default vs. user-supplied
if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote
import-before))) {
$self->_lei_store(1)->write_prepare($self);
+ if ($opt->{'mail-sync'}) {
+ my $lms = $l2m->{-lms_rw} = $self->lms(1);
+ $lms->lms_write_prepare->lms_pause; # just create
+ }
}
- $l2m and $l2m->{-wq_nr_workers} = $mj // do {
- $mj = int($nproc * 0.75 + 0.5); # keep some CPU for git
+ $l2m and $l2m->{-wq_nr_workers} //= $mj // do {
+ # keep some CPU for git, and don't overload IMAP destinations
+ my $n = int($nproc * 0.75 + 0.5);
+ $self->{net} && $n > 4 ? 4 : $n;
};
# descending docid order is cheapest, MUA controls sorting order
@@ -55,19 +61,23 @@ sub _start_query { # used by "lei q" and "lei up"
}
sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin
- my ($self) = @_; # $_[1] = $rbuf
- if (defined($_[1])) {
- $_[1] eq '' and return eval {
- $self->fchdir or return;
- $self->{mset_opt}->{q_raw} = $self->{mset_opt}->{qstr};
- $self->{lse}->query_approxidate($self->{lse}->git,
- $self->{mset_opt}->{qstr});
- _start_query($self);
- };
- $self->{mset_opt}->{qstr} .= $_[1];
- } else {
- $self->fail("error reading stdin: $!");
- }
+ my ($lei) = @_; # $_[1] = $rbuf
+ $_[1] // $lei->fail("error reading stdin: $!");
+ return $lei->{mset_opt}->{qstr} .= $_[1] if $_[1] ne '';
+ eval {
+ $lei->fchdir;
+ $lei->{mset_opt}->{q_raw} = $lei->{mset_opt}->{qstr};
+ $lei->{lse}->query_approxidate($lei->{lse}->git,
+ $lei->{mset_opt}->{qstr});
+ _start_query($lei);
+ };
+ $lei->fail($@) if $@;
+}
+
+# make the URI||PublicInbox::{Inbox,ExtSearch} a config-file friendly string
+sub cfg_ext ($) {
+ my ($x) = @_;
+ $x->isa('URI') ? "$x" : ($x->{inboxdir} // $x->{topdir});
}
sub lxs_prepare {
@@ -85,36 +95,40 @@ sub lxs_prepare {
$lxs->prepare_external($self->{lse});
}
if (@only) {
+ my $only;
for my $loc (@only) {
my @loc = $self->get_externals($loc) or return;
- $lxs->prepare_external($_) for @loc;
+ for (@loc) {
+ my $x = $lxs->prepare_external($_);
+ push(@$only, cfg_ext($x)) if $x;
+ }
}
+ $opt->{only} = $only if $only;
} else {
- my (@ilocals, @iremotes);
+ my (@ilocals, @iremotes, $incl);
for my $loc (@{$opt->{include} // []}) {
my @loc = $self->get_externals($loc) or return;
- $lxs->prepare_external($_) for @loc;
+ for (@loc) {
+ my $x = $lxs->prepare_external($_);
+ push(@$incl, cfg_ext($x)) if $x;
+ }
@ilocals = @{$lxs->{locals} // []};
@iremotes = @{$lxs->{remotes} // []};
}
+ $opt->{include} = $incl if $incl;
# --external is enabled by default, but allow --no-external
if ($opt->{external} //= 1) {
- my %x;
- for my $loc (@{$opt->{exclude} // []}) {
- my @l = $self->get_externals($loc, 1) or return;
- $x{$_} = 1 for @l;
- }
- my $ne = $self->externals_each(\&prep_ext, $lxs, \%x);
+ my $ex = $self->canonicalize_excludes($opt->{exclude});
+ my @excl = keys %$ex;
+ $opt->{exclude} = \@excl if scalar(@excl);
+ $self->externals_each(\&prep_ext, $lxs, $ex);
$opt->{remote} //= !($lxs->locals - $opt->{'local'});
- if ($opt->{'local'}) {
- $lxs->{remotes} = \@iremotes if !$opt->{remote};
- } else {
- $lxs->{locals} = \@ilocals;
- }
+ $lxs->{locals} = \@ilocals if !$opt->{'local'};
+ $lxs->{remotes} = \@iremotes if !$opt->{remote};
}
}
($lxs->locals || $lxs->remotes) ? ($self->{lxs} = $lxs) :
- $self->fail('no local or remote inboxes to search');
+ die("no local or remote inboxes to search\n");
}
# the main "lei q SEARCH_TERMS" method
@@ -149,6 +163,7 @@ no query allowed on command-line with --stdin
PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self);
return;
}
+ chomp(@argv) and $self->qerr("# trailing `\\n' removed");
$mset_opt{q_raw} = [ @argv ]; # copy
$mset_opt{qstr} =
$self->{lse}->query_argv_to_string($self->{lse}->git, \@argv);
@@ -158,11 +173,11 @@ no query allowed on command-line with --stdin
# shell completion helper called by lei__complete
sub _complete_q {
my ($self, @argv) = @_;
- my $ext = qr/\A(?:-I|(?:--(?:include|exclude|only)))\z/;
my @cur;
+ my $cb = $self->lazy_cb(qw(forget-external _complete_));
while (@argv) {
- if ($argv[-1] =~ $ext) {
- my @c = $self->_complete_forget_external(@cur);
+ if ($argv[-1] =~ /\A(?:-I|(?:--(?:include|exclude|only)))\z/) {
+ my @c = $cb->($self, @cur);
# try basename match:
if (scalar(@cur) == 1 && index($cur[0], '/') < 0) {
my $all = $self->externals_each;
@@ -192,7 +207,7 @@ sub _complete_q {
# FIXME: Getopt::Long doesn't easily let us support support options with
# '.' in them (e.g. --http1.1)
# TODO: should we depend on "-c http.*" options for things which have
-# analogues in git(1)? that would reduce likelyhood of conflicts with
+# analogues in git(1)? that would reduce likelihood of conflicts with
# our other CLI options
# Note: some names are renamed to avoid potential conflicts,
# see %lei2curl in lib/PublicInbox/LeiCurl.pm