X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiQuery.pm;h=cb5ac8fb84a72a06907075c7a65c2a7d94c3aec6;hb=c1cfe2c2ff17bca3cafb9dea4d7a7697eeab9683;hp=eaf91f2ec90b02647bc935788dd8aa6ff62a342c;hpb=fb8b16ff2b40ecd22ebbdea0d27069749e800077;p=public-inbox.git diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index eaf91f2e..cb5ac8fb 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -1,7 +1,8 @@ # Copyright (C) 2021 all contributors # License: AGPL-3.0+ -# handles lei commands +# handles "lei q" command and provides internals for +# several other sub-commands (up, lcat, ...) package PublicInbox::LeiQuery; use strict; use v5.10.1; @@ -11,24 +12,55 @@ sub prep_ext { # externals_each callback $lxs->prepare_external($loc) unless $exclude->{$loc}; } -sub _start_query { +sub _start_query { # used by "lei q" and "lei up" my ($self) = @_; - if (my $nwr = $self->{nwr}) { + require PublicInbox::LeiOverview; + PublicInbox::LeiOverview->new($self) or return; + my $opt = $self->{opt}; + my ($xj, $mj) = split(/,/, $opt->{jobs} // ''); + if (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) { + return $self->fail("`$xj' search jobs must be >= 1"); + } + my $lxs = $self->{lxs}; + $xj ||= $lxs->concurrency($opt); # allow: "--jobs ,$WRITER_ONLY" + my $nproc = $lxs->detect_nproc || 1; # don't memoize, schedtool(1) exists + $xj = $nproc if $xj > $nproc; + $lxs->{-wq_nr_workers} = $xj; + if (defined($mj) && $mj !~ /\A[1-9][0-9]*\z/) { + return $self->fail("`$mj' writer jobs must be >= 1"); + } + my $l2m = $self->{l2m}; + # we use \1 (a ref) to distinguish between default vs. user-supplied + if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote + import-before))) { + $self->_lei_store(1)->write_prepare($self); + } + $l2m and $l2m->{-wq_nr_workers} = $mj // do { + $mj = int($nproc * 0.75 + 0.5); # keep some CPU for git + }; + + # descending docid order is cheapest, MUA controls sorting order + $self->{mset_opt}->{relevance} //= -2 if $l2m || $opt->{threads}; + + my $tot = $self->{mset_opt}->{total} //= $self->{opt}->{limit} // 10000; + $self->{mset_opt}->{limit} = $tot > 10000 ? 10000 : $tot; + $self->{mset_opt}->{offset} //= 0; + $self->{mset_opt}->{threads} //= $opt->{threads}; + + if ($self->{net}) { require PublicInbox::LeiAuth; - my $auth = $self->{auth} = PublicInbox::LeiAuth->new($nwr); - my $lxs = $self->{lxs}; - $auth->auth_start($self, $lxs->can('do_query'), $lxs, $self); - } else { - $self->{lxs}->do_query($self); + $self->{auth} = PublicInbox::LeiAuth->new } + $lxs->do_query($self); } sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin my ($self) = @_; # $_[1] = $rbuf if (defined($_[1])) { $_[1] eq '' and return eval { - my $lse = delete $self->{lse}; - $lse->query_approxidate($lse->git, + $self->fchdir or return; + $self->{mset_opt}->{q_raw} = $self->{mset_opt}->{qstr}; + $self->{lse}->query_approxidate($self->{lse}->git, $self->{mset_opt}->{qstr}); _start_query($self); }; @@ -38,23 +70,19 @@ sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin } } -# the main "lei q SEARCH_TERMS" method -sub lei_q { - my ($self, @argv) = @_; +sub lxs_prepare { + my ($self) = @_; require PublicInbox::LeiXSearch; - require PublicInbox::LeiOverview; - require PublicInbox::V2Writable; - PublicInbox::Config->json; # preload before forking - my $opt = $self->{opt}; # prepare any number of LeiXSearch || LeiSearch || Inbox || URL my $lxs = $self->{lxs} = PublicInbox::LeiXSearch->new; + my $opt = $self->{opt}; my @only = @{$opt->{only} // []}; # --local is enabled by default unless --only is used # we'll allow "--only $LOCATION --local" my $sto = $self->_lei_store(1); - my $lse = $sto->search; + $self->{lse} = $sto->search; if ($opt->{'local'} //= scalar(@only) ? 0 : 1) { - $lxs->prepare_external($lse); + $lxs->prepare_external($self->{lse}); } if (@only) { for my $loc (@only) { @@ -62,9 +90,12 @@ sub lei_q { $lxs->prepare_external($_) for @loc; } } else { + my (@ilocals, @iremotes); for my $loc (@{$opt->{include} // []}) { my @loc = $self->get_externals($loc) or return; $lxs->prepare_external($_) for @loc; + @ilocals = @{$lxs->{locals} // []}; + @iremotes = @{$lxs->{remotes} // []}; } # --external is enabled by default, but allow --no-external if ($opt->{external} //= 1) { @@ -76,32 +107,25 @@ sub lei_q { my $ne = $self->externals_each(\&prep_ext, $lxs, \%x); $opt->{remote} //= !($lxs->locals - $opt->{'local'}); if ($opt->{'local'}) { - delete($lxs->{remotes}) if !$opt->{remote}; + $lxs->{remotes} = \@iremotes if !$opt->{remote}; } else { - delete($lxs->{locals}); + $lxs->{locals} = \@ilocals; } } } - unless ($lxs->locals || $lxs->remotes) { - return $self->fail('no local or remote inboxes to search'); - } - my ($xj, $mj) = split(/,/, $opt->{jobs} // ''); - if (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) { - return $self->fail("`$xj' search jobs must be >= 1"); - } - $xj ||= $lxs->concurrency($opt); # allow: "--jobs ,$WRITER_ONLY" - my $nproc = $lxs->detect_nproc; # don't memoize, schedtool(1) exists - $xj = $nproc if $xj > $nproc; - $lxs->{jobs} = $xj; - if (defined($mj) && $mj !~ /\A[1-9][0-9]*\z/) { - return $self->fail("`$mj' writer jobs must be >= 1"); - } - $self->{l2m}->{jobs} = ($mj // $nproc) if $self->{l2m}; - PublicInbox::LeiOverview->new($self) or return; + ($lxs->locals || $lxs->remotes) ? ($self->{lxs} = $lxs) : + $self->fail('no local or remote inboxes to search'); +} - my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset); +# the main "lei q SEARCH_TERMS" method +sub lei_q { + my ($self, @argv) = @_; + PublicInbox::Config->json; # preload before forking + my $lxs = lxs_prepare($self) or return; + $self->ale->refresh_externals($lxs, $self); + my $opt = $self->{opt}; + my %mset_opt; $mset_opt{asc} = $opt->{'reverse'} ? 1 : 0; - $mset_opt{limit} //= 10000; if (defined(my $sort = $opt->{'sort'})) { if ($sort eq 'relevance') { $mset_opt{relevance} = 1; @@ -112,9 +136,9 @@ sub lei_q { } else { die "unrecognized --sort=$sort\n"; } + $opt->{save} and return + $self->fail('--save and --sort are incompatible'); } - # descending docid order - $mset_opt{relevance} //= -2 if $opt->{threads}; $self->{mset_opt} = \%mset_opt; if ($opt->{stdin}) { @@ -122,11 +146,12 @@ sub lei_q { no query allowed on command-line with --stdin require PublicInbox::InputPipe; - $self->{lse} = $lse; # for query_approxidate PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self); return; } - $mset_opt{qstr} = $lse->query_argv_to_string($lse->git, \@argv); + $mset_opt{q_raw} = [ @argv ]; # copy + $mset_opt{qstr} = + $self->{lse}->query_argv_to_string($self->{lse}->git, \@argv); _start_query($self); } @@ -170,7 +195,7 @@ sub _complete_q { # analogues in git(1)? that would reduce likelyhood of conflicts with # our other CLI options # Note: some names are renamed to avoid potential conflicts, -# see %lei2mail in lib/PublicInbox/LeiCurl.pm +# see %lei2curl in lib/PublicInbox/LeiCurl.pm sub curl_opt { qw( curl-config=s@ abstract-unix-socket=s anyauth basic cacert=s capath=s