lei q|up: limit default write --jobs for IMAP(S)

[public-inbox.git] / lib / PublicInbox / LeiQuery.pm

diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 56b82acc8b5b7da13cfc3dace4712e7d5f44b468..0f8392362e4bc9ec2fb213a3a349484bf3794798 100644 (file)
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # handles "lei q" command and provides internals for
@@ -6,6 +6,7 @@
 package PublicInbox::LeiQuery;
 use strict;
 use v5.10.1;
+use PublicInbox::OverIdx;
 
 sub prep_ext { # externals_each callback
        my ($lxs, $exclude, $loc) = @_;
@@ -17,6 +18,7 @@ sub _start_query { # used by "lei q" and "lei up"
        require PublicInbox::LeiOverview;
        PublicInbox::LeiOverview->new($self) or return;
        my $opt = $self->{opt};
+       PublicInbox::OverIdx::fork_ok($opt);
        my ($xj, $mj) = split(/,/, $opt->{jobs} // '');
        (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) and
                die "`$xj' search jobs must be >= 1\n";
@@ -32,9 +34,15 @@ sub _start_query { # used by "lei q" and "lei up"
        if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote
                                                        import-before))) {
                $self->_lei_store(1)->write_prepare($self);
+               if ($opt->{'mail-sync'}) {
+                       my $lms = $l2m->{-lms_rw} = $self->lms(1);
+                       $lms->lms_write_prepare->lms_pause; # just create
+               }
        }
-       $l2m and $l2m->{-wq_nr_workers} = $mj // do {
-               $mj = int($nproc * 0.75 + 0.5); # keep some CPU for git
+       $l2m and $l2m->{-wq_nr_workers} //= $mj // do {
+               # keep some CPU for git, and don't overload IMAP destinations
+               my $n = int($nproc * 0.75 + 0.5);
+               $self->{net} && $n > 4 ? 4 : $n;
        };
 
        # descending docid order is cheapest, MUA controls sorting order
@@ -95,18 +103,11 @@ sub lxs_prepare {
                }
                # --external is enabled by default, but allow --no-external
                if ($opt->{external} //= 1) {
-                       my %x;
-                       for my $loc (@{$opt->{exclude} // []}) {
-                               my @l = $self->get_externals($loc, 1) or return;
-                               $x{$_} = 1 for @l;
-                       }
-                       my $ne = $self->externals_each(\&prep_ext, $lxs, \%x);
+                       my $ex = $self->canonicalize_excludes($opt->{exclude});
+                       $self->externals_each(\&prep_ext, $lxs, $ex);
                        $opt->{remote} //= !($lxs->locals - $opt->{'local'});
-                       if ($opt->{'local'}) {
-                               $lxs->{remotes} = \@iremotes if !$opt->{remote};
-                       } else {
-                               $lxs->{locals} = \@ilocals;
-                       }
+                       $lxs->{locals} = \@ilocals if !$opt->{'local'};
+                       $lxs->{remotes} = \@iremotes if !$opt->{remote};
                }
        }
        ($lxs->locals || $lxs->remotes) ? ($self->{lxs} = $lxs) :
@@ -145,6 +146,7 @@ no query allowed on command-line with --stdin
                PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self);
                return;
        }
+       chomp(@argv) and $self->qerr("# trailing `\\n' removed");
        $mset_opt{q_raw} = [ @argv ]; # copy
        $mset_opt{qstr} =
                $self->{lse}->query_argv_to_string($self->{lse}->git, \@argv);
@@ -188,7 +190,7 @@ sub _complete_q {
 # FIXME: Getopt::Long doesn't easily let us support support options with
 # '.' in them (e.g. --http1.1)
 # TODO: should we depend on "-c http.*" options for things which have
-# analogues in git(1)? that would reduce likelyhood of conflicts with
+# analogues in git(1)? that would reduce likelihood of conflicts with
 # our other CLI options
 # Note: some names are renamed to avoid potential conflicts,
 # see %lei2curl in lib/PublicInbox/LeiCurl.pm