lei: add-external --mirror support

[public-inbox.git] / lib / PublicInbox / LEI.pm

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8d5a921e1e0e1ee5df20d9f687130c180dac6571..bdeab7e39b74f4f90fb0b0331078859f6d30fdc7 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -98,6 +98,13 @@ sub _config_path ($) {
                .'/lei/config');
 }
 
+sub index_opt {
+       # TODO: drop underscore variants everywhere, they're undocumented
+       qw(fsync|sync! jobs|j=i indexlevel|index-level|L=s compact+
+       max_size|max-size=s sequential_shard|sequential-shard
+       batch_size|batch-size=s skip-docdata quiet|q verbose|v+)
+}
+
 # TODO: generate shell completion + help using %CMD and %OPTDESC
 # command => [ positional_args, 1-line description, Getopt::Long option spec ]
 our %CMD = ( # sorted in order of importance/use:
@@ -105,7 +112,7 @@ our %CMD = ( # sorted in order of importance/use:
        save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a
        sort|s=s reverse|r offset=i remote! local! external! pretty
        include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g stdin|
-       mua-cmd|mua=s no-torsocks torsocks=s verbose|v quiet|q
+       mua-cmd|mua=s no-torsocks torsocks=s verbose|v+ quiet|q
        received-after=s received-before=s sent-after=s sent-since=s),
        PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ],
 
@@ -115,7 +122,8 @@ our %CMD = ( # sorted in order of importance/use:
 
 'add-external' => [ 'URL_OR_PATHNAME',
        'add/set priority of a publicinbox|extindex for extra matches',
-       qw(boost=i quiet|q) ],
+       qw(boost=i c=s@ mirror=s no-torsocks torsocks=s inbox-version=i),
+       index_opt(), PublicInbox::LeiQuery::curl_opt() ],
 'ls-external' => [ '[FILTER...]', 'list publicinbox|extindex locations',
        qw(format|f=s z|0 local remote quiet|q) ],
 'forget-external' => [ 'URL_OR_PATHNAME...|--prune',
@@ -204,7 +212,7 @@ my %OPTDESC = (
 'help|h' => 'show this built-in help',
 'quiet|q' => 'be quiet',
 'globoff|g' => "do not match locations using '*?' wildcards and '[]' ranges",
-'verbose|v' => 'be more verbose',
+'verbose|v+' => 'be more verbose',
 'solve!' => 'do not attempt to reconstruct blobs from emails',
 'torsocks=s' => ['auto|no|yes',
                'whether or not to wrap git and curl commands with torsocks'],
@@ -286,6 +294,8 @@ my %CONFIG_KEYS = (
        'leistore.dir' => 'top-level storage location',
 );
 
+my @WQ_KEYS = qw(lxs l2m imp mrr); # internal workers
+
 # pronounced "exit": x_it(1 << 8) => exit(1); x_it(13) => SIGPIPE
 sub x_it ($$) {
        my ($self, $code) = @_;
@@ -296,7 +306,7 @@ sub x_it ($$) {
                send($s, "x_it $code", MSG_EOR);
        } elsif ($self->{oneshot}) {
                # don't want to end up using $? from child processes
-               for my $f (qw(lxs l2m)) {
+               for my $f (@WQ_KEYS) {
                        my $wq = delete $self->{$f} or next;
                        $wq->DESTROY;
                }
@@ -327,7 +337,7 @@ sub qerr ($;@) { $_[0]->{opt}->{quiet} or err(shift, @_) }
 
 sub fail_handler ($;$$) {
        my ($lei, $code, $io) = @_;
-       for my $f (qw(imp lxs l2m)) {
+       for my $f (@WQ_KEYS) {
                my $wq = delete $lei->{$f} or next;
                $wq->wq_wait_old($lei) if $wq->wq_kill_old; # lei-daemon
        }
@@ -335,7 +345,7 @@ sub fail_handler ($;$$) {
        $lei->x_it($code // (1 >> 8));
 }
 
-sub sigpipe_handler { # handles SIGPIPE from l2m/lxs workers
+sub sigpipe_handler { # handles SIGPIPE from @WQ_KEYS workers
        fail_handler($_[0], 13, delete $_[0]->{1});
 }
 
@@ -712,6 +722,7 @@ sub lei__complete {
                }
                puts $self, grep(/$re/, map { # generate short/long names
                        if (s/[:=].+\z//) { # req/optional args, e.g output|o=i
+                       } elsif (s/\+\z//) { # verbose|v+
                        } elsif (s/!\z//) {
                                # negation: solve! => no-solve|solve
                                s/([\w\-]+)/$1|no-$1/g
@@ -856,7 +867,7 @@ sub accept_dispatch { # Listener {post_accept} callback
 sub dclose {
        my ($self) = @_;
        delete $self->{-progress};
-       for my $f (qw(lxs l2m)) {
+       for my $f (@WQ_KEYS) {
                my $wq = delete $self->{$f} or next;
                if ($wq->wq_kill) {
                        $wq->wq_close