X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLEI.pm;h=320a2bfc28695e9f513a8bcb85a438ebdf4b90f7;hb=af0b0fb7a454470a32c452119d0392e0dedb3fe1;hp=fd412324be3b9d3eefbc73f983a0e11938fcd79f;hpb=f2c7b911a1c4a7520091ba7224773c30e409c337;p=public-inbox.git diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index fd412324..320a2bfc 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2020 all contributors +# Copyright (C) 2020-2021 all contributors # License: AGPL-3.0+ # Backend for `lei' (local email interface). Unlike the C10K-oriented @@ -8,7 +8,7 @@ package PublicInbox::LEI; use strict; use v5.10.1; -use parent qw(PublicInbox::DS); +use parent qw(PublicInbox::DS PublicInbox::LeiExternal); use Getopt::Long (); use Socket qw(AF_UNIX SOCK_STREAM pack_sockaddr_un); use Errno qw(EAGAIN ECONNREFUSED ENOENT); @@ -16,10 +16,10 @@ use POSIX (); use IO::Handle (); use Sys::Syslog qw(syslog openlog); use PublicInbox::Config; -use PublicInbox::Syscall qw($SFD_NONBLOCK EPOLLIN EPOLLONESHOT); +use PublicInbox::Syscall qw(SFD_NONBLOCK EPOLLIN EPOLLONESHOT); use PublicInbox::Sigfd; -use PublicInbox::DS qw(now); -use PublicInbox::Spawn qw(spawn); +use PublicInbox::DS qw(now dwaitpid); +use PublicInbox::Spawn qw(spawn run_die); use PublicInbox::OnDestroy; use Text::Wrap qw(wrap); use File::Path qw(mkpath); @@ -36,6 +36,21 @@ our %PATH2CFG; # persistent for socket daemon # (may) pass options through to another command: sub pass_through { $GLP_PASS } +my $OPT; +sub opt_dash { + my ($spec, $re_str) = @_; # 'limit|n=i', '([0-9]+)' + my ($key) = ($spec =~ m/\A([a-z]+)/g); + my $cb = sub { # Getopt::Long "<>" catch-all handler + my ($arg) = @_; + if ($arg =~ /\A-($re_str)\z/) { + $OPT->{$key} = $1; + } else { + die "bad argument for --$key: $arg\n"; + } + }; + ($spec, '<>' => $cb, $GLP_PASS) +} + sub _store_path ($) { my ($env) = @_; File::Spec->rel2abs(($env->{XDG_DATA_HOME} // @@ -53,23 +68,23 @@ sub _config_path ($) { # TODO: generate shell completion + help using %CMD and %OPTDESC # command => [ positional_args, 1-line description, Getopt::Long option spec ] our %CMD = ( # sorted in order of importance/use: -'query' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw( - save-as=s output|o=s format|f=s dedupe|d=s thread|t augment|a - limit|n=i sort|s=s@ reverse|r offset=i remote local! extinbox! - since|after=s until|before=s) ], +'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw( + save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a + sort|s=s@ reverse|r offset=i remote local! external! + since|after=s until|before=s), opt_dash('limit|n=i', '[0-9]+') ], 'show' => [ 'MID|OID', 'show a given object (Message-ID or object ID)', qw(type=s solve! format|f=s dedupe|d=s thread|t remote local!), pass_through('git show') ], -'add-extinbox' => [ 'URL_OR_PATHNAME', +'add-external' => [ 'URL_OR_PATHNAME', 'add/set priority of a publicinbox|extindex for extra matches', - qw(prio=i) ], -'ls-extinbox' => [ '[FILTER...]', 'list publicinbox|extindex locations', - qw(format|f=s z local remote) ], -'forget-extinbox' => [ '{URL_OR_PATHNAME|--prune}', + qw(boost=i quiet|q) ], +'ls-external' => [ '[FILTER...]', 'list publicinbox|extindex locations', + qw(format|f=s z|0 local remote quiet|q) ], +'forget-external' => [ '{URL_OR_PATHNAME|--prune}', 'exclude further results from a publicinbox|extindex', - qw(prune) ], + qw(prune quiet|q) ], 'ls-query' => [ '[FILTER...]', 'list saved search queries', qw(name-only format|f=s z) ], @@ -83,7 +98,7 @@ our %CMD = ( # sorted in order of importance/use: 'set/unset flags on message(s) from stdin', qw(stdin| oid=s exact by-mid|mid:s) ], 'forget' => [ '[--stdin|--oid=OID|--by-mid=MID]', - 'exclude message(s) on stdin from query results', + "exclude message(s) on stdin from `q' search results", qw(stdin| oid=s exact by-mid|mid:s quiet|q) ], 'purge-mailsource' => [ '{URL_OR_PATHNAME|--all}', @@ -92,7 +107,7 @@ our %CMD = ( # sorted in order of importance/use: # code repos are used for `show' to solve blobs from patch mails 'add-coderepo' => [ 'PATHNAME', 'add or set priority of a git code repo', - qw(prio=i) ], + qw(boost=i) ], 'ls-coderepo' => [ '[FILTER_TERMS...]', 'list known code repos', qw(format|f=s z) ], 'forget-coderepo' => [ 'PATHNAME', @@ -111,7 +126,7 @@ our %CMD = ( # sorted in order of importance/use: 'import' => [ '{URL_OR_PATHNAME|--stdin}', 'one-shot import/update from URL or filesystem', - qw(stdin| limit|n=i offset=i recursive|r exclude=s include=s !flags), + qw(stdin| offset=i recursive|r exclude=s include=s !flags), ], 'config' => [ '[...]', sub { @@ -121,7 +136,8 @@ our %CMD = ( # sorted in order of importance/use: 'init' => [ '[PATHNAME]', sub { 'initialize storage, default: '._store_path($_[0]); }, qw(quiet|q) ], -'daemon-stop' => [ '', 'stop the lei-daemon' ], +'daemon-kill' => [ '[-SIGNAL]', 'signal the lei-daemon', + opt_dash('signal|s=s', '[0-9]+|(?:[A-Z][A-Z0-9]+)') ], 'daemon-pid' => [ '', 'show the PID of the lei-daemon' ], 'daemon-env' => [ '[NAME=VALUE...]', 'set, unset, or show daemon environment', qw(clear| unset|u=s@ z|0) ], @@ -132,7 +148,11 @@ our %CMD = ( # sorted in order of importance/use: 'reorder-local-store-and-break-history' => [ '[REFNAME]', 'rewrite git history in an attempt to improve compression', - 'gc!' ] + 'gc!' ], + +# internal commands are prefixed with '_' +'_complete' => [ '[...]', 'internal shell completion helper', + pass_through('everything') ], ); # @CMD # switch descriptions, try to keep consistent across commands @@ -152,10 +172,10 @@ my %OPTDESC = ( 'type=s' => [ 'any|mid|git', 'disambiguate type' ], -'dedupe|d=s' => ['STRAT|content|oid|mid', +'dedupe|d=s' => ['STRAT|content|oid|mid|none', 'deduplication strategy'], 'show thread|t' => 'display entire thread a message belongs to', -'query thread|t' => +'q thread|t' => 'return all messages in the same thread as the actual match(es)', 'augment|a' => 'augment --output destination instead of clobbering', @@ -166,19 +186,18 @@ my %OPTDESC = ( 'message/object output format' ], 'mark format|f=s' => $stdin_formats, 'forget format|f=s' => $stdin_formats, -'query format|f=s' => [ 'OUT|maildir|mboxrd|mboxcl2|mboxcl|html|oid', +'q format|f=s' => [ 'OUT|maildir|mboxrd|mboxcl2|mboxcl|html|oid|json', 'specify output format, default depends on --output'], 'ls-query format|f=s' => $ls_format, -'ls-extinbox format|f=s' => $ls_format, +'ls-external format|f=s' => $ls_format, -'limit|n=i' => ['NUM', - 'limit on number of matches (default: 10000)' ], +'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ], 'offset=i' => ['OFF', 'search result offset (default: 0)'], 'sort|s=s@' => [ 'VAL|internaldate,date,relevance,docid', "order of results `--output'-dependent"], -'prio=i' => 'priority of query source', +'boost=i' => 'increase/decrease priority of results (default: 0)', 'local' => 'limit operations to the local filesystem', 'local!' => 'exclude results from the local filesystem', @@ -198,8 +217,7 @@ my %OPTDESC = ( 'by-mid|mid:s' => [ 'MID', 'match only by Message-ID, ignoring contents' ], 'jobs:i' => 'set parallelism level', -# xargs, env, use "-0", git(1) uses "-z". Should we support z|0 everywhere? -'z' => 'use NUL \\0 instead of newline (CR) to delimit lines', +# xargs, env, use "-0", git(1) uses "-z". We support z|0 everywhere 'z|0' => 'use NUL \\0 instead of newline (CR) to delimit lines', # note: no "--ignore-environment" / "-i" support like env(1) since that @@ -207,8 +225,14 @@ my %OPTDESC = ( 'clear|' => 'clear the daemon environment', 'unset|u=s@' => ['NAME', 'unset matching NAME, may be specified multiple times'], + +'signal|s=s' => [ 'SIG', 'signal to send lei-daemon (default: TERM)' ], ); # %OPTDESC +my %CONFIG_KEYS = ( + 'leistore.dir' => 'top-level storage location', +); + sub x_it ($$) { # pronounced "exit" my ($self, $code) = @_; if (my $sig = ($code & 127)) { @@ -223,18 +247,15 @@ sub x_it ($$) { # pronounced "exit" } } -sub emit { - my ($self, $channel) = @_; # $buf = $_[2] - print { $self->{$channel} } $_[2] or die "print FD[$channel]: $!"; -} +sub puts ($;@) { print { shift->{1} } map { "$_\n" } @_ } + +sub out ($;@) { print { shift->{1} } @_ } -sub err { - my ($self, $buf) = @_; - $buf .= "\n" unless $buf =~ /\n\z/s; - emit($self, 2, $buf); +sub err ($;@) { + print { shift->{2} } @_, (substr($_[-1], -1, 1) eq "\n" ? () : "\n"); } -sub qerr { $_[0]->{opt}->{quiet} or err(@_) } +sub qerr ($;@) { $_[0]->{opt}->{quiet} or err(shift, @_) } sub fail ($$;$) { my ($self, $buf, $exit_code) = @_; @@ -314,8 +335,7 @@ EOF $msg .= $rhs; $msg .= "\n"; } - my $channel = $errmsg ? 2 : 1; - emit($self, $channel, $msg); + print { $self->{$errmsg ? 2 : 1} } $msg; x_it($self, $errmsg ? 1 << 8 : 0); # stderr => failure undef; } @@ -323,23 +343,23 @@ EOF sub optparse ($$$) { my ($self, $cmd, $argv) = @_; $self->{cmd} = $cmd; - my $opt = $self->{opt} = {}; + $OPT = $self->{opt} = {}; my $info = $CMD{$cmd} // [ '[...]' ]; my ($proto, undef, @spec) = @$info; - my $glp = ref($spec[-1]) ? pop(@spec) : $GLP; # or $GLP_PASS + my $glp = ref($spec[-1]) eq ref($GLP) ? pop(@spec) : $GLP; push @spec, qw(help|h); my $lone_dash; if ($spec[0] =~ s/\|\z//s) { # "stdin|" or "clear|" allows "-" alias $lone_dash = $spec[0]; - $opt->{$spec[0]} = \(my $var); + $OPT->{$spec[0]} = \(my $var); push @spec, '' => \$var; } - $glp->getoptionsfromarray($argv, $opt, @spec) or + $glp->getoptionsfromarray($argv, $OPT, @spec) or return _help($self, "bad arguments or options for $cmd"); - return _help($self) if $opt->{help}; + return _help($self) if $OPT->{help}; # "-" aliases "stdin" or "clear" - $opt->{$lone_dash} = ${$opt->{$lone_dash}} if defined $lone_dash; + $OPT->{$lone_dash} = ${$OPT->{$lone_dash}} if defined $lone_dash; my $i = 0; my $POS_ARG = '[A-Z][A-Z0-9_]+'; @@ -355,14 +375,14 @@ sub optparse ($$$) { } elsif ($var =~ /\.\.\.\]\z/) { # optional args start $inf = 1; last; - } elsif ($var =~ /\A\[$POS_ARG\]\z/) { # one optional arg + } elsif ($var =~ /\A\[-?$POS_ARG\]\z/) { # one optional arg $i++; } elsif ($var =~ /\A.+?\|/) { # required FOO|--stdin my @or = split(/\|/, $var); my $ok; for my $o (@or) { if ($o =~ /\A--([a-z0-9\-]+)/) { - $ok = defined($opt->{$1}); + $ok = defined($OPT->{$1}); last; } elsif (defined($argv->[$i])) { $ok = 1; @@ -377,7 +397,6 @@ sub optparse ($$$) { } last if $err; } - # warn "inf=$inf ".scalar(@$argv). ' '.scalar(@args)."\n"; if (!$inf && scalar(@$argv) > scalar(@args)) { $err //= 'too many arguments'; } @@ -386,7 +405,7 @@ sub optparse ($$$) { sub dispatch { my ($self, $cmd, @argv) = @_; - local $SIG{__WARN__} = sub { err($self, "@_") }; + local $SIG{__WARN__} = sub { err($self, @_) }; return _help($self, 'no command given') unless defined($cmd); my $func = "lei_$cmd"; $func =~ tr/-/_/; @@ -435,7 +454,9 @@ sub _lei_store ($;$) { $cfg->{-lei_store} //= do { require PublicInbox::LeiStore; PublicInbox::SearchIdx::load_xapian_writable(); - defined(my $dir = $cfg->{'leistore.dir'}) or return; + my $dir = $cfg->{'leistore.dir'}; + $dir //= _store_path($self->{env}) if $creat; + return unless $dir; PublicInbox::LeiStore->new($dir, { creat => $creat }); }; } @@ -461,8 +482,7 @@ sub lei_config { my $cfg = _lei_cfg($self, 1); my $cmd = [ qw(git config -f), $cfg->{'-f'}, @argv ]; my %rdr = map { $_ => $self->{$_} } (0..2); - require PublicInbox::Import; - PublicInbox::Import::run_die($cmd, $env, \%rdr); + run_die($cmd, $env, \%rdr); } sub lei_init { @@ -498,9 +518,13 @@ E: leistore.dir=$cur already initialized and it is not $dir return qerr($self, $exists); } -sub lei_daemon_pid { emit($_[0], 1, "$$\n") } +sub lei_daemon_pid { puts shift, $$ } -sub lei_daemon_stop { $quit->(0) } +sub lei_daemon_kill { + my ($self) = @_; + my $sig = $self->{opt}->{signal} // 'TERM'; + kill($sig, $$) or fail($self, "kill($sig, $$): $!"); +} sub lei_daemon_env { my ($self, @argv) = @_; @@ -516,12 +540,61 @@ sub lei_daemon_env { my $eor = $opt->{z} ? "\0" : "\n"; my $buf = ''; while (my ($k, $v) = each %ENV) { $buf .= "$k=$v$eor" } - emit($self, 1, $buf) + out $self, $buf; } } sub lei_help { _help($_[0]) } +# Shell completion helper. Used by lei-completion.bash and hopefully +# other shells. Try to do as much here as possible to avoid redundancy +# and improve maintainability. +sub lei__complete { + my ($self, @argv) = @_; # argv = qw(lei and any other args...) + shift @argv; # ignore "lei", the entire command is sent + @argv or return puts $self, grep(!/^_/, keys %CMD), qw(--help -h); + my $cmd = shift @argv; + my $info = $CMD{$cmd} // do { # filter matching commands + @argv or puts $self, grep(/\A\Q$cmd\E/, keys %CMD); + return; + }; + my ($proto, undef, @spec) = @$info; + my $cur = pop @argv; + my $re = defined($cur) ? qr/\A\Q$cur\E/ : qr/./; + if (substr($cur // '-', 0, 1) eq '-') { # --switches + # gross special case since the only git-config options + # Consider moving to a table if we need more special cases + # we use Getopt::Long for are the ones we reject, so these + # are the ones we don't reject: + if ($cmd eq 'config') { + puts $self, grep(/$re/, keys %CONFIG_KEYS); + @spec = qw(add z|null get get-all unset unset-all + replace-all get-urlmatch + remove-section rename-section + name-only list|l edit|e + get-color-name get-colorbool); + # fall-through + } + # TODO: arg support + puts $self, grep(/$re/, map { # generate short/long names + my $eq = ''; + if (s/=.+\z//) { # required arg, e.g. output|o=i + $eq = '='; + } elsif (s/:.+\z//) { # optional arg, e.g. mid:s + } else { # negation: solve! => no-solve|solve + s/\A(.+)!\z/no-$1|$1/; + } + map { + length > 1 ? "--$_$eq" : "-$_" + } split(/\|/, $_, -1) # help|h + } grep { $OPTDESC{"$cmd\t$_"} || $OPTDESC{$_} } @spec); + } elsif ($cmd eq 'config' && !@argv && !$CONFIG_KEYS{$cur}) { + puts $self, grep(/$re/, keys %CONFIG_KEYS); + } + # TODO: URLs, pathnames, OIDs, MIDs, etc... See optparse() for + # proto parsing. +} + sub reap_exec { # dwaitpid callback my ($self, $pid) = @_; x_it($self, $?); @@ -531,7 +604,7 @@ sub lei_git { # support passing through random git commands my ($self, @argv) = @_; my %rdr = map { $_ => $self->{$_} } (0..2); my $pid = spawn(['git', @argv], $self->{env}, \%rdr); - PublicInbox::DS::dwaitpid($pid, \&reap_exec, $self); + dwaitpid($pid, \&reap_exec, $self); } sub accept_dispatch { # Listener {post_accept} callback @@ -602,7 +675,7 @@ sub lazy_start { require IO::FDPass; require PublicInbox::Listener; require PublicInbox::EOFpipe; - (-p STDOUT && -p STDERR) or die "E: stdout+stderr must be pipes\n"; + (-p STDOUT) or die "E: stdout must be a pipe\n"; open(STDIN, '+<', '/dev/null') or die "redirect stdin failed: $!"; POSIX::setsid() > 0 or die "setsid: $!"; my $pid = fork // die "fork: $!"; @@ -631,7 +704,7 @@ sub lazy_start { USR1 => \&noop, USR2 => \&noop, }; - my $sigfd = PublicInbox::Sigfd->new($sig, $SFD_NONBLOCK); + my $sigfd = PublicInbox::Sigfd->new($sig, SFD_NONBLOCK); local %SIG = (%SIG, %$sig) if !$sigfd; if ($sigfd) { # TODO: use inotify/kqueue to detect unlinked sockets PublicInbox::DS->SetLoopTimeout(5000); @@ -667,17 +740,16 @@ sub lazy_start { $n; # true: continue, false: stop }); - # STDIN was redirected to /dev/null above, closing STDOUT and - # STDERR will cause the calling `lei' client process to finish - # reading <$daemon> pipe. - open STDOUT, '>&STDIN' or die "redirect stdout failed: $!"; + # STDIN was redirected to /dev/null above, closing STDERR and + # STDOUT will cause the calling `lei' client process to finish + # reading the <$daemon> pipe. openlog($path, 'pid', 'user'); local $SIG{__WARN__} = sub { syslog('warning', "@_") }; - my $owner_pid = $$; - my $on_destroy = PublicInbox::OnDestroy->new(sub { - syslog('crit', "$@") if $@ && $$ == $owner_pid; + my $on_destroy = PublicInbox::OnDestroy->new($$, sub { + syslog('crit', "$@") if $@; }); open STDERR, '>&STDIN' or die "redirect stderr failed: $!"; + open STDOUT, '>&STDIN' or die "redirect stdout failed: $!"; # $daemon pipe to `lei' closed, main loop begins: PublicInbox::DS->EventLoop; @$on_destroy = (); # cancel on_destroy if we get here