use Errno qw(EAGAIN EINTR ECONNREFUSED ENOENT ECONNRESET);
use POSIX ();
use IO::Handle ();
+use Fcntl qw(SEEK_SET);
use Sys::Syslog qw(syslog openlog);
use PublicInbox::Config;
use PublicInbox::Syscall qw(SFD_NONBLOCK EPOLLIN EPOLLET);
use PublicInbox::Sigfd;
use PublicInbox::DS qw(now dwaitpid);
-use PublicInbox::Spawn qw(spawn run_die popen_rd);
+use PublicInbox::Spawn qw(spawn popen_rd);
use PublicInbox::OnDestroy;
use Text::Wrap qw(wrap);
use File::Path qw(mkpath);
use File::Spec;
our $quit = \&CORE::exit;
-our $current_lei;
+our ($current_lei, $errors_log, $listener);
my ($recv_cmd, $send_cmd);
my $GLP = Getopt::Long::Parser->new;
$GLP->configure(qw(gnu_getopt no_ignore_case auto_abbrev));
$GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
our %PATH2CFG; # persistent for socket daemon
-our @TO_CLOSE_ATFORK_CHILD;
# TBD: this is a documentation mechanism to show a subcommand
# (may) pass options through to another command:
'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw(
save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a
sort|s=s reverse|r offset=i remote local! external! pretty mua-cmd=s
+ verbose|v
since|after=s until|before=s), opt_dash('limit|n=i', '[0-9]+') ],
'show' => [ 'MID|OID', 'show a given object (Message-ID or object ID)',
qw(boost=i quiet|q) ],
'ls-external' => [ '[FILTER...]', 'list publicinbox|extindex locations',
qw(format|f=s z|0 local remote quiet|q) ],
-'forget-external' => [ '{URL_OR_PATHNAME|--prune}',
+'forget-external' => [ 'URL_OR_PATHNAME...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune quiet|q) ],
"exclude message(s) on stdin from `q' search results",
qw(stdin| oid=s exact by-mid|mid:s quiet|q) ],
-'purge-mailsource' => [ '{URL_OR_PATHNAME|--all}',
+'purge-mailsource' => [ 'URL_OR_PATHNAME|--all',
'remove imported messages from IMAP, Maildirs, and MH',
qw(exact! all jobs:i indexed) ],
'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch',
qw(prune) ],
-'import' => [ '{URL_OR_PATHNAME|--stdin}',
+'import' => [ 'URL_OR_PATHNAME|--stdin',
'one-shot import/update from URL or filesystem',
qw(stdin| offset=i recursive|r exclude=s include=s !flags),
],
my ($self, $code) = @_;
# make sure client sees stdout before exit
$self->{1}->autoflush(1) if $self->{1};
+ dump_and_clear_log();
if (my $sock = $self->{sock}) {
send($sock, "x_it $code", MSG_EOR);
} elsif (!($code & 127)) { # oneshot, ignore signals
+ # don't want to end up using $? from child processes
+ for my $f (qw(lxs l2m)) {
+ my $wq = delete $self->{$f} or next;
+ $wq->DESTROY;
+ }
$quit->($code >> 8);
}
}
sub err ($;@) {
my $self = shift;
- my $err = $self->{2} // *STDERR{IO};
+ my $err = $self->{2} // ($self->{pgr} // [])->[2] // *STDERR{IO};
print $err @_, (substr($_[-1], -1, 1) eq "\n" ? () : "\n");
}
undef;
}
+sub child_error { # passes non-fatal curl exit codes to user
+ my ($self, $child_error) = @_; # child_error is $?
+ if (my $sock = $self->{sock}) { # send to lei(1) client
+ send($sock, "child_error $child_error", MSG_EOR);
+ } else { # oneshot
+ $self->{child_error} = $child_error;
+ }
+ undef;
+}
+
sub atfork_prepare_wq {
my ($self, $wq) = @_;
- my $tcafc = $wq->{-ipc_atfork_child_close};
- push @$tcafc, @TO_CLOSE_ATFORK_CHILD;
+ my $tcafc = $wq->{-ipc_atfork_child_close} //= [ $listener // () ];
if (my $sock = $self->{sock}) {
push @$tcafc, @$self{qw(0 1 2)}, $sock;
}
+ if (my $pgr = $self->{pgr}) {
+ push @$tcafc, @$pgr[1,2];
+ }
+ if (my $old_1 = $self->{old_1}) {
+ push @$tcafc, $old_1;
+ }
+ for my $f (qw(lxs l2m)) {
+ my $ipc = $self->{$f} or next;
+ push @$tcafc, grep { defined }
+ @$ipc{qw(-wq_s1 -wq_s2 -ipc_req -ipc_res)};
+ }
}
# usage: my %sig = $lei->atfork_child_wq($wq);
# local @SIG{keys %sig} = values %sig;
sub atfork_child_wq {
my ($self, $wq) = @_;
- @$self{qw(0 1 2 sock)} = delete(@$wq{0..3});
+ my ($sock, $l2m_wq_s1);
+ (@$self{qw(0 1 2)}, $sock, $l2m_wq_s1) = delete(@$wq{0..4});
+ $self->{sock} = $sock if -S $sock;
+ $self->{l2m}->{-wq_s1} = $l2m_wq_s1 if $l2m_wq_s1 && -S $l2m_wq_s1;
%PATH2CFG = ();
+ undef $errors_log;
$quit = \&CORE::exit;
- @TO_CLOSE_ATFORK_CHILD = ();
(__WARN__ => sub { err($self, @_) },
PIPE => sub {
$self->x_it(13); # SIGPIPE = 13
# we need to close explicitly to avoid Perl warning on SIGPIPE
- close(delete $self->{1});
- # regular files and /dev/null (-c) won't trigger SIGPIPE
- close(delete $self->{2}) unless (-f $self->{2} || -c _);
- syswrite($self->{0}, '!') unless $self->{sock}; # for eof_wait
+ for my $i (1, 2) {
+ next unless $self->{$i} && (-p $self->{$i} || -S _);
+ close(delete $self->{$i});
+ }
+ # trigger the LeiXSearch $done OpPipe:
+ syswrite($self->{0}, '!') if $self->{0} && -p $self->{0};
+ $SIG{PIPE} = 'DEFAULT';
die bless(\"$_[0]", 'PublicInbox::SIGPIPE'),
});
}
# usage: ($lei, @io) = $lei->atfork_parent_wq($wq);
sub atfork_parent_wq {
my ($self, $wq) = @_;
- if ($wq->wq_workers) {
- my $env = delete $self->{env}; # env is inherited at fork
- my $ret = bless { %$self }, ref($self);
- $self->{env} = $env;
- delete @$ret{qw(-lei_store cfg pgr)};
- ($ret, delete @$ret{0..2}, delete($ret->{sock}) // ());
- } else {
- ($self, @$self{0..2}, $self->{sock} // ());
+ my $env = delete $self->{env}; # env is inherited at fork
+ my $ret = bless { %$self }, ref($self);
+ if (my $dedupe = delete $ret->{dedupe}) {
+ $ret->{dedupe} = $wq->deep_clone($dedupe);
+ }
+ $self->{env} = $env;
+ delete @$ret{qw(-lei_store cfg old_1 pgr lxs)}; # keep l2m
+ my @io = delete @$ret{0..2};
+ $io[3] = delete($ret->{sock}) // $io[2];
+ my $l2m = $ret->{l2m};
+ if ($l2m && $l2m != $wq) { # $wq == lxs
+ $io[4] = $l2m->{-wq_s1} if $l2m->{-wq_s1};
+ $l2m->wq_close(1);
}
+ ($ret, @io);
}
sub _help ($;$) {
last;
} # else continue looping
}
+ last if $ok;
my $last = pop @or;
$err = join(', ', @or) . " or $last must be set";
} else {
sub dispatch {
my ($self, $cmd, @argv) = @_;
local $current_lei = $self; # for __WARN__
+ dump_and_clear_log("from previous run\n");
return _help($self, 'no command given') unless defined($cmd);
my $func = "lei_$cmd";
$func =~ tr/-/_/;
my ($self, @argv) = @_;
}
-sub lei_config {
+sub _config {
my ($self, @argv) = @_;
- $self->{opt}->{'config-file'} and return fail $self,
- "config file switches not supported by `lei config'";
my $env = $self->{env};
delete local $env->{GIT_CONFIG};
+ delete local $ENV{GIT_CONFIG};
my $cfg = _lei_cfg($self, 1);
my $cmd = [ qw(git config -f), $cfg->{'-f'}, @argv ];
my %rdr = map { $_ => $self->{$_} } (0..2);
- run_die($cmd, $env, \%rdr);
+ waitpid(spawn($cmd, $env, \%rdr), 0);
+}
+
+sub lei_config {
+ my ($self, @argv) = @_;
+ $self->{opt}->{'config-file'} and return fail $self,
+ "config file switches not supported by `lei config'";
+ _config(@_);
+ x_it($self, $?) if $?;
}
sub lei_init {
} elsif ($cmd eq 'config' && !@argv && !$CONFIG_KEYS{$cur}) {
puts $self, grep(/$re/, keys %CONFIG_KEYS);
}
+ $cmd =~ tr/-/_/;
+ if (my $sub = $self->can("_complete_$cmd")) {
+ puts $self, $sub->($self, @argv, $cur);
+ }
# TODO: URLs, pathnames, OIDs, MIDs, etc... See optparse() for
# proto parsing.
}
sub dclose {
my ($self) = @_;
- delete $self->{lxs}; # stops LeiXSearch queries
+ for my $f (qw(lxs l2m)) {
+ my $wq = delete $self->{$f} or next;
+ if ($wq->wq_kill) {
+ $self->wq_close
+ } elsif ($wq->wq_kill_old) {
+ $wq->wq_wait_old;
+ }
+ }
+ close(delete $self->{1}) if $self->{1}; # may reap_compress
$self->close if $self->{sock}; # PublicInbox::DS::close
}
my ($self) = @_;
local %ENV = %{$self->{env}};
my $sock = $self->{sock};
+ local $current_lei = $self;
eval {
while (my @fds = $recv_cmd->($sock, my $buf, 4096)) {
if (scalar(@fds) == 1 && !defined($fds[0])) {
our $oldset; sub oldset { $oldset }
+sub dump_and_clear_log {
+ if (defined($errors_log) && -s STDIN && seek(STDIN, 0, SEEK_SET)) {
+ my @pfx = @_;
+ unshift(@pfx, "$errors_log ") if @pfx;
+ warn @pfx, do { local $/; <STDIN> };
+ truncate(STDIN, 0) or warn "ftruncate ($errors_log): $!";
+ }
+}
+
# lei(1) calls this when it can't connect
sub lazy_start {
my ($path, $errno, $narg) = @_;
die "connect($path): $!";
}
umask(077) // die("umask(077): $!");
- socket(my $l, AF_UNIX, SOCK_SEQPACKET, 0) or die "socket: $!";
- bind($l, pack_sockaddr_un($path)) or die "bind($path): $!";
- listen($l, 1024) or die "listen: $!";
+ local $listener;
+ socket($listener, AF_UNIX, SOCK_SEQPACKET, 0) or die "socket: $!";
+ bind($listener, pack_sockaddr_un($path)) or die "bind($path): $!";
+ listen($listener, 1024) or die "listen: $!";
my @st = stat($path) or die "stat($path): $!";
my $dev_ino_expect = pack('dd', $st[0], $st[1]); # dev+ino
- pipe(my ($eof_r, $eof_w)) or die "pipe: $!";
local $oldset = PublicInbox::DS::block_signals();
if ($narg == 5) {
$send_cmd = PublicInbox::Spawn->can('send_cmd4');
require PublicInbox::Listener;
require PublicInbox::EOFpipe;
(-p STDOUT) or die "E: stdout must be a pipe\n";
- open(STDIN, '+<', '/dev/null') or die "redirect stdin failed: $!";
+ local $errors_log;
+ ($errors_log) = ($path =~ m!\A(.+?/)[^/]+\z!);
+ $errors_log .= 'errors.log';
+ open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!";
+ STDIN->autoflush(1);
+ dump_and_clear_log("from previous daemon process:\n");
POSIX::setsid() > 0 or die "setsid: $!";
my $pid = fork // die "fork: $!";
return if $pid;
$0 = "lei-daemon $path";
local %PATH2CFG;
- local @TO_CLOSE_ATFORK_CHILD = ($l, $eof_w);
- $l->blocking(0);
- $l = PublicInbox::Listener->new($l, \&accept_dispatch, $l);
+ $listener->blocking(0);
my $exit_code;
- local $quit = sub {
- $exit_code //= shift;
- my $listener = $l or exit($exit_code);
- # closing eof_w triggers \&noop wakeup
- $eof_w = $l = $path = undef;
- $listener->close; # DS::close
- PublicInbox::DS->SetLoopTimeout(1000);
+ my $pil = PublicInbox::Listener->new($listener, \&accept_dispatch);
+ local $quit = do {
+ pipe(my ($eof_r, $eof_w)) or die "pipe: $!";
+ PublicInbox::EOFpipe->new($eof_r, \&noop, undef);
+ sub {
+ $exit_code //= shift;
+ my $lis = $pil or exit($exit_code);
+ # closing eof_w triggers \&noop wakeup
+ $listener = $eof_w = $pil = $path = undef;
+ $lis->close; # DS::close
+ PublicInbox::DS->SetLoopTimeout(1000);
+ };
};
- PublicInbox::EOFpipe->new($eof_r, \&noop, undef);
- undef $eof_r;
my $sig = {
CHLD => \&PublicInbox::DS::enqueue_reap,
QUIT => $quit,
exit($exit_code // 0);
}
-# for users w/o Socket::Msghdr
+# for users w/o Socket::Msghdr installed or Inline::C enabled
sub oneshot {
my ($main_pkg) = @_;
my $exit = $main_pkg->can('exit'); # caller may override exit()
local $quit = $exit if $exit;
local %PATH2CFG;
umask(077) // die("umask(077): $!");
- dispatch((bless {
+ my $self = bless {
0 => *STDIN{GLOB},
1 => *STDOUT{GLOB},
2 => *STDERR{GLOB},
env => \%ENV
- }, __PACKAGE__), @ARGV);
+ }, __PACKAGE__;
+ dispatch($self, @ARGV);
+ x_it($self, $self->{child_error}) if $self->{child_error};
}
# ensures stdout hits the FS before sock disconnects so a client