use File::Path qw(mkpath);
use File::Spec;
our $quit = \&CORE::exit;
-our ($current_lei, $errors_log, $listener, $oldset);
+our ($current_lei, $errors_log, $listener, $oldset, $dir_idle);
my ($recv_cmd, $send_cmd);
my $GLP = Getopt::Long::Parser->new;
$GLP->configure(qw(gnu_getopt no_ignore_case auto_abbrev));
my ($self, $p) = @_;
if (index($p, '/') == 0) { # already absolute
$p =~ tr!/!/!s; # squeeze redundant slashes
+ chop($p) if substr($p, -1, 1) eq '/';
return $p;
}
my $pwd = $self->{env}->{PWD};
import-remote! no-torsocks torsocks=s),
PublicInbox::LeiQuery::curl_opt());
+# we don't support -C as an alias for --find-copies since it's already
+# used for chdir
+our @diff_opt = qw(unified|U=i output-indicator-new=s output-indicator-old=s
+ output-indicator-context=s indent-heuristic!
+ minimal patience histogram anchored=s@ diff-algorithm=s
+ color-moved:s color-moved-ws=s no-color-moved no-color-moved-ws
+ word-diff:s word-diff-regex=s color-words:s no-renames
+ rename-empty! check ws-error-highlight=s full-index binary
+ abbrev:i break-rewrites|B:s find-renames|M:s find-copies:s
+ find-copies-harder irreversible-delete|D l=i diff-filter=s
+ S=s G=s find-object=s pickaxe-all pickaxe-regex O=s R
+ relative:s text|a ignore-cr-at-eol ignore-space-at-eol
+ ignore-space-change|b ignore-all-space|w ignore-blank-lines
+ inter-hunk-context=i function-context|W exit-code ext-diff
+ no-ext-diff textconv! src-prefix=s dst-prefix=s no-prefix
+ line-prefix=s);
+
# we generate shell completion + help using %CMD and %OPTDESC,
# see lei__complete() and PublicInbox::LeiHelp
# command => [ positional_args, 1-line description, Getopt::Long option spec ]
qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
- color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
+ shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'up' => [ 'OUTPUT|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
-'lcat' => [ '--stdin|MSGID_OR_URL..', 'display local copy of message(s)',
+'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
'stdin|', # /|\z/ must be first for lone dash
# some of these options are ridiculous for lcat
@lxs_opt, qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
@lxs_opt, @c_opt ],
+'rediff' => [ '--stdin|LOCATION...',
+ 'regenerate a diff with different options',
+ 'stdin|', # /|\z/ must be first for lone dash
+ qw(git-dir=s@ cwd! verbose|v+ color:s no-color),
+ @diff_opt, @lxs_opt, @c_opt ],
+
'add-external' => [ 'LOCATION',
'add/set priority of a publicinbox|extindex for extra matches',
qw(boost=i mirror=s no-torsocks torsocks=s inbox-version=i
'ls-external' => [ '[FILTER]', 'list publicinbox|extindex locations',
qw(format|f=s z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
-'ls-sync' => [ '', 'list sync folders',
- qw(z|0 z|0 globoff|g invert-match|v local remote), @c_opt ],
+'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
+ qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
qw(verbose|v+), @c_opt ],
'edit-search' => [ 'OUTPUT', "edit saved search via `git config --edit'",
@c_opt ],
-
+'rm' => [ '--stdin|LOCATION...',
+ 'remove a message from the index and prevent reindexing',
+ 'stdin|', # /|\z/ must be first for lone dash
+ @c_opt ],
'plonk' => [ '--threads|--from=IDENT',
'exclude mail matching From: or threads from non-Message-ID searches',
qw(stdin| threads|t from|f=s mid=s oid=s), @c_opt ],
'tag' => [ 'KEYWORDS...',
'set/unset keywords and/or labels on message(s)',
- qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@), @c_opt,
+ qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@),
+ qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt,
pass_through('-kw:foo for delete') ],
-'forget' => [ '[--stdin|--oid=OID|--by-mid=MID]',
- "exclude message(s) on stdin from `q' search results",
- qw(stdin| oid=s exact by-mid|mid:s), @c_opt ],
'purge-mailsource' => [ 'LOCATION|--all',
'remove imported messages from IMAP, Maildirs, and MH',
'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch',
qw(prune), @c_opt ],
+'index' => [ 'LOCATION...', 'one-time index from URL or filesystem',
+ qw(in-format|F=s kw! offset=i recursive|r exclude=s include|I=s
+ verbose|v+ incremental!),
+ PublicInbox::LeiQuery::curl_opt(), # mainly for --proxy=
+ @c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s
- lock=s@ in-format|F=s kw! verbose|v+ incremental! sync!), @c_opt ],
+ qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s
+ lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
+ qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
+'forget-mail-sync' => [ 'LOCATION...',
+ 'forget sync information for a mail folder', @c_opt ],
+'export-kw' => [ 'LOCATION...|--all',
+ 'one-time export of keywords of sync sources',
+ qw(all:s mode=s), @c_opt ],
'convert' => [ 'LOCATION...|--stdin',
'one-time conversion from URL or filesystem to another format',
- qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s
- lock=s@ kw!), @c_opt ],
+ qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
+ qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'p2q' => [ 'FILE|COMMIT_OID|--stdin',
"use a patch to generate a query for `lei q --stdin'",
qw(stdin| want|w=s@ uri debug), @c_opt ],
'daemon-pid' => [ '', 'show the PID of the lei-daemon' ],
'help' => [ '[SUBCOMMAND]', 'show help' ],
-# XXX do we need this?
-# 'git' => [ '[ANYTHING...]', 'git(1) wrapper', pass_through('git') ],
-
-'reorder-local-store-and-break-history' => [ '[REFNAME]',
- 'rewrite git history in an attempt to improve compression',
- qw(gc!), @c_opt ],
-
+# TODO
+#'reorder-local-store-and-break-history' => [ '[REFNAME]',
+# 'rewrite git history in an attempt to improve compression',
+# qw(gc!), @c_opt ],
+#'fuse-mount' => [ 'PATHNAME', 'expose lei/store as Maildir(s)', @c_opt ],
+#
# internal commands are prefixed with '_'
'_complete' => [ '[...]', 'internal shell completion helper',
pass_through('everything') ],
'incremental! import' => 'import already seen IMAP and NNTP articles',
'globoff|g' => "do not match locations using '*?' wildcards ".
"and\xa0'[]'\x{a0}ranges",
+'invert-match|v' => 'select non-matching lines',
'color!' => 'disable color (for --format=text)',
'verbose|v+' => 'be more verbose',
'external!' => 'do not use externals',
'path-a|a=s' => 'pre-image pathname associated with OID',
'path-b|b=s' => 'post-image pathname associated with OID',
'git-dir=s@' => 'additional git repository to scan',
+'proxy=s' => [ 'PROTO://HOST[:PORT]', # shared with curl(1)
+ "proxy for (e.g. `socks5h://0:9050')" ],
'torsocks=s' => ['VAL|auto|no|yes',
'whether or not to wrap git and curl commands with torsocks'],
'no-torsocks' => 'alias for --torsocks=no',
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m wq1); # internal workers
+my @WQ_KEYS = qw(lxs l2m ikw pmd wq1); # internal workers
sub _drop_wq {
my ($self) = @_;
- for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) { $wq->DESTROY }
+ for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) {
+ if ($wq->wq_kill) {
+ $wq->wq_close(0, undef, $self);
+ } elsif ($wq->wq_kill_old) {
+ $wq->wq_wait_old(undef, $self);
+ }
+ $wq->DESTROY;
+ }
}
# pronounced "exit": x_it(1 << 8) => exit(1); x_it(13) => SIGPIPE
# make sure client sees stdout before exit
$self->{1}->autoflush(1) if $self->{1};
dump_and_clear_log();
- if (my $s = $self->{pkt_op_p} // $self->{sock}) {
- send($s, "x_it $code", MSG_EOR);
- } elsif ($self->{oneshot}) {
- # don't want to end up using $? from child processes
- _drop_wq($self);
- # cleanup anything that has tempfiles or open file handles
- %PATH2CFG = ();
- delete @$self{qw(ovv dedupe sto cfg)};
- if (my $signum = ($code & 127)) { # usually SIGPIPE (13)
- $SIG{PIPE} = 'DEFAULT'; # $SIG{$signum} doesn't work
- kill $signum, $$;
- sleep(1) while 1; # wait for signal
- } else {
- $quit->($code >> 8);
- }
+ if ($self->{pkt_op_p}) { # to top lei-daemon
+ $self->{pkt_op_p}->pkt_do('x_it', $code);
+ } elsif ($self->{sock}) { # to lei(1) client
+ send($self->{sock}, "x_it $code", MSG_EOR);
} # else ignore if client disconnected
}
sub fail ($$;$) {
my ($self, $buf, $exit_code) = @_;
+ $self->{failed}++;
err($self, $buf) if defined $buf;
# calls fail_handler:
- send($self->{pkt_op_p}, '!', MSG_EOR) if $self->{pkt_op_p};
+ $self->{pkt_op_p}->pkt_do('!') if $self->{pkt_op_p};
x_it($self, ($exit_code // 1) << 8);
undef;
}
sub child_error { # passes non-fatal curl exit codes to user
my ($self, $child_error, $msg) = @_; # child_error is $?
$self->err($msg) if $msg;
- if (my $s = $self->{pkt_op_p} // $self->{sock}) {
- # send to the parent lei-daemon or to lei(1) client
- send($s, "child_error $child_error", MSG_EOR);
- } elsif (!$PublicInbox::DS::in_loop) {
- $self->{child_error} = $child_error;
+ if ($self->{pkt_op_p}) { # to top lei-daemon
+ $self->{pkt_op_p}->pkt_do('child_error', $child_error);
+ } elsif ($self->{sock}) { # to lei(1) client
+ send($self->{sock}, "child_error $child_error", MSG_EOR);
} # else noop if client disconnected
}
sub note_sigpipe { # triggers sigpipe_handler
my ($self, $fd) = @_;
close(delete($self->{$fd})); # explicit close silences Perl warning
- send($self->{pkt_op_p}, '|', MSG_EOR) if $self->{pkt_op_p};
+ $self->{pkt_op_p}->pkt_do('|') if $self->{pkt_op_p};
x_it($self, 13);
}
}
} else { # worker, Net::NNTP (Net::Cmd) uses STDERR directly
open STDERR, '+>&='.fileno($self->{2}) or warn "open $!";
+ STDERR->autoflush(1);
}
close($_) for (grep(defined, delete @$self{qw(3 old_1 au_done)}));
if (my $op_c = delete $self->{pkt_op_c}) {
}
close $listener if $listener;
undef $listener;
+ undef $dir_idle;
%PATH2CFG = ();
undef $errors_log;
$quit = \&CORE::exit;
if (my $op = delete $self->{pkt_op_c}) { # in case of die
$op->close; # PublicInbox::PktOp::close
}
- my $unclosed_after_die = delete($self->{pkt_op_p}) or return;
- close $unclosed_after_die;
+ my $pkt_op_p = delete($self->{pkt_op_p}) or return;
+ close $pkt_op_p->{op_p};
}
sub pkt_op_pair {
}
sub workers_start {
- my ($lei, $wq, $ident, $jobs, $ops) = @_;
+ my ($lei, $wq, $jobs, $ops, $flds) = @_;
$ops = {
'!' => [ \&fail_handler, $lei ],
'|' => [ \&sigpipe_handler, $lei ],
};
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
- $wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
+ my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
+ $flds->{lei} = $lei;
+ $wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
+ # {-lei_sock} persists script/lei process until ops->{''} EOF callback
+ $op_c->{-lei_sock} = $lei->{sock};
@$end = ();
$lei->event_step_init;
($op_c, $ops);
}
+# call this when we're ready to wait on events and yield to other clients
+sub wait_wq_events {
+ my ($lei, $op_c, $ops) = @_;
+ for my $wq (grep(defined, @$lei{qw(ikw pmd)})) { # auxiliary WQs
+ $wq->wq_close(1);
+ }
+ $op_c->{ops} = $ops;
+}
+
sub _help {
require PublicInbox::LeiHelp;
PublicInbox::LeiHelp::call($_[0], $_[1], \%CMD, \%OPTDESC);
my $ok;
for my $o (@or) {
if ($o =~ /\A--([a-z0-9\-]+)/) {
- $ok = defined($OPT->{$1});
+ my $sw = $1;
+ # assume pipe/regular file on stdin
+ # w/o args means stdin
+ if ($sw eq 'stdin' && !@$argv &&
+ (-p $self->{0} ||
+ -f _) && -r _) {
+ $OPT->{stdin} //= 1;
+ }
+ $ok = defined($OPT->{$sw});
last if $ok;
} elsif (defined($argv->[$i])) {
$ok = 1;
sub dispatch {
my ($self, $cmd, @argv) = @_;
local $current_lei = $self; # for __WARN__
+ $self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
dump_and_clear_log("from previous run\n");
return _help($self, 'no command given') unless defined($cmd);
# do not support Getopt bundling for this
}
push @cmd, $mfolder unless defined($replaced);
if ($self->{sock}) { # lei(1) client process runs it
- # restore terminal: echo $query | lei q -stdin --mua=...
+ # restore terminal: echo $query | lei q --stdin --mua=...
my $io = [];
$io->[0] = $self->{1} if $self->{opt}->{stdin} && -t $self->{1};
send_exec_cmd($self, $io, \@cmd, {});
- } elsif ($self->{oneshot}) {
- my $pid = fork // die "fork: $!";
- if ($pid > 0) { # original process
- if ($self->{opt}->{stdin} && -t STDOUT) {
- open STDIN, '+<&', \*STDOUT or die "dup2: $!";
- }
- exec(@cmd);
- warn "exec @cmd: $!\n";
- POSIX::_exit(1);
- }
- POSIX::setsid() > 0 or die "setsid: $!";
}
if ($self->{lxs} && $self->{au_done}) { # kick wait_startq
syswrite($self->{au_done}, 'q' x ($self->{lxs}->{jobs} // 0));
sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
my ($self) = @_;
my $alerts = $self->{opt}->{alert} // return;
+ my $sock = $self->{sock};
while (my $op = shift(@$alerts)) {
if ($op eq ':WINCH') {
# hit the process group that started the MUA
- if ($self->{sock}) {
- send($self->{sock}, '-WINCH', MSG_EOR);
- } elsif ($self->{oneshot}) {
- kill('-WINCH', $$);
- }
+ send($sock, '-WINCH', MSG_EOR) if $sock;
} elsif ($op eq ':bell') {
out($self, "\a");
} elsif ($op =~ /(?<!\\),/) { # bare ',' (not ',,')
my $cmd = $1; # run an arbitrary command
require Text::ParseWords;
$cmd = [ Text::ParseWords::shellwords($cmd) ];
- if (my $s = $self->{sock}) {
- send($s, exec_buf($cmd, {}), MSG_EOR);
- } elsif ($self->{oneshot}) {
- $self->{"pid.$self.$$"}->{spawn($cmd)} = $cmd;
- }
+ send($sock, exec_buf($cmd, {}), MSG_EOR) if $sock;
} else {
err($self, "W: unsupported --alert=$op"); # non-fatal
}
if ($self->{sock}) { # lei(1) process runs it
delete @$new_env{keys %$env}; # only set iff unset
send_exec_cmd($self, [ @$rdr{0..2} ], [$pager], $new_env);
- } elsif ($self->{oneshot}) {
- my $cmd = [$pager];
- $self->{"pid.$self.$$"}->{spawn($cmd, $new_env, $rdr)} = $cmd;
} else {
die 'BUG: start_pager w/o socket';
}
}
$i == 4 or return send($sock, 'not enough FDs='.($i-1), MSG_EOR)
}
- $self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
# $ENV_STR = join('', map { "\0$_=$ENV{$_}" } keys %ENV);
# $buf = "$argc\0".join("\0", @ARGV).$ENV_STR."\0\0";
substr($buf, -2, 2, '') eq "\0\0" or # s/\0\0\z//
sub dclose {
my ($self) = @_;
delete $self->{-progress};
- _drop_wq($self);
+ _drop_wq($self) if $self->{failed};
close(delete $self->{1}) if $self->{1}; # may reap_compress
$self->close if $self->{-event_init_done}; # PublicInbox::DS::close
}
sub lazy_start {
my ($path, $errno, $narg) = @_;
local ($errors_log, $listener);
- ($errors_log) = ($path =~ m!\A(.+?/)[^/]+\z!);
- $errors_log .= 'errors.log';
+ my ($sock_dir) = ($path =~ m!\A(.+?)/[^/]+\z!);
+ $errors_log = "$sock_dir/errors.log";
my $addr = pack_sockaddr_un($path);
my $lk = bless { lock_path => $errors_log }, 'PublicInbox::Lock';
$lk->lock_acquire;
(Socket::MsgHdr || Inline::C) missing/unconfigured (narg=$narg);
require PublicInbox::Listener;
- require PublicInbox::EOFpipe;
+ require PublicInbox::PktOp;
(-p STDOUT) or die "E: stdout must be a pipe\n";
open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!";
STDIN->autoflush(1);
my $exit_code;
my $pil = PublicInbox::Listener->new($listener, \&accept_dispatch);
local $quit = do {
- pipe(my ($eof_r, $eof_w)) or die "pipe: $!";
- PublicInbox::EOFpipe->new($eof_r, \&noop, undef);
+ my (undef, $eof_p) = PublicInbox::PktOp->pair;
sub {
$exit_code //= shift;
my $lis = $pil or exit($exit_code);
- # closing eof_w triggers \&noop wakeup
- $listener = $eof_w = $pil = $path = undef;
+ # closing eof_p triggers \&noop wakeup
+ $listener = $eof_p = $pil = $path = undef;
$lis->close; # DS::close
PublicInbox::DS->SetLoopTimeout(1000);
};
local @SIG{keys %$sig} = values(%$sig) unless $sigfd;
undef $sig;
local $SIG{PIPE} = 'IGNORE';
- if ($sigfd) { # TODO: use inotify/kqueue to detect unlinked sockets
- undef $sigfd;
- PublicInbox::DS->SetLoopTimeout(5000);
+ require PublicInbox::DirIdle;
+ local $dir_idle = PublicInbox::DirIdle->new([$sock_dir], sub {
+ # just rely on wakeup ot hit PostLoopCallback set below
+ _dir_idle_handler(@_) if $_[0]->fullname ne $path;
+ }, 1);
+ if ($sigfd) {
+ undef $sigfd; # unref, already in DS::DescriptorMap
} else {
# wake up every second to accept signals if we don't
# have signalfd or IO::KQueue:
sub busy { 1 } # prevent daemon-shutdown if client is connected
-# for users w/o Socket::Msghdr installed or Inline::C enabled
-sub oneshot {
- my ($main_pkg) = @_;
- my $exit = $main_pkg->can('exit'); # caller may override exit()
- local $quit = $exit if $exit;
- local %PATH2CFG;
- umask(077) // die("umask(077): $!");
- my $self = bless { oneshot => 1, env => \%ENV }, __PACKAGE__;
- for (0..2) { open($self->{$_}, '+<&=', $_) or die "open fd=$_: $!" }
- dispatch($self, @ARGV);
- x_it($self, $self->{child_error}) if $self->{child_error};
-}
-
# ensures stdout hits the FS before sock disconnects so a client
# can immediately reread it
sub DESTROY {
my ($self) = @_;
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
- my $err = $?;
- my $oneshot_pids = delete $self->{"pid.$self.$$"} or return;
- waitpid($_, 0) for keys %$oneshot_pids;
- $? = $err if $err; # preserve ->fail or ->x_it code
+ # preserve $? for ->fail or ->x_it code
}
sub wq_done_wait { # dwaitpid callback
$lei->dclose;
}
+sub fchdir {
+ my ($lei) = @_;
+ my $dh = $lei->{3} // die 'BUG: lei->{3} (CWD) gone';
+ chdir($dh) || $lei->fail("fchdir: $!");
+}
+
sub wq_eof { # EOF callback for main daemon
my ($lei) = @_;
my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed