-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Backend for `lei' (local email interface). Unlike the C10K-oriented
PublicInbox::LeiQuery);
use Getopt::Long ();
use Socket qw(AF_UNIX SOCK_SEQPACKET MSG_EOR pack_sockaddr_un);
-use Errno qw(EPIPE EAGAIN EINTR ECONNREFUSED ENOENT ECONNRESET);
+use Errno qw(EPIPE EAGAIN ECONNREFUSED ENOENT ECONNRESET);
use Cwd qw(getcwd);
use POSIX qw(strftime);
use IO::Handle ();
use Fcntl qw(SEEK_SET);
use PublicInbox::Config;
-use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
-use PublicInbox::DS qw(now dwaitpid);
+use PublicInbox::Syscall qw(EPOLLIN);
+use PublicInbox::DS qw(dwaitpid);
use PublicInbox::Spawn qw(spawn popen_rd);
use PublicInbox::Lock;
use PublicInbox::Eml;
sub ale {
my ($self) = @_;
- $self->{ale} //= do {
+ $self->{ale} // do {
require PublicInbox::LeiALE;
- $self->_lei_cfg(1)->{ale} //= PublicInbox::LeiALE->new($self);
+ my $cfg = $self->_lei_cfg(1);
+ $self->{ale} = $cfg->{ale} //= PublicInbox::LeiALE->new($self);
};
}
shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'up' => [ 'OUTPUT...|--all', 'update saved search',
- qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+
- remote-fudge-time=s all:s), @c_opt ],
+ qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ exclude=s@
+ remote-fudge-time=s all:s remote! local! external!), @net_opt, @c_opt ],
'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
'stdin|', # /|\z/ must be first for lone dash
'mail-diff' => [ '--stdin|LOCATION...', 'diff the contents of emails',
'stdin|', # /|\z/ must be first for lone dash
- qw(verbose|v+ color:s no-color raw-header),
+ qw(verbose|v+ in-format|F=s color:s no-color raw-header),
@diff_opt, @net_opt, @c_opt ],
'add-external' => [ 'LOCATION',
'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
- qw(z|0 ascii l pretty url), @c_opt ],
+ qw(z|0 ascii l pretty url), @net_opt, @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
'ls-search' => [ '[PREFIX]', 'list saved search queries',
qw(format|f=s pretty l ascii z|0), @c_opt ],
-'forget-search' => [ 'OUTPUT...', 'forget a saved search',
- qw(verbose|v+), @c_opt ],
+'forget-search' => [ 'OUTPUT...|--prune', 'forget a saved search',
+ qw(verbose|v+ prune:s), @c_opt ],
'edit-search' => [ 'OUTPUT', "edit saved search via `git config --edit'",
@c_opt ],
'rm' => [ '--stdin|LOCATION...',
'forget-mail-sync' => [ 'LOCATION...',
'forget sync information for a mail folder', @c_opt ],
'refresh-mail-sync' => [ 'LOCATION...|--all',
- 'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
+ 'prune dangling sync data for a mail folder', 'all:s',
+ @net_opt, @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
qw(all:s mode=s), @net_opt, @c_opt ],
'one-time conversion from URL or filesystem to another format',
qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
@net_opt, @c_opt ],
-'p2q' => [ 'FILE|COMMIT_OID|--stdin',
+'p2q' => [ 'LOCATION_OR_COMMIT...|--stdin',
"use a patch to generate a query for `lei q --stdin'",
- qw(stdin| want|w=s@ uri debug), @c_opt ],
+ qw(stdin| in-format|F=s want|w=s@ uri debug), @net_opt, @c_opt ],
'config' => [ '[...]', sub {
'git-config(1) wrapper for '._config_path($_[0]);
}, qw(config-file|system|global|file|f=s), # for conflict detection
'want|w=s@' => [ 'PREFIX|dfpost|dfn', # common ones in help...
'search prefixes to extract (default: dfpost7)' ],
+'uri p2q' => [ 'URI escape output' ],
'alert=s@' => ['CMD,:WINCH,:bell,<any command>',
'run command(s) or perform ops when done writing to output ' .
'url ls-mail-source' => 'show full URL of newsgroup or IMAP folder',
'format|f=s ls-external' => $ls_format,
+'prune:s forget-search' =>
+ ['TYPE|local|remote', 'prune all, remote or local folders' ],
+
'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
'offset=i' => ['OFF', 'search result offset (default: 0)'],
}
sub fail ($$;$) {
- my ($self, $buf, $exit_code) = @_;
+ my ($self, $msg, $exit_code) = @_;
local $current_lei = $self;
$self->{failed}++;
- warn($buf, "\n") if defined $buf;
+ warn(substr($msg, -1, 1) eq "\n" ? $msg : "$msg\n") if defined $msg;
$self->{pkt_op_p}->pkt_do('fail_handler') if $self->{pkt_op_p};
x_it($self, ($exit_code // 1) << 8);
undef;
my ($self, $child_error, $msg) = @_; # child_error is $?
local $current_lei = $self;
$child_error ||= 1 << 8;
- warn($msg, "\n") if defined $msg;
+ warn(substr($msg, -1, 1) eq "\n" ? $msg : "$msg\n") if defined $msg;
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('child_error', $child_error);
} elsif ($self->{sock}) { # to lei(1) client
# we need to explicitly close things which are on stack
if ($persist) {
open $self->{3}, '<', '/' or die "open(/) $!";
- fchdir($self) or die;
+ fchdir($self);
close($_) for (grep(defined, delete @$self{qw(0 1 2 sock)}));
if (my $cfg = $self->{cfg}) {
delete @$cfg{qw(-lei_store -watches -lei_note_event)};
sub workers_start {
my ($lei, $wq, $jobs, $ops, $flds) = @_;
- $ops = pkt_ops($lei, { ($ops ? %$ops : ()) });
+ $ops //= {};
+ ($wq->can('net_merge_all_done') && $lei->{auth}) and
+ $lei->{auth}->op_merge($ops, $wq, $lei);
+ pkt_ops($lei, $ops);
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
# call this when we're ready to wait on events and yield to other clients
sub wait_wq_events {
my ($lei, $op_c, $ops) = @_;
+ my $wq1 = $lei->{wq1};
+ ($wq1 && $wq1->can('net_merge_all_done') && !$lei->{auth}) and
+ $wq1->net_merge_all_done;
for my $wq (grep(defined, @$lei{qw(ikw pmd)})) { # auxiliary WQs
$wq->wq_close;
}
+ $wq1->{lei_sock} = $lei->{sock} if $wq1;
$op_c->{ops} = $ops;
}
+sub wq1_start {
+ my ($lei, $wq, $jobs) = @_;
+ my ($op_c, $ops) = workers_start($lei, $wq, $jobs // 1);
+ $lei->{wq1} = $wq;
+ wait_wq_events($lei, $op_c, $ops); # net_merge_all_done if !{auth}
+}
+
sub _help {
require PublicInbox::LeiHelp;
PublicInbox::LeiHelp::call($_[0], $_[1], \%CMD, \%OPTDESC);
sub dispatch {
my ($self, $cmd, @argv) = @_;
- fchdir($self) or return;
+ fchdir($self);
local %ENV = %{$self->{env}};
local $current_lei = $self; # for __WARN__
$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
next if $d eq ''; # same as git(1)
chdir $d or return fail($self, "cd $d: $!");
}
- open $self->{3}, '.' or return fail($self, "open . $!");
+ open $self->{3}, '<', '.' or
+ return fail($self, "open . $!");
}
$cb->($self, @argv);
} elsif (grep(/\A-/, $cmd, @argv)) { # --help or -h only
my %env = map { split(/=/, $_, 2) } splice(@argv, $argc);
$self->{env} = \%env;
eval { dispatch($self, @argv) };
- send($sock, $@, MSG_EOR) if $@;
+ $self->fail($@) if $@;
}
sub dclose {
local %ENV = %{$self->{env}};
local $current_lei = $self;
eval {
- my $buf;
- while (my @fds = $recv_cmd->($self->{sock}, $buf, 4096)) {
- if (scalar(@fds) == 1 && !defined($fds[0])) {
- return if $! == EAGAIN;
- next if $! == EINTR;
- last if $! == ECONNRESET;
- die "recvmsg: $!";
- }
- for (@fds) { open my $rfh, '+<&=', $_ }
+ my @fds = $recv_cmd->($self->{sock} // return, my $buf, 4096);
+ if (scalar(@fds) == 1 && !defined($fds[0])) {
+ return if $! == EAGAIN;
+ die "recvmsg: $!" if $! != ECONNRESET;
+ $buf = '';
+ @fds = (); # for open loop below:
}
+ for (@fds) { open my $rfh, '+<&=', $_ }
if ($buf eq '') {
_drop_wq($self); # EOF, client disconnected
dclose($self);
sub event_step_init {
my ($self) = @_;
my $sock = $self->{sock} or return;
- $self->{-event_init_done} //= do { # persist til $ops done
+ $self->{-event_init_done} // do { # persist til $ops done
$sock->blocking(0);
- $self->SUPER::new($sock, EPOLLIN|EPOLLET);
- $sock;
+ $self->SUPER::new($sock, EPOLLIN);
+ $self->{-event_init_done} = $sock;
};
}
my $fn = $ev->fullname;
if ($fn =~ m!\A(.+)/(new|cur)/([^/]+)\z!) { # Maildir file
my ($mdir, $nc, $bn) = ($1, $2, $3);
- $nc = '' if $ev->IN_DELETE;
+ $nc = '' if $ev->IN_DELETE || $ev->IN_MOVED_FROM;
for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
my $cfg = $PATH2CFG{$f} // next;
eval {
require PublicInbox::CmdIPC4;
$send_cmd = PublicInbox::CmdIPC4->can('send_cmd4');
PublicInbox::CmdIPC4->can('recv_cmd4');
+ } // do {
+ $send_cmd = PublicInbox::Syscall->can('send_cmd4');
+ PublicInbox::Syscall->can('recv_cmd4');
};
}
$recv_cmd or die <<"";
$quit->();
}
return 1 if defined($path);
- my $now = now();
my $n = 0;
for my $s (values %$dmap) {
$s->can('busy') or next;
- if ($s->busy($now)) {
+ if ($s->busy) {
++$n;
} else {
$s->close;
sub fchdir {
my ($lei) = @_;
my $dh = $lei->{3} // die 'BUG: lei->{3} (CWD) gone';
- chdir($dh) || $lei->fail("fchdir: $!");
+ chdir($dh) || die "fchdir: $!";
}
sub wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
+ my ($lei, $wq_fld) = @_;
local $current_lei = $lei;
- delete $lei->{wq1} // return $lei->fail; # already failed
+ my $wq = delete $lei->{$wq_fld // 'wq1'};
+ $lei->sto_done_request($wq);
+ $wq // $lei->fail; # already failed
}
sub watch_state_ok ($) {
}
sub lms {
- my ($lei, $rw) = @_;
+ my ($lei, $creat) = @_;
my $sto = $lei->{sto} // _lei_store($lei) // return;
require PublicInbox::LeiMailSync;
my $f = "$sto->{priv_eidx}->{topdir}/mail_sync.sqlite3";
- (-f $f || $rw) ? PublicInbox::LeiMailSync->new($f) : undef;
+ (-f $f || $creat) ? PublicInbox::LeiMailSync->new($f) : undef;
}
sub sto_done_request {
- my ($lei, $sock) = @_;
+ my ($lei, $wq) = @_;
+ return unless $lei->{sto};
local $current_lei = $lei;
+ my $sock = $wq ? $wq->{lei_sock} : undef;
eval {
if ($sock //= $lei->{sock}) { # issue, async wait
$lei->{sto}->wq_io_do('done', [ $sock ]);