use PublicInbox::DS qw(now dwaitpid);
use PublicInbox::Spawn qw(spawn popen_rd);
use PublicInbox::Lock;
+use PublicInbox::Eml;
use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
($spec, '<>' => $cb, $GLP_PASS) # for Getopt::Long
}
-sub rel2abs ($$) {
+# rel2abs preserves symlinks in parent, unlike abs_path
+sub rel2abs {
my ($self, $p) = @_;
- return $p if index($p, '/') == 0; # already absolute
+ if (index($p, '/') == 0) { # already absolute
+ $p =~ tr!/!/!s; # squeeze redundant slashes
+ return $p;
+ }
my $pwd = $self->{env}->{PWD};
+ my $cwd;
if (defined $pwd) {
- my $cwd = $self->{3} // getcwd() // die "getcwd(PWD=$pwd): $!";
+ my $xcwd = $self->{3} //
+ ($cwd = getcwd() // die "getcwd(PWD=$pwd): $!");
if (my @st_pwd = stat($pwd)) {
- my @st_cwd = stat($cwd) or die "stat($cwd): $!";
+ my @st_cwd = stat($xcwd) or die "stat($xcwd): $!";
"@st_pwd[1,0]" eq "@st_cwd[1,0]" or
- $self->{env}->{PWD} = $pwd = $cwd;
+ $self->{env}->{PWD} = $pwd = undef;
} else { # PWD was invalid
- delete $self->{env}->{PWD};
- undef $pwd;
+ $self->{env}->{PWD} = $pwd = undef;
}
}
- $pwd //= $self->{env}->{PWD} = getcwd() // die "getcwd(PWD=$pwd): $!";
+ $pwd //= $self->{env}->{PWD} = $cwd // getcwd() // die "getcwd: $!";
File::Spec->rel2abs($p, $pwd);
}
-sub store_path ($) {
+# abs_path resolves symlinks in parent iff all parents exist
+sub abs_path { Cwd::abs_path($_[1]) // rel2abs(@_) }
+
+sub share_path ($) { # $HOME/.local/share/lei/$FOO
my ($self) = @_;
rel2abs($self, ($self->{env}->{XDG_DATA_HOME} //
($self->{env}->{HOME} // '/nonexistent').'/.local/share')
- .'/lei/store');
+ .'/lei');
}
+sub store_path ($) { share_path($_[0]) . '/store' }
+
sub _config_path ($) {
my ($self) = @_;
rel2abs($self, ($self->{env}->{XDG_CONFIG_HOME} //
import-remote! no-torsocks torsocks=s),
PublicInbox::LeiQuery::curl_opt());
+# we don't support -C as an alias for --find-copies since it's already
+# used for chdir
+our @diff_opt = qw(unified|U=i output-indicator-new=s output-indicator-old=s
+ output-indicator-context=s indent-heuristic!
+ minimal patience histogram anchored=s@ diff-algorithm=s
+ color-moved=s color-moved-ws=s no-color-moved no-color-moved-ws
+ word-diff:s word-diff-regex=s color-words:s no-renames
+ rename-empty! check ws-error-highlight=s full-index binary
+ abbrev:i break-rewrites|B:s find-renames|M:s find-copies:s
+ find-copies-harder irreversible-delete|D l=i diff-filter=s
+ S=s G=s find-object=s pickaxe-all pickaxe-regex O=s R
+ relative:s text|a ignore-cr-at-eol ignore-space-at-eol
+ ignore-space-change|b ignore-all-space|w ignore-blank-lines
+ inter-hunk-context=i function-context|W exit-code ext-diff
+ no-ext-diff textconv! src-prefix=s dst-prefix=s no-prefix
+ line-prefix=s);
+
# we generate shell completion + help using %CMD and %OPTDESC,
# see lei__complete() and PublicInbox::LeiHelp
# command => [ positional_args, 1-line description, Getopt::Long option spec ]
'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms',
'stdin|', # /|\z/ must be first for lone dash
@lxs_opt,
- qw(save-as=s output|mfolder|o=s format|f=s dedupe|d=s threads|t+
+ qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
- import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+), @c_opt,
- opt_dash('limit|n=i', '[0-9]+') ],
+ import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
+ color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
+
+'up' => [ 'OUTPUT|--all', 'update saved search',
+ qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
+
+'lcat' => [ '--stdin|MSGID_OR_URL..', 'display local copy of message(s)',
+ 'stdin|', # /|\z/ must be first for lone dash
+ # some of these options are ridiculous for lcat
+ @lxs_opt, qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
+ sort|s=s reverse|r offset=i jobs|j=s globoff|g augment|a
+ import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
+ color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'blob' => [ 'OID', 'show a git blob, reconstructing from mail if necessary',
qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
@lxs_opt, @c_opt ],
+'rediff' => [ '[--stdin|LOCATION...]',
+ 'regenerate a diff with different options',
+ qw(git-dir=s@ cwd! verbose|v+ color:s no-color),
+ @diff_opt, @lxs_opt, @c_opt ],
+
'add-external' => [ 'LOCATION',
'add/set priority of a publicinbox|extindex for extra matches',
qw(boost=i mirror=s no-torsocks torsocks=s inbox-version=i
'ls-external' => [ '[FILTER]', 'list publicinbox|extindex locations',
qw(format|f=s z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
+'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
+ qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
-'ls-query' => [ '[FILTER...]', 'list saved search queries',
- qw(name-only format|f=s), @c_opt ],
-'rm-query' => [ 'QUERY_NAME', 'remove a saved search', @c_opt ],
-'mv-query' => [ qw(OLD_NAME NEW_NAME), 'rename a saved search', @c_opt ],
+'ls-search' => [ '[PREFIX]', 'list saved search queries',
+ qw(format|f=s pretty l ascii z|0), @c_opt ],
+'forget-search' => [ 'OUTPUT', 'forget a saved search',
+ qw(verbose|v+), @c_opt ],
+'edit-search' => [ 'OUTPUT', "edit saved search via `git config --edit'",
+ @c_opt ],
'plonk' => [ '--threads|--from=IDENT',
'exclude mail matching From: or threads from non-Message-ID searches',
qw(stdin| threads|t from|f=s mid=s oid=s), @c_opt ],
'tag' => [ 'KEYWORDS...',
'set/unset keywords and/or labels on message(s)',
- qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@), @c_opt,
+ qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@),
+ qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt,
pass_through('-kw:foo for delete') ],
'forget' => [ '[--stdin|--oid=OID|--by-mid=MID]',
"exclude message(s) on stdin from `q' search results",
qw(exact! all jobs:i indexed), @c_opt ],
'add-watch' => [ 'LOCATION', 'watch for new messages and flag changes',
- qw(import! kw|keywords|flags! interval=s recursive|r
+ qw(import! kw! interval=s recursive|r
exclude=s include=s), @c_opt ],
'ls-watch' => [ '[FILTER...]', 'list active watches with numbers and status',
qw(format|f=s z), @c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
qw(stdin| offset=i recursive|r exclude=s include|I=s
- lock=s@ in-format|F=s kw|keywords|flags! verbose|v+), @c_opt ],
+ lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
+ qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'convert' => [ 'LOCATION...|--stdin',
'one-time conversion from URL or filesystem to another format',
- qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s
- lock=s@ kw|keywords|flags!), @c_opt ],
+ qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
+ qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'p2q' => [ 'FILE|COMMIT_OID|--stdin',
"use a patch to generate a query for `lei q --stdin'",
qw(stdin| want|w=s@ uri debug), @c_opt ],
'git-config(1) wrapper for '._config_path($_[0]);
}, qw(config-file|system|global|file|f=s), # for conflict detection
qw(c=s@ C=s@), pass_through('git config') ],
+'inspect' => [ 'ITEMS...', 'inspect lei/store and/or local external',
+ qw(pretty ascii dir=s), @c_opt ],
+
'init' => [ '[DIRNAME]', sub {
"initialize storage, default: ".store_path($_[0]);
}, @c_opt ],
'lock=s@' => [ 'METHOD|dotlock|fcntl|flock|none',
'mbox(5) locking method(s) to use (default: fcntl,dotlock)' ],
+'incremental! import' => 'import already seen IMAP and NNTP articles',
'globoff|g' => "do not match locations using '*?' wildcards ".
"and\xa0'[]'\x{a0}ranges",
+'invert-match|v' => 'select non-matching lines',
+'color!' => 'disable color (for --format=text)',
'verbose|v+' => 'be more verbose',
'external!' => 'do not use externals',
'mail!' => 'do not look in mail storage for OID',
'path-a|a=s' => 'pre-image pathname associated with OID',
'path-b|b=s' => 'post-image pathname associated with OID',
'git-dir=s@' => 'additional git repository to scan',
+'proxy=s' => [ 'PROTO://HOST[:PORT]', # shared with curl(1)
+ "proxy for (e.g. `socks5h://0:9050')" ],
'torsocks=s' => ['VAL|auto|no|yes',
'whether or not to wrap git and curl commands with torsocks'],
'no-torsocks' => 'alias for --torsocks=no',
-'save-as=s' => ['NAME', 'save a search terms by given name'],
+'save' => "save a search for `lei up'",
'import-remote!' => 'do not memoize remote messages into local store',
'type=s' => [ 'any|mid|git', 'disambiguate type' ],
'jobs|j=i add-external' => 'set parallelism when indexing after --mirror',
'in-format|F=s' => $stdin_formats,
-'format|f=s ls-query' => $ls_format,
+'format|f=s ls-search' => ['OUT|json|jsonl|concatjson',
+ 'listing output format' ],
+'l ls-search' => 'long listing format',
'format|f=s ls-external' => $ls_format,
'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
'remote' => 'limit operations to those requiring network access',
'remote!' => 'prevent operations requiring network access',
+'all:s up' => ['local', 'update all (local) saved searches' ],
+
'mid=s' => 'specify the Message-ID of a message',
'oid=s' => 'specify the git object ID of a message',
'by-mid|mid:s' => [ 'MID', 'match only by Message-ID, ignoring contents' ],
-'kw|keywords|flags!' => 'disable/enable importing flags',
+'kw!' => 'disable/enable importing keywords (aka "flags")',
# xargs, env, use "-0", git(1) uses "-z". We support z|0 everywhere
'z|0' => 'use NUL \\0 instead of newline (CR) to delimit lines',
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q tag sol); # internal workers
+my @WQ_KEYS = qw(lxs l2m wq1); # internal workers
sub _drop_wq {
my ($self) = @_;
- for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) { $wq->DESTROY }
+ for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) {
+ if ($wq->wq_kill) {
+ $wq->wq_close(0, undef, $self);
+ } elsif ($wq->wq_kill_old) {
+ $wq->wq_wait_old(undef, $self);
+ }
+ $wq->DESTROY;
+ }
}
# pronounced "exit": x_it(1 << 8) => exit(1); x_it(13) => SIGPIPE
# we need to explicitly close things which are on stack
if ($persist) {
chdir '/' or die "chdir(/): $!";
- my @io = delete @$self{qw(0 1 2 sock)};
- unless ($self->{oneshot}) {
- close($_) for @io;
- }
+ close($_) for (grep(defined, delete @$self{qw(0 1 2 sock)}));
if (my $cfg = $self->{cfg}) {
delete $cfg->{-lei_store};
}
} else { # worker, Net::NNTP (Net::Cmd) uses STDERR directly
open STDERR, '+>&='.fileno($self->{2}) or warn "open $!";
+ STDERR->autoflush(1);
}
- for (delete @$self{qw(3 old_1 au_done)}) {
- close($_) if defined($_);
- }
+ close($_) for (grep(defined, delete @$self{qw(3 old_1 au_done)}));
if (my $op_c = delete $self->{pkt_op_c}) {
close(delete $op_c->{sock});
}
%PATH2CFG = ();
undef $errors_log;
$quit = \&CORE::exit;
+ $self->{-eml_noisy} or # only "lei import" sets this atm
+ $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$current_lei = $persist ? undef : $self; # for SIG{__WARN__}
}
}
sub workers_start {
- my ($lei, $wq, $ident, $jobs, $ops) = @_;
+ my ($lei, $wq, $jobs, $ops) = @_;
$ops = {
'!' => [ \&fail_handler, $lei ],
'|' => [ \&sigpipe_handler, $lei ],
'child_error' => [ \&child_error, $lei ],
($ops ? %$ops : ()),
};
- $ops->{''} //= [ \&dclose, $lei ];
+ $ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
+ my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
$wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
sub dispatch {
my ($self, $cmd, @argv) = @_;
local $current_lei = $self; # for __WARN__
+ $self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
dump_and_clear_log("from previous run\n");
return _help($self, 'no command given') unless defined($cmd);
# do not support Getopt bundling for this
$cur_st = pack('dd', $st[10], $st[7]);
qerr($self, "# $f created") if $self->{cmd} ne 'config';
}
- my $cfg = PublicInbox::Config::git_config_dump($f);
- bless $cfg, 'PublicInbox::Config';
+ my $cfg = PublicInbox::Config->git_config_dump($f);
$cfg->{-st} = $cur_st;
$cfg->{'-f'} = $f;
if ($sto && File::Spec->canonpath($sto_dir // store_path($self))
@cmd = map { $_ eq '%f' ? ($replaced = $mfolder) : $_ } @cmd;
}
push @cmd, $mfolder unless defined($replaced);
- if (my $sock = $self->{sock}) { # lei(1) client process runs it
- send($sock, exec_buf(\@cmd, {}), MSG_EOR);
+ if ($self->{sock}) { # lei(1) client process runs it
+ # restore terminal: echo $query | lei q -stdin --mua=...
+ my $io = [];
+ $io->[0] = $self->{1} if $self->{opt}->{stdin} && -t $self->{1};
+ send_exec_cmd($self, $io, \@cmd, {});
} elsif ($self->{oneshot}) {
my $pid = fork // die "fork: $!";
if ($pid > 0) { # original process
+ if ($self->{opt}->{stdin} && -t STDOUT) {
+ open STDIN, '+<&', \*STDOUT or die "dup2: $!";
+ }
exec(@cmd);
warn "exec @cmd: $!\n";
POSIX::_exit(1);
open($self->{$i++}, '+<&=', $fd) and next;
send($sock, "open(+<&=$fd) (FD=$i): $!", MSG_EOR);
}
- return if scalar(@fds) != 4;
+ $i == 4 or return send($sock, 'not enough FDs='.($i-1), MSG_EOR)
}
- $self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
# $ENV_STR = join('', map { "\0$_=$ENV{$_}" } keys %ENV);
# $buf = "$argc\0".join("\0", @ARGV).$ENV_STR."\0\0";
substr($buf, -2, 2, '') eq "\0\0" or # s/\0\0\z//
local $quit = $exit if $exit;
local %PATH2CFG;
umask(077) // die("umask(077): $!");
- my $self = bless {
- oneshot => 1,
- 0 => *STDIN{GLOB},
- 1 => *STDOUT{GLOB},
- 2 => *STDERR{GLOB},
- env => \%ENV
- }, __PACKAGE__;
+ my $self = bless { oneshot => 1, env => \%ENV }, __PACKAGE__;
+ for (0..2) { open($self->{$_}, '+<&=', $_) or die "open fd=$_: $!" }
dispatch($self, @ARGV);
x_it($self, $self->{child_error}) if $self->{child_error};
}
$? = $err if $err; # preserve ->fail or ->x_it code
}
+sub wq_done_wait { # dwaitpid callback
+ my ($arg, $pid) = @_;
+ my ($wq, $lei) = @$arg;
+ my $err_type = $lei->{-err_type};
+ $? and $lei->child_error($?,
+ $err_type ? "$err_type errors during $lei->{cmd}" : ());
+ $lei->dclose;
+}
+
+sub wq_eof { # EOF callback for main daemon
+ my ($lei) = @_;
+ my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed
+ $wq1->wq_wait_old(\&wq_done_wait, $lei);
+}
+
1;