lib/PublicInbox/Inbox.pm
lib/PublicInbox/InboxIdle.pm
lib/PublicInbox/InboxWritable.pm
+lib/PublicInbox/InputPipe.pm
lib/PublicInbox/Isearch.pm
lib/PublicInbox/KQNotify.pm
lib/PublicInbox/LEI.pm
--- /dev/null
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# for reading pipes and sockets off the DS event loop
+package PublicInbox::InputPipe;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::DS);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+
+sub consume {
+ my ($in, $cb, @args) = @_;
+ my $self = bless { cb => $cb, sock => $in, args => \@args },__PACKAGE__;
+ if ($PublicInbox::DS::in_loop) {
+ eval { $self->SUPER::new($in, EPOLLIN|EPOLLET) };
+ return $in->blocking(0) unless $@; # regular file sets $@
+ }
+ event_step($self) while $self->{sock};
+}
+
+sub event_step {
+ my ($self) = @_;
+ my ($r, $rbuf);
+ while (($r = sysread($self->{sock}, $rbuf, 65536))) {
+ $self->{cb}->(@{$self->{args} // []}, $rbuf);
+ }
+ if (defined($r)) { # EOF
+ $self->{cb}->(@{$self->{args} // []}, '');
+ } elsif ($!{EAGAIN}) {
+ return;
+ } else {
+ $self->{cb}->(@{$self->{args} // []}, undef)
+ }
+ $self->{sock}->blocking ? delete($self->{sock}) : $self->close
+}
+
+1;
# TODO: generate shell completion + help using %CMD and %OPTDESC
# command => [ positional_args, 1-line description, Getopt::Long option spec ]
our %CMD = ( # sorted in order of importance/use:
-'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw(
+'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms', qw(
save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a
sort|s=s reverse|r offset=i remote! local! external! pretty
- include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g
+ include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g stdin|
mua-cmd|mua=s no-torsocks torsocks=s verbose|v quiet|q
received-after=s received-before=s sent-after=s sent-since=s),
PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ],
} elsif ($var =~ /\A\[-?$POS_ARG\]\z/) { # one optional arg
$i++;
} elsif ($var =~ /\A.+?\|/) { # required FOO|--stdin
+ $inf = 1 if index($var, '...') > 0;
my @or = split(/\|/, $var);
my $ok;
for my $o (@or) {
if ($o =~ /\A--([a-z0-9\-]+)/) {
$ok = defined($OPT->{$1});
- last;
+ last if $ok;
} elsif (defined($argv->[$i])) {
$ok = 1;
$i++;
my ($isatty, $seekable);
if ($dst eq '/dev/stdout') {
$isatty = -t $lei->{1};
- $lei->start_pager if $isatty;
$opt->{pretty} //= $isatty;
if (!$isatty && -f _) {
my $fl = fcntl($lei->{1}, F_GETFL, 0) //
$lxs->prepare_external($loc) unless $exclude->{$loc};
}
+sub qstr_add { # for --stdin
+ my ($self) = @_; # $_[1] = $rbuf
+ if (defined($_[1])) {
+ return eval { $self->{lxs}->do_query($self) } if $_[1] eq '';
+ $self->{mset_opt}->{qstr} .= $_[1];
+ } else {
+ $self->fail("error reading stdin: $!");
+ }
+}
+
# the main "lei q SEARCH_TERMS" method
sub lei_q {
my ($self, @argv) = @_;
my %mset_opt = map { $_ => $opt->{$_} } qw(thread limit offset);
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
$mset_opt{limit} //= 10000;
- $mset_opt{qstr} = join(' ', map {;
- # Consider spaces in argv to be for phrase search in Xapian.
- # In other words, the users should need only care about
- # normal shell quotes and not have to learn Xapian quoting.
- /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_
- } @argv);
if (defined(my $sort = $opt->{'sort'})) {
if ($sort eq 'relevance') {
$mset_opt{relevance} = 1;
# descending docid order
$mset_opt{relevance} //= -2 if $opt->{thread};
$self->{mset_opt} = \%mset_opt;
- $self->{ovv}->ovv_begin($self);
+
+ if ($opt->{stdin}) {
+ return $self->fail(<<'') if @argv;
+no query allowed on command-line with --stdin
+
+ require PublicInbox::InputPipe;
+ PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self);
+ return;
+ }
+ # Consider spaces in argv to be for phrase search in Xapian.
+ # In other words, the users should need only care about
+ # normal shell quotes and not have to learn Xapian quoting.
+ $mset_opt{qstr} = join(' ', map {;
+ /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_
+ } @argv);
$lxs->do_query($self);
}
sub do_query {
my ($self, $lei) = @_;
$lei->{1}->autoflush(1);
+ $lei->start_pager if -t $lei->{1};
+ $lei->{ovv}->ovv_begin($lei);
my ($au_done, $zpipe);
my $l2m = $lei->{l2m};
if ($l2m) {
my $pretty = $json->decode($out);
is_deeply($res, $pretty, '--pretty is identical after decode');
+ {
+ open my $fh, '+>', undef or BAIL_OUT $!;
+ $fh->autoflush(1);
+ print $fh 's:use' or BAIL_OUT $!;
+ seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
+ ok($lei->([qw(q -q --stdin)], undef, { %$opt, 0 => $fh }),
+ '--stdin on regular file works');
+ like($out, qr/use boolean prefix/, '--stdin on regular file');
+ }
+ {
+ pipe(my ($r, $w)) or BAIL_OUT $!;
+ print $w 's:use' or BAIL_OUT $!;
+ close $w or BAIL_OUT $!;
+ ok($lei->([qw(q -q --stdin)], undef, { %$opt, 0 => $r }),
+ '--stdin on pipe file works');
+ like($out, qr/use boolean prefix/, '--stdin on pipe');
+ }
+ ok(!$lei->(qw(q -q --stdin s:use)), "--stdin and argv don't mix");
+
for my $fmt (qw(ldjson ndjson jsonl)) {
$lei->('q', '-f', $fmt, 's:use boolean prefix');
is($out, $json->encode($pretty->[0])."\n", "-f $fmt");