lei q: emit progress and counting via PktOp

[public-inbox.git] / lib / PublicInbox / LEI.pm

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 22cd20f6edbbf1d0a5fdfd5731a5e7250b1a70b5..6c2515dc0eb74cf37530445e27acb44134d44702 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -14,10 +14,9 @@ use Getopt::Long ();
 use Socket qw(AF_UNIX SOCK_SEQPACKET MSG_EOR pack_sockaddr_un);
 use Errno qw(EPIPE EAGAIN EINTR ECONNREFUSED ENOENT ECONNRESET);
 use Cwd qw(getcwd);
-use POSIX ();
+use POSIX qw(strftime);
 use IO::Handle ();
 use Fcntl qw(SEEK_SET);
-use Sys::Syslog qw(syslog openlog);
 use PublicInbox::Config;
 use PublicInbox::Syscall qw(SFD_NONBLOCK EPOLLIN EPOLLET);
 use PublicInbox::Sigfd;
@@ -105,7 +104,7 @@ our %CMD = ( # sorted in order of importance/use:
 'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw(
        save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a
        sort|s=s reverse|r offset=i remote! local! external! pretty
-       mua-cmd|mua=s no-torsocks torsocks=s verbose|v
+       mua-cmd|mua=s no-torsocks torsocks=s verbose|v quiet|q
        received-after=s received-before=s sent-after=s sent-since=s),
        PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ],
 
@@ -306,7 +305,8 @@ sub qerr ($;@) { $_[0]->{opt}->{quiet} or err(shift, @_) }
 
 sub fail ($$;$) {
        my ($self, $buf, $exit_code) = @_;
-       err($self, $buf);
+       err($self, $buf) if defined $buf;
+       send($self->{pkt_op}, '!', MSG_EOR) if $self->{pkt_op}; # fail_handler
        x_it($self, ($exit_code // 1) << 8);
        undef;
 }
@@ -366,18 +366,17 @@ sub io_restore ($$) {
        }
 }
 
-# triggers sigpipe_handler
-sub note_sigpipe {
+sub note_sigpipe { # triggers sigpipe_handler
        my ($self, $fd) = @_;
        close(delete($self->{$fd})); # explicit close silences Perl warning
-       syswrite($self->{op_pipe}, '!') if $self->{op_pipe};
+       send($self->{pkt_op}, '|', MSG_EOR) if $self->{pkt_op};
        x_it($self, 13);
 }
 
 sub atfork_child_wq {
        my ($self, $wq) = @_;
        io_restore($self, $wq);
-       -p $self->{op_pipe} or die 'BUG: {op_pipe} expected';
+       -S $self->{pkt_op} or die 'BUG: {pkt_op} expected';
        io_restore($self->{l2m}, $wq);
        %PATH2CFG = ();
        undef $errors_log;
@@ -402,13 +401,14 @@ sub atfork_parent_wq {
        my ($self, $wq) = @_;
        my $env = delete $self->{env}; # env is inherited at fork
        my $lei = bless { %$self }, ref($self);
-       if (my $dedupe = delete $lei->{dedupe}) {
-               $lei->{dedupe} = $wq->deep_clone($dedupe);
+       for my $f (qw(dedupe ovv)) {
+               my $tmp = delete($lei->{$f}) or next;
+               $lei->{$f} = $wq->deep_clone($tmp);
        }
        $self->{env} = $env;
        delete @$lei{qw(3 -lei_store cfg old_1 pgr lxs)}; # keep l2m
        my @io = (delete(@$lei{qw(0 1 2)}),
-                       io_extract($lei, qw(sock op_pipe startq)));
+                       io_extract($lei, qw(sock pkt_op startq)));
        my $l2m = $lei->{l2m};
        if ($l2m && $l2m != $wq) { # $wq == lxs
                if (my $wq_s1 = $l2m->{-wq_s1}) {
@@ -824,7 +824,7 @@ sub accept_dispatch { # Listener {post_accept} callback
        $sock->autoflush(1);
        my $self = bless { sock => $sock }, __PACKAGE__;
        vec(my $rvec = '', fileno($sock), 1) = 1;
-       select($rvec, undef, undef, 1) or
+       select($rvec, undef, undef, 60) or
                return send($sock, 'timed out waiting to recv FDs', MSG_EOR);
        my @fds = $recv_cmd->($sock, my $buf, 4096 * 33); # >MAX_ARG_STRLEN
        if (scalar(@fds) == 4) {
@@ -834,7 +834,9 @@ sub accept_dispatch { # Listener {post_accept} callback
                        send($sock, "open(+<&=$fd) (FD=$i): $!", MSG_EOR);
                }
        } else {
-               return send($sock, "recv_cmd failed: $!", MSG_EOR);
+               my $msg = "recv_cmd failed: $!";
+               warn $msg;
+               return send($sock, $msg, MSG_EOR);
        }
        $self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
        # $ENV_STR = join('', map { "\0$_=$ENV{$_}" } keys %ENV);
@@ -861,7 +863,7 @@ sub dclose {
                if ($wq->wq_kill) {
                        $wq->wq_close
                } elsif ($wq->wq_kill_old) {
-                       $wq->wq_wait_old;
+                       $wq->wq_wait_old($self);
                }
        }
        close(delete $self->{1}) if $self->{1}; # may reap_compress
@@ -898,7 +900,6 @@ sub event_step {
 sub event_step_init {
        my ($self) = @_;
        if (my $sock = $self->{sock}) { # using DS->EventLoop
-               $sock->blocking(0);
                $self->SUPER::new($sock, EPOLLIN|EPOLLET);
        }
 }
@@ -1007,9 +1008,9 @@ sub lazy_start {
                                warn "$path dev/ino changed, quitting\n";
                                $path = undef;
                        }
-               } elsif (defined($path)) {
-                       warn "stat($path): $!, quitting ...\n";
-                       undef $path; # don't unlink
+               } elsif (defined($path)) { # ENOENT is common
+                       warn "stat($path): $!, quitting ...\n" if $! != ENOENT;
+                       undef $path;
                        $quit->();
                }
                return 1 if defined($path);
@@ -1029,18 +1030,14 @@ sub lazy_start {
        # STDIN was redirected to /dev/null above, closing STDERR and
        # STDOUT will cause the calling `lei' client process to finish
        # reading the <$daemon> pipe.
-       openlog($path, 'pid', 'user');
        local $SIG{__WARN__} = sub {
-               $current_lei ? err($current_lei, @_) : syslog('warning', "@_");
+               $current_lei ? err($current_lei, @_) : warn(
+                 strftime('%Y-%m-%dT%H:%M:%SZ', gmtime(time))," $$ ", @_);
        };
-       my $on_destroy = PublicInbox::OnDestroy->new($$, sub {
-               syslog('crit', "$@") if $@;
-       });
        open STDERR, '>&STDIN' or die "redirect stderr failed: $!";
        open STDOUT, '>&STDIN' or die "redirect stdout failed: $!";
        # $daemon pipe to `lei' closed, main loop begins:
        PublicInbox::DS->EventLoop;
-       @$on_destroy = (); # cancel on_destroy if we get here
        exit($exit_code // 0);
 }