X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FQspawn.pm;h=aebcb1f7db091ea8d3283407a2c0cbd559d407f8;hb=740db9e58da0cb548d6c9f6534e0047a96298c47;hp=9e16123424b962e7c2fbc2fc30c6e692d99fd666;hpb=1971bbe4cd599b3d9583084145266369525bfca2;p=public-inbox.git diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm index 9e161234..aebcb1f7 100644 --- a/lib/PublicInbox/Qspawn.pm +++ b/lib/PublicInbox/Qspawn.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2019 all contributors +# Copyright (C) 2016-2020 all contributors # License: AGPL-3.0+ # Like most Perl modules in public-inbox, this is internal and @@ -27,7 +27,6 @@ package PublicInbox::Qspawn; use strict; use warnings; use PublicInbox::Spawn qw(popen_rd); -require Plack::Util; # n.b.: we get EAGAIN with public-inbox-httpd, and EINTR on other PSGI servers use Errno qw(EAGAIN EINTR); @@ -46,27 +45,25 @@ sub new ($$$;) { sub _do_spawn { my ($self, $start_cb, $limiter) = @_; my $err; - my ($cmd, $cmd_env, $opts) = @{$self->{args}}; - my %opts = %{$opts || {}}; + my ($cmd, $cmd_env, $opt) = @{delete $self->{args}}; + my %o = %{$opt || {}}; $self->{limiter} = $limiter; foreach my $k (PublicInbox::Spawn::RLIMITS()) { if (defined(my $rlimit = $limiter->{$k})) { - $opts{$k} = $rlimit; + $o{$k} = $rlimit; } } + $self->{cmd} = $o{quiet} ? undef : $cmd; + eval { + # popen_rd may die on EMFILE, ENFILE + ($self->{rpipe}, $self->{pid}) = popen_rd($cmd, $cmd_env, \%o); - ($self->{rpipe}, $self->{pid}) = popen_rd($cmd, $cmd_env, \%opts); + die "E: $!" unless defined($self->{pid}); - # drop any IO handles opt was holding open via $opt->{hold} - # No need to hold onto the descriptor once the child process has it. - $self->{args} = $cmd; # keep this around for logging - - if (defined $self->{pid}) { $limiter->{running}++; - } else { - $self->{err} = $!; - } - $start_cb->($self); + $start_cb->($self); # EPOLL_CTL_ADD may ENOSPC/ENOMEM + }; + finish($self, $@) if $@; } sub child_err ($) { @@ -83,16 +80,8 @@ sub log_err ($$) { $env->{'psgi.errors'}->print($msg, "\n"); } -# callback for dwaitpid -sub waitpid_err ($$) { - my ($self, $pid) = @_; - my $xpid = delete $self->{pid}; - my $err; - if ($pid > 0) { # success! - $err = child_err($?); - } elsif ($pid < 0) { # ??? does this happen in our case? - $err = "W: waitpid($xpid, 0) => $pid: $!"; - } # else should not be called with pid == 0 +sub finalize ($$) { + my ($self, $err) = @_; my ($env, $qx_cb, $qx_arg, $qx_buf) = delete @$self{qw(psgi_env qx_cb qx_arg qx_buf)}; @@ -108,12 +97,37 @@ sub waitpid_err ($$) { } if ($err) { - $self->{err} = $err; - if ($env && !$env->{'qspawn.quiet'}) { - log_err($env, join(' ', @{$self->{args}}) . ": $err"); + if (defined $self->{err}) { + $self->{err} .= "; $err"; + } else { + $self->{err} = $err; } + if ($env && $self->{cmd}) { + log_err($env, join(' ', @{$self->{cmd}}) . ": $err"); + } + } + if ($qx_cb) { + eval { $qx_cb->($qx_buf, $qx_arg) }; + } elsif (my $wcb = delete $env->{'qspawn.wcb'}) { + # have we started writing, yet? + require PublicInbox::WwwStatic; + $wcb->(PublicInbox::WwwStatic::r(500)); } - eval { $qx_cb->($qx_buf, $qx_arg) } if $qx_cb; +} + +# callback for dwaitpid +sub waitpid_err ($$) { + my ($self, $pid) = @_; + my $xpid = delete $self->{pid}; + my $err; + if (defined $pid) { + if ($pid > 0) { # success! + $err = child_err($?); + } elsif ($pid < 0) { # ??? does this happen in our case? + $err = "W: waitpid($xpid, 0) => $pid: $!"; + } # else should not be called with pid == 0 + } + finalize($self, $err); } sub do_waitpid ($) { @@ -129,14 +143,12 @@ sub do_waitpid ($) { } } -sub finish ($) { - my ($self) = @_; +sub finish ($;$) { + my ($self, $err) = @_; if (delete $self->{rpipe}) { do_waitpid($self); } else { - my ($env, $qx_cb, $qx_arg, $qx_buf) = - delete @$self{qw(psgi_env qx_cb qx_arg qx_buf)}; - eval { $qx_cb->($qx_buf, $qx_arg) } if $qx_cb; + finalize($self, $err); } } @@ -160,7 +172,7 @@ reread: $async->async_pass($self->{psgi_env}->{'psgix.io'}, $qx_fh, \$buf); } elsif (defined $r) { - $r ? $qx_fh->write($buf) : event_step($self, undef); + $r ? (print $qx_fh $buf) : event_step($self, undef); } else { return if $! == EAGAIN; # try again when notified goto reread if $! == EINTR; @@ -197,19 +209,6 @@ sub psgi_qx { start($self, $limiter, \&psgi_qx_start); } -# create a filter for "push"-based streaming PSGI writes used by HTTPD::Async -sub filter_fh ($$) { - my ($fh, $filter) = @_; - Plack::Util::inline_object( - close => sub { - $fh->write($filter->(undef)); - $fh->close; - }, - write => sub { - $fh->write($filter->($_[0])); - }); -} - # this is called on pipe EOF to reap the process, may be called # via PublicInbox::DS event loop OR via GetlineBody for generic # PSGI servers. @@ -229,12 +228,17 @@ sub rd_hdr ($) { my $ret; my $total_rd = 0; my $hdr_buf = $self->{hdr_buf}; + my ($ph_cb, $ph_arg) = @{$self->{parse_hdr}}; do { my $r = sysread($self->{rpipe}, $$hdr_buf, 4096, length($$hdr_buf)); if (defined($r)) { $total_rd += $r; - $ret = $self->{parse_hdr}->($total_rd, $hdr_buf); + eval { $ret = $ph_cb->($total_rd, $hdr_buf, $ph_arg) }; + if ($@) { + log_err($self->{psgi_env}, "parse_hdr: $@"); + $ret = [ 500, [], [ "Internal error\n" ] ]; + } } else { # caller should notify us when it's ready: return if $! == EAGAIN; @@ -266,7 +270,7 @@ sub psgi_return_init_cb { } elsif ($async) { # done reading headers, handoff to read body my $fh = $wcb->($r); # scalar @$r == 2 - $fh = filter_fh($fh, $filter) if $filter; + $fh = $filter->attach($fh) if $filter; $self->{fh} = $fh; $async->async_pass($env->{'psgix.io'}, $fh, delete($self->{hdr_buf})); @@ -304,8 +308,9 @@ sub psgi_return_start { # may run later, much later... # psgi_return will return an anonymous # sub for the PSGI server to call # -# $env->{'qspawn.filter'} - filter callback, receives a string as input, -# undef on EOF +# $env->{'qspawn.filter'} - filter object, responds to ->attach for +# pi-httpd.async and ->translate for generic +# PSGI servers # # $limiter - the Limiter object to use (uses the def_limiter if not given) # @@ -317,10 +322,10 @@ sub psgi_return_start { # may run later, much later... # psgix.io. 3-element arrays means the body is available # immediately (or streamed via ->getline (pull-based)). sub psgi_return { - my ($self, $env, $limiter, $parse_hdr) = @_; + my ($self, $env, $limiter, $parse_hdr, $hdr_arg) = @_; $self->{psgi_env} = $env; $self->{hdr_buf} = \(my $hdr_buf = ''); - $self->{parse_hdr} = $parse_hdr; + $self->{parse_hdr} = [ $parse_hdr, $hdr_arg ]; $limiter ||= $def_limiter ||= PublicInbox::Qspawn::Limiter->new(32); # the caller already captured the PSGI write callback from