X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FQspawn.pm;h=5e4fd5cb8541a2a8202d188ae1fd73c88fe64188;hb=686fb938a2a27beca406af6e8ab819aad608b06e;hp=53d0ad55ee84d47ccc10bd2d7516136fd38207e8;hpb=bb8a6a435a4b7707e9fb0329c49671f495262016;p=public-inbox.git diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm index 53d0ad55..5e4fd5cb 100644 --- a/lib/PublicInbox/Qspawn.pm +++ b/lib/PublicInbox/Qspawn.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2021 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # Like most Perl modules in public-inbox, this is internal and @@ -25,9 +25,10 @@ # processes such as git-apply(1). package PublicInbox::Qspawn; -use strict; +use v5.12; use PublicInbox::Spawn qw(popen_rd); use PublicInbox::GzipFilter; +use Scalar::Util qw(blessed); # n.b.: we get EAGAIN with public-inbox-httpd, and EINTR on other PSGI servers use Errno qw(EAGAIN EINTR); @@ -38,7 +39,8 @@ my $def_limiter; # $cmd is the command to spawn # $cmd_env is the environ for the child process (not PSGI env) # $opt can include redirects and perhaps other process spawning options -sub new ($$$;) { +# {qsp_err} is an optional error buffer callers may access themselves +sub new { my ($class, $cmd, $cmd_env, $opt) = @_; bless { args => [ $cmd, $cmd_env, $opt ] }, $class; } @@ -54,36 +56,23 @@ sub _do_spawn { $o{$k} = $rlimit; } } - $self->{cmd} = $o{quiet} ? undef : $cmd; + $self->{cmd} = $cmd; + $self->{-quiet} = 1 if $o{quiet}; + $o{cb_arg} = [ \&waitpid_err, $self ]; eval { # popen_rd may die on EMFILE, ENFILE - $self->{rpipe} = popen_rd($cmd, $cmd_env, \%o); - - die "E: $!" unless defined($self->{rpipe}); - + $self->{rpipe} = popen_rd($cmd, $cmd_env, \%o) // die "E: $!"; $limiter->{running}++; $start_cb->($self); # EPOLL_CTL_ADD may ENOSPC/ENOMEM }; finish($self, $@) if $@; } -sub child_err ($) { - my ($child_error) = @_; # typically $? - my $exitstatus = ($child_error >> 8) or return; - my $sig = $child_error & 127; - my $msg = "exit status=$exitstatus"; - $msg .= " signal=$sig" if $sig; - $msg; -} - -sub finalize ($$) { - my ($self, $err) = @_; - - my ($env, $qx_cb, $qx_arg, $qx_buf) = - delete @$self{qw(psgi_env qx_cb qx_arg qx_buf)}; +sub finalize ($) { + my ($self) = @_; - # done, spawn whatever's in the queue - my $limiter = $self->{limiter}; + # process is done, spawn whatever's in the queue + my $limiter = delete $self->{limiter} or return; my $running = --$limiter->{running}; if ($running < $limiter->{max}) { @@ -91,34 +80,54 @@ sub finalize ($$) { _do_spawn(@$next, $limiter); } } - - if ($err) { - if (defined $self->{err}) { - $self->{err} .= "; $err"; - } else { - $self->{err} = $err; - } - if ($env && $self->{cmd}) { - warn join(' ', @{$self->{cmd}}) . ": $err"; + if (my $err = $self->{_err}) { # set by finish or waitpid_err + utf8::decode($err); + if (my $dst = $self->{qsp_err}) { + $$dst .= $$dst ? " $err" : "; $err"; } + warn "@{$self->{cmd}}: $err\n" if !$self->{-quiet}; } + + my ($env, $qx_cb, $qx_arg, $qx_buf) = + delete @$self{qw(psgi_env qx_cb qx_arg qx_buf)}; if ($qx_cb) { eval { $qx_cb->($qx_buf, $qx_arg) }; - } elsif (my $wcb = delete $env->{'qspawn.wcb'}) { + return unless $@; + warn "E: $@"; # hope qspawn.wcb can handle it + } + return if $self->{passed}; # another command chained it + if (my $wcb = delete $env->{'qspawn.wcb'}) { # have we started writing, yet? + my $code = delete $env->{'qspawn.fallback'} // 500; require PublicInbox::WwwStatic; - $wcb->(PublicInbox::WwwStatic::r(500)); + $wcb->(PublicInbox::WwwStatic::r($code)); } } -# callback for dwaitpid or ProcessPipe -sub waitpid_err { finalize($_[0], child_err($?)) } +sub DESTROY { finalize($_[0]) } # ->finalize is idempotent + +sub waitpid_err { # callback for awaitpid + my (undef, $self) = @_; # $_[0]: pid + $self->{_err} = ''; # for defined check in ->finish + if ($?) { + my $status = $? >> 8; + my $sig = $? & 127; + $self->{_err} .= "exit status=$status"; + $self->{_err} .= " signal=$sig" if $sig; + } + finalize($self) if !$self->{rpipe}; +} sub finish ($;$) { my ($self, $err) = @_; - my $tied_pp = delete($self->{rpipe}) or return finalize($self, $err); - my PublicInbox::ProcessPipe $pp = tied *$tied_pp; - @$pp{qw(cb arg)} = (\&waitpid_err, $self); # for ->DESTROY + $self->{_err} //= $err; # only for $@ + + # we can safely finalize if pipe was closed before, or if + # {_err} is defined by waitpid_err. Deleting {rpipe} will + # trigger PublicInbox::ProcessPipe::DESTROY -> waitpid_err, + # but it may not fire right away if inside the event loop. + my $closed_before = !delete($self->{rpipe}); + finalize($self) if $closed_before || defined($self->{_err}); } sub start ($$$) { @@ -130,22 +139,22 @@ sub start ($$$) { } } -sub psgi_qx_init_cb { +sub psgi_qx_init_cb { # this may be PublicInbox::HTTPD::Async {cb} my ($self) = @_; - my $async = delete $self->{async}; my ($r, $buf); - my $qx_fh = $self->{qx_fh}; reread: $r = sysread($self->{rpipe}, $buf, 65536); - if ($async) { - $async->async_pass($self->{psgi_env}->{'psgix.io'}, - $qx_fh, \$buf); - } elsif (defined $r) { - $r ? (print $qx_fh $buf) : event_step($self, undef); - } else { + if (!defined($r)) { return if $! == EAGAIN; # try again when notified goto reread if $! == EINTR; event_step($self, $!); + } elsif (my $as = delete $self->{async}) { # PublicInbox::HTTPD::Async + $as->async_pass($self->{psgi_env}->{'psgix.io'}, + $self->{qx_fh}, \$buf); + } elsif ($r) { # generic PSGI: + print { $self->{qx_fh} } $buf; + } else { # EOF + event_step($self, undef); } } @@ -185,7 +194,7 @@ sub event_step { my ($self, $err) = @_; # $err: $! warn "psgi_{return,qx} $err" if defined($err); finish($self); - my ($fh, $qx_fh) = delete(@$self{qw(fh qx_fh)}); + my ($fh, $qx_fh) = delete(@$self{qw(qfh qx_fh)}); $fh->close if $fh; # async-only (psgi_return) } @@ -197,7 +206,7 @@ sub rd_hdr ($) { my $total_rd = 0; my $hdr_buf = $self->{hdr_buf}; my ($ph_cb, $ph_arg) = @{$self->{parse_hdr}}; - do { + until (defined($ret)) { my $r = sysread($self->{rpipe}, $$hdr_buf, 4096, length($$hdr_buf)); if (defined($r)) { @@ -206,6 +215,11 @@ sub rd_hdr ($) { if ($@) { warn "parse_hdr: $@"; $ret = [ 500, [], [ "Internal error\n" ] ]; + } elsif (!defined($ret) && !$r) { + warn <{cmd}} ($self->{psgi_env}->{REQUEST_URI}) +EOM + $ret = [ 500, [], [ "Internal error\n" ] ]; } } else { # caller should notify us when it's ready: @@ -214,50 +228,61 @@ sub rd_hdr ($) { warn "error reading header: $!"; $ret = [ 500, [], [ "Internal error\n" ] ]; } - } until (defined $ret); + } delete $self->{parse_hdr}; # done parsing headers $ret; } -sub psgi_return_init_cb { +sub psgi_return_init_cb { # this may be PublicInbox::HTTPD::Async {cb} my ($self) = @_; my $r = rd_hdr($self) or return; my $env = $self->{psgi_env}; - my $filter = delete $env->{'qspawn.filter'} // - PublicInbox::GzipFilter::qsp_maybe($r->[1], $env); + my $filter; + + # this is for RepoAtom since that can fire after parse_cgi_headers + if (ref($r) eq 'ARRAY' && blessed($r->[2]) && $r->[2]->can('attach')) { + $filter = pop @$r; + } + $filter //= delete($env->{'qspawn.filter'}) // (ref($r) eq 'ARRAY' ? + PublicInbox::GzipFilter::qsp_maybe($r->[1], $env) : undef); my $wcb = delete $env->{'qspawn.wcb'}; - my $async = delete $self->{async}; - if (scalar(@$r) == 3) { # error - if ($async) { - # calls rpipe->close && ->event_step - $async->close; - } else { - $self->{rpipe}->close; + my $async = delete $self->{async}; # PublicInbox::HTTPD::Async + if (ref($r) ne 'ARRAY' || scalar(@$r) == 3) { # error + if ($async) { # calls rpipe->close && ->event_step + $async->close; # PublicInbox::HTTPD::Async::close + } else { # generic PSGI, use PublicInbox::ProcessPipe::CLOSE + delete($self->{rpipe})->close; event_step($self); } - $wcb->($r); + if (ref($r) eq 'ARRAY') { # error + $wcb->($r) + } elsif (ref($r) eq 'CODE') { # chain another command + $r->($wcb); + $self->{passed} = 1; + } + # else do nothing } elsif ($async) { # done reading headers, handoff to read body my $fh = $wcb->($r); # scalar @$r == 2 $fh = $filter->attach($fh) if $filter; - $self->{fh} = $fh; + $self->{qfh} = $fh; $async->async_pass($env->{'psgix.io'}, $fh, delete($self->{hdr_buf})); } else { # for synchronous PSGI servers require PublicInbox::GetlineBody; + my $buf = delete $self->{hdr_buf}; $r->[2] = PublicInbox::GetlineBody->new($self->{rpipe}, - \&event_step, $self, - ${$self->{hdr_buf}}, $filter); + \&event_step, $self, $$buf, $filter); $wcb->($r); } } sub psgi_return_start { # may run later, much later... my ($self) = @_; - if (my $async = $self->{psgi_env}->{'pi-httpd.async'}) { + if (my $cb = $self->{psgi_env}->{'pi-httpd.async'}) { # PublicInbox::HTTPD::Async->new(rpipe, $cb, $cb_arg, $end_obj) - $self->{async} = $async->($self->{rpipe}, + $self->{async} = $cb->($self->{rpipe}, \&psgi_return_init_cb, $self, $self); } else { # generic PSGI psgi_return_init_cb($self) while $self->{parse_hdr}; @@ -310,8 +335,7 @@ sub psgi_return { } package PublicInbox::Qspawn::Limiter; -use strict; -use warnings; +use v5.12; sub new { my ($class, $max) = @_;