X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FQspawn.pm;h=da7cd74fe8cb1628f6884b36d29b61c9c230ea7e;hb=372d6e03debe13cab1ed00a3a59e834566c7a709;hp=31a1583dfebcef7d6f9a25749493f63fce563f9a;hpb=79585642a30f54622541496a9a0090f0aa591692;p=public-inbox.git diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm index 31a1583d..da7cd74f 100644 --- a/lib/PublicInbox/Qspawn.pm +++ b/lib/PublicInbox/Qspawn.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2019 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # Like most Perl modules in public-inbox, this is internal and @@ -12,12 +12,13 @@ # operate in. This can be useful to ensure smaller inboxes can # be cloned while cloning of large inboxes is maxed out. # -# This does not depend on PublicInbox::DS or any other external -# scheduling mechanism, you just need to call start() and finish() -# appropriately. However, public-inbox-httpd (which uses PublicInbox::DS) -# will be able to schedule this based on readability of stdout from -# the spawned process. See GitHTTPBackend.pm and SolverGit.pm for -# usage examples. It does not depend on any form of threading. +# This does not depend on the PublicInbox::DS::event_loop or any +# other external scheduling mechanism, you just need to call +# start() and finish() appropriately. However, public-inbox-httpd +# (which uses PublicInbox::DS) will be able to schedule this +# based on readability of stdout from the spawned process. +# See GitHTTPBackend.pm and SolverGit.pm for usage examples. +# It does not depend on any form of threading. # # This is useful for scheduling CGI execution of both long-lived # git-http-backend(1) process (for "git clone") as well as short-lived @@ -25,8 +26,9 @@ package PublicInbox::Qspawn; use strict; -use warnings; +use v5.10.1; use PublicInbox::Spawn qw(popen_rd); +use PublicInbox::GzipFilter; # n.b.: we get EAGAIN with public-inbox-httpd, and EINTR on other PSGI servers use Errno qw(EAGAIN EINTR); @@ -37,6 +39,7 @@ my $def_limiter; # $cmd is the command to spawn # $cmd_env is the environ for the child process (not PSGI env) # $opt can include redirects and perhaps other process spawning options +# {qsp_err} is an optional error buffer callers may access themselves sub new ($$$;) { my ($class, $cmd, $cmd_env, $opt) = @_; bless { args => [ $cmd, $cmd_env, $opt ] }, $class; @@ -45,28 +48,25 @@ sub new ($$$;) { sub _do_spawn { my ($self, $start_cb, $limiter) = @_; my $err; - my ($cmd, $cmd_env, $opt) = @{$self->{args}}; + my ($cmd, $cmd_env, $opt) = @{delete $self->{args}}; my %o = %{$opt || {}}; $self->{limiter} = $limiter; - foreach my $k (PublicInbox::Spawn::RLIMITS()) { + foreach my $k (@PublicInbox::Spawn::RLIMITS) { if (defined(my $rlimit = $limiter->{$k})) { $o{$k} = $rlimit; } } + $self->{cmd} = $o{quiet} ? undef : $cmd; eval { # popen_rd may die on EMFILE, ENFILE - ($self->{rpipe}, $self->{pid}) = popen_rd($cmd, $cmd_env, \%o); - $self->{args} = $o{quiet} ? undef : $cmd; + $self->{rpipe} = popen_rd($cmd, $cmd_env, \%o); - die "E: $!" unless defined($self->{pid}); + die "E: $!" unless defined($self->{rpipe}); $limiter->{running}++; $start_cb->($self); # EPOLL_CTL_ADD may ENOSPC/ENOMEM }; - if ($@) { - $self->{err} = $@; - finish($self); - } + finish($self, $@) if $@; } sub child_err ($) { @@ -78,21 +78,8 @@ sub child_err ($) { $msg; } -sub log_err ($$) { - my ($env, $msg) = @_; - $env->{'psgi.errors'}->print($msg, "\n"); -} - -# callback for dwaitpid -sub waitpid_err ($$) { - my ($self, $pid) = @_; - my $xpid = delete $self->{pid}; - my $err; - if ($pid > 0) { # success! - $err = child_err($?); - } elsif ($pid < 0) { # ??? does this happen in our case? - $err = "W: waitpid($xpid, 0) => $pid: $!"; - } # else should not be called with pid == 0 +sub finalize ($$) { + my ($self, $err) = @_; my ($env, $qx_cb, $qx_arg, $qx_buf) = delete @$self{qw(psgi_env qx_cb qx_arg qx_buf)}; @@ -108,40 +95,32 @@ sub waitpid_err ($$) { } if ($err) { - if ($self->{err}) { - $self->{err} .= "; $err"; - } else { - $self->{err} = $err; - } - if ($env && $self->{args}) { - log_err($env, join(' ', @{$self->{args}}) . ": $err"); + utf8::decode($err); + if (my $dst = $self->{qsp_err}) { + $$dst .= $$dst ? " $err" : "; $err"; } + warn "@{$self->{cmd}}: $err" if $self->{cmd}; } - eval { $qx_cb->($qx_buf, $qx_arg) } if $qx_cb; -} - -sub do_waitpid ($) { - my ($self) = @_; - my $pid = $self->{pid}; - # PublicInbox::DS may not be loaded - eval { PublicInbox::DS::dwaitpid($pid, \&waitpid_err, $self) }; - # done if we're running in PublicInbox::DS::EventLoop - if ($@) { - # non public-inbox-{httpd,nntpd} callers may block: - my $ret = waitpid($pid, 0); - waitpid_err($self, $ret); + if ($qx_cb) { + eval { $qx_cb->($qx_buf, $qx_arg) }; + return unless $@; + warn "E: $@"; # hope qspawn.wcb can handle it + } + if (my $wcb = delete $env->{'qspawn.wcb'}) { + # have we started writing, yet? + require PublicInbox::WwwStatic; + $wcb->(PublicInbox::WwwStatic::r(500)); } } -sub finish ($) { - my ($self) = @_; - if (delete $self->{rpipe}) { - do_waitpid($self); - } else { - my ($env, $qx_cb, $qx_arg, $qx_buf) = - delete @$self{qw(psgi_env qx_cb qx_arg qx_buf)}; - eval { $qx_cb->($qx_buf, $qx_arg) } if $qx_cb; - } +# callback for dwaitpid or ProcessPipe +sub waitpid_err { finalize($_[0], child_err($?)) } + +sub finish ($;$) { + my ($self, $err) = @_; + my $tied_pp = delete($self->{rpipe}) or return finalize($self, $err); + my PublicInbox::ProcessPipe $pp = tied *$tied_pp; + @$pp{qw(cb arg)} = (\&waitpid_err, $self); # for ->DESTROY } sub start ($$$) { @@ -155,7 +134,7 @@ sub start ($$$) { sub psgi_qx_init_cb { my ($self) = @_; - my $async = delete $self->{async}; + my $async = delete $self->{async}; # PublicInbox::HTTPD::Async my ($r, $buf); my $qx_fh = $self->{qx_fh}; reread: @@ -164,7 +143,7 @@ reread: $async->async_pass($self->{psgi_env}->{'psgix.io'}, $qx_fh, \$buf); } elsif (defined $r) { - $r ? $qx_fh->write($buf) : event_step($self, undef); + $r ? (print $qx_fh $buf) : event_step($self, undef); } else { return if $! == EAGAIN; # try again when notified goto reread if $! == EINTR; @@ -206,7 +185,7 @@ sub psgi_qx { # PSGI servers. sub event_step { my ($self, $err) = @_; # $err: $! - log_err($self->{psgi_env}, "psgi_{return,qx} $err") if defined($err); + warn "psgi_{return,qx} $err" if defined($err); finish($self); my ($fh, $qx_fh) = delete(@$self{qw(fh qx_fh)}); $fh->close if $fh; # async-only (psgi_return) @@ -215,7 +194,6 @@ sub event_step { sub rd_hdr ($) { my ($self) = @_; # typically used for reading CGI headers - # we must loop until EAGAIN for EPOLLET in HTTPD/Async.pm # We also need to check EINTR for generic PSGI servers. my $ret; my $total_rd = 0; @@ -226,12 +204,16 @@ sub rd_hdr ($) { length($$hdr_buf)); if (defined($r)) { $total_rd += $r; - $ret = $ph_cb->($total_rd, $hdr_buf, $ph_arg); + eval { $ret = $ph_cb->($total_rd, $hdr_buf, $ph_arg) }; + if ($@) { + warn "parse_hdr: $@"; + $ret = [ 500, [], [ "Internal error\n" ] ]; + } } else { # caller should notify us when it's ready: return if $! == EAGAIN; next if $! == EINTR; # immediate retry - log_err($self->{psgi_env}, "error reading header: $!"); + warn "error reading header: $!"; $ret = [ 500, [], [ "Internal error\n" ] ]; } } until (defined $ret); @@ -243,41 +225,40 @@ sub psgi_return_init_cb { my ($self) = @_; my $r = rd_hdr($self) or return; my $env = $self->{psgi_env}; + my $filter = delete($env->{'qspawn.filter'}) // (ref($r) eq 'ARRAY' ? + PublicInbox::GzipFilter::qsp_maybe($r->[1], $env) : undef); + my $wcb = delete $env->{'qspawn.wcb'}; - my $async = delete $self->{async}; - if (scalar(@$r) == 3) { # error - if ($async) { - # calls rpipe->close && ->event_step - $async->close; + my $async = delete $self->{async}; # PublicInbox::HTTPD::Async + if (ref($r) ne 'ARRAY' || scalar(@$r) == 3) { # error + if ($async) { # calls rpipe->close && ->event_step + $async->close; # PublicInbox::HTTPD::Async::close } else { $self->{rpipe}->close; event_step($self); } - $wcb->($r); + $wcb->($r) if ref($r) eq 'ARRAY'; } elsif ($async) { # done reading headers, handoff to read body my $fh = $wcb->($r); # scalar @$r == 2 + $fh = $filter->attach($fh) if $filter; $self->{fh} = $fh; $async->async_pass($env->{'psgix.io'}, $fh, delete($self->{hdr_buf})); } else { # for synchronous PSGI servers require PublicInbox::GetlineBody; + my $buf = delete $self->{hdr_buf}; $r->[2] = PublicInbox::GetlineBody->new($self->{rpipe}, - \&event_step, $self, - ${$self->{hdr_buf}}); + \&event_step, $self, $$buf, $filter); $wcb->($r); } - - # Workaround a leak under Perl 5.16.3 when combined with - # Plack::Middleware::Deflater: - $wcb = undef; } sub psgi_return_start { # may run later, much later... my ($self) = @_; - if (my $async = $self->{psgi_env}->{'pi-httpd.async'}) { + if (my $cb = $self->{psgi_env}->{'pi-httpd.async'}) { # PublicInbox::HTTPD::Async->new(rpipe, $cb, $cb_arg, $end_obj) - $self->{async} = $async->($self->{rpipe}, + $self->{async} = $cb->($self->{rpipe}, \&psgi_return_init_cb, $self, $self); } else { # generic PSGI psgi_return_init_cb($self) while $self->{parse_hdr}; @@ -294,6 +275,10 @@ sub psgi_return_start { # may run later, much later... # psgi_return will return an anonymous # sub for the PSGI server to call # +# $env->{'qspawn.filter'} - filter object, responds to ->attach for +# pi-httpd.async and ->translate for generic +# PSGI servers +# # $limiter - the Limiter object to use (uses the def_limiter if not given) # # $parse_hdr - Initial read function; often for parsing CGI header output. @@ -343,12 +328,12 @@ sub new { } sub setup_rlimit { - my ($self, $name, $config) = @_; - foreach my $rlim (PublicInbox::Spawn::RLIMITS()) { + my ($self, $name, $cfg) = @_; + foreach my $rlim (@PublicInbox::Spawn::RLIMITS) { my $k = lc($rlim); $k =~ tr/_//d; $k = "publicinboxlimiter.$name.$k"; - defined(my $v = $config->{$k}) or next; + defined(my $v = $cfg->{$k}) or next; my @rlimit = split(/\s*,\s*/, $v); if (scalar(@rlimit) == 1) { push @rlimit, $rlimit[0];