qspawn|getlinebody: support streaming filters

[public-inbox.git] / lib / PublicInbox / Qspawn.pm

diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index 4950da25f0012aecac5a34e5256926e858ada1df..3247cd078cba71aaaa058798cff4840309ca7932 100644 (file)
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Limits the number of processes spawned
@@ -9,6 +9,8 @@ package PublicInbox::Qspawn;
 use strict;
 use warnings;
 use PublicInbox::Spawn qw(popen_rd);
+require Plack::Util;
+my $def_limiter;
 
 sub new ($$$;) {
        my ($class, $cmd, $env, $opt) = @_;
@@ -59,6 +61,79 @@ sub start {
        }
 }
 
+# create a filter for "push"-based streaming PSGI writes used by HTTPD::Async
+sub filter_fh ($$) {
+       my ($fh, $filter) = @_;
+       Plack::Util::inline_object(
+               close => sub {
+                       $fh->write($filter->(undef));
+                       $fh->close;
+               },
+               write => sub {
+                       $fh->write($filter->($_[0]));
+               });
+}
+
+sub psgi_return {
+       my ($self, $env, $limiter, $parse_hdr) = @_;
+       my ($fh, $rpipe);
+       my $end = sub {
+               my $err = $self->finish;
+               if ($err && !$env->{'qspawn.quiet'}) {
+                       $err = join(' ', @{$self->{args}->[0]}).": $err\n";
+                       $env->{'psgi.errors'}->print($err);
+               }
+               $fh->close if $fh; # async-only
+       };
+
+       # Danga::Socket users, we queue up the read_enable callback to
+       # fire after pending writes are complete:
+       my $buf = '';
+       my $rd_hdr = sub {
+               my $r = sysread($rpipe, $buf, 1024, length($buf));
+               return if !defined($r) && ($!{EINTR} || $!{EAGAIN});
+               $parse_hdr->($r, \$buf);
+       };
+       my $res;
+       my $async = $env->{'pi-httpd.async'};
+       my $cb = sub {
+               my $r = $rd_hdr->() or return;
+               $rd_hdr = undef;
+               my $filter = delete $env->{'qspawn.filter'};
+               if (scalar(@$r) == 3) { # error
+                       if ($async) {
+                               $async->close; # calls rpipe->close
+                       } else {
+                               $rpipe->close;
+                               $end->();
+                       }
+                       $res->($r);
+               } elsif ($async) {
+                       $fh = $res->($r); # scalar @$r == 2
+                       $fh = filter_fh($fh, $filter) if $filter;
+                       $async->async_pass($env->{'psgix.io'}, $fh, \$buf);
+               } else { # for synchronous PSGI servers
+                       require PublicInbox::GetlineBody;
+                       $r->[2] = PublicInbox::GetlineBody->new($rpipe, $end,
+                                                               $buf, $filter);
+                       $res->($r);
+               }
+       };
+       $limiter ||= $def_limiter ||= PublicInbox::Qspawn::Limiter->new(32);
+       sub {
+               ($res) = @_;
+               $self->start($limiter, sub { # may run later, much later...
+                       ($rpipe) = @_;
+                       if ($async) {
+                       # PublicInbox::HTTPD::Async->new($rpipe, $cb, $end)
+                               $async = $async->($rpipe, $cb, $end);
+                       } else { # generic PSGI
+                               $cb->() while $rd_hdr;
+                       }
+               });
+       };
+}
+
 package PublicInbox::Qspawn::Limiter;
 use strict;
 use warnings;