http: use bigger, but shorter-lived buffers for pipes

Linux pipes default to 65536 bytes in size, and we want to read
external processes as fast as possible now that we don't use
Danga::Socket or buffer to heap.

However, drop the buffer ASAP if we need to wait on anything;
since idle buffers can be idle for eons.  This lets other
execution contexts can reuse that memory right away.

diff --git a/lib/PublicInbox/HTTPD/Async.pm b/lib/PublicInbox/HTTPD/Async.pm
index 1ffc2565a01c5902030732a351d3269a96b3612c..584db8d4ba5bd14c2e16408d5e41840fb1b1cad6 100644 (file)
--- a/lib/PublicInbox/HTTPD/Async.pm
+++ b/lib/PublicInbox/HTTPD/Async.pm
@@ -33,13 +33,13 @@ sub new {
        $self;
 }
 
-sub main_cb ($$$) {
-       my ($http, $fh, $bref) = @_;
+sub main_cb ($$) {
+       my ($http, $fh) = @_;
        sub {
                my ($self) = @_;
-               my $r = sysread($self->{sock}, $$bref, 8192);
+               my $r = sysread($self->{sock}, my $buf, 65536);
                if ($r) {
-                       $fh->write($$bref); # may call $http->close
+                       $fh->write($buf); # may call $http->close
                        if ($http->{sock}) { # !closed
                                $self->requeue;
                                # let other clients get some work done, too
@@ -64,7 +64,8 @@ sub async_pass {
        # will automatically close this ($self) object.
        $http->{forward} = $self;
        $fh->write($$bref); # PublicInbox:HTTP::{chunked,identity}_wcb
-       my $cb = $self->{cb} = main_cb($http, $fh, $bref);
+       $$bref = undef; # we're done with this
+       my $cb = $self->{cb} = main_cb($http, $fh);
        $cb->($self); # either hit EAGAIN or ->requeue to keep EPOLLET happy
 }
 

diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index f2630a0f0109a624690edf38b849423b9b0af8dd..8f0b9fe240ebc378a81ce86a7207f73d71ec4f63 100644 (file)
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -128,7 +128,7 @@ sub psgi_qx {
        my $rpipe; # comes from popen_rd
        my $async = $env->{'pi-httpd.async'};
        my $cb = sub {
-               my $r = sysread($rpipe, my $buf, 8192);
+               my $r = sysread($rpipe, my $buf, 65536);
                if ($async) {
                        $async->async_pass($env->{'psgix.io'}, $qx, \$buf);
                } elsif (defined $r) {