githttpbackend: clamp to one smart HTTP request at-a-time

Server admins may not be able to afford to have too many
git-pack-objects processes running at once.  Since PSGI
HTTP servers should already be configured to use multiple
processes for other requests; limit concurrency of smart
backends to one; and fall back to dumb responses if we're
already generating a pack.

diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm
index c44c67d5baba41df5bf6645442b21d3baea85af0..a7cac100d02631c2d683456cd3eaa59b1caae69c 100644 (file)
--- a/lib/PublicInbox/GitHTTPBackend.pm
+++ b/lib/PublicInbox/GitHTTPBackend.pm
@@ -10,6 +10,14 @@ use Fcntl qw(:seek);
 use IO::File;
 use PublicInbox::Spawn qw(spawn);
 
+# TODO: make configurable, but keep in mind it's better to have
+# multiple -httpd worker processes which are already scaled to
+# the proper number of CPUs and memory.  git-pack-objects(1) may
+# also use threads and bust memory limits, too, so I recommend
+# limiting threads to 1 (via `pack.threads` knob in git) for serving.
+my $LIMIT = 1;
+my $nr_running = 0;
+
 # n.b. serving "description" and "cloneurl" should be innocuous enough to
 # not cause problems.  serving "config" might...
 my @text = qw[HEAD info/refs
@@ -31,6 +39,8 @@ sub r {
 
 sub serve {
        my ($cgi, $git, $path) = @_;
+       return serve_dumb($cgi, $git, $path) if $nr_running >= $LIMIT;
+
        my $service = $cgi->param('service') || '';
        if ($service =~ /\Agit-\w+-pack\z/ || $path =~ /\Agit-\w+-pack\z/) {
                my $ok = serve_smart($cgi, $git, $path);
@@ -174,6 +184,7 @@ sub serve_smart {
        $wpipe = $in = undef;
        $buf = '';
        my ($vin, $fh, $res);
+       $nr_running++;
        my $end = sub {
                if ($fh) {
                        $fh->close;
@@ -182,6 +193,7 @@ sub serve_smart {
                if ($rpipe) {
                        $rpipe->close; # _may_ be Danga::Socket::close
                        $rpipe = undef;
+                       $nr_running--;
                }
                if (defined $pid && $pid != waitpid($pid, 0)) {
                        $err->print("git http-backend ($git_dir): $?\n");

diff --git a/t/httpd.t b/t/httpd.t
index 037903117320ba9e49dfd1824c6d4326f60b8fc3..781fe03a83df3791f8d39459b5ddd0a34ef09945 100644 (file)
--- a/t/httpd.t
+++ b/t/httpd.t
@@ -104,7 +104,7 @@ EOF
 
        is(system(qw(git clone -q --mirror),
                        "http://$host:$port/$group", "$tmpdir/clone.git"),
-               0, 'clone successful');
+               0, 'smart clone successful');
 
        # ensure dumb cloning works, too:
        is(system('git', "--git-dir=$maindir",