X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FQspawn.pm;h=3500f8a4a86fcd35fe21e6b0fcded3ceeea7f87d;hb=3d41aa23f35501ca92aab8aa42980fa73f7fa74f;hp=9e4c8e081903ff183aec7a2a8ad4a2cd35289d2b;hpb=3f779258173530ca88f31e1dc5332f951d2c44cd;p=public-inbox.git
diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index 9e4c8e08..3500f8a4 100644
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -1,12 +1,14 @@
-# Copyright (C) 2016 all contributors
+# Copyright (C) 2016-2018 all contributors
# License: AGPL-3.0+
+
+# Limits the number of processes spawned
+# This does not depend on Danga::Socket or any other external
+# scheduling mechanism, you just need to call start and finish
+# appropriately
package PublicInbox::Qspawn;
use strict;
use warnings;
use PublicInbox::Spawn qw(popen_rd);
-our $LIMIT = 1;
-my $running = 0;
-my @run_queue;
sub new ($$$;) {
my ($class, $cmd, $env, $opt) = @_;
@@ -16,9 +18,10 @@ sub new ($$$;) {
sub _do_spawn {
my ($self, $cb) = @_;
my $err;
+
($self->{rpipe}, $self->{pid}) = popen_rd(@{$self->{args}});
- if ($self->{pid}) {
- $running++;
+ if (defined $self->{pid}) {
+ $self->{limiter}->{running}++;
} else {
$self->{err} = $!;
}
@@ -27,26 +30,47 @@ sub _do_spawn {
sub finish ($) {
my ($self) = @_;
+ my $limiter = $self->{limiter};
+ my $running;
if (delete $self->{rpipe}) {
my $pid = delete $self->{pid};
$self->{err} = $pid == waitpid($pid, 0) ? $? :
"PID:$pid still running?";
- $running--;
+ $running = --$limiter->{running};
}
- if (my $next = shift @run_queue) {
- _do_spawn(@$next);
+
+ # limiter->{max} may change dynamically
+ if (($running || $limiter->{running}) < $limiter->{max}) {
+ if (my $next = shift @{$limiter->{run_queue}}) {
+ _do_spawn(@$next);
+ }
}
$self->{err};
}
-sub start ($$) {
- my ($self, $cb) = @_;
+sub start {
+ my ($self, $limiter, $cb) = @_;
+ $self->{limiter} = $limiter;
- if ($running < $LIMIT) {
+ if ($limiter->{running} < $limiter->{max}) {
_do_spawn($self, $cb);
} else {
- push @run_queue, [ $self, $cb ];
+ push @{$limiter->{run_queue}}, [ $self, $cb ];
}
}
+package PublicInbox::Qspawn::Limiter;
+use strict;
+use warnings;
+
+sub new {
+ my ($class, $max) = @_;
+ bless {
+ # 32 is same as the git-daemon connection limit
+ max => $max || 32,
+ running => 0,
+ run_queue => [],
+ }, $class;
+}
+
1;