From: Eric Wong <e@80x24.org>
Date: Thu, 23 May 2019 09:36:45 +0000 (+0000)
Subject: xapcmd: support spawn options
X-Git-Tag: v1.2.0~255
X-Git-Url: http://www.git.stargrave.org/?a=commitdiff_plain;h=f0cc8e6795077bc7a04e579655a1c5e90c80a4e8;p=public-inbox.git

xapcmd: support spawn options

copydatabase(1) is exceptionally noisy and it's output is
confusing when run in parallel.  Support redirects at least, and
env while we're at it to give us future options.

We can also stuff a -jobs parameter into the options to limit
parallelism since it can be useful for low-priority upgrade
jobs.
---

diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 586d7e6a..999ddd1c 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -25,7 +25,8 @@ sub commit_changes ($$$) {
 }
 
 sub run {
-	my ($ibx, $cmd) = @_;
+	my ($ibx, $cmd, $env, $opt) = @_;
+	$opt ||= {};
 	my $dir = $ibx->{mainrepo} or die "no mainrepo in inbox\n";
 	which($cmd->[0]) or die "$cmd->[0] not found in PATH\n";
 	$ibx->umask_prepare;
@@ -50,13 +51,21 @@ sub run {
 		die "No Xapian parts found in $old\n" unless @cmds;
 	}
 	my $im = $ibx->importer(0);
+	my $max = $opt->{jobs} || scalar(@cmds);
 	$ibx->with_umask(sub {
 		$im->lock_acquire;
-		my %pids = map {; spawn($_) => join(' ', @$_) } @cmds;
-		while (scalar keys %pids) {
-			my $pid = waitpid(-1, 0);
-			my $desc = delete $pids{$pid};
-			die "$desc failed: $?\n" if $?;
+		my %pids;
+		while (@cmds) {
+			while (scalar(keys(%pids)) < $max && scalar(@cmds)) {
+				my $x = shift @cmds;
+				$pids{spawn($x, $env, $opt)} = $x;
+			}
+
+			while (scalar keys %pids) {
+				my $pid = waitpid(-1, 0);
+				my $x = delete $pids{$pid};
+				die join(' ', @$x)." failed: $?\n" if $?;
+			}
 		}
 		commit_changes($im, $old, $new);
 	});