In case there's unbalanced shards AND we're limiting parallelism
while using many shards, spawn the next task in the queue ASAP
once a task is done, instead of waiting for all tasks to finish
before spawning the next batch.
Unbalanced shards probably isn't a big issue for most users;
however many smaller shards with few jobs can be useful for HDD
users to reduce the effect of random writes.
use File::Temp 0.19 (); # ->newdir
use File::Path qw(remove_tree);
use File::Basename qw(dirname);
use File::Temp 0.19 (); # ->newdir
use File::Path qw(remove_tree);
use File::Basename qw(dirname);
# support testing with dev versions of Xapian which installs
# commands with a version number suffix (e.g. "xapian-compact-1.5")
# support testing with dev versions of Xapian which installs
# commands with a version number suffix (e.g. "xapian-compact-1.5")
$pids{cb_spawn($cb, $args, $opt)} = $args;
}
$pids{cb_spawn($cb, $args, $opt)} = $args;
}
while (scalar keys %pids) {
while (scalar keys %pids) {
- my $pid = waitpid(-1, 0);
+ my $pid = waitpid(-1, $flags) or last;
+ last if $pid < 0;
my $args = delete $pids{$pid};
if ($args) {
die join(' ', @$args)." failed: $?\n" if $?;
} else {
warn "unknown PID($pid) reaped: $?\n";
}
my $args = delete $pids{$pid};
if ($args) {
die join(' ', @$args)." failed: $?\n" if $?;
} else {
warn "unknown PID($pid) reaped: $?\n";
}
+ $flags = WNOHANG if scalar(@$queue);