2 # Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
3 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
5 # Parallel test runner which preloads code and reuses worker processes
6 # to give a nice speedup over prove(1). It also generates per-test
7 # .log files (similar to automake tests).
9 # *.t files run by this should not rely on global state.
11 # Usage: $PERL -I lib -w t/run.perl -j4
12 # Or via prove(1): prove -lvw t/run.perl :: -j4
15 use IO::Handle; # ->autoflush
16 use PublicInbox::TestCommon;
17 use PublicInbox::Spawn;
18 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
21 use POSIX qw(_POSIX_PIPE_BUF WNOHANG);
26 our $log_suffix = '.log';
27 my ($shuffle, %pids, @err);
28 GetOptions('j|jobs=i' => \$jobs,
29 'repeat=i' => \$repeat,
30 'log=s' => \$log_suffix,
31 's|shuffle' => \$shuffle,
32 ) or die "Usage: $0 [-j JOBS] [--log=SUFFIX] [--repeat RUNS]";
33 if (($ENV{TEST_RUN_MODE} // 2) == 0) {
34 die "$0 is not compatible with TEST_RUN_MODE=0\n";
36 my @tests = scalar(@ARGV) ? @ARGV : glob('t/*.t');
37 open my $cwd_fh, '<', '.' or die "open .: $!";
38 open my $OLDOUT, '>&STDOUT' or die "dup STDOUT: $!";
39 open my $OLDERR, '>&STDERR' or die "dup STDERR: $!";
40 $OLDOUT->autoflush(1);
41 $OLDERR->autoflush(1);
43 my ($run_log, $tmp_rl);
44 my $rl = $ENV{TEST_RUN_LOG};
46 $tmp_rl = File::Temp->new(CLEANUP => 1);
47 $rl = $tmp_rl->filename;
49 open $run_log, '+>>', $rl or die "open $rl: $!";
50 $run_log->autoflush(1); # one reader, many writers
52 key2sub($_) for @tests; # precache
54 my ($for_destroy, $lei_env, $lei_daemon_pid, $owner_pid);
56 # I get ECONNRESET from lei on FreeBSD 11.3 even with
57 # kern.ipc.soacceptqueue=1073741823, so persistent lei-daemon in tests
58 # is Linux-only for now:
59 if ($^O eq 'linux' && !$ENV{TEST_LEI_DAEMON_PERSIST_DIR} &&
60 (PublicInbox::Spawn->can('recv_cmd4') ||
61 eval { require Socket::MsgHdr })) {
63 ($lei_env->{XDG_RUNTIME_DIR}, $for_destroy) = tmpdir;
64 $ENV{TEST_LEI_DAEMON_PERSIST_DIR} = $lei_env->{XDG_RUNTIME_DIR};
65 run_script([qw(lei daemon-pid)], $lei_env, { 1 => \$lei_daemon_pid });
66 chomp $lei_daemon_pid;
67 $lei_daemon_pid =~ /\A[0-9]+\z/ or die "no daemon pid: $lei_daemon_pid";
68 kill(0, $lei_daemon_pid) or die "kill $lei_daemon_pid: $!";
69 if (my $t = $ENV{GNU_TAIL}) {
70 system("$t --pid=$lei_daemon_pid -F " .
71 "$lei_env->{XDG_RUNTIME_DIR}/lei/errors.log >&2 &");
73 if (my $strace_cmd = $ENV{STRACE_CMD}) {
74 system("$strace_cmd -p $lei_daemon_pid &");
81 } elsif (open(my $prove_state, '<', '.prove') && eval { require YAML::XS }) {
82 # reuse "prove --state=save" data to start slowest tests, first
83 my $state = YAML::XS::Load(do { local $/; <$prove_state> });
84 my $t = $state->{tests};
86 ($t->{$b}->{elapsed} // 0) <=> ($t->{$a}->{elapsed} // 0)
90 our $tb = Test::More->builder;
97 our ($worker, $worker_test);
100 $? = 255 if $? == 0 && !$tb->is_passing;
101 my $status = $? ? 'not ok' : 'ok';
102 chdir($cwd_fh) or DIE "fchdir: $!";
103 if ($log_suffix ne '') {
104 my $log = $worker_test;
105 $log =~ s/\.t\z/$log_suffix/;
107 if (open my $fh, '<', $log) {
108 my @not_ok = grep(!/^(?:ok |[ \t]*#)/ms, <$fh>);
109 my $last = $not_ok[-1] // '';
110 pop @not_ok if $last =~ /^[0-9]+\.\.[0-9]+$/;
111 my $pfx = "# $log: ";
112 print $OLDERR map { $pfx.$_ } @not_ok;
113 seek($fh, 0, SEEK_SET) or die "seek: $!";
115 # show unique skip texts and the number of times
116 # each text was skipped
118 my @sk = (<$fh> =~ m/^ok [0-9]+ (# skip [^\n]+)/mgs);
121 my @err = grep { !$nr{$_}++ } @sk;
122 print $OLDERR "$pfx$_ ($nr{$_})\n" for @err;
123 $skip = ' # total skipped: '.scalar(@sk);
126 print $OLDERR "could not open: $log: $!\n";
128 print $OLDOUT "$status $worker_test$skip\n";
132 # Test::Builder or Test2::Hub may call exit() from plan(skip_all => ...)
133 END { test_status() if (defined($worker_test) && $worker == $$) }
137 syswrite($run_log, "$$ $test\n");
139 if ($log_suffix ne '') {
141 $log =~ s/\.[^\.]+\z/$log_suffix/ or DIE "can't log for $test";
142 open $log_fh, '>', $log or DIE "open $log: $!";
143 $log_fh->autoflush(1);
144 $tb->output($log_fh);
145 $tb->failure_output($log_fh);
146 $tb->todo_output($log_fh);
147 open STDOUT, '>&', $log_fh or DIE "1>$log: $!";
148 open STDERR, '>&', $log_fh or DIE "2>$log: $!";
150 $worker_test = $test;
153 $worker_test = undef;
154 push @err, "$test ($?)" if $?;
157 sub UINT_SIZE () { 4 }
159 # worker processes will SIGUSR1 the producer process when it
160 # sees EOF on the pipe. On FreeBSD 11.2 and Perl 5.30.0,
161 # sys/ioctl.ph gives the wrong value for FIONREAD().
163 my $eof; # we stop respawning if true
165 my $start_worker = sub {
166 my ($j, $rd, $wr, $todo) = @_;
167 my $pid = fork // DIE "fork: $!";
172 my $r = sysread($rd, my $buf, UINT_SIZE);
178 DIE "short read $r" if $r != UINT_SIZE;
179 my $t = unpack('I', $buf);
180 run_test($todo->[$t]);
183 kill 'USR1', $producer if !$eof; # sets $eof in $producer
184 DIE join('', map { "E: $_\n" } @err) if @err;
191 # negative $repeat means loop forever:
192 for (my $i = $repeat; $i != 0; $i--) {
193 my @todo = $shuffle ? List::Util::shuffle(@tests) : @tests;
195 # single-producer, multi-consumer queue relying on POSIX pipe semantics
196 # POSIX.1-2008 stipulates a regular file should work, but Linux <3.14
197 # had broken read(2) semantics according to the read(2) manpage
198 pipe(my ($rd, $wr)) or DIE "pipe: $!";
200 # fill the queue before forking so children can start earlier
201 my $n = (_POSIX_PIPE_BUF / UINT_SIZE);
203 print $wr join('', map { pack('I', $_) } (0..$#todo)) or DIE;
205 } else { # write what we can...
207 print $wr join('', map { pack('I', $_) } (0..$n)) or DIE;
208 $n += 1; # and send more ($n..$#todo), later
211 local $SIG{USR1} = sub { $eof = 1 };
214 my $flags = $sig ? WNOHANG : 0;
216 my $pid = waitpid(-1, $flags) or return;
218 my $j = delete $pids{$pid};
220 push @err, "reaped unknown $pid ($?)";
224 seek($run_log, 0, SEEK_SET);
225 chomp(my @t = grep(/^$pid /, <$run_log>));
226 $t[0] //= "$pid unknown";
227 push @err, "job[$j] ($?) PID=$t[-1]";
229 # skip_all can exit(0), respawn if needed:
231 print $OLDERR "# respawning job[$j]\n";
232 $start_worker->($j, $rd, $wr, \@todo);
237 # start the workers to consume the queue
238 for (my $j = 0; $j < $jobs; $j++) {
239 $start_worker->($j, $rd, $wr, \@todo);
242 local $SIG{CHLD} = $sigchld;
243 # too many tests to fit in the pipe before starting workers,
244 # send the rest now the workers are running
245 print $wr join('', map { pack('I', $_) } ($n..$#todo)) or DIE;
249 $sigchld->(0) while scalar(keys(%pids));
250 DIE join('', map { "E: $_\n" } @err) if @err;
253 print $OLDOUT "1..".($repeat * scalar(@tests))."\n" if $repeat >= 0;
254 if ($lei_env && $$ == $owner_pid) {
255 my $opt = { 1 => $OLDOUT, 2 => $OLDERR };
256 run_script([qw(lei daemon-kill)], $lei_env, $opt);