use PublicInbox::Spawn qw(popen_rd spawn);
use PublicInbox::PktOp;
+sub do_finish_mirror { # dwaitpid callback
+ my ($arg, $pid) = @_;
+ my ($mrr, $lei) = @$arg;
+ if ($? == 0 && unlink("$mrr->{dst}/mirror.done")) {
+ $lei->add_external_finish($mrr->{dst});
+ }
+ $lei->dclose;
+}
+
sub mirror_done { # EOF callback for main daemon
my ($lei) = @_;
- my $mrr = delete $lei->{mrr};
- $mrr->wq_wait_old($lei) if $mrr;
- # FIXME: check $? before finish
- $lei->add_external_finish($mrr->{dst});
- $lei->dclose;
+ my $mrr = delete $lei->{mrr} or return;
+ $mrr->wq_wait_old(\&do_finish_mirror, $lei);
}
# for old installations without manifest.js.gz
}
sub clone_cmd {
- my ($lei) = @_;
+ my ($lei, $opt) = @_;
my @cmd = qw(git);
+ $opt->{$_} = $lei->{$_} for (0..2);
# we support "-c $key=$val" for arbitrary git config options
# e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
my $f = "$ce-$$.tmp";
open(my $fh, '+>', $f) or return $lei->err("open $f: $! (non-fatal)");
my $opt = { 0 => $lei->{0}, 1 => $fh, 2 => $lei->{2} };
- $lei->qerr("# @$cmd");
- my $pid = spawn($cmd, $lei->{env}, $opt);
- waitpid($pid, 0) == $pid or return $lei->err("waitpid @$cmd: $!");
- if (($? >> 8) == 22) { # 404 missing
+ my $cerr = run_reap($lei, $cmd, $opt) // return;
+ if (($cerr >> 8) == 22) { # 404 missing
unlink($f) if -s $fh == 0;
return;
}
- return $lei->err("# @$cmd failed (non-fatal)") if $?;
+ return $lei->err("# @$cmd failed (non-fatal)") if $cerr;
rename($f, $ce) or return $lei->err("link($f, $ce): $! (non-fatal)");
my $cfg = PublicInbox::Config::git_config_dump($f);
my $ibx = $self->{ibx} = {};
local %ENV = (%ENV, %$env) if $env;
PublicInbox::Admin::progress_prepare($opt, $lei->{2});
PublicInbox::Admin::index_inbox($ibx, undef, $opt);
+ open my $x, '>', "$self->{dst}/mirror.done"; # for do_finish_mirror
+}
+
+sub run_reap {
+ my ($lei, $cmd, $opt) = @_;
+ $lei->qerr("# @$cmd");
+ $opt->{pgid} = 0;
+ my $pid = spawn($cmd, $lei->{env}, $opt);
+ my $reap = PublicInbox::OnDestroy->new($lei->can('sigint_reap'), $pid);
+ my $err = waitpid($pid, 0) == $pid ? undef : "waitpid @$cmd: $!";
+ @$reap = (); # cancel reap
+ $err ? $lei->err($err) : $?
}
sub clone_v1 {
my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
my $uri = URI->new($self->{src});
my $pfx = $curl->torsocks($lei, $uri) or return;
- my $cmd = [ @$pfx, clone_cmd($lei), $uri->as_string, $self->{dst} ];
- $lei->qerr("# @$cmd");
- my $pid = spawn($cmd, $lei->{env}, $lei);
- waitpid($pid, 0) == $pid or die "BUG: waitpid @$cmd: $!";
- $? == 0 or return $lei->child_error($?, "@$cmd failed");
+ my $cmd = [ @$pfx, clone_cmd($lei, my $opt = {}),
+ $uri->as_string, $self->{dst} ];
+ my $cerr = run_reap($lei, $cmd, $opt) // return;
+ return $lei->child_error($cerr, "@$cmd failed") if $cerr;
_try_config($self);
index_cloned_inbox($self, 1);
}
my $lk = bless { lock_path => "$dst/inbox.lock" }, 'PublicInbox::Lock';
_try_config($self);
my $on_destroy = $lk->lock_for_scope($$);
- my @cmd = clone_cmd($lei);
+ my @cmd = clone_cmd($lei, my $opt = {});
while (my $pair = shift(@src_edst)) {
my $cmd = [ @$pfx, @cmd, @$pair ];
- $lei->qerr("# @$cmd");
- my $pid = spawn($cmd, $lei->{env}, $lei);
- waitpid($pid, 0) == $pid or die "BUG: waitpid @$cmd: $!";
- $? == 0 or return $lei->child_error($?, "@$cmd failed");
+ my $cerr = run_reap($lei, $cmd, $opt) // return;
+ return $lei->child_error($cerr, "@$cmd failed") if $cerr;
}
undef $on_destroy; # unlock
index_cloned_inbox($self, 2);
my $cmd = $curl->for_uri($lei, $uri);
$lei->qerr("# @$cmd");
my $opt = { 0 => $lei->{0}, 2 => $lei->{2} };
- my $fh = popen_rd($cmd, $lei->{env}, $opt);
+ my ($fh, $pid) = popen_rd($cmd, $lei->{env}, $opt);
+ my $reap = PublicInbox::OnDestroy->new($lei->can('sigint_reap'), $pid);
my $gz = do { local $/; <$fh> } // die "read(curl $uri): $!";
- unless (close $fh) {
+ close $fh;
+ my $err = waitpid($pid, 0) == $pid ? undef : "waitpid @$cmd: $!";
+ @$reap = ();
+ return $lei->err($err) if $err;
+ if ($?) {
return try_scrape($self) if ($? >> 8) == 22; # 404 missing
return $lei->child_error($?, "@$cmd failed");
}
sub ipc_atfork_child {
my ($self) = @_;
$self->{lei}->lei_atfork_child;
+ $SIG{TERM} = sub { exit(128 + 15) }; # trigger OnDestroy $reap
$self->SUPER::ipc_atfork_child;
}
$each_smsg->($smsg, undef, $eml);
}
-# PublicInbox::OnDestroy callback
-sub kill_reap {
- my ($pid) = @_;
- kill('KILL', $pid); # spawn() blocks other signals
- waitpid($pid, 0);
-}
-
sub query_remote_mboxrd {
my ($self, $uris) = @_;
local $0 = "$0 query_remote_mboxrd";
my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
push(@qform, t => 1) if $opt->{thread};
my $verbose = $opt->{verbose};
- my $reap;
+ my ($reap_tail, $reap_curl);
my $cerr = File::Temp->new(TEMPLATE => 'curl.err-XXXX', TMPDIR => 1);
fcntl($cerr, F_SETFL, O_APPEND|O_RDWR) or warn "set O_APPEND: $!";
- my $rdr = { 2 => $cerr };
+ my $rdr = { 2 => $cerr, pgid => 0 };
my $coff = 0;
+ my $sigint_reap = $lei->can('sigint_reap');
if ($verbose) {
# spawn a process to force line-buffering, otherwise curl
# will write 1 character at-a-time and parallel outputs
# mmmaaayyy llloookkk llliiikkkeee ttthhhiiisss
- my $o = { 1 => $lei->{2}, 2 => $lei->{2} };
+ my $o = { 1 => $lei->{2}, 2 => $lei->{2}, pgid => 0 };
my $pid = spawn(['tail', '-f', $cerr->filename], undef, $o);
- $reap = PublicInbox::OnDestroy->new(\&kill_reap, $pid);
+ $reap_tail = PublicInbox::OnDestroy->new($sigint_reap, $pid);
}
my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
push @$curl, '-s', '-d', '';
my $cmd = $curl->for_uri($lei, $uri);
$lei->err("# @$cmd") if $verbose;
my ($fh, $pid) = popen_rd($cmd, $env, $rdr);
+ $reap_curl = PublicInbox::OnDestroy->new($sigint_reap, $pid);
$fh = IO::Uncompress::Gunzip->new($fh);
PublicInbox::MboxReader->mboxrd($fh, \&each_eml, $self,
$lei, $each_smsg);
- waitpid($pid, 0) == $pid or die "BUG: waitpid (curl): $!";
+ my $err = waitpid($pid, 0) == $pid ? undef : "BUG: waitpid: $!";
+ @$reap_curl = (); # cancel OnDestroy
+ die $err if $err;
if ($? == 0) {
my $nr = $lei->{-nr_remote_eml};
mset_progress($lei, $lei->{-current_url}, $nr, $nr);
my $t1 = "$home/t1-mirror";
ok($lei->('add-external', $t1, '--mirror', "$http/t1/"), '--mirror v1');
ok(-f "$t1/public-inbox/msgmap.sqlite3", 't1-mirror indexed');
+
+ ok($lei->('ls-external'), 'ls-external');
+ like($lei_out, qr!\Q$t1\E!, 't1 added to ls-externals');
+
my $t2 = "$home/t2-mirror";
ok($lei->('add-external', $t2, '--mirror', "$http/t2/"), '--mirror v2');
ok(-f "$t2/msgmap.sqlite3", 't2-mirror indexed');
+ ok($lei->('ls-external'), 'ls-external');
+ like($lei_out, qr!\Q$t2\E!, 't2 added to ls-externals');
+
ok(!$lei->('add-external', $t2, '--mirror', "$http/t2/"),
'--mirror fails if reused');
+ ok($lei->('ls-external'), 'ls-external');
+ like($lei_out, qr!\Q$t2\E!, 'still in ls-externals');
+
ok(!$lei->('add-external', "$t2-fail", '-Lmedium'), '--mirror v2');
ok(!-d "$t2-fail", 'destination not created on failure');
+ ok($lei->('ls-external'), 'ls-external');
+ unlike($lei_out, qr!\Q$t2-fail\E!, 'not added to ls-external');
});
ok($td->kill, 'killed -httpd');