-# Copyright (C) 2014-2020 all contributors <meta@public-inbox.org>
+# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
# License: GPLv2 or later <https://www.gnu.org/licenses/gpl-2.0.txt>
#
# Used to read files from a git repository without excessive forking.
use parent qw(Exporter);
use POSIX ();
use IO::Handle; # ->autoflush
-use Errno qw(EINTR);
+use Errno qw(EINTR EAGAIN);
use File::Glob qw(bsd_glob GLOB_NOSORT);
use File::Spec ();
use Time::HiRes qw(stat);
use PublicInbox::Spawn qw(popen_rd);
use PublicInbox::Tmpfile;
+use IO::Poll qw(POLLIN);
use Carp qw(croak);
use Digest::SHA ();
+use PublicInbox::DS qw(dwaitpid);
our @EXPORT_OK = qw(git_unquote git_quote);
our $PIPE_BUFSIZ = 65536; # Linux default
our $in_cleanup;
+our $RDTIMEO = 60_000; # milliseconds
use constant MAX_INFLIGHT =>
(($^O eq 'linux' ? 4096 : POSIX::_POSIX_PIPE_BUF()) * 3)
sub last_check_err {
my ($self) = @_;
my $fh = $self->{err_c} or return;
- sysseek($fh, 0, 0) or fail($self, "sysseek failed: $!");
+ sysseek($fh, 0, 0) or $self->fail("sysseek failed: $!");
defined(sysread($fh, my $buf, -s $fh)) or
- fail($self, "sysread failed: $!");
+ $self->fail("sysread failed: $!");
$buf;
}
if ($self->{$pid}) {
if (defined $err) { # "err_c"
my $fh = $self->{$err};
- sysseek($fh, 0, 0) or fail($self, "sysseek failed: $!");
- truncate($fh, 0) or fail($self, "truncate failed: $!");
+ sysseek($fh, 0, 0) or $self->fail("sysseek failed: $!");
+ truncate($fh, 0) or $self->fail("truncate failed: $!");
}
return;
}
my ($out_r, $out_w);
- pipe($out_r, $out_w) or fail($self, "pipe failed: $!");
+ pipe($out_r, $out_w) or $self->fail("pipe failed: $!");
my @cmd = (qw(git), "--git-dir=$self->{git_dir}",
qw(-c core.abbrev=40 cat-file), $batch);
my $redir = { 0 => $out_r };
if ($err) {
my $id = "git.$self->{git_dir}$batch.err";
- my $fh = tmpfile($id) or fail($self, "tmpfile($id): $!");
+ my $fh = tmpfile($id) or $self->fail("tmpfile($id): $!");
$self->{$err} = $fh;
$redir->{2} = $fh;
}
my ($in_r, $p) = popen_rd(\@cmd, undef, $redir);
$self->{$pid} = $p;
+ $self->{"$pid.owner"} = $$;
$out_w->autoflush(1);
if ($^O eq 'linux') { # 1031: F_SETPIPE_SZ
fcntl($out_w, 1031, 4096);
$self->{$in} = $in_r;
}
+sub poll_in ($) { IO::Poll::_poll($RDTIMEO, fileno($_[0]), my $ev = POLLIN) }
+
sub my_read ($$$) {
my ($fh, $rbuf, $len) = @_;
my $left = $len - length($$rbuf);
$r = sysread($fh, $$rbuf, $PIPE_BUFSIZ, length($$rbuf));
if ($r) {
$left -= $r;
+ } elsif (defined($r)) { # EOF
+ return 0;
} else {
- next if (!defined($r) && $! == EINTR);
- return $r;
+ next if ($! == EAGAIN and poll_in($fh));
+ next if $! == EINTR; # may be set by sysread or poll_in
+ return; # unrecoverable error
}
}
\substr($$rbuf, 0, $len, '');
if ((my $n = index($$rbuf, "\n")) >= 0) {
return substr($$rbuf, 0, $n + 1, '');
}
- my $r = sysread($fh, $$rbuf, $PIPE_BUFSIZ, length($$rbuf));
- next if $r || (!defined($r) && $! == EINTR);
- return defined($r) ? '' : undef; # EOF or error
+ my $r = sysread($fh, $$rbuf, $PIPE_BUFSIZ, length($$rbuf))
+ and next;
+
+ # return whatever's left on EOF
+ return substr($$rbuf, 0, length($$rbuf)+1, '') if defined($r);
+
+ next if ($! == EAGAIN and poll_in($fh));
+ next if $! == EINTR; # may be set by sysread or poll_in
+ return; # unrecoverable error
}
}
for (my $i = 0; $i < @$inflight; $i += 3) {
$buf .= "$inflight->[$i]\n";
}
- print { $self->{out} } $buf or fail($self, "write error: $!");
+ print { $self->{out} } $buf or $self->fail("write error: $!");
unshift(@$inflight, \$req, $cb, $arg); # \$ref to indicate retried
cat_async_step($self, $inflight); # take one step
$type = 'missing';
$oid = ref($req) ? $$req : $req if $oid eq '';
} else {
- $self->fail("Unexpected result from async git cat-file: $head");
+ my $err = $! ? " ($!)" : '';
+ $self->fail("bad result from async cat-file: $head$err");
}
$self->{cat_rbuf} = $rbuf if $$rbuf ne '';
eval { $cb->($bref, $oid, $type, $size, $arg) };
# https://public-inbox.org/git/20190118033845.s2vlrb3wd3m2jfzu@dcvr/T/
if ($hex eq 'dangling' || $hex eq 'notdir' || $hex eq 'loop') {
my $ret = my_read($self->{in_c}, $rbuf, $type + 1);
- fail($self, defined($ret) ? 'read EOF' : "read: $!") if !$ret;
+ $self->fail(defined($ret) ? 'read EOF' : "read: $!") if !$ret;
}
$self->{chk_rbuf} = $rbuf if $$rbuf ne '';
eval { $cb->($hex, $type, $size, $arg, $self) };
while (scalar(@$inflight_c) >= MAX_INFLIGHT) {
check_async_step($self, $inflight_c);
}
- print { $self->{out_c} } $oid, "\n" or fail($self, "write error: $!");
+ print { $self->{out_c} } $oid, "\n" or $self->fail("write error: $!");
push(@$inflight_c, $oid, $cb, $arg);
}
# GitAsyncCat::event_step may delete {pid}
my $p = delete $self->{$pid} or return;
-
- # PublicInbox::DS may not be loaded
- eval { PublicInbox::DS::dwaitpid($p, undef, undef) };
- waitpid($p, 0) if $@; # wait synchronously if not in event loop
+ dwaitpid($p) if $$ == $self->{"$pid.owner"};
}
sub cat_async_abort ($) {
cleanup($self);
}
-sub fail {
+sub fail { # may be augmented in subclasses
my ($self, $msg) = @_;
cat_async_abort($self);
croak(ref($self) . ' ' . ($self->{git_dir} // '') . ": $msg");
sub qx {
my ($self, @cmd) = @_;
my $fh = $self->popen(@cmd);
- local $/ = "\n";
- return <$fh> if wantarray;
- local $/;
- <$fh>
+ local $/ = wantarray ? "\n" : undef;
+ <$fh>;
}
# check_async and cat_async may trigger the other, so ensure they're
while (scalar(@$inflight) >= MAX_INFLIGHT) {
cat_async_step($self, $inflight);
}
- print { $self->{out} } $oid, "\n" or fail($self, "write error: $!");
+ print { $self->{out} } $oid, "\n" or $self->fail("write error: $!");
push(@$inflight, $oid, $cb, $arg);
}
# but lets not allow one client to monopolize a git process
if (scalar(@$inflight) < int(MAX_INFLIGHT/2)) {
print { $self->{out} } $oid, "\n" or
- fail($self, "write error: $!");
+ $self->fail("write error: $!");
return push(@$inflight, $oid, $cb, $arg);
}
}