-# Copyright (C) 2014-2020 all contributors <meta@public-inbox.org>
+# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
# License: GPLv2 or later <https://www.gnu.org/licenses/gpl-2.0.txt>
#
# Used to read files from a git repository without excessive forking.
use parent qw(Exporter);
use POSIX ();
use IO::Handle; # ->autoflush
-use Errno qw(EINTR EAGAIN);
+use Errno qw(EINTR EAGAIN ENOENT);
use File::Glob qw(bsd_glob GLOB_NOSORT);
use File::Spec ();
use Time::HiRes qw(stat);
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Spawn qw(popen_rd spawn);
use PublicInbox::Tmpfile;
use IO::Poll qw(POLLIN);
use Carp qw(croak);
use Digest::SHA ();
+use PublicInbox::DS qw(dwaitpid);
our @EXPORT_OK = qw(git_unquote git_quote);
our $PIPE_BUFSIZ = 65536; # Linux default
our $in_cleanup;
our $RDTIMEO = 60_000; # milliseconds
-use constant MAX_INFLIGHT =>
- (($^O eq 'linux' ? 4096 : POSIX::_POSIX_PIPE_BUF()) * 3)
- /
+use constant MAX_INFLIGHT => (POSIX::PIPE_BUF * 3) /
65; # SHA-256 hex size + "\n" in preparation for git using non-SHA1
my %GIT_ESC = (
sub git_unquote ($) {
return $_[0] unless ($_[0] =~ /\A"(.*)"\z/);
$_[0] = $1;
- $_[0] =~ s/\\([\\"abfnrtv])/$GIT_ESC{$1}/g;
- $_[0] =~ s/\\([0-7]{1,3})/chr(oct($1))/ge;
+ $_[0] =~ s!\\([\\"abfnrtv]|[0-3][0-7]{2})!$GIT_ESC{$1}//chr(oct($1))!ge;
$_[0];
}
sub git_quote ($) {
if ($_[0] =~ s/([\\"\a\b\f\n\r\t\013]|[^[:print:]])/
- '\\'.($ESC_GIT{$1}||sprintf("%0o",ord($1)))/egs) {
+ '\\'.($ESC_GIT{$1}||sprintf("%03o",ord($1)))/egs) {
return qq{"$_[0]"};
}
$_[0];
}
my ($in_r, $p) = popen_rd(\@cmd, undef, $redir);
$self->{$pid} = $p;
+ $self->{"$pid.owner"} = $$;
$out_w->autoflush(1);
if ($^O eq 'linux') { # 1031: F_SETPIPE_SZ
fcntl($out_w, 1031, 4096);
}
sub _cat_file_cb {
- my ($bref, undef, undef, $size, $result) = @_;
- @$result = ($bref, $size);
+ my ($bref, $oid, $type, $size, $result) = @_;
+ @$result = ($bref, $oid, $type, $size);
}
sub cat_file {
- my ($self, $oid, $sizeref) = @_;
+ my ($self, $oid) = @_;
my $result = [];
cat_async($self, $oid, \&_cat_file_cb, $result);
cat_async_wait($self);
- $$sizeref = $result->[1] if $sizeref;
- $result->[0];
+ wantarray ? @$result : $result->[0];
}
sub check_async_step ($$) {
# GitAsyncCat::event_step may delete {pid}
my $p = delete $self->{$pid} or return;
-
- # PublicInbox::DS may not be loaded
- eval { PublicInbox::DS::dwaitpid($p, undef, undef) };
- waitpid($p, 0) if $@; # wait synchronously if not in event loop
+ dwaitpid($p) if $$ == $self->{"$pid.owner"};
}
sub cat_async_abort ($) {
croak(ref($self) . ' ' . ($self->{git_dir} // '') . ": $msg");
}
+# $git->popen(qw(show f00)); # or
+# $git->popen(qw(show f00), { GIT_CONFIG => ... }, { 2 => ... });
sub popen {
- my ($self, @cmd) = @_;
- @cmd = ('git', "--git-dir=$self->{git_dir}", @cmd);
- popen_rd(\@cmd);
+ my ($self, $cmd) = splice(@_, 0, 2);
+ $cmd = [ 'git', "--git-dir=$self->{git_dir}",
+ ref($cmd) ? @$cmd : ($cmd, grep { defined && !ref } @_) ];
+ popen_rd($cmd, grep { !defined || ref } @_); # env and opt
}
+# same args as popen above
sub qx {
- my ($self, @cmd) = @_;
- my $fh = $self->popen(@cmd);
- local $/ = wantarray ? "\n" : undef;
- <$fh>;
+ my $fh = popen(@_);
+ if (wantarray) {
+ my @ret = <$fh>;
+ close $fh; # caller should check $?
+ @ret;
+ } else {
+ local $/;
+ my $ret = <$fh>;
+ close $fh; # caller should check $?
+ $ret;
+ }
+}
+
+sub date_parse {
+ my $self = shift;
+ map {
+ substr($_, length('--max-age='), -1)
+ } $self->qx('rev-parse', map { "--since=$_" } @_);
}
# check_async and cat_async may trigger the other, so ensure they're
delete $self->{inflight_c};
_destroy($self, qw(cat_rbuf in out pid));
_destroy($self, qw(chk_rbuf in_c out_c pid_c err_c));
- !!($self->{pid} || $self->{pid_c});
+ defined($self->{pid}) || defined($self->{pid_c});
}
my ($self) = @_;
my $ret = '???';
# don't show full FS path, basename should be OK:
- if ($self->{git_dir} =~ m!/([^/]+)(?:/\.git)?\z!) {
- $ret = "/path/to/$1";
+ if ($self->{git_dir} =~ m!/([^/]+)(?:/*\.git/*)?\z!) {
+ $ret = "$1.git";
}
wantarray ? ($ret) : $ret;
}
push(@$inflight, $oid, $cb, $arg);
}
-sub async_prefetch {
- my ($self, $oid, $cb, $arg) = @_;
- if (my $inflight = $self->{inflight}) {
- # we could use MAX_INFLIGHT here w/o the halving,
- # but lets not allow one client to monopolize a git process
- if (scalar(@$inflight) < int(MAX_INFLIGHT/2)) {
- print { $self->{out} } $oid, "\n" or
- $self->fail("write error: $!");
- return push(@$inflight, $oid, $cb, $arg);
- }
- }
- undef;
-}
-
-sub extract_cmt_time {
- my ($bref, undef, undef, undef, $modified) = @_;
-
- if ($$bref =~ /^committer .*?> ([0-9]+) [\+\-]?[0-9]+/sm) {
- my $cmt_time = $1 + 0;
- $$modified = $cmt_time if $cmt_time > $$modified;
- }
-}
-
# returns the modified time of a git repo, same as the "modified" field
# of a grokmirror manifest
sub modified ($) {
- my ($self) = @_;
- my $modified = 0;
- my $fh = popen($self, qw(rev-parse --branches));
- local $/ = "\n";
- while (my $oid = <$fh>) {
- chomp $oid;
- cat_async($self, $oid, \&extract_cmt_time, \$modified);
- }
- cat_async_wait($self);
- $modified || time;
+ # committerdate:unix is git 2.9.4+ (2017-05-05), so using raw instead
+ my $fh = popen($_[0], qw[for-each-ref --sort=-committerdate
+ --format=%(committerdate:raw) --count=1]);
+ (split(/ /, <$fh> // time))[0] + 0; # integerize for JSON
}
# for grokmirror, which doesn't read gitweb.description
# templates/this--description in git.git
sub manifest_entry {
my ($self, $epoch, $default_desc) = @_;
- my ($fh, $pid) = $self->popen('show-ref');
+ my $fh = $self->popen('show-ref');
my $dig = Digest::SHA->new(1);
while (read($fh, my $buf, 65536)) {
$dig->add($buf);
}
- close $fh;
- waitpid($pid, 0);
- return if $?; # empty, uninitialized git repo
+ close $fh or return; # empty, uninitialized git repo
+ undef $fh; # for open, below
my $git_dir = $self->{git_dir};
my $ent = {
fingerprint => $dig->hexdigest,
$ent;
}
+# returns true if there are pending cat-file processes
+sub cleanup_if_unlinked {
+ my ($self) = @_;
+ return cleanup($self) if $^O ne 'linux';
+ # Linux-specific /proc/$PID/maps access
+ # TODO: support this inside git.git
+ my $ret = 0;
+ for my $fld (qw(pid pid_c)) {
+ my $pid = $self->{$fld} // next;
+ open my $fh, '<', "/proc/$pid/maps" or return cleanup($self);
+ while (<$fh>) {
+ # n.b. we do not restart for unlinked multi-pack-index
+ # since it's not too huge, and the startup cost may
+ # be higher.
+ return cleanup($self) if /\.(?:idx|pack) \(deleted\)$/;
+ }
+ ++$ret;
+ }
+ $ret;
+}
+
1;
__END__
=pod