daemons: revamp periodic cleanup task

[public-inbox.git] / lib / PublicInbox / Git.pm

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 2ae5eff9c01614a13f0781561c9aed64d1a52838..3c577ab317b29339271f468f3538345ed14690ca 100644 (file)
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -12,11 +12,11 @@ use v5.10.1;
 use parent qw(Exporter);
 use POSIX ();
 use IO::Handle; # ->autoflush
-use Errno qw(EINTR EAGAIN);
+use Errno qw(EINTR EAGAIN ENOENT);
 use File::Glob qw(bsd_glob GLOB_NOSORT);
 use File::Spec ();
 use Time::HiRes qw(stat);
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Spawn qw(popen_rd spawn);
 use PublicInbox::Tmpfile;
 use IO::Poll qw(POLLIN);
 use Carp qw(croak);
@@ -27,9 +27,7 @@ our $PIPE_BUFSIZ = 65536; # Linux default
 our $in_cleanup;
 our $RDTIMEO = 60_000; # milliseconds
 
-use constant MAX_INFLIGHT =>
-       (($^O eq 'linux' ? 4096 : POSIX::_POSIX_PIPE_BUF()) * 3)
-       /
+use constant MAX_INFLIGHT => (POSIX::PIPE_BUF * 3) /
        65; # SHA-256 hex size + "\n" in preparation for git using non-SHA1
 
 my %GIT_ESC = (
@@ -50,14 +48,13 @@ my %ESC_GIT = map { $GIT_ESC{$_} => $_ } keys %GIT_ESC;
 sub git_unquote ($) {
        return $_[0] unless ($_[0] =~ /\A"(.*)"\z/);
        $_[0] = $1;
-       $_[0] =~ s/\\([\\"abfnrtv])/$GIT_ESC{$1}/g;
-       $_[0] =~ s/\\([0-7]{1,3})/chr(oct($1))/ge;
+       $_[0] =~ s!\\([\\"abfnrtv]|[0-3][0-7]{2})!$GIT_ESC{$1}//chr(oct($1))!ge;
        $_[0];
 }
 
 sub git_quote ($) {
        if ($_[0] =~ s/([\\"\a\b\f\n\r\t\013]|[^[:print:]])/
-                     '\\'.($ESC_GIT{$1}||sprintf("%0o",ord($1)))/egs) {
+                     '\\'.($ESC_GIT{$1}||sprintf("%03o",ord($1)))/egs) {
                return qq{"$_[0]"};
        }
        $_[0];
@@ -403,7 +400,7 @@ sub cleanup {
        delete $self->{inflight_c};
        _destroy($self, qw(cat_rbuf in out pid));
        _destroy($self, qw(chk_rbuf in_c out_c pid_c err_c));
-       !!($self->{pid} || $self->{pid_c});
+       defined($self->{pid}) || defined($self->{pid_c});
 }
 
 
@@ -426,7 +423,7 @@ sub local_nick ($) {
        my ($self) = @_;
        my $ret = '???';
        # don't show full FS path, basename should be OK:
-       if ($self->{git_dir} =~ m!/([^/]+)(?:/\.git)?\z!) {
+       if ($self->{git_dir} =~ m!/([^/]+)(?:/*\.git/*)?\z!) {
                $ret = "$1.git";
        }
        wantarray ? ($ret) : $ret;
@@ -467,28 +464,13 @@ sub cat_async ($$$;$) {
        push(@$inflight, $oid, $cb, $arg);
 }
 
-sub extract_cmt_time {
-       my ($bref, undef, undef, undef, $modified) = @_;
-
-       if ($$bref =~ /^committer .*?> ([0-9]+) [\+\-]?[0-9]+/sm) {
-               my $cmt_time = $1 + 0;
-               $$modified = $cmt_time if $cmt_time > $$modified;
-       }
-}
-
 # returns the modified time of a git repo, same as the "modified" field
 # of a grokmirror manifest
 sub modified ($) {
-       my ($self) = @_;
-       my $modified = 0;
-       my $fh = popen($self, qw(rev-parse --branches));
-       local $/ = "\n";
-       while (my $oid = <$fh>) {
-               chomp $oid;
-               cat_async($self, $oid, \&extract_cmt_time, \$modified);
-       }
-       cat_async_wait($self);
-       $modified || time;
+       # committerdate:unix is git 2.9.4+ (2017-05-05), so using raw instead
+       my $fh = popen($_[0], qw[for-each-ref --sort=-committerdate
+                               --format=%(committerdate:raw) --count=1]);
+       (split(/ /, <$fh> // time))[0] + 0; # integerize for JSON
 }
 
 # for grokmirror, which doesn't read gitweb.description
@@ -541,6 +523,27 @@ sub manifest_entry {
        $ent;
 }
 
+# returns true if there are pending cat-file processes
+sub cleanup_if_unlinked {
+       my ($self) = @_;
+       return cleanup($self) if $^O ne 'linux';
+       # Linux-specific /proc/$PID/maps access
+       # TODO: support this inside git.git
+       my $ret = 0;
+       for my $fld (qw(pid pid_c)) {
+               my $pid = $self->{$fld} // next;
+               open my $fh, '<', "/proc/$pid/maps" or return cleanup($self);
+               while (<$fh>) {
+                       # n.b. we do not restart for unlinked multi-pack-index
+                       # since it's not too huge, and the startup cost may
+                       # be higher.
+                       return cleanup($self) if /\.(?:idx|pack) \(deleted\)$/;
+               }
+               ++$ret;
+       }
+       $ret;
+}
+
 1;
 __END__
 =pod