manifest.js.gz: avoid long-lived per-epoch cat-file processes

When generating per-inbox manifests, we were forgetting to
cleanup per-epoch "git cat-file --batch" processes.  Our
previous method of generating modified times was also stupidly
inefficient, so replace the pipeline with a single
"git for-each-ref" invocation.

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index e557f6d62631c68dff1cee535dc962452bab718a..219a173246a8cc08e197b45691a7427cee71e2a9 100644 (file)
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -16,7 +16,7 @@ use Errno qw(EINTR EAGAIN);
 use File::Glob qw(bsd_glob GLOB_NOSORT);
 use File::Spec ();
 use Time::HiRes qw(stat);
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Spawn qw(popen_rd spawn);
 use PublicInbox::Tmpfile;
 use IO::Poll qw(POLLIN);
 use Carp qw(croak);
@@ -466,28 +466,13 @@ sub cat_async ($$$;$) {
        push(@$inflight, $oid, $cb, $arg);
 }
 
-sub extract_cmt_time {
-       my ($bref, undef, undef, undef, $modified) = @_;
-
-       if ($$bref =~ /^committer .*?> ([0-9]+) [\+\-]?[0-9]+/sm) {
-               my $cmt_time = $1 + 0;
-               $$modified = $cmt_time if $cmt_time > $$modified;
-       }
-}
-
 # returns the modified time of a git repo, same as the "modified" field
 # of a grokmirror manifest
 sub modified ($) {
-       my ($self) = @_;
-       my $modified = 0;
-       my $fh = popen($self, qw(rev-parse --branches));
-       local $/ = "\n";
-       while (my $oid = <$fh>) {
-               chomp $oid;
-               cat_async($self, $oid, \&extract_cmt_time, \$modified);
-       }
-       cat_async_wait($self);
-       $modified || time;
+       # committerdate:unix is git 2.9.4+ (2017-05-05), so using raw instead
+       my $fh = popen($_[0], qw[for-each-ref --sort=-committerdate
+                               --format=%(committerdate:raw) --count=1]);
+       (split(/ /, <$fh> // time))[0] + 0; # integerize for JSON
 }
 
 # for grokmirror, which doesn't read gitweb.description

diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index 69d81fa1ace13dc634625e86abd4c15935c57230..cde60245428af739834ac2b85436dbd659215943 100644 (file)
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -27,7 +27,7 @@ sub inject_entry ($$$;$) {
        $ctx->{manifest}->{$url_path} = $ent;
 }
 
-sub manifest_add ($$;$$) { # slow path w/o extindex "all"
+sub manifest_add ($$;$$) { # slow path w/o extindex "all" (or per-inbox)
        my ($ctx, $ibx, $epoch, $default_desc) = @_;
        my $url_path = "/$ibx->{name}";
        my $git;