shrink low-bandwidth pipes under Linux

[public-inbox.git] / lib / PublicInbox / Git.pm

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index f47bc439080749c25100aca9cb1bb0f32b2d7c13..16117277079e86b6aecfc4291a3cf90e25928c0a 100644 (file)
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -1,5 +1,5 @@
-# Copyright (C) 2014-2015 all contributors <meta@public-inbox.org>
-# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
+# Copyright (C) 2014-2018 all contributors <meta@public-inbox.org>
+# License: GPLv2 or later <https://www.gnu.org/licenses/gpl-2.0.txt>
 #
 # Used to read files from a git repository without excessive forking.
 # Used in our web interfaces as well as our -nntpd server.
@@ -15,7 +15,19 @@ use PublicInbox::Spawn qw(spawn popen_rd);
 
 sub new {
        my ($class, $git_dir) = @_;
-       bless { git_dir => $git_dir }, $class
+       my @st;
+       $st[7] = $st[10] = 0;
+       bless { git_dir => $git_dir, st => \@st }, $class
+}
+
+sub alternates_changed {
+       my ($self) = @_;
+       my $alt = "$self->{git_dir}/objects/info/alternates";
+       my @st = stat($alt) or return 0;
+       my $old_st = $self->{st};
+       # 10 - ctime, 7 - size
+       return 0 if ($st[10] == $old_st->[10] && $st[7] == $old_st->[7]);
+       $self->{st} = \@st;
 }
 
 sub _bidi_pipe {
@@ -25,6 +37,10 @@ sub _bidi_pipe {
 
        pipe($in_r, $in_w) or fail($self, "pipe failed: $!");
        pipe($out_r, $out_w) or fail($self, "pipe failed: $!");
+       if ($^O eq 'linux') { # 1031: F_SETPIPE_SZ
+               fcntl($out_w, 1031, 4096);
+               fcntl($in_w, 1031, 4096) if $batch eq '--batch-check';
+       }
 
        my @cmd = ('git', "--git-dir=$self->{git_dir}", qw(cat-file), $batch);
        my $redir = { 0 => fileno($out_r), 1 => fileno($in_w) };
@@ -38,14 +54,23 @@ sub _bidi_pipe {
 
 sub cat_file {
        my ($self, $obj, $ref) = @_;
+       my ($retried, $in, $head);
 
-       $self->_bidi_pipe(qw(--batch in out pid));
+again:
+       batch_prepare($self);
        $self->{out}->print($obj, "\n") or fail($self, "write error: $!");
 
-       my $in = $self->{in};
+       $in = $self->{in};
        local $/ = "\n";
-       my $head = $in->getline;
-       $head =~ / missing$/ and return undef;
+       $head = $in->getline;
+       if ($head =~ / missing$/) {
+               if (!$retried && alternates_changed($self)) {
+                       $retried = 1;
+                       cleanup($self);
+                       goto again;
+               }
+               return;
+       }
        $head =~ /^[0-9a-f]{40} \S+ (\d+)$/ or
                fail($self, "Unexpected result from git cat-file: $head");
 
@@ -89,6 +114,8 @@ sub cat_file {
        $rv;
 }
 
+sub batch_prepare ($) { _bidi_pipe($_[0], qw(--batch in out pid)) }
+
 sub check {
        my ($self, $obj) = @_;
        $self->_bidi_pipe(qw(--batch-check in_c out_c pid_c));
@@ -137,6 +164,18 @@ sub cleanup {
        _destroy($self, qw(in_c out_c pid_c));
 }
 
+# assuming a well-maintained repo, this should be a somewhat
+# accurate estimation of its size
+# TODO: show this in the WWW UI as a hint to potential cloners
+sub packed_bytes {
+       my ($self) = @_;
+       my $n = 0;
+       foreach my $p (glob("$self->{git_dir}/objects/pack/*.pack")) {
+               $n += -s $p;
+       }
+       $n
+}
+
 sub DESTROY { cleanup(@_) }
 
 1;