1 # Copyright (C) all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
4 # backend for a git-cat-file-workalike based on libgit2,
5 # other libgit2 stuff may go here, too.
6 package PublicInbox::Gcf2;
9 use PublicInbox::Spawn qw(which popen_rd); # may set PERL_INLINE_DIRECTORY
10 use Fcntl qw(LOCK_EX SEEK_SET);
11 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
12 use IO::Handle; # autoflush
13 use File::Path qw(make_path);
18 # PublicInbox::Spawn will set PERL_INLINE_DIRECTORY
19 # to ~/.cache/public-inbox/inline-c if it exists
20 my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //
21 die 'PERL_INLINE_DIRECTORY not defined';
22 make_path($inline_dir);
23 my $f = "$inline_dir/.public-inbox.lock";
24 open my $fh, '+>', $f or die "open($f): $!";
26 # CentOS 7.x ships Inline 0.53, 0.64+ has built-in locking
27 flock($fh, LOCK_EX) or die "LOCK_EX($f): $!\n";
29 my $pc = which($ENV{PKG_CONFIG} // 'pkg-config') //
30 die "pkg-config missing for libgit2";
31 my ($dir) = (__FILE__ =~ m!\A(.+?)/[^/]+\z!);
32 my $ef = "$inline_dir/.public-inbox.pkg-config.err";
33 open my $err, '+>', $ef or die "open($ef): $!";
34 for my $x (qw(libgit2)) {
35 my $rdr = { 2 => $err };
36 my ($l, $pid) = popen_rd([$pc, '--libs', $x], undef, $rdr);
37 $l = do { local $/; <$l> };
40 (my $c, $pid) = popen_rd([$pc, '--cflags', $x], undef, $rdr);
41 $c = do { local $/; <$c> };
45 # note: we name C source files .h to prevent
46 # ExtUtils::MakeMaker from automatically trying to
48 my $f = "$dir/gcf2_$x.h";
49 open(my $src, '<', $f) or die "E: open($f): $!";
52 defined($c_src = <$src>) or die "read $f: $!";
58 seek($err, 0, SEEK_SET);
59 $err = do { local $/; <$err> };
60 die "E: libgit2 not installed: $err\n";
62 open my $oldout, '>&', \*STDOUT or die "dup(1): $!";
63 open my $olderr, '>&', \*STDERR or die "dup(2): $!";
64 open STDOUT, '>&', $fh or die "1>$f: $!";
65 open STDERR, '>&', $fh or die "2>$f: $!";
69 # we use Capitalized and ALLCAPS for compatibility with old Inline::C
71 use Inline C => Config => %CFG, BOOT => q[git_libgit2_init();];
72 use Inline C => $c_src, BUILD_NOISY => 1;
75 open(STDERR, '>&', $olderr) or warn "restore stderr: $!";
76 open(STDOUT, '>&', $oldout) or warn "restore stdout: $!";
78 seek($fh, 0, SEEK_SET);
80 die "Inline::C Gcf2 build failed:\n", $err, "\n", @msg;
85 my ($gcf2, $git_dir) = @_;
86 my $objdir = PublicInbox::Git->new($git_dir)->git_path('objects');
88 # libgit2 (tested 0.27.7+dfsg.1-0.2 and 0.28.3+dfsg.1-1~bpo10+1
89 # in Debian) doesn't handle relative epochs properly when nested
90 # multiple levels. Add all the absolute paths to workaround it,
91 # since $EXTINDEX_DIR/ALL.git/objects/info/alternates uses absolute
92 # paths to reference $V2INBOX_DIR/all.git/objects and
93 # $V2INBOX_DIR/all.git/objects/info/alternates uses relative paths
94 # to refer to $V2INBOX_DIR/git/$EPOCH.git/objects
96 # See https://bugs.debian.org/975607
97 if (open(my $fh, '<', "$objdir/info/alternates")) {
98 chomp(my @abs_alt = grep(m!^/!, <$fh>));
99 $gcf2->add_alternate($_) for @abs_alt;
101 $gcf2->add_alternate($objdir);
105 sub have_unlinked_files () {
106 # FIXME: port gcf2-like over to git.git so we won't need to
108 return 1 if $^O ne 'linux';
109 open my $fh, '<', "/proc/$$/maps" or return;
110 while (<$fh>) { return 1 if /\.(?:idx|pack) \(deleted\)$/ }
114 # Usage: $^X -MPublicInbox::Gcf2 -e PublicInbox::Gcf2::loop [EXPIRE-TIMEOUT]
115 # (see lib/PublicInbox/Gcf2Client.pm)
117 my $exp = $_[0] || $ARGV[0] || 60; # seconds
119 my (%seen, $check_at);
120 STDERR->autoflush(1);
121 STDOUT->autoflush(1);
125 my ($oid, $git_dir) = split(/ /, $_, 2);
126 $seen{$git_dir} //= add_alt($gcf2, $git_dir);
127 if (!$gcf2->cat_oid(1, $oid)) {
128 # retry once if missing. We only get unabbreviated OIDs
129 # from SQLite or Xapian DBs, here, so malicious clients
130 # can't trigger excessive retries:
131 warn "# $$ $oid missing, retrying in $git_dir\n";
134 %seen = ($git_dir => add_alt($gcf2, $git_dir));
135 $check_at = clock_gettime(CLOCK_MONOTONIC) + $exp;
137 if ($gcf2->cat_oid(1, $oid)) {
138 warn "# $$ $oid found after retry\n";
140 warn "W: $$ $oid missing after retry\n";
141 print "$oid missing\n"; # mimic git-cat-file
143 } else { # check expiry to deal with deleted pack files
144 my $now = clock_gettime(CLOCK_MONOTONIC);
145 $check_at //= $now + $exp;
146 if ($now > $check_at) {
148 if (have_unlinked_files()) {