1 # Copyright (C) all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
3 # Wrapper to "git fetch" remote public-inboxes
4 package PublicInbox::Fetch;
7 use parent qw(PublicInbox::IPC);
9 use PublicInbox::Spawn qw(popen_rd);
10 use PublicInbox::Admin;
12 use PublicInbox::LeiCurl;
13 use PublicInbox::LeiMirror;
14 use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
17 sub new { bless {}, __PACKAGE__ }
22 $opt->{$_} = $lei->{$_} for (0..2);
23 # we support "-c $key=$val" for arbitrary git config options
24 # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
25 push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
27 push @cmd, '-q' if $lei->{opt}->{quiet};
28 push @cmd, '-v' if $lei->{opt}->{verbose};
33 my ($lei, $dir) = @_; # TODO: support non-"origin"?
34 my $cmd = [ qw(git config remote.origin.url) ];
35 my $fh = popen_rd($cmd, undef, { -C => $dir, 2 => $lei->{2} });
43 my ($cls, $lei, $cd) = @_;
45 my $curl = PublicInbox::LeiCurl->new($lei) or return;
46 my $dir = PublicInbox::Admin::resolve_inboxdir($cd, \$ibx_ver);
48 my $url = remote_url($lei, $dir) //
49 die "E: $dir missing remote.origin.url\n";
50 my $uri = URI->new($url);
51 my $torsocks = $curl->torsocks($lei, $uri);
52 my $opt = { -C => $dir };
53 my $cmd = [ @$torsocks, fetch_cmd($lei, $opt) ];
54 my $cerr = PublicInbox::LeiMirror::run_reap($lei, $cmd, $opt);
55 $lei->child_error($cerr, "@$cmd failed") if $cerr;
59 opendir my $dh, "$dir/git" or die "opendir $dir/git: $!";
60 my @epochs = sort { $b <=> $a } map { substr($_, 0, -4) + 0 }
61 grep(/\A[0-9]+\.git\z/, readdir($dh));
62 my ($git_url, $epoch);
63 for my $nr (@epochs) { # try newest epoch, first
64 my $edir = "$dir/git/$nr.git";
65 if (defined(my $url = remote_url($lei, $edir))) {
70 warn "W: $edir missing remote.origin.url\n";
73 $git_url or die "Unable to determine git URL\n";
74 my $inbox_url = $git_url;
75 $inbox_url =~ s!/git/$epoch(?:\.git)?/?\z!! or
76 $inbox_url =~ s!/$epoch(?:\.git)?/?\z!! or die <<EOM;
77 Unable to infer inbox URL from <$git_url>
79 $lei->qerr("# inbox URL: $inbox_url/");
80 my $muri = URI->new("$inbox_url/manifest.js.gz");
81 my $ft = File::Temp->new(TEMPLATE => 'manifest-XXXX',
82 UNLINK => 1, DIR => $dir);
83 my $fn = $ft->filename;
84 my @opt = (qw(-R -o), $fn);
85 my $mf = "$dir/manifest.js.gz";
86 my $m0; # current manifest.js.gz contents
87 if (open my $fh, '<', $mf) {
89 PublicInbox::LeiMirror::decode_manifest($fh, $mf, $mf)
92 push @opt, '-z', $mf if defined($m0);
94 my $curl_cmd = $curl->for_uri($lei, $muri, @opt);
96 $opt->{$_} = $lei->{$_} for (0..2);
97 my $cerr = PublicInbox::LeiMirror::run_reap($lei, $curl_cmd, $opt);
98 return $lei->child_error($cerr, "@$curl_cmd failed") if $cerr;
99 return if !-s $ft; # 304 Not Modified via curl -z
101 my $m1 = PublicInbox::LeiMirror::decode_manifest($ft, $fn, $muri);
102 my $mdiff = { %$m1 };
104 # filter out unchanged entries
105 while (my ($k, $v0) = each %{$m0 // {}}) {
106 my $cur = $m1->{$k} // next;
107 my $f0 = $v0->{fingerprint} // next;
108 my $f1 = $cur->{fingerprint} // next;
109 my $t0 = $v0->{modified} // next;
110 my $t1 = $cur->{modified} // next;
111 delete($mdiff->{$k}) if $f0 eq $f1 && $t0 == $t1;
113 my $ibx_uri = URI->new("$inbox_url/");
114 my ($path_pfx, $v1_bare, @v2_epochs) =
115 PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path);
116 defined($v1_bare) and die <<EOM;
117 E: got v1 `$v1_bare' when expecting v2 epoch(s) in <$muri>, WTF?
119 my @epoch_nr = sort { $a <=> $b }
120 map { my ($nr) = (m!/([0-9]+)\.git\z!g) } @v2_epochs;
122 # n.b. this expects all epochs are from the same host
123 my $torsocks = $curl->torsocks($lei, $muri);
124 for my $nr (@epoch_nr) {
125 my $dir = "$dir/git/$nr.git";
130 $cmd = [ @$torsocks, fetch_cmd($lei, $opt) ];
132 my $e_uri = $ibx_uri->clone;
133 $e_uri->path($ibx_uri->path."git/$nr.git");
135 PublicInbox::LeiMirror::clone_cmd($lei, $opt),
138 my $cerr = PublicInbox::LeiMirror::run_reap($lei, $cmd, $opt);
139 return $lei->child_error($cerr, "@$cmd failed") if $cerr;
141 rename($fn, $mf) or die "E: rename($fn, $mf): $!\n";
142 $ft->unlink_on_destroy(0);