X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FFetch.pm;h=198e2a605a0940b7299aa95d6cee6e117e6e8b48;hb=d28167f0a78a2010a752bdc16b3b8a62e8fc91ae;hp=5ada1f49e4dc28ba39e2c0278208869575e4456a;hpb=55c8753c483d697fa0574b45ae77dec40509bb7f;p=public-inbox.git

diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 5ada1f49..198e2a60 100644
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -2,8 +2,7 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Wrapper to "git fetch" remote public-inboxes
 package PublicInbox::Fetch;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::IPC);
 use URI ();
 use PublicInbox::Spawn qw(popen_rd run_die spawn);
@@ -12,32 +11,36 @@ use PublicInbox::LEI;
 use PublicInbox::LeiCurl;
 use PublicInbox::LeiMirror;
 use File::Temp ();
-use PublicInbox::Config;
-use IO::Compress::Gzip qw(gzip $GzipError);
 
 sub new { bless {}, __PACKAGE__ }
 
-sub fetch_args ($$) {
-	my ($lei, $opt) = @_;
-	my @cmd; # (git --git-dir=...) to be added by caller
-	$opt->{$_} = $lei->{$_} for (0..2);
-	# we support "-c $key=$val" for arbitrary git config options
-	# e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
-	push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
-	push @cmd, 'fetch';
-	push @cmd, '-q' if $lei->{opt}->{quiet};
-	push @cmd, '-v' if $lei->{opt}->{verbose};
-	@cmd;
+sub remote_url ($$) {
+	my ($lei, $dir) = @_;
+	my $rn = $lei->{opt}->{'try-remote'} // [ 'origin', '_grokmirror' ];
+	for my $r (@$rn) {
+		my $cmd = [ qw(git config), "remote.$r.url" ];
+		my $fh = popen_rd($cmd, undef, { -C => $dir, 2 => $lei->{2} });
+		my $url = <$fh>;
+		close $fh or next;
+		$url =~ s!/*\n!!s;
+		return $url;
+	}
+	undef
 }
 
-sub remote_url ($$) {
-	my ($lei, $dir) = @_; # TODO: support non-"origin"?
-	my $cmd = [ qw(git config remote.origin.url) ];
-	my $fh = popen_rd($cmd, undef, { -C => $dir, 2 => $lei->{2} });
-	my $url = <$fh>;
-	close $fh or return;
-	$url =~ s!/*\n!!s;
-	$url;
+# PSGI mount prefixes and manifest.js.gz prefixes don't always align...
+# TODO: remove, handle multi-inbox fetch
+sub deduce_epochs ($$) {
+	my ($m, $path) = @_;
+	my ($v1_ent, @v2_epochs);
+	my $path_pfx = '';
+	$path =~ s!/+\z!!;
+	do {
+		$v1_ent = $m->{$path};
+		@v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
+	} while (!defined($v1_ent) && !@v2_epochs &&
+		$path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1);
+	($path_pfx, $v1_ent ? $path : undef, @v2_epochs);
 }
 
 sub do_manifest ($$$) {
@@ -45,16 +48,15 @@ sub do_manifest ($$$) {
 	my $muri = URI->new("$ibx_uri/manifest.js.gz");
 	my $ft = File::Temp->new(TEMPLATE => 'm-XXXX',
 				UNLINK => 1, DIR => $dir, SUFFIX => '.tmp');
-	my $fn = $ft->filename;
 	my $mf = "$dir/manifest.js.gz";
 	my $m0; # current manifest.js.gz contents
 	if (open my $fh, '<', $mf) {
 		$m0 = eval {
 			PublicInbox::LeiMirror::decode_manifest($fh, $mf, $mf)
 		};
-		$lei->err($@) if $@;
+		warn($@) if $@;
 	}
-	my ($bn) = ($fn =~ m!/([^/]+)\z!);
+	my ($bn) = ($ft->filename =~ m!/([^/]+)\z!);
 	my $curl_cmd = $lei->{curl}->for_uri($lei, $muri, qw(-R -o), $bn);
 	my $opt = { -C => $dir };
 	$opt->{$_} = $lei->{$_} for (0..2);
@@ -65,7 +67,7 @@ sub do_manifest ($$$) {
 		return;
 	}
 	my $m1 = eval {
-		PublicInbox::LeiMirror::decode_manifest($ft, $fn, $muri);
+		PublicInbox::LeiMirror::decode_manifest($ft, $ft, $muri);
 	} or return [ 404, $muri ];
 	my $mdiff = { %$m1 };
 
@@ -84,7 +86,7 @@ sub do_manifest ($$$) {
 		return;
 	}
 	my (undef, $v1_path, @v2_epochs) =
-		PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path);
+		deduce_epochs($mdiff, $ibx_uri->path);
 	[ 200, $muri, $v1_path, \@v2_epochs, $ft, $mf, $m1 ];
 }
 
@@ -110,7 +112,7 @@ sub do_fetch { # main entry point
 	my ($ibx_uri, @git_dir, @epochs, $mg, @new_epoch, $skip);
 	if ($ibx_ver == 1) {
 		my $url = remote_url($lei, $dir) //
-			die "E: $dir missing remote.origin.url\n";
+			die "E: $dir missing remote.*.url\n";
 		$ibx_uri = URI->new($url);
 	} else { # v2:
 		require PublicInbox::MultiGit;
@@ -128,7 +130,7 @@ sub do_fetch { # main entry point
 				$git_url = $url;
 				$epoch = $nr;
 			} else {
-				warn "W: $edir missing remote.origin.url\n";
+				warn "W: $edir missing remote.*.url\n";
 				my $pid = spawn([qw(git config -l)], undef,
 					{ 1 => $lei->{2}, 2 => $lei->{2} });
 				waitpid($pid, 0);
@@ -188,7 +190,7 @@ EOM
 		if (-d $d) {
 			$fp2->[0] = get_fingerprint2($d) if $fp2;
 			$cmd = [ @$torsocks, 'git', "--git-dir=$d",
-				fetch_args($lei, $opt) ];
+			       PublicInbox::LeiMirror::fetch_args($lei, $opt)];
 		} else {
 			my $e_uri = $ibx_uri->clone;
 			my ($epath) = ($d =~ m!(/git/[0-9]+\.git)\z!);
@@ -214,13 +216,8 @@ EOM
 	}
 	for my $i (@new_epoch) { $mg->epoch_cfg_set($i) }
 	if ($ft) {
-		my $fn = $ft->filename;
-		if ($mculled) {
-			my $json = PublicInbox::Config->json->encode($m1);
-			gzip(\$json => $fn) or die "gzip: $GzipError";
-		}
-		rename($fn, $mf) or die "E: rename($fn, $mf): $!\n";
-		$ft->unlink_on_destroy(0);
+		PublicInbox::LeiMirror::dump_manifest($m1 => $ft) if $mculled;
+		PublicInbox::LeiMirror::ft_rename($ft, $mf, 0666);
 	}
 	$lei->child_error($xit << 8) if $fp2 && $xit;
 }