X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FFetch.pm;h=3b6aa389f383405981a17c047d8a9bcaddddfedf;hb=d922fc44d1afb5e4400f2606c475e42aee087c24;hp=7f60b619e0a4c7b2270cc59882f9410dd86cbb86;hpb=718d054be8b2cc23635b42e8ce880424492d3a84;p=public-inbox.git

diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 7f60b619..3b6aa389 100644
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -6,7 +6,7 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC);
 use URI ();
-use PublicInbox::Spawn qw(popen_rd run_die);
+use PublicInbox::Spawn qw(popen_rd run_die spawn);
 use PublicInbox::Admin;
 use PublicInbox::LEI;
 use PublicInbox::LeiCurl;
@@ -31,13 +31,32 @@ sub fetch_args ($$) {
 }
 
 sub remote_url ($$) {
-	my ($lei, $dir) = @_; # TODO: support non-"origin"?
-	my $cmd = [ qw(git config remote.origin.url) ];
-	my $fh = popen_rd($cmd, undef, { -C => $dir, 2 => $lei->{2} });
-	my $url = <$fh>;
-	close $fh or return;
-	$url =~ s!/*\n!!s;
-	$url;
+	my ($lei, $dir) = @_;
+	my $rn = $lei->{opt}->{'try-remote'} // [ 'origin', '_grokmirror' ];
+	for my $r (@$rn) {
+		my $cmd = [ qw(git config), "remote.$r.url" ];
+		my $fh = popen_rd($cmd, undef, { -C => $dir, 2 => $lei->{2} });
+		my $url = <$fh>;
+		close $fh or next;
+		$url =~ s!/*\n!!s;
+		return $url;
+	}
+	undef
+}
+
+# PSGI mount prefixes and manifest.js.gz prefixes don't always align...
+# TODO: remove, handle multi-inbox fetch
+sub deduce_epochs ($$) {
+	my ($m, $path) = @_;
+	my ($v1_ent, @v2_epochs);
+	my $path_pfx = '';
+	$path =~ s!/+\z!!;
+	do {
+		$v1_ent = $m->{$path};
+		@v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
+	} while (!defined($v1_ent) && !@v2_epochs &&
+		$path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1);
+	($path_pfx, $v1_ent ? $path : undef, @v2_epochs);
 }
 
 sub do_manifest ($$$) {
@@ -52,7 +71,7 @@ sub do_manifest ($$$) {
 		$m0 = eval {
 			PublicInbox::LeiMirror::decode_manifest($fh, $mf, $mf)
 		};
-		$lei->err($@) if $@;
+		warn($@) if $@;
 	}
 	my ($bn) = ($fn =~ m!/([^/]+)\z!);
 	my $curl_cmd = $lei->{curl}->for_uri($lei, $muri, qw(-R -o), $bn);
@@ -60,11 +79,13 @@ sub do_manifest ($$$) {
 	$opt->{$_} = $lei->{$_} for (0..2);
 	my $cerr = PublicInbox::LeiMirror::run_reap($lei, $curl_cmd, $opt);
 	if ($cerr) {
-		return [ 404 ] if ($cerr >> 8) == 22; # 404 Missing
+		return [ 404, $muri ] if ($cerr >> 8) == 22; # 404 Missing
 		$lei->child_error($cerr, "@$curl_cmd failed");
 		return;
 	}
-	my $m1 = PublicInbox::LeiMirror::decode_manifest($ft, $fn, $muri);
+	my $m1 = eval {
+		PublicInbox::LeiMirror::decode_manifest($ft, $fn, $muri);
+	} or return [ 404, $muri ];
 	my $mdiff = { %$m1 };
 
 	# filter out unchanged entries.  We check modified, too, since
@@ -82,8 +103,8 @@ sub do_manifest ($$$) {
 		return;
 	}
 	my (undef, $v1_path, @v2_epochs) =
-		PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path);
-	[ 200, $v1_path, \@v2_epochs, $muri, $ft, $mf, $m1 ];
+		deduce_epochs($mdiff, $ibx_uri->path);
+	[ 200, $muri, $v1_path, \@v2_epochs, $ft, $mf, $m1 ];
 }
 
 sub get_fingerprint2 {
@@ -93,6 +114,13 @@ sub get_fingerprint2 {
 	Digest::SHA::sha256(do { local $/; <$rd> });
 }
 
+sub writable_dir ($) {
+	my ($dir) = @_;
+	return unless -d $dir && -w _;
+	my @st = stat($dir);
+	$st[2] & 0222; # any writable bits set? (in case of root)
+}
+
 sub do_fetch { # main entry point
 	my ($cls, $lei, $cd) = @_;
 	my $ibx_ver;
@@ -101,16 +129,16 @@ sub do_fetch { # main entry point
 	my ($ibx_uri, @git_dir, @epochs, $mg, @new_epoch, $skip);
 	if ($ibx_ver == 1) {
 		my $url = remote_url($lei, $dir) //
-			die "E: $dir missing remote.origin.url\n";
+			die "E: $dir missing remote.*.url\n";
 		$ibx_uri = URI->new($url);
 	} else { # v2:
 		require PublicInbox::MultiGit;
 		$mg = PublicInbox::MultiGit->new($dir, 'all.git', 'git');
-		my @epochs = $mg->git_epochs;
+		@epochs = $mg->git_epochs;
 		my ($git_url, $epoch);
 		for my $nr (@epochs) { # try newest epoch, first
 			my $edir = "$dir/git/$nr.git";
-			unless (-d $edir && -w _) { # must be writable dir
+			if (!writable_dir($edir)) {
 				$skip->{$nr} = 1;
 				next;
 			}
@@ -119,7 +147,11 @@ sub do_fetch { # main entry point
 				$git_url = $url;
 				$epoch = $nr;
 			} else {
-				warn "W: $edir missing remote.origin.url\n";
+				warn "W: $edir missing remote.*.url\n";
+				my $pid = spawn([qw(git config -l)], undef,
+					{ 1 => $lei->{2}, 2 => $lei->{2} });
+				waitpid($pid, 0);
+				$lei->child_error($?) if $?;
 			}
 		}
 		@epochs = grep { !$skip->{$_} } @epochs if $skip;
@@ -135,7 +167,7 @@ EOM
 	PublicInbox::LeiMirror::write_makefile($dir, $ibx_ver);
 	$lei->qerr("# inbox URL: $ibx_uri/");
 	my $res = do_manifest($lei, $dir, $ibx_uri) or return;
-	my ($code, $v1_path, $v2_epochs, $muri, $ft, $mf, $m1) = @$res;
+	my ($code, $muri, $v1_path, $v2_epochs, $ft, $mf, $m1) = @$res;
 	if ($code == 404) {
 		# any pre-manifest.js.gz instances running? Just fetch all
 		# existing ones and unconditionally try cloning the next
@@ -201,13 +233,14 @@ EOM
 	}
 	for my $i (@new_epoch) { $mg->epoch_cfg_set($i) }
 	if ($ft) {
-		my $fn = $ft->filename;
 		if ($mculled) {
 			my $json = PublicInbox::Config->json->encode($m1);
+			my $fn = $ft->filename;
+			my $mtime = (stat($fn))[9];
 			gzip(\$json => $fn) or die "gzip: $GzipError";
+			utime($mtime, $mtime, $fn) or die "utime(..., $fn): $!";
 		}
-		rename($fn, $mf) or die "E: rename($fn, $mf): $!\n";
-		$ft->unlink_on_destroy(0);
+		PublicInbox::LeiMirror::ft_rename($ft, $mf, 0666);
 	}
 	$lei->child_error($xit << 8) if $fp2 && $xit;
 }