X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiMirror.pm;h=afe12bb30dd719e545224b348575ff52b82b3f92;hb=0830817c132cb105291104937a645d30efe06db1;hp=d6d2c69eb90e981b57eeafbf7f8f8bf7576c769d;hpb=40df0363872f9b34556703221c12bb36ece714d8;p=public-inbox.git

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index d6d2c69e..afe12bb3 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -16,23 +16,24 @@ use Carp qw(croak);
 use URI;
 use PublicInbox::Config;
 use PublicInbox::Inbox;
+use PublicInbox::Git;
 use PublicInbox::LeiCurl;
 use PublicInbox::OnDestroy;
-use Digest::SHA qw(sha256_hex sha1_hex);
+use PublicInbox::SHA qw(sha256_hex sha1_hex);
 use POSIX qw(strftime);
 
 our $LIVE; # pid => callback
 our $FGRP_TODO; # objstore -> [ fgrp mirror objects ]
 our $TODO; # reference => [ non-fgrp mirror objects ]
+our @PUH; # post-update hooks
 
 sub keep_going ($) {
 	$LIVE && (!$_[0]->{lei}->{child_error} ||
 		$_[0]->{lei}->{opt}->{'keep-going'});
 }
 
-sub _wq_done_wait { # dwaitpid callback (via wq_eof)
-	my ($arg, $pid) = @_;
-	my ($mrr, $lei) = @$arg;
+sub _wq_done_wait { # awaitpid cb (via wq_eof)
+	my ($pid, $mrr, $lei) = @_;
 	if ($?) {
 		$lei->child_error($?);
 	} elsif (!$lei->{child_error}) {
@@ -235,7 +236,7 @@ sub index_cloned_inbox {
 			my ($k) = ($sw =~ /\A([\w-]+)/);
 			$opt->{$k} = $lei->{opt}->{$k};
 		}
-		# force synchronous dwaitpid for v2:
+		# force synchronous awaitpid for v2:
 		local $PublicInbox::DS::in_loop = 0;
 		my $cfg = PublicInbox::Config->new(undef, $lei->{2});
 		my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
@@ -258,7 +259,8 @@ sub run_reap {
 sub start_cmd {
 	my ($self, $cmd, $opt, $fini) = @_;
 	do_reap($self);
-	$self->{lei}->qerr("# @$cmd");
+	utf8::decode(my $msg = "# @$cmd");
+	$self->{lei}->qerr($msg);
 	return if $self->{dry_run};
 	$LIVE->{spawn($cmd, undef, $opt)} = [ \&reap_cmd, $self, $cmd, $fini ]
 }
@@ -275,6 +277,8 @@ sub fetch_args ($$) {
 			($lei->{opt}->{jobs} // 1) > 1;
 	push @cmd, '-v' if $lei->{opt}->{verbose};
 	push(@cmd, '-p') if $lei->{opt}->{prune};
+	PublicInbox::Git::version() >= ((2 << 24) | (29 << 16)) and
+		push(@cmd, '--no-write-fetch-head');
 	@cmd;
 }
 
@@ -299,7 +303,7 @@ sub fgrp_update {
 	my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
 		qw(update-ref --stdin -z) ];
 	my $lei = $fgrp->{lei};
-	my $pack = PublicInbox::OnDestroy->new($$, \&pack_dst, $fgrp);
+	my $pack = PublicInbox::OnDestroy->new($$, \&satellite_done, $fgrp);
 	start_cmd($fgrp, $cmd, { 0 => $r, 2 => $lei->{2} }, $pack);
 	close $r or die "close(r): $!";
 	return if $fgrp->{dry_run};
@@ -319,9 +323,10 @@ sub fgrp_update {
 	close($w) or warn "E: close(update-ref --stdin): $! (need git 1.8.5+)\n";
 }
 
-sub pack_dst { # packs lightweight satellite repos
+sub satellite_done {
 	my ($fgrp) = @_;
 	pack_refs($fgrp, $fgrp->{cur_dst});
+	run_puh($fgrp);
 }
 
 sub pack_refs {
@@ -461,7 +466,11 @@ EOM
 }
 
 sub fp_done {
-	my ($self, $cb, @arg) = @_;
+	my ($self, $cmd, $cb, @arg) = @_;
+	if ($?) {
+		$self->{lei}->err("@$cmd failed (\$?=$?) (non-fatal)");
+		$? = 0; # don't let it influence normal exit
+	}
 	return if !keep_going($self);
 	my $fh = delete $self->{-show_ref} // die 'BUG: no show-ref output';
 	seek($fh, SEEK_SET, 0) or die "seek(show_ref): $!";
@@ -487,8 +496,10 @@ sub cmp_fp_do {
 	my $opt = { 2 => $self->{lei}->{2} };
 	open($opt->{1}, '+>', undef) or die "open(tmp): $!";
 	$self->{-show_ref} = $opt->{1};
-	my $done = PublicInbox::OnDestroy->new($$, \&fp_done, $self, $cb, @arg);
-	start_cmd($self, $cmd, $opt, $done);
+	do_reap($self);
+	$self->{lei}->qerr("# @$cmd");
+	$LIVE->{spawn($cmd, undef, $opt)} = [ \&fp_done, $self, $cmd,
+						$cb, @arg ];
 }
 
 sub resume_fetch {
@@ -497,17 +508,16 @@ sub resume_fetch {
 	my $dst = $self->{cur_dst} // $self->{dst};
 	my @git = ('git', "--git-dir=$dst");
 	my $opt = { 2 => $self->{lei}->{2} };
-	my $rn = 'origin'; # configurable?
+	my $rn = 'random'.int(rand(1 << 30));
 	for ("url=$uri", "fetch=+refs/*:refs/*", 'mirror=true') {
-		my @kv = split(/=/, $_, 2);
-		$kv[0] = "remote.$rn.$kv[0]";
-		next if $self->{dry_run};
-		run_die([@git, 'config', @kv], undef, $opt);
+		push @git, '-c', "remote.$rn.$_";
 	}
 	my $cmd = [ @{$self->{-torsocks}}, @git,
 			fetch_args($self->{lei}, $opt), $rn ];
 	push @$cmd, '-P' if $self->{lei}->{prune}; # --prune-tags implied
-	start_cmd($self, $cmd, $opt, $fini);
+	my $run_puh = PublicInbox::OnDestroy->new($$, \&run_puh, $self, $fini);
+	++$self->{chg}->{nr_chg};
+	start_cmd($self, $cmd, $opt, $run_puh);
 }
 
 sub fgrp_enqueue {
@@ -524,6 +534,7 @@ sub fgrp_enqueue {
 		$fgrp->{dry_run} ? $fgrp->{lei}->qerr("# @cmd @kv") :
 				run_die([@cmd, @kv], undef, $opt);
 	}
+	++$fgrp->{chg}->{nr_chg};
 	push @{$FGRP_TODO->{$fgrp->{-osdir}}}, $fgrp;
 }
 
@@ -556,7 +567,9 @@ sub clone_v1 {
 						"$self->{dst}$ref";
 			}
 		}
-		start_cmd($self, $cmd, $opt, $fini);
+		++$self->{chg}->{nr_chg};
+		start_cmd($self, $cmd, $opt, PublicInbox::OnDestroy->new($$,
+						\&run_puh, $self, $fini));
 	}
 	if (!$self->{-is_epoch} && $lei->{opt}->{'inbox-config'} =~
 				/\A(?:always|v1)\z/s) {
@@ -675,6 +688,22 @@ sub atomic_write ($$$) {
 	ft_rename($ft, "$dn/$bn", 0666);
 }
 
+sub run_next_puh {
+	my ($self) = @_;
+	my $puh = shift @{$self->{-puh_todo}} // return delete($self->{-fini});
+	my $fini = PublicInbox::OnDestroy->new($$, \&run_next_puh, $self);
+	my $cmd = [ @$puh, ($self->{cur_dst} // $self->{dst}) ];
+	my $opt = +{ map { $_ => $self->{lei}->{$_} } (0..2) };
+	start_cmd($self, $cmd, undef, $opt, $fini);
+}
+
+sub run_puh {
+	my ($self, $fini) = @_;
+	$self->{-fini} = $fini;
+	@{$self->{-puh_todo}} = @PUH;
+	run_next_puh($self);
+}
+
 # modifies the to-be-written manifest entry, and sets values from it, too
 sub update_ent {
 	my ($self) = @_;
@@ -724,6 +753,7 @@ sub update_ent {
 				}
 			}
 			symlink($tgt, $ln) or die "symlink($tgt, $ln): $!";
+			++$self->{chg}->{nr_chg};
 		}
 	}
 	if (defined(my $t = $self->{-ent}->{modified})) {
@@ -960,9 +990,13 @@ sub clone_all {
 			# resolve multi-level references
 			while ($m && defined($nxt = $m->{$x}->{reference})) {
 				exists($todo->{$nxt}) or last;
-				die <<EOM if ++$nr > 1000;
-E: dependency loop detected (`$x' => `$nxt')
+				if (++$nr > 1000) {
+					$m->{$x}->{reference} = undef;
+					$m->{$nxt}->{reference} = undef;
+					warn <<EOM
+E: dependency loop detected (`$x' => `$nxt'), breaking
 EOM
+				}
 				$x = $nxt;
 			}
 			my $y = delete $todo->{$x} // next; # already done
@@ -1021,7 +1055,9 @@ EOM
 	warn "\t", $_, "\n" for @local;
 
 	my (undef, $dn, $bn) = File::Spec->splitpath($f);
-	atomic_write($dn, $bn, join("\n", @list, ''));
+	$self->{chg}->{nr_chg} += scalar(@remote) + scalar(@local);
+	my $new = join("\n", @list, '');
+	atomic_write($dn, $bn, $new) if $new ne $old;
 }
 
 # FIXME: this gets confused by single inbox instance w/ global manifest.js.gz
@@ -1052,7 +1088,10 @@ sub try_manifest {
 	}
 
 	# bail out if curl -z/--timecond hit 304 Not Modified, $ft will be empty
-	return $lei->qerr("# $manifest unchanged") if -f $manifest && !-s $ft;
+	if (-f $manifest && !-s $ft) {
+		$lei->child_error(127 << 8) if $lei->{opt}->{'exit-code'};
+		return $lei->qerr("# $manifest unchanged");
+	}
 
 	my $m = eval { decode_manifest($ft, $ft, $uri) };
 	if ($@) {
@@ -1140,6 +1179,8 @@ W: The following exist and have not been converted to symlinks
 EOM
 	dump_project_list($self, $m);
 	ft_rename($ft, $manifest, 0666);
+	!$self->{chg}->{nr_chg} && $lei->{opt}->{'exit-code'} and
+		$lei->child_error(127 << 8);
 }
 
 sub start_clone_url {
@@ -1154,6 +1195,11 @@ sub do_mirror { # via wq_io_do or public-inbox-clone
 	$self->{dry_run} = 1 if $lei->{opt}->{'dry-run'};
 	umask($lei->{client_umask}) if defined $lei->{client_umask};
 	$self->{-initial_clone} = 1 if !-d $self->{dst};
+	local @PUH;
+	if (defined(my $puh = $lei->{opt}->{'post-update-hook'})) {
+		require Text::ParseWords;
+		@PUH = map { [ Text::ParseWords::shellwords($_) ] } @$puh;
+	}
 	eval {
 		my $ic = $lei->{opt}->{'inbox-config'} //= 'always';
 		$ic =~ /\A(?:v1|v2|always|never)\z/s or die <<"";