X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiMirror.pm;h=d959b6b61ac4cd018b1142681d663261846596b6;hb=8a1708f40df5f8d367f032eef746e5390f614a50;hp=9843d1a67e3ab964d8e08a78d583a5ec4afcf527;hpb=f1817699f3f3feb964653eaf3586477abbf7947d;p=public-inbox.git

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 9843d1a6..d959b6b6 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -16,23 +16,24 @@ use Carp qw(croak);
 use URI;
 use PublicInbox::Config;
 use PublicInbox::Inbox;
+use PublicInbox::Git;
 use PublicInbox::LeiCurl;
 use PublicInbox::OnDestroy;
-use Digest::SHA qw(sha256_hex sha1_hex);
+use PublicInbox::SHA qw(sha256_hex sha1_hex);
 use POSIX qw(strftime);
 
 our $LIVE; # pid => callback
 our $FGRP_TODO; # objstore -> [ fgrp mirror objects ]
 our $TODO; # reference => [ non-fgrp mirror objects ]
+our @PUH; # post-update hooks
 
 sub keep_going ($) {
 	$LIVE && (!$_[0]->{lei}->{child_error} ||
 		$_[0]->{lei}->{opt}->{'keep-going'});
 }
 
-sub _wq_done_wait { # dwaitpid callback (via wq_eof)
-	my ($arg, $pid) = @_;
-	my ($mrr, $lei) = @$arg;
+sub _wq_done_wait { # awaitpid cb (via wq_eof)
+	my ($pid, $mrr, $lei) = @_;
 	if ($?) {
 		$lei->child_error($?);
 	} elsif (!$lei->{child_error}) {
@@ -235,7 +236,7 @@ sub index_cloned_inbox {
 			my ($k) = ($sw =~ /\A([\w-]+)/);
 			$opt->{$k} = $lei->{opt}->{$k};
 		}
-		# force synchronous dwaitpid for v2:
+		# force synchronous awaitpid for v2:
 		local $PublicInbox::DS::in_loop = 0;
 		my $cfg = PublicInbox::Config->new(undef, $lei->{2});
 		my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
@@ -258,7 +259,8 @@ sub run_reap {
 sub start_cmd {
 	my ($self, $cmd, $opt, $fini) = @_;
 	do_reap($self);
-	$self->{lei}->qerr("# @$cmd");
+	utf8::decode(my $msg = "# @$cmd");
+	$self->{lei}->qerr($msg);
 	return if $self->{dry_run};
 	$LIVE->{spawn($cmd, undef, $opt)} = [ \&reap_cmd, $self, $cmd, $fini ]
 }
@@ -275,6 +277,8 @@ sub fetch_args ($$) {
 			($lei->{opt}->{jobs} // 1) > 1;
 	push @cmd, '-v' if $lei->{opt}->{verbose};
 	push(@cmd, '-p') if $lei->{opt}->{prune};
+	PublicInbox::Git::version() >= ((2 << 24) | (29 << 16)) and
+		push(@cmd, '--no-write-fetch-head');
 	@cmd;
 }
 
@@ -284,6 +288,19 @@ sub upr { # feed `git update-ref --stdin -z' verbosely
 	print $w "$op ", join("\0", @rest, '') or die "print(w): $!";
 }
 
+sub start_update_ref {
+	my ($fgrp) = @_;
+	pipe(my ($r, $w)) or die "pipe: $!";
+	my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
+		qw(update-ref --stdin -z) ];
+	my $pack = PublicInbox::OnDestroy->new($$, \&satellite_done, $fgrp);
+	start_cmd($fgrp, $cmd, { 0 => $r, 2 => $fgrp->{lei}->{2} }, $pack);
+	close $r or die "close(r): $!";
+	$fgrp->{dry_run} ? undef : $w;
+}
+
+sub upref_warn { warn "E: close(update-ref --stdin): $! (need git 1.8.5+)\n" }
+
 sub fgrp_update {
 	my ($fgrp) = @_;
 	return if !keep_going($fgrp);
@@ -295,14 +312,9 @@ sub fgrp_update {
 	close $srcfh;
 	my %dst = map { chomp; split(/\0/) } (<$dstfh>);
 	close $dstfh;
-	pipe(my ($r, $w)) or die "pipe: $!";
-	my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
-		qw(update-ref --stdin -z) ];
+	my $w = start_update_ref($fgrp) or return;
 	my $lei = $fgrp->{lei};
-	my $pack = PublicInbox::OnDestroy->new($$, \&pack_dst, $fgrp);
-	start_cmd($fgrp, $cmd, { 0 => $r, 2 => $lei->{2} }, $pack);
-	close $r or die "close(r): $!";
-	return if $fgrp->{dry_run};
+	my $ndel;
 	for my $ref (keys %dst) {
 		my $new = delete $src{$ref};
 		my $old = $dst{$ref};
@@ -311,17 +323,33 @@ sub fgrp_update {
 				upr($lei, $w, 'update', $ref, $new, $old);
 		} else {
 			upr($lei, $w, 'delete', $ref, $old);
+			++$ndel;
 		}
 	}
-	while (my ($ref, $oid) = each %src) {
-		upr($lei, $w, 'create', $ref, $oid);
+	# git's ref files backend doesn't allow directory/file conflicts
+	# between `delete' and `create' ops:
+	if ($ndel && scalar(keys %src)) {
+		$fgrp->{-create_refs} = \%src;
+	} else {
+		while (my ($ref, $oid) = each %src) {
+			upr($lei, $w, 'create', $ref, $oid);
+		}
 	}
-	close($w) or warn "E: close(update-ref --stdin): $! (need git 1.8.5+)\n";
+	close($w) or upref_warn();
 }
 
-sub pack_dst { # packs lightweight satellite repos
+sub satellite_done {
 	my ($fgrp) = @_;
-	pack_refs($fgrp, $fgrp->{cur_dst});
+	if (my $create = delete $fgrp->{-create_refs}) {
+		my $w = start_update_ref($fgrp) or return;
+		while (my ($ref, $oid) = each %$create) {
+			upr($fgrp->{lei}, $w, 'create', $ref, $oid);
+		}
+		close($w) or upref_warn();
+	} else {
+		pack_refs($fgrp, $fgrp->{cur_dst});
+		run_puh($fgrp);
+	}
 }
 
 sub pack_refs {
@@ -493,7 +521,6 @@ sub cmp_fp_do {
 	$self->{-show_ref} = $opt->{1};
 	do_reap($self);
 	$self->{lei}->qerr("# @$cmd");
-	return if $self->{dry_run};
 	$LIVE->{spawn($cmd, undef, $opt)} = [ \&fp_done, $self, $cmd,
 						$cb, @arg ];
 }
@@ -504,17 +531,16 @@ sub resume_fetch {
 	my $dst = $self->{cur_dst} // $self->{dst};
 	my @git = ('git', "--git-dir=$dst");
 	my $opt = { 2 => $self->{lei}->{2} };
-	my $rn = 'origin'; # configurable?
+	my $rn = 'random'.int(rand(1 << 30));
 	for ("url=$uri", "fetch=+refs/*:refs/*", 'mirror=true') {
-		my @kv = split(/=/, $_, 2);
-		$kv[0] = "remote.$rn.$kv[0]";
-		next if $self->{dry_run};
-		run_die([@git, 'config', @kv], undef, $opt);
+		push @git, '-c', "remote.$rn.$_";
 	}
 	my $cmd = [ @{$self->{-torsocks}}, @git,
 			fetch_args($self->{lei}, $opt), $rn ];
 	push @$cmd, '-P' if $self->{lei}->{prune}; # --prune-tags implied
-	start_cmd($self, $cmd, $opt, $fini);
+	my $run_puh = PublicInbox::OnDestroy->new($$, \&run_puh, $self, $fini);
+	++$self->{chg}->{nr_chg};
+	start_cmd($self, $cmd, $opt, $run_puh);
 }
 
 sub fgrp_enqueue {
@@ -531,6 +557,7 @@ sub fgrp_enqueue {
 		$fgrp->{dry_run} ? $fgrp->{lei}->qerr("# @cmd @kv") :
 				run_die([@cmd, @kv], undef, $opt);
 	}
+	++$fgrp->{chg}->{nr_chg};
 	push @{$FGRP_TODO->{$fgrp->{-osdir}}}, $fgrp;
 }
 
@@ -563,7 +590,9 @@ sub clone_v1 {
 						"$self->{dst}$ref";
 			}
 		}
-		start_cmd($self, $cmd, $opt, $fini);
+		++$self->{chg}->{nr_chg};
+		start_cmd($self, $cmd, $opt, PublicInbox::OnDestroy->new($$,
+						\&run_puh, $self, $fini));
 	}
 	if (!$self->{-is_epoch} && $lei->{opt}->{'inbox-config'} =~
 				/\A(?:always|v1)\z/s) {
@@ -682,6 +711,22 @@ sub atomic_write ($$$) {
 	ft_rename($ft, "$dn/$bn", 0666);
 }
 
+sub run_next_puh {
+	my ($self) = @_;
+	my $puh = shift @{$self->{-puh_todo}} // return delete($self->{-fini});
+	my $fini = PublicInbox::OnDestroy->new($$, \&run_next_puh, $self);
+	my $cmd = [ @$puh, ($self->{cur_dst} // $self->{dst}) ];
+	my $opt = +{ map { $_ => $self->{lei}->{$_} } (0..2) };
+	start_cmd($self, $cmd, undef, $opt, $fini);
+}
+
+sub run_puh {
+	my ($self, $fini) = @_;
+	$self->{-fini} = $fini;
+	@{$self->{-puh_todo}} = @PUH;
+	run_next_puh($self);
+}
+
 # modifies the to-be-written manifest entry, and sets values from it, too
 sub update_ent {
 	my ($self) = @_;
@@ -731,6 +776,7 @@ sub update_ent {
 				}
 			}
 			symlink($tgt, $ln) or die "symlink($tgt, $ln): $!";
+			++$self->{chg}->{nr_chg};
 		}
 	}
 	if (defined(my $t = $self->{-ent}->{modified})) {
@@ -1032,7 +1078,9 @@ EOM
 	warn "\t", $_, "\n" for @local;
 
 	my (undef, $dn, $bn) = File::Spec->splitpath($f);
-	atomic_write($dn, $bn, join("\n", @list, ''));
+	$self->{chg}->{nr_chg} += scalar(@remote) + scalar(@local);
+	my $new = join("\n", @list, '');
+	atomic_write($dn, $bn, $new) if $new ne $old;
 }
 
 # FIXME: this gets confused by single inbox instance w/ global manifest.js.gz
@@ -1063,7 +1111,10 @@ sub try_manifest {
 	}
 
 	# bail out if curl -z/--timecond hit 304 Not Modified, $ft will be empty
-	return $lei->qerr("# $manifest unchanged") if -f $manifest && !-s $ft;
+	if (-f $manifest && !-s $ft) {
+		$lei->child_error(127 << 8) if $lei->{opt}->{'exit-code'};
+		return $lei->qerr("# $manifest unchanged");
+	}
 
 	my $m = eval { decode_manifest($ft, $ft, $uri) };
 	if ($@) {
@@ -1151,6 +1202,8 @@ W: The following exist and have not been converted to symlinks
 EOM
 	dump_project_list($self, $m);
 	ft_rename($ft, $manifest, 0666);
+	!$self->{chg}->{nr_chg} && $lei->{opt}->{'exit-code'} and
+		$lei->child_error(127 << 8);
 }
 
 sub start_clone_url {
@@ -1165,6 +1218,11 @@ sub do_mirror { # via wq_io_do or public-inbox-clone
 	$self->{dry_run} = 1 if $lei->{opt}->{'dry-run'};
 	umask($lei->{client_umask}) if defined $lei->{client_umask};
 	$self->{-initial_clone} = 1 if !-d $self->{dst};
+	local @PUH;
+	if (defined(my $puh = $lei->{opt}->{'post-update-hook'})) {
+		require Text::ParseWords;
+		@PUH = map { [ Text::ParseWords::shellwords($_) ] } @$puh;
+	}
 	eval {
 		my $ic = $lei->{opt}->{'inbox-config'} //= 'always';
 		$ic =~ /\A(?:v1|v2|always|never)\z/s or die <<"";