X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiMirror.pm;h=1138a82d04f7c03d92de5134358650d8bd09b317;hb=550d283867c0f8a39207f368c49b5d7496045320;hp=163f45ee24bbe740eec8914808cba1e5596b184f;hpb=f75f1c1033ed0474e22247a41a517cc8b46aed05;p=public-inbox.git

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 163f45ee..1138a82d 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -113,9 +113,9 @@ sub clone_cmd {
 	@cmd;
 }
 
-sub ft_rename ($$$) {
-	my ($ft, $dst, $open_mode) = @_;
-	my @st = stat($dst);
+sub ft_rename ($$$;$) {
+	my ($ft, $dst, $open_mode, $fh) = @_;
+	my @st = stat($fh // $dst);
 	my $mode = @st ? ($st[2] & 07777) : ($open_mode & ~umask);
 	chmod($mode, $ft) or croak "E: chmod($ft): $!";
 	require File::Copy;
@@ -218,8 +218,6 @@ sub set_description ($) {
 sub index_cloned_inbox {
 	my ($self, $iv) = @_;
 	my $lei = $self->{lei};
-	eval { set_description($self) };
-	warn $@ if $@;
 
 	# n.b. public-inbox-clone works w/o (SQLite || Xapian)
 	# lei is useless without Xapian + SQLite
@@ -283,7 +281,7 @@ sub fetch_args ($$) {
 
 sub fgrp_update_old ($) { # for git <1.8.5
 	my ($fgrp) = @_;
-	my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
+	my $cmd = [ @{$fgrp->{-torsocks}}, 'git', "--git-dir=$fgrp->{cur_dst}",
 		fetch_args($fgrp->{lei}, my $opt = {}) ];
 	$fgrp->{lei}->qerr("# @$cmd");
 	do_reap($fgrp);
@@ -291,8 +289,8 @@ sub fgrp_update_old ($) { # for git <1.8.5
 }
 
 sub upr { # feed `git update-ref --stdin -z' verbosely
-	my ($fgrp, $w, $op, $ref, $oid) = @_;
-	$fgrp->{lei}->qerr("# $op $ref $oid");
+	my ($lei, $w, $op, $ref, $oid) = @_;
+	$lei->qerr("# $op $ref $oid") if $lei->{opt}->{verbose};
 	print $w "$op $ref\0$oid\0" or die "print(w): $!";
 }
 
@@ -309,21 +307,22 @@ sub fgrp_update {
 	pipe(my ($r, $w)) or die "pipe: $!";
 	my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
 		qw(update-ref --stdin -z) ];
-	$fgrp->{lei}->qerr("# @$cmd");
-	my $opt = { 0 => $r, 1 => $fgrp->{lei}->{1}, 2 => $fgrp->{lei}->{2} };
+	my $lei = $fgrp->{lei};
+	$lei->qerr("# @$cmd");
+	my $opt = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2} };
 	my $pid = spawn($cmd, undef, $opt);
 	close $r or die "close(r): $!";
 	for my $ref (keys %dst) {
 		my $new = delete $src{$ref};
 		my $old = $dst{$ref};
 		if (defined $new) {
-			upr($fgrp, $w, 'update', $ref, $new) if $new ne $old;
+			upr($lei, $w, 'update', $ref, $new) if $new ne $old;
 		} else {
-			upr($fgrp, $w, 'delete', $ref, $old);
+			upr($lei, $w, 'delete', $ref, $old);
 		}
 	}
 	while (my ($ref, $oid) = each %src) {
-		upr($fgrp, $w, 'create', $ref, $oid);
+		upr($lei, $w, 'create', $ref, $oid);
 	}
 	if (close($w)) { # git >= 1.8.5
 		$LIVE->{$pid} = [ \&reap_cmd, $fgrp, $cmd ];
@@ -336,16 +335,21 @@ sub fgrp_update {
 	}
 }
 
+sub pack_refs {
+	my ($self, $git_dir) = @_;
+	do_reap($self);
+	my $cmd = [ 'git', "--git-dir=$git_dir", qw(pack-refs --all --prune) ];
+	$self->{lei}->qerr("# @$cmd");
+	my $opt = { 1 => $self->{lei}->{1}, 2 => $self->{lei}->{2} };
+	$LIVE->{spawn($cmd, undef, $opt)} = [ \&reap_cmd, $self, $cmd ];
+}
+
 sub fgrp_fetched {
 	my ($fgrp) = @_;
 	return if $fgrp->{dry_run} || !$LIVE;
 	my $rn = $fgrp->{-remote};
 	my %opt = map { $_ => $fgrp->{lei}->{$_} } (0..2);
-	my $cmd = [ 'git', "--git-dir=$fgrp->{-osdir}",
-			qw(pack-refs --all --prune) ];
-	do_reap($fgrp);
-	$fgrp->{lei}->qerr("# @$cmd");
-	$LIVE->{spawn($cmd, undef, \%opt)} = [ \&reap_cmd, $fgrp, $cmd ];
+	pack_refs($fgrp, $fgrp->{-osdir}); # objstore refs always packed
 
 	$update_ref_stdin or return fgrp_update_old($fgrp);
 
@@ -370,9 +374,9 @@ sub fgrp_fetched {
 }
 
 sub fgrp_fetch {
-	my ($fgrp, $pfx, $fini) = @_;
-	my $cmd = [ @$pfx, 'git', "--git-dir=$fgrp->{-osdir}",
-			fetch_args($fgrp->{lei}, my $opt = {}),
+	my ($fgrp, $fini) = @_;
+	my $cmd = [ @{$fgrp->{-torsocks}}, 'git', "--git-dir=$fgrp->{-osdir}",
+			fetch_args($fgrp->{lei}, my $opt = {}), '--no-tags',
 			$fgrp->{-remote} ];
 	$fgrp->{-fini} = $fini;
 	do_reap($fgrp);
@@ -402,11 +406,14 @@ sub forkgroup_prep {
 	my $key = $self->{-key} // die 'BUG: no -key';
 	my ($bn) = ($key =~ m{/([a-z0-9_,;=!\+\{\}\|][^/]*)(?:\.git)?\z}i);
 	my $rn = "$bn-".substr(sha256_hex($key), 0, 16);
-	for ("url=$uri", "fetch=+refs/*:refs/remotes/$rn/*") {
+	# --no-tags is required to avoid conflicts
+	for ("url=$uri", "fetch=+refs/*:refs/remotes/$rn/*",
+			'tagopt=--no-tags') {
 		my @kv = split(/=/, $_, 2);
 		$kv[0] = "remote.$rn.$kv[0]";
 		run_die([@cmd, @kv], undef, $opt);
 	}
+	$self->{-do_pack_refs} = 1; # likely coderepo
 	if (!-d $self->{cur_dst}) {
 		my $alt = File::Spec->rel2abs("$dir/objects");
 		PublicInbox::Import::init_bare($self->{cur_dst});
@@ -436,10 +443,11 @@ sub clone_v1 {
 	my $uri = URI->new($self->{cur_src} // $self->{src});
 	defined($lei->{opt}->{epoch}) and
 		die "$uri is a v1 inbox, --epoch is not supported\n";
-	my $pfx = $curl->torsocks($lei, $uri) or return;
+	$self->{-torsocks} //= $curl->torsocks($lei, $uri) or return;
 	my $dst = $self->{cur_dst} // $self->{dst};
 	my $fini = PublicInbox::OnDestroy->new($$, \&v1_done, $self);
-	my $cmd = [ @$pfx, clone_cmd($lei, my $opt = {}), "$uri", $dst ];
+	my $cmd = [ @{$self->{-torsocks}}, clone_cmd($lei, my $opt = {}),
+		"$uri", $dst ];
 	my $fgrp = forkgroup_prep($self, $uri);
 	if (!defined($fgrp) && defined($self->{-ent})) {
 		if (defined(my $ref = $self->{-ent}->{reference})) {
@@ -447,7 +455,7 @@ sub clone_v1 {
 				push @$cmd, '--reference', "$self->{dst}$ref";
 		}
 	}
-	$fgrp ? fgrp_fetch($fgrp, $pfx, $fini) :
+	$fgrp ? fgrp_fetch($fgrp, $fini) :
 		start_clone($self, $cmd, $opt, $fini);
 
 	if (!$self->{-is_epoch} && $lei->{opt}->{'inbox-config'} =~
@@ -558,20 +566,25 @@ sub v1_done { # called via OnDestroy
 		run_die([qw(git config -f), "$dst/config", 'gitweb.owner', $o]);
 	}
 	my $o = "$dst/objects";
-	if (open(my $fh, '<', "$o/info/alternates")) {
+	if (open(my $fh, '<', my $fn = "$o/info/alternates")) {;
 		my $base = File::Spec->rel2abs($o);
-		chomp(my @l = <$fh>);
+		my @l = <$fh>;
+		my $ft;
 		for (@l) {
-			$_ = File::Spec->abs2rel($_, $base) if m!\A/!;
-			$_ .= "\n";
+			next unless m!\A/!;
+			$_ = File::Spec->abs2rel($_, $base);
+			$ft //= File::Temp->new(TEMPLATE => '.XXXX',
+						DIR => "$o/info");
+		}
+		if ($ft) {
+			print $ft @l or die "print($ft): $!";
+			$ft->flush or die "flush($ft): $!";
+			ft_rename($ft, $fn, 0666, $fh);
 		}
-		my $f = File::Temp->new(TEMPLATE => '.XXXX', DIR => "$o/info");
-		print $f @l;
-		$f->flush or die "flush($f): $!";
-		rename($f->filename, "$o/info/alternates") or
-			die "rename($f, $o/info/alternates): $!";
-		$f->unlink_on_destroy(0);
 	}
+	pack_refs($self, $dst) if delete $self->{-do_pack_refs};
+	eval { set_description($self) };
+	warn $@ if $@;
 	return if ($self->{-is_epoch} ||
 		$self->{lei}->{opt}->{'inbox-config'} ne 'always');
 	write_makefile($dst, 1);
@@ -596,6 +609,8 @@ sub v2_done { # called via OnDestroy
 	}
 	write_makefile($dst, 2);
 	undef $lck; # unlock
+	eval { set_description($self) };
+	warn $@ if $@;
 	index_cloned_inbox($self, 2);
 }
 
@@ -604,7 +619,7 @@ sub clone_v2_prep ($$;$) {
 	my $lei = $self->{lei};
 	my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
 	my $first_uri = (map { $_->[0] } values %$v2_epochs)[0];
-	my $pfx = $curl->torsocks($lei, $first_uri) or return;
+	$self->{-torsocks} //= $curl->torsocks($lei, $first_uri) or return;
 	my $dst = $self->{cur_dst} // $self->{dst};
 	my $want = parse_epochs($lei->{opt}->{epoch}, $v2_epochs);
 	my $task = $m ? bless { %$self }, __PACKAGE__ : $self;
@@ -881,7 +896,8 @@ sub do_mirror { # via wq_io_do or public-inbox-clone
 
 		if (defined(my $os = $lei->{opt}->{objstore})) {
 			$os = 'objstore' if $os eq ''; # --objstore w/o args
-			$self->{-objstore} = "$self->{dst}/$os";
+			$os = "$self->{dst}/$os" if $os !~ m!\A/!;
+			$self->{-objstore} = $os;
 		}
 		local $LIVE;
 		my $iv = $lei->{opt}->{'inbox-version'} //