use URI;
use PublicInbox::Config;
use PublicInbox::Inbox;
+use PublicInbox::Git;
use PublicInbox::LeiCurl;
use PublicInbox::OnDestroy;
-use Digest::SHA qw(sha256_hex sha1_hex);
+use PublicInbox::SHA qw(sha256_hex sha1_hex);
use POSIX qw(strftime);
our $LIVE; # pid => callback
-our $FGRP_TODO; # objstore -> [ fgrp mirror objects ]
+our $FGRP_TODO; # objstore -> [[ to resume ], [ to clone ]]
our $TODO; # reference => [ non-fgrp mirror objects ]
our @PUH; # post-update hooks
$_[0]->{lei}->{opt}->{'keep-going'});
}
-sub _wq_done_wait { # dwaitpid callback (via wq_eof)
- my ($arg, $pid) = @_;
- my ($mrr, $lei) = @$arg;
+sub _wq_done_wait { # awaitpid cb (via wq_eof)
+ my ($pid, $mrr, $lei) = @_;
if ($?) {
$lei->child_error($?);
} elsif (!$lei->{child_error}) {
my ($k) = ($sw =~ /\A([\w-]+)/);
$opt->{$k} = $lei->{opt}->{$k};
}
- # force synchronous dwaitpid for v2:
+ # force synchronous awaitpid for v2:
local $PublicInbox::DS::in_loop = 0;
my $cfg = PublicInbox::Config->new(undef, $lei->{2});
my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
sub start_cmd {
my ($self, $cmd, $opt, $fini) = @_;
do_reap($self);
- $self->{lei}->qerr("# @$cmd");
+ utf8::decode(my $msg = "# @$cmd");
+ $self->{lei}->qerr($msg);
return if $self->{dry_run};
$LIVE->{spawn($cmd, undef, $opt)} = [ \&reap_cmd, $self, $cmd, $fini ]
}
($lei->{opt}->{jobs} // 1) > 1;
push @cmd, '-v' if $lei->{opt}->{verbose};
push(@cmd, '-p') if $lei->{opt}->{prune};
+ PublicInbox::Git::version() >= ((2 << 24) | (29 << 16)) and
+ push(@cmd, '--no-write-fetch-head');
@cmd;
}
print $w "$op ", join("\0", @rest, '') or die "print(w): $!";
}
+sub start_update_ref {
+ my ($fgrp) = @_;
+ pipe(my ($r, $w)) or die "pipe: $!";
+ my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
+ qw(update-ref --stdin -z) ];
+ my $pack = PublicInbox::OnDestroy->new($$, \&satellite_done, $fgrp);
+ start_cmd($fgrp, $cmd, { 0 => $r, 2 => $fgrp->{lei}->{2} }, $pack);
+ close $r or die "close(r): $!";
+ $fgrp->{dry_run} ? undef : $w;
+}
+
+sub upref_warn { warn "E: close(update-ref --stdin): $! (need git 1.8.5+)\n" }
+
sub fgrp_update {
my ($fgrp) = @_;
return if !keep_going($fgrp);
close $srcfh;
my %dst = map { chomp; split(/\0/) } (<$dstfh>);
close $dstfh;
- pipe(my ($r, $w)) or die "pipe: $!";
- my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
- qw(update-ref --stdin -z) ];
+ my $w = start_update_ref($fgrp) or return;
my $lei = $fgrp->{lei};
- my $pack = PublicInbox::OnDestroy->new($$, \&satellite_done, $fgrp);
- start_cmd($fgrp, $cmd, { 0 => $r, 2 => $lei->{2} }, $pack);
- close $r or die "close(r): $!";
- return if $fgrp->{dry_run};
+ my $ndel;
for my $ref (keys %dst) {
my $new = delete $src{$ref};
my $old = $dst{$ref};
upr($lei, $w, 'update', $ref, $new, $old);
} else {
upr($lei, $w, 'delete', $ref, $old);
+ ++$ndel;
}
}
- while (my ($ref, $oid) = each %src) {
- upr($lei, $w, 'create', $ref, $oid);
+ # git's ref files backend doesn't allow directory/file conflicts
+ # between `delete' and `create' ops:
+ if ($ndel && scalar(keys %src)) {
+ $fgrp->{-create_refs} = \%src;
+ } else {
+ while (my ($ref, $oid) = each %src) {
+ upr($lei, $w, 'create', $ref, $oid);
+ }
}
- close($w) or warn "E: close(update-ref --stdin): $! (need git 1.8.5+)\n";
+ close($w) or upref_warn();
}
sub satellite_done {
my ($fgrp) = @_;
- pack_refs($fgrp, $fgrp->{cur_dst});
- run_puh($fgrp);
+ if (my $create = delete $fgrp->{-create_refs}) {
+ my $w = start_update_ref($fgrp) or return;
+ while (my ($ref, $oid) = each %$create) {
+ upr($fgrp->{lei}, $w, 'create', $ref, $oid);
+ }
+ close($w) or upref_warn();
+ } else {
+ pack_refs($fgrp, $fgrp->{cur_dst});
+ run_puh($fgrp);
+ }
}
sub pack_refs {
my $opt = {};
my @fetch = do {
local $self->{lei}->{opt}->{jobs} = 1;
- (fetch_args($self->{lei}, $opt),
- qw(--no-tags --multiple));
+ (fetch_args($self->{lei}, $opt), qw(--no-tags --multiple));
};
push(@fetch, "-j$j") if $j;
- while (my ($osdir, $fgrpv) = each %$todo) {
+ while (my ($osdir, $fgrp_old_new) = each %$todo) {
my $f = "$osdir/config";
return if !keep_going($self);
+ my ($fgrpv, $new) = @$fgrp_old_new;
+ @$fgrpv = sort { $b->{-sort} <=> $a->{-sort} } @$fgrpv;
+ push @$fgrpv, @$new; # $new is ordered by references
+ my $cmd = ['git', "--git-dir=$osdir", qw(config -f), $f ];
# clobber group from previous run atomically
- my $cmd = ['git', "--git-dir=$osdir", qw(config -f),
- $f, '--unset-all', "remotes.$grp"];
- $self->{lei}->qerr("# @$cmd");
- if (!$self->{dry_run}) {
- my $pid = spawn($cmd, undef, { 2 => $self->{lei}->{2} });
+ for ("remotes.$grp") { # TODO: hideRefs
+ my $c = [ @$cmd, '--unset-all', $_ ];
+ $self->{lei}->qerr("# @$c");
+ next if $self->{dry_run};
+ my $pid = spawn($c, undef, $opt);
waitpid($pid, 0) // die "waitpid: $!";
- die "E: @$cmd: \$?=$?" if ($? && ($? >> 8) != 5);
+ die "E: @$c \$?=$?" if ($? && ($? >> 8) != 5);
+ }
+ # permanent configs:
+ my $cfg = PublicInbox::Config->git_config_dump($f);
+ for my $fgrp (@$fgrpv) {
+ my $u = $fgrp->{-uri} // die 'BUG: no {-uri}';
+ my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}';
+ for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*",
+ 'tagopt=--no-tags') {
+ my ($k, $v) = split(/=/, $_, 2);
+ $k = "remote.$rn.$k";
+ next if ($cfg->{$k} // '') eq $v;
+ my $c = [@$cmd, $k, $v];
+ $fgrp->{lei}->qerr("# @$c");
+ next if $fgrp->{dry_run};
+ run_die($c, undef, $opt);
+ }
+ }
+
+ if (!$self->{dry_run}) {
# update the config atomically via O_APPEND while
# respecting git-config locking
sysopen(my $lk, "$f.lock", O_CREAT|O_EXCL|O_WRONLY)
close $fh or die "close($f): $!";
unlink("$f.lock") or die "unlink($f.lock): $!";
}
-
$cmd = [ @git, "--git-dir=$osdir", @fetch, $grp ];
my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $fgrpv);
start_cmd($self, $cmd, $opt, $end);
my $dir = "$os/$fg.git";
if (!-d $dir && !$self->{dry_run}) {
PublicInbox::Import::init_bare($dir);
- my @cmd = ('git', "--git-dir=$dir", 'config');
- my $opt = { 2 => $self->{lei}->{2} };
- for ('repack.useDeltaIslands=true',
- 'pack.island=refs/remotes/([^/]+)/') {
- run_die([@cmd, split(/=/, $_, 2)], undef, $opt);
- }
+ my $f = "$dir/config";
+ open my $fh, '+>>', $f or die "open:($f): $!";
+ print $fh <<EOM or die "print($f): $!";
+[repack]
+ useDeltaIslands = true
+[pack]
+ island = refs/remotes/([^/]+)/
+EOM
+ close $fh or die "close($f): $!";
}
my $key = $self->{-key} // die 'BUG: no -key';
my $rn = substr(sha256_hex($key), 0, 16);
my $dst = $self->{cur_dst} // $self->{dst};
my @git = ('git', "--git-dir=$dst");
my $opt = { 2 => $self->{lei}->{2} };
- my $rn = 'origin'; # configurable?
+ my $rn = 'random'.int(rand(1 << 30));
for ("url=$uri", "fetch=+refs/*:refs/*", 'mirror=true') {
- my @kv = split(/=/, $_, 2);
- $kv[0] = "remote.$rn.$kv[0]";
- next if $self->{dry_run};
- run_die([@git, 'config', @kv], undef, $opt);
+ push @git, '-c', "remote.$rn.$_";
}
my $cmd = [ @{$self->{-torsocks}}, @git,
fetch_args($self->{lei}, $opt), $rn ];
sub fgrp_enqueue {
my ($fgrp, $end) = @_; # $end calls fgrp_fetch_all
return if !keep_going($fgrp);
- my $opt = { 2 => $fgrp->{lei}->{2} };
- # --no-tags is required to avoid conflicts
- my $u = $fgrp->{-uri} // die 'BUG: no {-uri}';
- my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}';
- my @cmd = ('git', "--git-dir=$fgrp->{-osdir}", 'config');
- for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*", 'tagopt=--no-tags') {
- my @kv = split(/=/, $_, 2);
- $kv[0] = "remote.$rn.$kv[0]";
- $fgrp->{dry_run} ? $fgrp->{lei}->qerr("# @cmd @kv") :
- run_die([@cmd, @kv], undef, $opt);
- }
++$fgrp->{chg}->{nr_chg};
- push @{$FGRP_TODO->{$fgrp->{-osdir}}}, $fgrp;
+ my $dst = $FGRP_TODO->{$fgrp->{-osdir}} //= [ [], [] ]; # [ old, new ]
+ push @{$dst->[defined($fgrp->{-sort} ? 0 : 1)]}, $fgrp;
}
sub clone_v1 {
my $resume = -d $dst;
if (my $fgrp = forkgroup_prep($self, $uri)) {
$fgrp->{-fini} = $fini;
- $resume ? cmp_fp_do($fgrp, \&fgrp_enqueue, $end)
- : fgrp_enqueue($fgrp, $end);
+ if ($resume) {
+ $fgrp->{-sort} = $fgrp->{-ent}->{modified};
+ cmp_fp_do($fgrp, \&fgrp_enqueue, $end);
+ } else { # new repo, save for last
+ fgrp_enqueue($fgrp, $end);
+ }
} elsif ($resume) {
cmp_fp_do($self, \&resume_fetch, $uri, $fini);
} else { # normal clone