use URI;
use PublicInbox::Config;
use PublicInbox::Inbox;
+use PublicInbox::Git;
use PublicInbox::LeiCurl;
use PublicInbox::OnDestroy;
-use Digest::SHA qw(sha256_hex sha1_hex);
+use PublicInbox::SHA qw(sha256_hex sha1_hex);
use POSIX qw(strftime);
our $LIVE; # pid => callback
our $FGRP_TODO; # objstore -> [ fgrp mirror objects ]
our $TODO; # reference => [ non-fgrp mirror objects ]
+our @PUH; # post-update hooks
sub keep_going ($) {
$LIVE && (!$_[0]->{lei}->{child_error} ||
$_[0]->{lei}->{opt}->{'keep-going'});
}
-sub _wq_done_wait { # dwaitpid callback (via wq_eof)
- my ($arg, $pid) = @_;
- my ($mrr, $lei) = @$arg;
+sub _wq_done_wait { # awaitpid cb (via wq_eof)
+ my ($pid, $mrr, $lei) = @_;
if ($?) {
$lei->child_error($?);
} elsif (!$lei->{child_error}) {
my ($k) = ($sw =~ /\A([\w-]+)/);
$opt->{$k} = $lei->{opt}->{$k};
}
- # force synchronous dwaitpid for v2:
+ # force synchronous awaitpid for v2:
local $PublicInbox::DS::in_loop = 0;
my $cfg = PublicInbox::Config->new(undef, $lei->{2});
my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
sub start_cmd {
my ($self, $cmd, $opt, $fini) = @_;
do_reap($self);
- $self->{lei}->qerr("# @$cmd");
+ utf8::decode(my $msg = "# @$cmd");
+ $self->{lei}->qerr($msg);
return if $self->{dry_run};
$LIVE->{spawn($cmd, undef, $opt)} = [ \&reap_cmd, $self, $cmd, $fini ]
}
($lei->{opt}->{jobs} // 1) > 1;
push @cmd, '-v' if $lei->{opt}->{verbose};
push(@cmd, '-p') if $lei->{opt}->{prune};
+ PublicInbox::Git::version() >= ((2 << 24) | (29 << 16)) and
+ push(@cmd, '--no-write-fetch-head');
@cmd;
}
print $w "$op ", join("\0", @rest, '') or die "print(w): $!";
}
+sub start_update_ref {
+ my ($fgrp) = @_;
+ pipe(my ($r, $w)) or die "pipe: $!";
+ my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
+ qw(update-ref --stdin -z) ];
+ my $pack = PublicInbox::OnDestroy->new($$, \&satellite_done, $fgrp);
+ start_cmd($fgrp, $cmd, { 0 => $r, 2 => $fgrp->{lei}->{2} }, $pack);
+ close $r or die "close(r): $!";
+ $fgrp->{dry_run} ? undef : $w;
+}
+
+sub upref_warn { warn "E: close(update-ref --stdin): $! (need git 1.8.5+)\n" }
+
sub fgrp_update {
my ($fgrp) = @_;
return if !keep_going($fgrp);
close $srcfh;
my %dst = map { chomp; split(/\0/) } (<$dstfh>);
close $dstfh;
- pipe(my ($r, $w)) or die "pipe: $!";
- my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
- qw(update-ref --stdin -z) ];
+ my $w = start_update_ref($fgrp) or return;
my $lei = $fgrp->{lei};
- my $pack = PublicInbox::OnDestroy->new($$, \&pack_dst, $fgrp);
- start_cmd($fgrp, $cmd, { 0 => $r, 2 => $lei->{2} }, $pack);
- close $r or die "close(r): $!";
- return if $fgrp->{dry_run};
+ my $ndel;
for my $ref (keys %dst) {
my $new = delete $src{$ref};
my $old = $dst{$ref};
upr($lei, $w, 'update', $ref, $new, $old);
} else {
upr($lei, $w, 'delete', $ref, $old);
+ ++$ndel;
}
}
- while (my ($ref, $oid) = each %src) {
- upr($lei, $w, 'create', $ref, $oid);
+ # git's ref files backend doesn't allow directory/file conflicts
+ # between `delete' and `create' ops:
+ if ($ndel && scalar(keys %src)) {
+ $fgrp->{-create_refs} = \%src;
+ } else {
+ while (my ($ref, $oid) = each %src) {
+ upr($lei, $w, 'create', $ref, $oid);
+ }
}
- close($w) or warn "E: close(update-ref --stdin): $! (need git 1.8.5+)\n";
+ close($w) or upref_warn();
}
-sub pack_dst { # packs lightweight satellite repos
+sub satellite_done {
my ($fgrp) = @_;
- pack_refs($fgrp, $fgrp->{cur_dst});
+ if (my $create = delete $fgrp->{-create_refs}) {
+ my $w = start_update_ref($fgrp) or return;
+ while (my ($ref, $oid) = each %$create) {
+ upr($fgrp->{lei}, $w, 'create', $ref, $oid);
+ }
+ close($w) or upref_warn();
+ } else {
+ pack_refs($fgrp, $fgrp->{cur_dst});
+ run_puh($fgrp);
+ }
}
sub pack_refs {
}
sub fp_done {
- my ($self, $cb, @arg) = @_;
+ my ($self, $cmd, $cb, @arg) = @_;
+ if ($?) {
+ $self->{lei}->err("@$cmd failed (\$?=$?) (non-fatal)");
+ $? = 0; # don't let it influence normal exit
+ }
return if !keep_going($self);
my $fh = delete $self->{-show_ref} // die 'BUG: no show-ref output';
seek($fh, SEEK_SET, 0) or die "seek(show_ref): $!";
my $opt = { 2 => $self->{lei}->{2} };
open($opt->{1}, '+>', undef) or die "open(tmp): $!";
$self->{-show_ref} = $opt->{1};
- my $done = PublicInbox::OnDestroy->new($$, \&fp_done, $self, $cb, @arg);
- start_cmd($self, $cmd, $opt, $done);
+ do_reap($self);
+ $self->{lei}->qerr("# @$cmd");
+ $LIVE->{spawn($cmd, undef, $opt)} = [ \&fp_done, $self, $cmd,
+ $cb, @arg ];
}
sub resume_fetch {
my $dst = $self->{cur_dst} // $self->{dst};
my @git = ('git', "--git-dir=$dst");
my $opt = { 2 => $self->{lei}->{2} };
- my $rn = 'origin'; # configurable?
+ my $rn = 'random'.int(rand(1 << 30));
for ("url=$uri", "fetch=+refs/*:refs/*", 'mirror=true') {
- my @kv = split(/=/, $_, 2);
- $kv[0] = "remote.$rn.$kv[0]";
- next if $self->{dry_run};
- run_die([@git, 'config', @kv], undef, $opt);
+ push @git, '-c', "remote.$rn.$_";
}
my $cmd = [ @{$self->{-torsocks}}, @git,
fetch_args($self->{lei}, $opt), $rn ];
push @$cmd, '-P' if $self->{lei}->{prune}; # --prune-tags implied
- start_cmd($self, $cmd, $opt, $fini);
+ my $run_puh = PublicInbox::OnDestroy->new($$, \&run_puh, $self, $fini);
+ ++$self->{chg}->{nr_chg};
+ start_cmd($self, $cmd, $opt, $run_puh);
}
sub fgrp_enqueue {
$fgrp->{dry_run} ? $fgrp->{lei}->qerr("# @cmd @kv") :
run_die([@cmd, @kv], undef, $opt);
}
+ ++$fgrp->{chg}->{nr_chg};
push @{$FGRP_TODO->{$fgrp->{-osdir}}}, $fgrp;
}
"$self->{dst}$ref";
}
}
- start_cmd($self, $cmd, $opt, $fini);
+ ++$self->{chg}->{nr_chg};
+ start_cmd($self, $cmd, $opt, PublicInbox::OnDestroy->new($$,
+ \&run_puh, $self, $fini));
}
if (!$self->{-is_epoch} && $lei->{opt}->{'inbox-config'} =~
/\A(?:always|v1)\z/s) {
ft_rename($ft, "$dn/$bn", 0666);
}
+sub run_next_puh {
+ my ($self) = @_;
+ my $puh = shift @{$self->{-puh_todo}} // return delete($self->{-fini});
+ my $fini = PublicInbox::OnDestroy->new($$, \&run_next_puh, $self);
+ my $cmd = [ @$puh, ($self->{cur_dst} // $self->{dst}) ];
+ my $opt = +{ map { $_ => $self->{lei}->{$_} } (0..2) };
+ start_cmd($self, $cmd, undef, $opt, $fini);
+}
+
+sub run_puh {
+ my ($self, $fini) = @_;
+ $self->{-fini} = $fini;
+ @{$self->{-puh_todo}} = @PUH;
+ run_next_puh($self);
+}
+
# modifies the to-be-written manifest entry, and sets values from it, too
sub update_ent {
my ($self) = @_;
}
if (my $symlinks = $self->{-ent}->{symlinks}) {
my $top = File::Spec->rel2abs($self->{dst});
+ push @{$self->{-new_symlinks}}, @$symlinks;
for my $p (@$symlinks) {
my $ln = "$top/$p";
$ln =~ tr!/!/!s;
}
}
symlink($tgt, $ln) or die "symlink($tgt, $ln): $!";
+ ++$self->{chg}->{nr_chg};
}
}
if (defined(my $t = $self->{-ent}->{modified})) {
# resolve multi-level references
while ($m && defined($nxt = $m->{$x}->{reference})) {
exists($todo->{$nxt}) or last;
- die <<EOM if ++$nr > 1000;
-E: dependency loop detected (`$x' => `$nxt')
+ if (++$nr > 1000) {
+ $m->{$x}->{reference} = undef;
+ $m->{$nxt}->{reference} = undef;
+ warn <<EOM
+E: dependency loop detected (`$x' => `$nxt'), breaking
EOM
+ }
$x = $nxt;
}
my $y = delete $todo->{$x} // next; # already done
$new{substr($_, 1)} = 1 for keys %$m; # drop leading '/'
my @list = sort keys %new;
my @remote = grep { !defined($new{$_}) } @list;
+ my %lnk = map { substr($_, 1) => undef } @{$self->{-new_symlinks}};
+ @remote = grep { !exists($lnk{$_}) } @remote;
warn <<EOM if @remote;
The following local repositories are ignored/gone from $self->{src}:
warn "\t", $_, "\n" for @local;
my (undef, $dn, $bn) = File::Spec->splitpath($f);
- atomic_write($dn, $bn, join("\n", @list, ''));
+ $self->{chg}->{nr_chg} += scalar(@remote) + scalar(@local);
+ my $new = join("\n", @list, '');
+ atomic_write($dn, $bn, $new) if $new ne $old;
}
# FIXME: this gets confused by single inbox instance w/ global manifest.js.gz
}
# bail out if curl -z/--timecond hit 304 Not Modified, $ft will be empty
- return $lei->qerr("# $manifest unchanged") if -f $manifest && !-s $ft;
+ if (-f $manifest && !-s $ft) {
+ $lei->child_error(127 << 8) if $lei->{opt}->{'exit-code'};
+ return $lei->qerr("# $manifest unchanged");
+ }
my $m = eval { decode_manifest($ft, $ft, $uri) };
if ($@) {
}
local $self->{chg} = {};
local $self->{-local_manifest} = load_current_manifest($self);
+ local $self->{-new_symlinks} = [];
my ($path_pfx, $n, $multi) = multi_inbox($self, \$path, $m);
return $lei->child_error(1, $multi) if !ref($multi);
my $v2 = delete $multi->{v2};
EOM
dump_project_list($self, $m);
ft_rename($ft, $manifest, 0666);
+ !$self->{chg}->{nr_chg} && $lei->{opt}->{'exit-code'} and
+ $lei->child_error(127 << 8);
}
sub start_clone_url {
$self->{dry_run} = 1 if $lei->{opt}->{'dry-run'};
umask($lei->{client_umask}) if defined $lei->{client_umask};
$self->{-initial_clone} = 1 if !-d $self->{dst};
+ local @PUH;
+ if (defined(my $puh = $lei->{opt}->{'post-update-hook'})) {
+ require Text::ParseWords;
+ @PUH = map { [ Text::ParseWords::shellwords($_) ] } @$puh;
+ }
eval {
my $ic = $lei->{opt}->{'inbox-config'} //= 'always';
$ic =~ /\A(?:v1|v2|always|never)\z/s or die <<"";