+sub start_cmd {
+ my ($self, $cmd, $opt, $fini) = @_;
+ do_reap($self);
+ utf8::decode(my $msg = "# @$cmd");
+ $self->{lei}->qerr($msg);
+ return if $self->{dry_run};
+ $LIVE->{spawn($cmd, undef, $opt)} = [ \&reap_cmd, $self, $cmd, $fini ]
+}
+
+sub fetch_args ($$) {
+ my ($lei, $opt) = @_;
+ my @cmd; # (git --git-dir=...) to be added by caller
+ $opt->{$_} = $lei->{$_} for (0..2);
+ # we support "-c $key=$val" for arbitrary git config options
+ # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
+ push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
+ push @cmd, 'fetch';
+ push @cmd, '-q' if $lei->{opt}->{quiet} ||
+ ($lei->{opt}->{jobs} // 1) > 1;
+ push @cmd, '-v' if $lei->{opt}->{verbose};
+ push(@cmd, '-p') if $lei->{opt}->{prune};
+ PublicInbox::Git::version() >= ((2 << 24) | (29 << 16)) and
+ push(@cmd, '--no-write-fetch-head');
+ @cmd;
+}
+
+sub upr { # feed `git update-ref --stdin -z' verbosely
+ my ($lei, $w, $op, @rest) = @_; # ($ref, $oid) = @rest
+ $lei->qerr("# $op @rest") if $lei->{opt}->{verbose};
+ print $w "$op ", join("\0", @rest, '') or die "print(w): $!";
+}
+
+sub start_update_ref {
+ my ($fgrp) = @_;
+ pipe(my ($r, $w)) or die "pipe: $!";
+ my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
+ qw(update-ref --stdin -z) ];
+ my $pack = PublicInbox::OnDestroy->new($$, \&satellite_done, $fgrp);
+ start_cmd($fgrp, $cmd, { 0 => $r, 2 => $fgrp->{lei}->{2} }, $pack);
+ close $r or die "close(r): $!";
+ $fgrp->{dry_run} ? undef : $w;
+}
+
+sub upref_warn { warn "E: close(update-ref --stdin): $! (need git 1.8.5+)\n" }
+
+sub fgrp_update {
+ my ($fgrp) = @_;
+ return if !keep_going($fgrp);
+ my $srcfh = delete $fgrp->{srcfh} or return;
+ my $dstfh = delete $fgrp->{dstfh} or return;
+ seek($srcfh, SEEK_SET, 0) or die "seek(src): $!";
+ seek($dstfh, SEEK_SET, 0) or die "seek(dst): $!";
+ my %src = map { chomp; split(/\0/) } (<$srcfh>);
+ close $srcfh;
+ my %dst = map { chomp; split(/\0/) } (<$dstfh>);
+ close $dstfh;
+ my $w = start_update_ref($fgrp) or return;
+ my $lei = $fgrp->{lei};
+ my $ndel;
+ for my $ref (keys %dst) {
+ my $new = delete $src{$ref};
+ my $old = $dst{$ref};
+ if (defined $new) {
+ $new eq $old or
+ upr($lei, $w, 'update', $ref, $new, $old);
+ } else {
+ upr($lei, $w, 'delete', $ref, $old);
+ ++$ndel;
+ }
+ }
+ # git's ref files backend doesn't allow directory/file conflicts
+ # between `delete' and `create' ops:
+ if ($ndel && scalar(keys %src)) {
+ $fgrp->{-create_refs} = \%src;
+ } else {
+ while (my ($ref, $oid) = each %src) {
+ upr($lei, $w, 'create', $ref, $oid);
+ }
+ }
+ close($w) or upref_warn();
+}
+
+sub satellite_done {
+ my ($fgrp) = @_;
+ if (my $create = delete $fgrp->{-create_refs}) {
+ my $w = start_update_ref($fgrp) or return;
+ while (my ($ref, $oid) = each %$create) {
+ upr($fgrp->{lei}, $w, 'create', $ref, $oid);
+ }
+ close($w) or upref_warn();
+ } else {
+ pack_refs($fgrp, $fgrp->{cur_dst});
+ run_puh($fgrp);
+ }
+}
+
+sub pack_refs {
+ my ($self, $git_dir) = @_;
+ my $cmd = [ 'git', "--git-dir=$git_dir", qw(pack-refs --all --prune) ];
+ start_cmd($self, $cmd, { 2 => $self->{lei}->{2} });
+}
+
+sub fgrpv_done {
+ my ($fgrpv) = @_;
+ return if !$LIVE;
+ my $first = $fgrpv->[0] // die 'BUG: no fgrpv->[0]';
+ return if !keep_going($first);
+ pack_refs($first, $first->{-osdir}); # objstore refs always packed
+ for my $fgrp (@$fgrpv) {
+ my $rn = $fgrp->{-remote};
+ my %opt = ( 2 => $fgrp->{lei}->{2} );
+
+ my $update_ref = PublicInbox::OnDestroy->new($$,
+ \&fgrp_update, $fgrp);
+
+ my $src = [ 'git', "--git-dir=$fgrp->{-osdir}", 'for-each-ref',
+ "--format=refs/%(refname:lstrip=3)%00%(objectname)",
+ "refs/remotes/$rn/" ];
+ open(my $sfh, '+>', undef) or die "open(src): $!";
+ $fgrp->{srcfh} = $sfh;
+ start_cmd($fgrp, $src, { %opt, 1 => $sfh }, $update_ref);
+ my $dst = [ 'git', "--git-dir=$fgrp->{cur_dst}", 'for-each-ref',
+ '--format=%(refname)%00%(objectname)' ];
+ open(my $dfh, '+>', undef) or die "open(dst): $!";
+ $fgrp->{dstfh} = $dfh;
+ start_cmd($fgrp, $dst, { %opt, 1 => $dfh }, $update_ref);
+ }
+}
+
+sub fgrp_fetch_all {
+ my ($self) = @_;
+ my $todo = $FGRP_TODO;
+ $FGRP_TODO = \'BUG on further use';
+ keys(%$todo) or return;
+
+ # Rely on the fgrptmp remote groups in the config file rather
+ # than listing all remotes since the remote name list may exceed
+ # system argv limits:
+ my $grp = 'fgrptmp';
+
+ my @git = (@{$self->{-torsocks}}, 'git');
+ my $j = $self->{lei}->{opt}->{jobs};
+ my $opt = {};
+ my @fetch = do {
+ local $self->{lei}->{opt}->{jobs} = 1;
+ (fetch_args($self->{lei}, $opt),
+ qw(--no-tags --multiple));
+ };
+ push(@fetch, "-j$j") if $j;
+ while (my ($osdir, $fgrpv) = each %$todo) {
+ my $f = "$osdir/config";
+ return if !keep_going($self);
+
+ # clobber group from previous run atomically
+ my $cmd = ['git', "--git-dir=$osdir", qw(config -f),
+ $f, '--unset-all', "remotes.$grp"];
+ $self->{lei}->qerr("# @$cmd");
+ if (!$self->{dry_run}) {
+ my $pid = spawn($cmd, undef, { 2 => $self->{lei}->{2} });
+ waitpid($pid, 0) // die "waitpid: $!";
+ die "E: @$cmd: \$?=$?" if ($? && ($? >> 8) != 5);
+
+ # update the config atomically via O_APPEND while
+ # respecting git-config locking
+ sysopen(my $lk, "$f.lock", O_CREAT|O_EXCL|O_WRONLY)
+ or die "open($f.lock): $!";
+ open my $fh, '>>', $f or die "open(>>$f): $!";
+ $fh->autoflush(1);
+ my $buf = join('', "[remotes]\n",
+ map { "\t$grp = $_->{-remote}\n" } @$fgrpv);
+ print $fh $buf or die "print($f): $!";
+ close $fh or die "close($f): $!";
+ unlink("$f.lock") or die "unlink($f.lock): $!";
+ }
+
+ $cmd = [ @git, "--git-dir=$osdir", @fetch, $grp ];
+ my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $fgrpv);
+ start_cmd($self, $cmd, $opt, $end);
+ }
+}
+
+# keep this idempotent for future use by public-inbox-fetch
+sub forkgroup_prep {
+ my ($self, $uri) = @_;
+ $self->{-ent} // return;
+ my $os = $self->{-objstore} // return;
+ my $fg = $self->{-ent}->{forkgroup} // return;
+ my $dir = "$os/$fg.git";
+ if (!-d $dir && !$self->{dry_run}) {
+ PublicInbox::Import::init_bare($dir);
+ my @cmd = ('git', "--git-dir=$dir", 'config');
+ my $opt = { 2 => $self->{lei}->{2} };
+ for ('repack.useDeltaIslands=true',
+ 'pack.island=refs/remotes/([^/]+)/') {
+ run_die([@cmd, split(/=/, $_, 2)], undef, $opt);
+ }
+ }
+ my $key = $self->{-key} // die 'BUG: no -key';
+ my $rn = substr(sha256_hex($key), 0, 16);
+ if (!-d $self->{cur_dst} && !$self->{dry_run}) {
+ PublicInbox::Import::init_bare($self->{cur_dst});
+ my $f = "$self->{cur_dst}/config";
+ open my $fh, '+>>', $f or die "open:($f): $!";
+ print $fh <<EOM or die "print($f): $!";
+; rely on the "$rn" remote in the
+; $fg fork group for fetches
+; only uncomment the following iff you detach from fork groups
+; [remote "origin"]
+; url = $uri
+; fetch = +refs/*:refs/*
+; mirror = true
+EOM
+ close $fh or die "close($f): $!";
+ }
+ if (!$self->{dry_run}) {
+ my $alt = File::Spec->rel2abs("$dir/objects");
+ my $o = "$self->{cur_dst}/objects";
+ my $f = "$o/info/alternates";
+ my $l = File::Spec->abs2rel($alt, File::Spec->rel2abs($o));
+ open my $fh, '+>>', $f or die "open($f): $!";
+ seek($fh, SEEK_SET, 0) or die "seek($f): $!";
+ chomp(my @cur = <$fh>);
+ if (!grep(/\A\Q$l\E\z/, @cur)) {
+ say $fh $l or die "say($f): $!";
+ }
+ close $fh or die "close($f): $!";
+ }
+ bless {
+ %$self, -osdir => $dir, -remote => $rn, -uri => $uri
+ }, __PACKAGE__;
+}
+
+sub fp_done {
+ my ($self, $cmd, $cb, @arg) = @_;
+ if ($?) {
+ $self->{lei}->err("@$cmd failed (\$?=$?) (non-fatal)");
+ $? = 0; # don't let it influence normal exit
+ }
+ return if !keep_going($self);
+ my $fh = delete $self->{-show_ref} // die 'BUG: no show-ref output';
+ seek($fh, SEEK_SET, 0) or die "seek(show_ref): $!";
+ $self->{-ent} // die 'BUG: no -ent';
+ my $A = $self->{-ent}->{fingerprint} // die 'BUG: no fingerprint';
+ my $B = sha1_hex(do { local $/; <$fh> } // die("read(show_ref): $!"));
+ return $cb->($self, @arg) if $A ne $B;
+ $self->{lei}->qerr("# $self->{-key} up-to-date");
+}
+
+sub cmp_fp_do {
+ my ($self, $cb, @arg) = @_;
+ # $cb is either resume_fetch or fgrp_enqueue
+ $self->{-ent} // return $cb->($self, @arg);
+ my $new = $self->{-ent}->{fingerprint} // return $cb->($self, @arg);
+ my $key = $self->{-key} // die 'BUG: no -key';
+ if (my $cur_ent = $self->{-local_manifest}->{$key}) {
+ # runs go_fetch->DESTROY run if eq
+ return if $cur_ent->{fingerprint} eq $new;
+ }
+ my $dst = $self->{cur_dst} // $self->{dst};
+ my $cmd = ['git', "--git-dir=$dst", 'show-ref'];
+ my $opt = { 2 => $self->{lei}->{2} };
+ open($opt->{1}, '+>', undef) or die "open(tmp): $!";
+ $self->{-show_ref} = $opt->{1};
+ do_reap($self);
+ $self->{lei}->qerr("# @$cmd");
+ $LIVE->{spawn($cmd, undef, $opt)} = [ \&fp_done, $self, $cmd,
+ $cb, @arg ];
+}
+
+sub resume_fetch {
+ my ($self, $uri, $fini) = @_;
+ return if !keep_going($self);
+ my $dst = $self->{cur_dst} // $self->{dst};
+ my @git = ('git', "--git-dir=$dst");
+ my $opt = { 2 => $self->{lei}->{2} };
+ my $rn = 'random'.int(rand(1 << 30));
+ for ("url=$uri", "fetch=+refs/*:refs/*", 'mirror=true') {
+ push @git, '-c', "remote.$rn.$_";
+ }
+ my $cmd = [ @{$self->{-torsocks}}, @git,
+ fetch_args($self->{lei}, $opt), $rn ];
+ push @$cmd, '-P' if $self->{lei}->{prune}; # --prune-tags implied
+ my $run_puh = PublicInbox::OnDestroy->new($$, \&run_puh, $self, $fini);
+ ++$self->{chg}->{nr_chg};
+ start_cmd($self, $cmd, $opt, $run_puh);
+}
+
+sub fgrp_enqueue {
+ my ($fgrp, $end) = @_; # $end calls fgrp_fetch_all
+ return if !keep_going($fgrp);
+ my $opt = { 2 => $fgrp->{lei}->{2} };
+ # --no-tags is required to avoid conflicts
+ my $u = $fgrp->{-uri} // die 'BUG: no {-uri}';
+ my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}';
+ my @cmd = ('git', "--git-dir=$fgrp->{-osdir}", 'config');
+ for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*", 'tagopt=--no-tags') {
+ my @kv = split(/=/, $_, 2);
+ $kv[0] = "remote.$rn.$kv[0]";
+ $fgrp->{dry_run} ? $fgrp->{lei}->qerr("# @cmd @kv") :
+ run_die([@cmd, @kv], undef, $opt);
+ }
+ ++$fgrp->{chg}->{nr_chg};
+ push @{$FGRP_TODO->{$fgrp->{-osdir}}}, $fgrp;
+}
+