lei_mirror: support --objstore and forkgroups

[public-inbox.git] / lib / PublicInbox / Fetch.pm

diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index e5756fb664e44cd774a6daa20214cd6c0291a943..3dbb0b55953912e1129a566004de0a711595b562 100644 (file)
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -12,32 +12,36 @@ use PublicInbox::LEI;
 use PublicInbox::LeiCurl;
 use PublicInbox::LeiMirror;
 use File::Temp ();
-use PublicInbox::Config;
-use IO::Compress::Gzip qw(gzip $GzipError);
 
 sub new { bless {}, __PACKAGE__ }
 
-sub fetch_args ($$) {
-       my ($lei, $opt) = @_;
-       my @cmd; # (git --git-dir=...) to be added by caller
-       $opt->{$_} = $lei->{$_} for (0..2);
-       # we support "-c $key=$val" for arbitrary git config options
-       # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
-       push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
-       push @cmd, 'fetch';
-       push @cmd, '-q' if $lei->{opt}->{quiet};
-       push @cmd, '-v' if $lei->{opt}->{verbose};
-       @cmd;
+sub remote_url ($$) {
+       my ($lei, $dir) = @_;
+       my $rn = $lei->{opt}->{'try-remote'} // [ 'origin', '_grokmirror' ];
+       for my $r (@$rn) {
+               my $cmd = [ qw(git config), "remote.$r.url" ];
+               my $fh = popen_rd($cmd, undef, { -C => $dir, 2 => $lei->{2} });
+               my $url = <$fh>;
+               close $fh or next;
+               $url =~ s!/*\n!!s;
+               return $url;
+       }
+       undef
 }
 
-sub remote_url ($$) {
-       my ($lei, $dir) = @_; # TODO: support non-"origin"?
-       my $cmd = [ qw(git config remote.origin.url) ];
-       my $fh = popen_rd($cmd, undef, { -C => $dir, 2 => $lei->{2} });
-       my $url = <$fh>;
-       close $fh or return;
-       $url =~ s!/*\n!!s;
-       $url;
+# PSGI mount prefixes and manifest.js.gz prefixes don't always align...
+# TODO: remove, handle multi-inbox fetch
+sub deduce_epochs ($$) {
+       my ($m, $path) = @_;
+       my ($v1_ent, @v2_epochs);
+       my $path_pfx = '';
+       $path =~ s!/+\z!!;
+       do {
+               $v1_ent = $m->{$path};
+               @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
+       } while (!defined($v1_ent) && !@v2_epochs &&
+               $path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1);
+       ($path_pfx, $v1_ent ? $path : undef, @v2_epochs);
 }
 
 sub do_manifest ($$$) {
@@ -84,7 +88,7 @@ sub do_manifest ($$$) {
                return;
        }
        my (undef, $v1_path, @v2_epochs) =
-               PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path);
+               deduce_epochs($mdiff, $ibx_uri->path);
        [ 200, $muri, $v1_path, \@v2_epochs, $ft, $mf, $m1 ];
 }
 
@@ -110,7 +114,7 @@ sub do_fetch { # main entry point
        my ($ibx_uri, @git_dir, @epochs, $mg, @new_epoch, $skip);
        if ($ibx_ver == 1) {
                my $url = remote_url($lei, $dir) //
-                       die "E: $dir missing remote.origin.url\n";
+                       die "E: $dir missing remote.*.url\n";
                $ibx_uri = URI->new($url);
        } else { # v2:
                require PublicInbox::MultiGit;
@@ -128,7 +132,7 @@ sub do_fetch { # main entry point
                                $git_url = $url;
                                $epoch = $nr;
                        } else {
-                               warn "W: $edir missing remote.origin.url\n";
+                               warn "W: $edir missing remote.*.url\n";
                                my $pid = spawn([qw(git config -l)], undef,
                                        { 1 => $lei->{2}, 2 => $lei->{2} });
                                waitpid($pid, 0);
@@ -188,7 +192,7 @@ EOM
                if (-d $d) {
                        $fp2->[0] = get_fingerprint2($d) if $fp2;
                        $cmd = [ @$torsocks, 'git', "--git-dir=$d",
-                               fetch_args($lei, $opt) ];
+                              PublicInbox::LeiMirror::fetch_args($lei, $opt)];
                } else {
                        my $e_uri = $ibx_uri->clone;
                        my ($epath) = ($d =~ m!(/git/[0-9]+\.git)\z!);
@@ -214,13 +218,8 @@ EOM
        }
        for my $i (@new_epoch) { $mg->epoch_cfg_set($i) }
        if ($ft) {
-               my $fn = $ft->filename;
-               if ($mculled) {
-                       my $json = PublicInbox::Config->json->encode($m1);
-                       gzip(\$json => $fn) or die "gzip: $GzipError";
-               }
-               rename($fn, $mf) or die "E: rename($fn, $mf): $!\n";
-               $ft->unlink_on_destroy(0);
+               PublicInbox::LeiMirror::dump_manifest($m1 => $ft) if $mculled;
+               PublicInbox::LeiMirror::ft_rename($ft, $mf, 0666);
        }
        $lei->child_error($xit << 8) if $fp2 && $xit;
 }