multi_git: hoist out common epoch/alternates handling

[public-inbox.git] / lib / PublicInbox / Fetch.pm

diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 9613a5829bad92a006811ecbff7cd2a48564136d..9ea55e9dcf9272d2ddd6e2715433babccc1a80f4 100644 (file)
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -6,12 +6,11 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC);
 use URI ();
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Spawn qw(popen_rd run_die);
 use PublicInbox::Admin;
 use PublicInbox::LEI;
 use PublicInbox::LeiCurl;
 use PublicInbox::LeiMirror;
-use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
 use File::Temp ();
 
 sub new { bless {}, __PACKAGE__ }
@@ -53,7 +52,6 @@ sub do_manifest ($$$) {
                        PublicInbox::LeiMirror::decode_manifest($fh, $mf, $mf)
                };
                $lei->err($@) if $@;
-               push @opt, '-z', $mf if defined($m0);
        }
        my $curl_cmd = $lei->{curl}->for_uri($lei, $muri, @opt);
        my $opt = {};
@@ -64,11 +62,11 @@ sub do_manifest ($$$) {
                $lei->child_error($cerr, "@$curl_cmd failed");
                return;
        }
-       return [ 304 ] if !-s $ft; # 304 Not Modified via curl -z
        my $m1 = PublicInbox::LeiMirror::decode_manifest($ft, $fn, $muri);
        my $mdiff = { %$m1 };
 
-       # filter out unchanged entries
+       # filter out unchanged entries.  We check modified, too, since
+       # fingerprints are SHA-1, so there's a teeny chance they'll collide
        while (my ($k, $v0) = each %{$m0 // {}}) {
                my $cur = $m1->{$k} // next;
                my $f0 = $v0->{fingerprint} // next;
@@ -77,9 +75,10 @@ sub do_manifest ($$$) {
                my $t1 = $cur->{modified} // next;
                delete($mdiff->{$k}) if $f0 eq $f1 && $t0 == $t1;
        }
-       my ($path_pfx, $v1_bare, @v2_epochs) =
+       return unless keys %$mdiff;
+       my (undef, $v1_path, @v2_epochs) =
                PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path);
-       [ 200, $path_pfx, $v1_bare, \@v2_epochs, $muri, $ft, $mf ];
+       [ 200, $v1_path, \@v2_epochs, $muri, $ft, $mf ];
 }
 
 sub do_fetch {
@@ -87,15 +86,15 @@ sub do_fetch {
        my $ibx_ver;
        $lei->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
        my $dir = PublicInbox::Admin::resolve_inboxdir($cd, \$ibx_ver);
-       my ($ibx_uri, @git_dir, @epochs);
+       my ($ibx_uri, @git_dir, @epochs, $mg, @new_epoch);
        if ($ibx_ver == 1) {
                my $url = remote_url($lei, $dir) //
                        die "E: $dir missing remote.origin.url\n";
                $ibx_uri = URI->new($url);
        } else { # v2:
-               opendir my $dh, "$dir/git" or die "opendir $dir/git: $!";
-               @epochs = sort { $b <=> $a } map { substr($_, 0, -4) + 0 }
-                                       grep(/\A[0-9]+\.git\z/, readdir($dh));
+               require PublicInbox::MultiGit;
+               $mg = PublicInbox::MultiGit->new($dir, 'all.git', 'git');
+               my @epochs = $mg->git_epochs;
                my ($git_url, $epoch);
                for my $nr (@epochs) { # try newest epoch, first
                        my $edir = "$dir/git/$nr.git";
@@ -117,27 +116,23 @@ EOM
        }
        $lei->qerr("# inbox URL: $ibx_uri/");
        my $res = do_manifest($lei, $dir, $ibx_uri) or return;
-       my ($code, $path_pfx, $v1_bare, $v2_epochs, $muri, $ft, $mf) = @$res;
-       return if $code == 304;
+       my ($code, $v1_path, $v2_epochs, $muri, $ft, $mf) = @$res;
        if ($code == 404) {
                # any pre-manifest.js.gz instances running? Just fetch all
                # existing ones and unconditionally try cloning the next
-               $v2_epochs = [ map {;
-                               "$dir/git/$_.git";
-                               } @epochs ];
+               $v2_epochs = [ map { "$dir/git/$_.git" } @epochs ];
                push @$v2_epochs, "$dir/git/".($epochs[-1] + 1) if @epochs;
        } else {
                $code == 200 or die "BUG unexpected code $code\n";
        }
        if ($ibx_ver == 2) {
-               defined($v1_bare) and warn <<EOM;
-E: got v1 `$v1_bare' when expecting v2 epoch(s) in <$muri>, WTF?
+               defined($v1_path) and warn <<EOM;
+E: got v1 `$v1_path' when expecting v2 epoch(s) in <$muri>, WTF?
 EOM
                @git_dir = map { "$dir/git/$_.git" } sort { $a <=> $b }
                        map { my ($nr) = (m!/([0-9]+)\.git\z!g) } @$v2_epochs;
        } else {
-               $v1_bare eq $dir or warn "$v1_bare != $dir";
-               $git_dir[0] = $v1_bare // $dir;
+               $git_dir[0] = $dir;
        }
        # n.b. this expects all epochs are from the same host
        my $torsocks = $lei->{curl}->torsocks($lei, $muri);
@@ -149,11 +144,14 @@ EOM
                        $cmd = [ @$torsocks, fetch_cmd($lei, $opt) ];
                } else {
                        my $e_uri = $ibx_uri->clone;
-                       my ($epath) = ($d =~ m!/(git/[0-9]+\.git)\z!);
+                       my ($epath) = ($d =~ m!(/git/[0-9]+\.git)\z!);
+                       defined($epath) or
+                               die "BUG: $d is not an epoch to clone\n";
                        $e_uri->path($ibx_uri->path.$epath);
                        $cmd = [ @$torsocks,
                                PublicInbox::LeiMirror::clone_cmd($lei, $opt),
                                $$e_uri, $d];
+                       push @new_epoch, substr($epath, 5, -4) + 0;
                }
                my $cerr = PublicInbox::LeiMirror::run_reap($lei, $cmd, $opt);
                # do not bail on clone failure if we didn't have a manifest
@@ -162,6 +160,7 @@ EOM
                        return;
                }
        }
+       for my $i (@new_epoch) { $mg->epoch_cfg_set($i) }
        if ($ft) {
                my $fn = $ft->filename;
                rename($fn, $mf) or die "E: rename($fn, $mf): $!\n";