lei_mirror: fetch most-recently-updated repos, first

Within the same forkgroup, we can assume the most recently updated
repo has the most data, so fetch those, first.  We'll save new clones
for last since we can preserve {reference} ordering for them.

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index dd6356bbc340d8fc754d781148351c5ce7a32c14..4dedac9b36c5be711a19844d3b471806726712ef 100644 (file)
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -23,7 +23,7 @@ use PublicInbox::SHA qw(sha256_hex sha1_hex);
 use POSIX qw(strftime);
 
 our $LIVE; # pid => callback
-our $FGRP_TODO; # objstore -> [ fgrp mirror objects ]
+our $FGRP_TODO; # objstore -> [[ to resume ], [ to clone ]]
 our $TODO; # reference => [ non-fgrp mirror objects ]
 our @PUH; # post-update hooks
 
@@ -404,9 +404,12 @@ sub fgrp_fetch_all {
                (fetch_args($self->{lei}, $opt), qw(--no-tags --multiple));
        };
        push(@fetch, "-j$j") if $j;
-       while (my ($osdir, $fgrpv) = each %$todo) {
+       while (my ($osdir, $fgrp_old_new) = each %$todo) {
                my $f = "$osdir/config";
                return if !keep_going($self);
+               my ($fgrpv, $new) = @$fgrp_old_new;
+               @$fgrpv = sort { $b->{-sort} <=> $a->{-sort} } @$fgrpv;
+               push @$fgrpv, @$new; # $new is ordered by references
 
                my $cmd = ['git', "--git-dir=$osdir", qw(config -f), $f ];
                # clobber group from previous run atomically
@@ -568,7 +571,8 @@ sub fgrp_enqueue {
        my ($fgrp, $end) = @_; # $end calls fgrp_fetch_all
        return if !keep_going($fgrp);
        ++$fgrp->{chg}->{nr_chg};
-       push @{$FGRP_TODO->{$fgrp->{-osdir}}}, $fgrp;
+       my $dst = $FGRP_TODO->{$fgrp->{-osdir}} //= [ [], [] ]; # [ old, new ]
+       push @{$dst->[defined($fgrp->{-sort} ? 0 : 1)]}, $fgrp;
 }
 
 sub clone_v1 {
@@ -586,8 +590,12 @@ sub clone_v1 {
        my $resume = -d $dst;
        if (my $fgrp = forkgroup_prep($self, $uri)) {
                $fgrp->{-fini} = $fini;
-               $resume ? cmp_fp_do($fgrp, \&fgrp_enqueue, $end)
-                       : fgrp_enqueue($fgrp, $end);
+               if ($resume) {
+                       $fgrp->{-sort} = $fgrp->{-ent}->{modified};
+                       cmp_fp_do($fgrp, \&fgrp_enqueue, $end);
+               } else { # new repo, save for last
+                       fgrp_enqueue($fgrp, $end);
+               }
        } elsif ($resume) {
                cmp_fp_do($self, \&resume_fetch, $uri, $fini);
        } else { # normal clone