From: Eric Wong <e@80x24.org>
Date: Sun, 12 Feb 2023 23:18:28 +0000 (+0000)
Subject: lei_mirror: fetch most-recently-updated repos, first
X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=commitdiff_plain;h=431bb226ccd7b219ad79bd52090eac10d2aa1bb6

lei_mirror: fetch most-recently-updated repos, first

Within the same forkgroup, we can assume the most recently updated
repo has the most data, so fetch those, first.  We'll save new clones
for last since we can preserve {reference} ordering for them.
---

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index dd6356bb..4dedac9b 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -23,7 +23,7 @@ use PublicInbox::SHA qw(sha256_hex sha1_hex);
 use POSIX qw(strftime);
 
 our $LIVE; # pid => callback
-our $FGRP_TODO; # objstore -> [ fgrp mirror objects ]
+our $FGRP_TODO; # objstore -> [[ to resume ], [ to clone ]]
 our $TODO; # reference => [ non-fgrp mirror objects ]
 our @PUH; # post-update hooks
 
@@ -404,9 +404,12 @@ sub fgrp_fetch_all {
 		(fetch_args($self->{lei}, $opt), qw(--no-tags --multiple));
 	};
 	push(@fetch, "-j$j") if $j;
-	while (my ($osdir, $fgrpv) = each %$todo) {
+	while (my ($osdir, $fgrp_old_new) = each %$todo) {
 		my $f = "$osdir/config";
 		return if !keep_going($self);
+		my ($fgrpv, $new) = @$fgrp_old_new;
+		@$fgrpv = sort { $b->{-sort} <=> $a->{-sort} } @$fgrpv;
+		push @$fgrpv, @$new; # $new is ordered by references
 
 		my $cmd = ['git', "--git-dir=$osdir", qw(config -f), $f ];
 		# clobber group from previous run atomically
@@ -568,7 +571,8 @@ sub fgrp_enqueue {
 	my ($fgrp, $end) = @_; # $end calls fgrp_fetch_all
 	return if !keep_going($fgrp);
 	++$fgrp->{chg}->{nr_chg};
-	push @{$FGRP_TODO->{$fgrp->{-osdir}}}, $fgrp;
+	my $dst = $FGRP_TODO->{$fgrp->{-osdir}} //= [ [], [] ]; # [ old, new ]
+	push @{$dst->[defined($fgrp->{-sort} ? 0 : 1)]}, $fgrp;
 }
 
 sub clone_v1 {
@@ -586,8 +590,12 @@ sub clone_v1 {
 	my $resume = -d $dst;
 	if (my $fgrp = forkgroup_prep($self, $uri)) {
 		$fgrp->{-fini} = $fini;
-		$resume ? cmp_fp_do($fgrp, \&fgrp_enqueue, $end)
-			: fgrp_enqueue($fgrp, $end);
+		if ($resume) {
+			$fgrp->{-sort} = $fgrp->{-ent}->{modified};
+			cmp_fp_do($fgrp, \&fgrp_enqueue, $end);
+		} else { # new repo, save for last
+			fgrp_enqueue($fgrp, $end);
+		}
 	} elsif ($resume) {
 		cmp_fp_do($self, \&resume_fetch, $uri, $fini);
 	} else { # normal clone