solver: check one git coderepo and inbox at a time

With public-inbox-httpd, this mitigates the effect of slow git
blob storage with multiple coderepos configured for an inbox.
It's still synchronous for now (and may need to remain that way
for ->last_check_err), but no longer monopolizes the event loop
when checking multiple coderepos.

We don't yet support multi-inbox scanning, yet; but this also
prepares us for a future where we do.

We'll also support >=40 char blob OIDs in preparation for future
git SHA-256 support, too.

diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm
index dd95f40043b14c3cf230393845ed812bf898d489..12024dbcc6dfa143b444e0e24959e1b405991a0c 100644 (file)
--- a/lib/PublicInbox/SolverGit.pm
+++ b/lib/PublicInbox/SolverGit.pm
@@ -70,34 +70,38 @@ sub ERR ($$) {
        die $err;
 }
 
-# look for existing objects already in git repos
+# look for existing objects already in git repos, returns arrayref
+# if found, number of remaining git coderepos to try if not.
 sub solve_existing ($$) {
        my ($self, $want) = @_;
+       my $try = $want->{try_gits} //= [ @{$self->{gits}} ]; # array copy
+       my $git = shift @$try or die 'BUG {try_gits} empty';
        my $oid_b = $want->{oid_b};
-       my $have_hints = scalar keys %$want > 1;
-       my @ambiguous; # Array of [ git, $oids]
-       foreach my $git (@{$self->{gits}}) {
-               my ($oid_full, $type, $size) = $git->check($oid_b);
-               if (defined($type) && (!$have_hints || $type eq 'blob')) {
-                       return [ $git, $oid_full, $type, int($size) ];
-               }
+       my ($oid_full, $type, $size) = $git->check($oid_b);
+
+       # other than {oid_b, try_gits, try_ibxs}
+       my $have_hints = scalar keys %$want > 3;
+       if (defined($type) && (!$have_hints || $type eq 'blob')) {
+               delete $want->{try_gits};
+               return [ $git, $oid_full, $type, int($size) ]; # done, success
+       }
 
-               next if length($oid_b) == 40;
+       # TODO: deal with 40-char "abbreviations" with future SHA-256 git
+       return scalar(@$try) if length($oid_b) >= 40;
 
-               # parse stderr of "git cat-file --batch-check"
-               my $err = $git->last_check_err;
-               my (@oids) = ($err =~ /\b([a-f0-9]{40})\s+blob\b/g);
-               next unless scalar(@oids);
+       # parse stderr of "git cat-file --batch-check"
+       my $err = $git->last_check_err;
+       my (@oids) = ($err =~ /\b([a-f0-9]{40,})\s+blob\b/g);
+       return scalar(@$try) unless scalar(@oids);
 
-               # TODO: do something with the ambiguous array?
-               # push @ambiguous, [ $git, @oids ];
+       # TODO: do something with the ambiguous array?
+       # push @ambiguous, [ $git, @oids ];
 
-               dbg($self, "`$oid_b' ambiguous in " .
-                               join("\n\t", $git->pub_urls($self->{psgi_env}))
-                                . "\n" .
-                               join('', map { "$_ blob\n" } @oids));
-       }
-       scalar(@ambiguous) ? \@ambiguous : undef;
+       dbg($self, "`$oid_b' ambiguous in " .
+                       join("\n\t", $git->pub_urls($self->{psgi_env}))
+                       . "\n" .
+                       join('', map { "$_ blob\n" } @oids));
+       scalar(@$try);
 }
 
 sub extract_diff ($$) {
@@ -523,10 +527,12 @@ sub resolve_patch ($$) {
 
        # see if we can find the blob in an existing git repo:
        my $cur_want = $want->{oid_b};
-       if ($self->{seen_oid}->{$cur_want}++) {
+       if (!$want->{try_ibxs} && $self->{seen_oid}->{$cur_want}++) {
                die "Loop detected solving $cur_want\n";
        }
-       if (my $existing = solve_existing($self, $want)) {
+       $want->{try_ibxs} //= [ @{$self->{inboxes}} ]; # array copy
+       my $existing = solve_existing($self, $want);
+       if (ref $existing) {
                my ($found_git, undef, $type, undef) = @$existing;
                dbg($self, "found $cur_want in " .
                        join(" ||\n\t",
@@ -539,13 +545,17 @@ sub resolve_patch ($$) {
                }
                mark_found($self, $cur_want, $existing);
                return next_step($self); # onto patch application
+       } elsif ($existing > 0) {
+               push @{$self->{todo}}, $want;
+               return next_step($self); # retry solve_existing
+       } else { # $existing == 0: we may retry if inbox scan (below) fails
+               delete $want->{try_gits};
        }
 
        # scan through inboxes to look for emails which results in
        # the oid we want:
-       foreach my $ibx (@{$self->{inboxes}}) {
-               my $diffs = find_extract_diffs($self, $ibx, $want) or next;
-
+       my $ibx = shift(@{$want->{try_ibxs}}) or die 'BUG: {try_ibxs} empty';
+       if (my $diffs = find_extract_diffs($self, $ibx, $want)) {
                unshift @{$self->{patches}}, @$diffs;
                dbg($self, "found $cur_want in ".
                        join(" ||\n\t", map { di_url($self, $_) } @$diffs));
@@ -562,7 +572,14 @@ sub resolve_patch ($$) {
                }
                return next_step($self); # onto the next todo item
        }
-       if (length($cur_want) > $OID_MIN) {
+
+       if (scalar @{$want->{try_ibxs}}) { # do we have more inboxes to try?
+               push @{$self->{todo}}, $want;
+               return next_step($self);
+       }
+
+       if (length($cur_want) > $OID_MIN) { # maybe a shorter OID will work
+               delete $want->{try_ibxs}; # drop empty arrayref
                chop($cur_want);
                dbg($self, "retrying $want->{oid_b} as $cur_want");
                $want->{oid_b} = $cur_want;

diff --git a/t/solver_git.t b/t/solver_git.t
index c162b60557cb540b578f7516c105e07cdfc02e55..6b0ed8d27677dcaecbf1ed5070f05ef481e43695 100644 (file)
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -35,6 +35,7 @@ my $deliver_patch = sub ($) {
 
 $deliver_patch->('t/solve/0001-simple-mod.patch');
 my $v1_0_0_tag = 'cb7c42b1e15577ed2215356a2bf925aef59cdd8d';
+my $v1_0_0_tag_short = substr($v1_0_0_tag, 0, 16);
 
 my $git = PublicInbox::Git->new($git_dir);
 $ibx->{-repo_objs} = [ $git ];
@@ -173,7 +174,9 @@ EOF
                is($res->code, 404, 'failure with null OID');
 
                $res = $cb->(GET("/$name/$v1_0_0_tag/s/"));
-               is($res->code, 200, 'shows commit');
+               is($res->code, 200, 'shows commit (unabbreviated)');
+               $res = $cb->(GET("/$name/$v1_0_0_tag_short/s/"));
+               is($res->code, 200, 'shows commit (abbreviated)');
                while (my ($label, $size) = each %bin) {
                        $res = $cb->(GET("/$name/$oid{$label}/s/"));
                        is($res->code, 200, "$label binary file");