X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSolverGit.pm;h=38fba0ca79340eb655c12a319ab28d54621644fa;hb=HEAD;hp=62b5a3439b2be772b004f13a5a8c2380a171f200;hpb=a34e62ec1d84eafb67cc63532a383d15a18dcc4d;p=public-inbox.git diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm index 62b5a343..38fba0ca 100644 --- a/lib/PublicInbox/SolverGit.pm +++ b/lib/PublicInbox/SolverGit.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2021 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # "Solve" blobs which don't exist in git code repositories by @@ -81,9 +81,8 @@ sub solve_existing ($$) { my $oid_b = $want->{oid_b}; my ($oid_full, $type, $size) = $git->check($oid_b); - # other than {oid_b, try_gits, try_ibxs} - my $have_hints = scalar keys %$want > 3; - if (defined($type) && (!$have_hints || $type eq 'blob')) { + if ($oid_b eq ($oid_full // '') || (defined($type) && + (!$self->{have_hints} || $type eq 'blob'))) { delete $want->{try_gits}; return [ $git, $oid_full, $type, int($size) ]; # done, success } @@ -106,6 +105,11 @@ sub solve_existing ($$) { scalar(@$try); } +sub _tmp { + $_[0]->{tmp} //= + File::Temp->newdir("solver.$_[0]->{oid_want}-XXXX", TMPDIR => 1); +} + sub extract_diff ($$) { my ($p, $arg) = @_; my ($self, $want, $smsg) = @$arg; @@ -193,8 +197,8 @@ sub extract_diff ($$) { my $path = ++$self->{tot}; $di->{n} = $path; - open(my $tmp, '>:utf8', $self->{tmp}->dirname . "/$path") or - die "open(tmp): $!"; + my $f = _tmp($self)->dirname."/$path"; + open(my $tmp, '>:utf8', $f) or die "open($f): $!"; print $tmp $di->{hdr_lines}, $patch or die "print(tmp): $!"; close $tmp or die "close(tmp): $!"; @@ -242,10 +246,8 @@ sub find_smsgs ($$$) { sub update_index_result ($$) { my ($bref, $self) = @_; - my ($qsp, $msg) = delete @$self{qw(-qsp -msg)}; - if (my $err = $qsp->{err}) { - ERR($self, "git update-index error: $err"); - } + my ($qsp_err, $msg) = delete @$self{qw(-qsp_err -msg)}; + ERR($self, "git update-index error:$qsp_err") if $qsp_err; dbg($self, $msg); next_step($self); # onto do_git_apply } @@ -278,7 +280,7 @@ sub prepare_index ($) { my $cmd = [ qw(git update-index -z --index-info) ]; my $qsp = PublicInbox::Qspawn->new($cmd, $self->{git_env}, $rdr); $path_a = git_quote($path_a); - $self->{-qsp} = $qsp; + $qsp->{qsp_err} = \($self->{-qsp_err} = ''); $self->{-msg} = "index prepared:\n$mode_a $oid_full\t$path_a"; $qsp->psgi_qx($self->{psgi_env}, undef, \&update_index_result, $self); } @@ -286,8 +288,7 @@ sub prepare_index ($) { # pure Perl "git init" sub do_git_init ($) { my ($self) = @_; - my $dir = $self->{tmp}->dirname; - my $git_dir = "$dir/git"; + my $git_dir = _tmp($self)->dirname.'/git'; foreach ('', qw(objects refs objects/info refs/heads)) { mkdir("$git_dir/$_") or die "mkdir $_: $!"; @@ -301,7 +302,6 @@ sub do_git_init ($) { repositoryFormatVersion = $v filemode = true bare = false - fsyncObjectfiles = false logAllRefUpdates = false EOF print $fh <{git_env} = { GIT_DIR => $git_dir, GIT_INDEX_FILE => "$git_dir/index", + GIT_TEST_FSYNC => 0, # undocumented git env }; prepare_index($self); } @@ -405,21 +406,18 @@ sub mark_found ($$$) { sub parse_ls_files ($$) { my ($self, $bref) = @_; - my ($qsp, $di) = delete @$self{qw(-qsp -cur_di)}; - if (my $err = $qsp->{err}) { - die "git ls-files error: $err"; - } + my ($qsp_err, $di) = delete @$self{qw(-qsp_err -cur_di)}; + die "git ls-files -s -z error:$qsp_err" if $qsp_err; - my ($line, @extra) = split(/\0/, $$bref); + my @ls = split(/\0/, $$bref); + my ($line, @extra) = grep(/\t\Q$di->{path_b}\E\z/, @ls); scalar(@extra) and die "BUG: extra files in index: <", - join('> <', @extra), ">"; - + join('> <', $line, @extra), ">"; + $line // die "no \Q$di->{path_b}\E in <",join('> <', @ls), '>'; my ($info, $file) = split(/\t/, $line, 2); my ($mode_b, $oid_b_full, $stage) = split(/ /, $info); - if ($file ne $di->{path_b}) { - die + $file eq $di->{path_b} or die "BUG: index mismatch: file=$file != path_b=$di->{path_b}"; - } my $tmp_git = $self->{tmp_git} or die 'no git working tree'; my (undef, undef, $size) = $tmp_git->check($oid_b_full); @@ -456,17 +454,18 @@ sub skip_identical ($$$) { sub apply_result ($$) { my ($bref, $self) = @_; - my ($qsp, $di) = delete @$self{qw(-qsp -cur_di)}; + my ($qsp_err, $di) = delete @$self{qw(-qsp_err -cur_di)}; dbg($self, $$bref); my $patches = $self->{patches}; - if (my $err = $qsp->{err}) { - my $msg = "git apply error: $err"; + if ($qsp_err) { + my $msg = "git apply error:$qsp_err"; my $nxt = $patches->[0]; if ($nxt && oids_same_ish($nxt->{oid_b}, $di->{oid_b})) { dbg($self, $msg); dbg($self, 'trying '.di_url($self, $nxt)); return do_git_apply($self); } else { + $msg .= " (no patches left to try for $di->{oid_b})\n"; ERR($self, $msg); } } else { @@ -474,30 +473,28 @@ sub apply_result ($$) { } my @cmd = qw(git ls-files -s -z); - $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env}); + my $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env}); $self->{-cur_di} = $di; - $self->{-qsp} = $qsp; + $qsp->{qsp_err} = \($self->{-qsp_err} = ''); $qsp->psgi_qx($self->{psgi_env}, undef, \&ls_files_result, $self); } sub do_git_apply ($) { my ($self) = @_; - my $dn = $self->{tmp}->dirname; my $patches = $self->{patches}; # we need --ignore-whitespace because some patches are CRLF my @cmd = (qw(git apply --cached --ignore-whitespace --unidiff-zero --whitespace=warn --verbose)); my $len = length(join(' ', @cmd)); - my $total = $self->{tot}; my $di; # keep track of the last one for "git ls-files" my $prv_oid_b; do { my $i = ++$self->{nr}; $di = shift @$patches; - dbg($self, "\napplying [$i/$total] " . di_url($self, $di) . - "\n" . $di->{hdr_lines}); + dbg($self, "\napplying [$i/$self->{nr_p}] " . + di_url($self, $di) . "\n" . $di->{hdr_lines}); my $path = $di->{n}; $len += length($path) + 1; push @cmd, $path; @@ -505,10 +502,10 @@ sub do_git_apply ($) { } while (@$patches && $len < $ARG_SIZE_MAX && !oids_same_ish($patches->[0]->{oid_b}, $prv_oid_b)); - my $opt = { 2 => 1, -C => $dn, quiet => 1 }; + my $opt = { 2 => 1, -C => _tmp($self)->dirname, quiet => 1 }; my $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env}, $opt); $self->{-cur_di} = $di; - $self->{-qsp} = $qsp; + $qsp->{qsp_err} = \($self->{-qsp_err} = ''); $qsp->psgi_qx($self->{psgi_env}, undef, \&apply_result, $self); } @@ -558,8 +555,10 @@ sub extract_diffs_done { my $diffs = delete $self->{tmp_diffs}; if (scalar @$diffs) { unshift @{$self->{patches}}, @$diffs; - dbg($self, "found $want->{oid_b} in " . join(" ||\n\t", - map { di_url($self, $_) } @$diffs)); + my %seen; # List::Util::uniq requires Perl 5.26+ :< + my @u = grep { !$seen{$_}++ } map { di_url($self, $_) } @$diffs; + dbg($self, "found $want->{oid_b} in " . join(" ||\n\t", @u)); + ++$self->{nr_p}; # good, we can find a path to the oid we $want, now # lets see if we need to apply more patches: @@ -641,7 +640,7 @@ sub resolve_patch ($$) { # scan through inboxes to look for emails which results in # the oid we want: - my $ibx = shift(@{$want->{try_ibxs}}) or die 'BUG: {try_ibxs} empty'; + my $ibx = shift(@{$want->{try_ibxs}}) or return done($self, undef); if (my $msgs = find_smsgs($self, $ibx, $want)) { $want->{try_smsgs} = $msgs; $want->{cur_ibx} = $ibx; @@ -656,14 +655,14 @@ sub resolve_patch ($$) { sub new { my ($class, $ibx, $user_cb, $uarg) = @_; - bless { - gits => $ibx->{-repo_objs}, + bless { # $ibx is undef if coderepo only (see WwwCoderepo) + gits => $ibx ? $ibx->{-repo_objs} : undef, user_cb => $user_cb, uarg => $uarg, - # -cur_di, -qsp, -msg => temporary fields for Qspawn callbacks + # -cur_di, -qsp_err, -msg => temp fields for Qspawn callbacks # TODO: config option for searching related inboxes - inboxes => [ $ibx ], + inboxes => $ibx ? [ $ibx ] : [], }, $class; } @@ -682,12 +681,12 @@ sub solve ($$$$$) { $self->{oid_want} = $oid_want; $self->{out} = $out; $self->{seen_oid} = {}; - $self->{tot} = 0; + $self->{tot} = $self->{nr_p} = 0; $self->{psgi_env} = $env; + $self->{have_hints} = 1 if scalar keys %$hints; $self->{todo} = [ { %$hints, oid_b => $oid_want } ]; $self->{patches} = []; # [ $di, $di, ... ] $self->{found} = {}; # { abbr => [ ::Git, oid, type, size, $di ] } - $self->{tmp} = File::Temp->newdir("solver.$oid_want-XXXX", TMPDIR => 1); dbg($self, "solving $oid_want ..."); if (my $async = $env->{'pi-httpd.async'}) {