qspawn: improve error reporting and handling

[public-inbox.git] / lib / PublicInbox / SolverGit.pm

diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm
index a53f28b105df291a1f64bae84d88c813114a95b0..dbb9b73907683361fa57cdc83558530f0b322f39 100644 (file)
--- a/lib/PublicInbox/SolverGit.pm
+++ b/lib/PublicInbox/SolverGit.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2020 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # "Solve" blobs which don't exist in git code repositories by
@@ -111,8 +111,6 @@ sub extract_diff ($$) {
        my ($self, $want, $smsg) = @$arg;
        my ($part) = @$p; # ignore $depth and @idx;
        my $ct = $part->content_type || 'text/plain';
-       my ($s, undef) = msg_part_text($part, $ct);
-       defined $s or return;
        my $post = $want->{oid_b};
        my $pre = $want->{oid_a};
        if (!defined($pre) || $pre !~ /\A[a-f0-9]+\z/) {
@@ -122,11 +120,12 @@ sub extract_diff ($$) {
        # Email::MIME::Encodings forces QP to be CRLF upon decoding,
        # change it back to LF:
        my $cte = $part->header('Content-Transfer-Encoding') || '';
+       my ($s, undef) = msg_part_text($part, $ct);
+       defined $s or return;
+       delete $part->{bdy};
        if ($cte =~ /\bquoted-printable\b/i && $part->crlf eq "\n") {
                $s =~ s/\r\n/\n/sg;
        }
-
-
        $s =~ m!( # $1 start header lines we save for debugging:
 
                # everything before ^index is optional, but we don't
@@ -169,6 +168,7 @@ sub extract_diff ($$) {
                # because git-apply(1) handles that case, too
                (?:^(?:[\@\+\x20\-\\][^\n]*|)$LF)+
        )!smx or return;
+       undef $s; # free memory
 
        my $di = {
                hdr_lines => $1,
@@ -242,10 +242,8 @@ sub find_smsgs ($$$) {
 
 sub update_index_result ($$) {
        my ($bref, $self) = @_;
-       my ($qsp, $msg) = delete @$self{qw(-qsp -msg)};
-       if (my $err = $qsp->{err}) {
-               ERR($self, "git update-index error: $err");
-       }
+       my ($qsp_err, $msg) = delete @$self{qw(-qsp_err -msg)};
+       ERR($self, "git update-index error:$qsp_err") if $qsp_err;
        dbg($self, $msg);
        next_step($self); # onto do_git_apply
 }
@@ -278,7 +276,7 @@ sub prepare_index ($) {
        my $cmd = [ qw(git update-index -z --index-info) ];
        my $qsp = PublicInbox::Qspawn->new($cmd, $self->{git_env}, $rdr);
        $path_a = git_quote($path_a);
-       $self->{-qsp} = $qsp;
+       $qsp->{qsp_err} = \($self->{-qsp_err} = '');
        $self->{-msg} = "index prepared:\n$mode_a $oid_full\t$path_a";
        $qsp->psgi_qx($self->{psgi_env}, undef, \&update_index_result, $self);
 }
@@ -293,14 +291,20 @@ sub do_git_init ($) {
                mkdir("$git_dir/$_") or die "mkdir $_: $!";
        }
        open my $fh, '>', "$git_dir/config" or die "open git/config: $!";
-       print $fh <<'EOF' or die "print git/config $!";
+       my $first = $self->{gits}->[0];
+       my $fmt = $first->object_format;
+       my $v = defined($$fmt) ? 1 : 0;
+       print $fh <<EOF or die "print git/config $!";
 [core]
-       repositoryFormatVersion = 0
+       repositoryFormatVersion = $v
        filemode = true
        bare = false
-       fsyncObjectfiles = false
        logAllRefUpdates = false
 EOF
+       print $fh <<EOM if defined($$fmt);
+[extensions]
+       objectformat = $$fmt
+EOM
        close $fh or die "close git/config: $!";
 
        open $fh, '>', "$git_dir/HEAD" or die "open git/HEAD: $!";
@@ -318,6 +322,7 @@ EOF
        $self->{git_env} = {
                GIT_DIR => $git_dir,
                GIT_INDEX_FILE => "$git_dir/index",
+               GIT_TEST_FSYNC => 0, # undocumented git env
        };
        prepare_index($self);
 }
@@ -398,10 +403,8 @@ sub mark_found ($$$) {
 
 sub parse_ls_files ($$) {
        my ($self, $bref) = @_;
-       my ($qsp, $di) = delete @$self{qw(-qsp -cur_di)};
-       if (my $err = $qsp->{err}) {
-               die "git ls-files error: $err";
-       }
+       my ($qsp_err, $di) = delete @$self{qw(-qsp_err -cur_di)};
+       die "git ls-files error:$qsp_err" if $qsp_err;
 
        my ($line, @extra) = split(/\0/, $$bref);
        scalar(@extra) and die "BUG: extra files in index: <",
@@ -449,11 +452,11 @@ sub skip_identical ($$$) {
 
 sub apply_result ($$) {
        my ($bref, $self) = @_;
-       my ($qsp, $di) = delete @$self{qw(-qsp -cur_di)};
+       my ($qsp_err, $di) = delete @$self{qw(-qsp_err -cur_di)};
        dbg($self, $$bref);
        my $patches = $self->{patches};
-       if (my $err = $qsp->{err}) {
-               my $msg = "git apply error: $err";
+       if ($qsp_err) {
+               my $msg = "git apply error:$qsp_err";
                my $nxt = $patches->[0];
                if ($nxt && oids_same_ish($nxt->{oid_b}, $di->{oid_b})) {
                        dbg($self, $msg);
@@ -467,9 +470,9 @@ sub apply_result ($$) {
        }
 
        my @cmd = qw(git ls-files -s -z);
-       $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env});
+       my $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env});
        $self->{-cur_di} = $di;
-       $self->{-qsp} = $qsp;
+       $qsp->{qsp_err} = \($self->{-qsp_err} = '');
        $qsp->psgi_qx($self->{psgi_env}, undef, \&ls_files_result, $self);
 }
 
@@ -501,7 +504,7 @@ sub do_git_apply ($) {
        my $opt = { 2 => 1, -C => $dn, quiet => 1 };
        my $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env}, $opt);
        $self->{-cur_di} = $di;
-       $self->{-qsp} = $qsp;
+       $qsp->{qsp_err} = \($self->{-qsp_err} = '');
        $qsp->psgi_qx($self->{psgi_env}, undef, \&apply_result, $self);
 }
 
@@ -593,8 +596,7 @@ sub resolve_patch ($$) {
        if (my $msgs = $want->{try_smsgs}) {
                my $smsg = shift @$msgs;
                if ($self->{psgi_env}->{'pi-httpd.async'}) {
-                       return git_async_cat($want->{cur_ibx}->git,
-                                               $smsg->{blob},
+                       return ibx_async_cat($want->{cur_ibx}, $smsg->{blob},
                                                \&extract_diff_async,
                                                [$self, $want, $smsg]);
                } else {
@@ -654,7 +656,7 @@ sub new {
                gits => $ibx->{-repo_objs},
                user_cb => $user_cb,
                uarg => $uarg,
-               # -cur_di, -qsp, -msg => temporary fields for Qspawn callbacks
+               # -cur_di, -qsp_err, -msg => temp fields for Qspawn callbacks
 
                # TODO: config option for searching related inboxes
                inboxes => [ $ibx ],
@@ -681,7 +683,7 @@ sub solve ($$$$$) {
        $self->{todo} = [ { %$hints, oid_b => $oid_want } ];
        $self->{patches} = []; # [ $di, $di, ... ]
        $self->{found} = {}; # { abbr => [ ::Git, oid, type, size, $di ] }
-       $self->{tmp} = File::Temp->newdir("solver.$oid_want-XXXXXXXX", TMPDIR => 1);
+       $self->{tmp} = File::Temp->newdir("solver.$oid_want-XXXX", TMPDIR => 1);
 
        dbg($self, "solving $oid_want ...");
        if (my $async = $env->{'pi-httpd.async'}) {