lei q: demangle and quiet curl output

curl(1) writes to stderr one byte-at-a-time (presumably for the
progress bar).  This ends up being unreadable on my terminal
when parallel processes are trying to write error messages.

So instead, we'll capture the output to a file and run
'tail -f' on it if --verbose is enabled.

Since HTTP 404s from non-existent results are a common response,
we'll ignore them and stay silent, matching behavior of local
searches.

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index fb608d00611455237a68b726becd6a76f4073da4..68be8ada5045508efe16ec4b36ea1971f217f863 100644 (file)
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -14,8 +14,9 @@ use PublicInbox::Import;
 use File::Temp 0.19 (); # 0.19 for ->newdir
 use File::Spec ();
 use PublicInbox::Search qw(xap_terms);
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Spawn qw(popen_rd spawn);
 use PublicInbox::MID qw(mids);
+use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
 
 sub new {
        my ($class) = @_;
@@ -176,6 +177,13 @@ sub each_eml { # callback for MboxReader->mboxrd
        $each_smsg->($smsg, undef, $eml);
 }
 
+# PublicInbox::OnDestroy callback
+sub kill_reap {
+       my ($pid) = @_;
+       kill('KILL', $pid); # spawn() blocks other signals
+       waitpid($pid, 0);
+}
+
 sub query_remote_mboxrd {
        my ($self, $lei, $uris) = @_;
        local $0 = "$0 query_remote_mboxrd";
@@ -186,7 +194,20 @@ sub query_remote_mboxrd {
        push(@qform, t => 1) if $opt->{thread};
        my @cmd = (qw(curl -sSf -d), '');
        my $verbose = $opt->{verbose};
-       push @cmd, '-v' if $verbose;
+       my $reap;
+       my $cerr = File::Temp->new(TEMPLATE => 'curl.err-XXXX', TMPDIR => 1);
+       fcntl($cerr, F_SETFL, O_APPEND|O_RDWR) or warn "set O_APPEND: $!";
+       my $rdr = { 2 => $cerr };
+       my $coff = 0;
+       if ($verbose) {
+               # spawn a process to force line-buffering, otherwise curl
+               # will write 1 character at-a-time and parallel outputs
+               # mmmaaayyy llloookkk llliiikkkeee ttthhhiiisss
+               push @cmd, '-v';
+               my $o = { 1 => $lei->{2}, 2 => $lei->{2} };
+               my $pid = spawn(['tail', '-f', $cerr->filename], undef, $o);
+               $reap = PublicInbox::OnDestroy->new(\&kill_reap, $pid);
+       }
        for my $o ($lei->curl_opt) {
                $o =~ s/\|[a-z0-9]\b//i; # remove single char short option
                if ($o =~ s/=[is]@\z//) {
@@ -213,21 +234,23 @@ sub query_remote_mboxrd {
                }
                $lei->err("# @$cmd") if $verbose;
                $? = 0;
-               my $fh = popen_rd($cmd, $env, { 2 => $lei->{2} });
+               my $fh = popen_rd($cmd, $env, $rdr);
                $fh = IO::Uncompress::Gunzip->new($fh);
                eval {
                        PublicInbox::MboxReader->mboxrd($fh, \&each_eml, $self,
                                                        $lei, $each_smsg);
                };
                return $lei->fail("E: @$cmd: $@") if $@;
-               if (($? >> 8) == 22) { # HTTP 404 from curl(1)
-                       $uri->query_form(q => $lei->{mset_opt}->{qstr});
-                       $lei->err('# no results from '.$uri->as_string);
-               } elsif ($?) {
-                       $uri->query_form(q => $lei->{mset_opt}->{qstr});
-                       $lei->err('E: '.$uri->as_string);
-                       $lei->child_error($?);
-               }
+               next unless $?;
+               seek($cerr, $coff, SEEK_SET) or warn "seek(curl stderr): $!\n";
+               my $e = do { local $/; <$cerr> } //
+                               die "read(curl stderr): $!\n";
+               $coff += length($e);
+               next if (($? >> 8) == 22 && $e =~ /\b404\b/);
+               $lei->child_error($?);
+               $uri->query_form(q => $lei->{mset_opt}->{qstr});
+               # --verbose already showed the error via tail(1)
+               $lei->err("E: $uri \$?=$?\n", $verbose ? () : $e);
        }
        undef $each_smsg;
        $lei->{ovv}->ovv_atexit_child($lei);

diff --git a/t/lei.t b/t/lei.t
index f826a96676a256b4832d80bdd0701bdcd54f6257..693382579faac97c87b55dfa5b17160641efc8bf 100644 (file)
--- a/t/lei.t
+++ b/t/lei.t
@@ -179,7 +179,7 @@ SKIP: {
        my $res = $json->decode($out);
        is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url");
        ok($lei->('q', "m:$mid", 'd:..20101002'), 'no results, no error');
-       like($err, qr/404/, 'noted 404');
+       is($err, '', 'no output on 404, matching local FS behavior');
        is($out, "[null]\n", 'got null results');
        $lei->('forget-external', $url);
 } # /SKIP