lei_xsearch: drop repeated "Xapian" in error message

[public-inbox.git] / lib / PublicInbox / LeiXSearch.pm

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index fb608d00611455237a68b726becd6a76f4073da4..9ea2b5f339066ad4e4425e77a94f5ac4bbb67bc6 100644 (file)
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -14,8 +14,9 @@ use PublicInbox::Import;
 use File::Temp 0.19 (); # 0.19 for ->newdir
 use File::Spec ();
 use PublicInbox::Search qw(xap_terms);
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Spawn qw(popen_rd spawn which);
 use PublicInbox::MID qw(mids);
+use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
 
 sub new {
        my ($class) = @_;
@@ -32,7 +33,7 @@ sub attach_external {
        my $srch = $ibxish->search or
                return warn("$desc not indexed for Xapian\n");
        my @shards = $srch->xdb_shards_flat or
-               return warn("$desc has no Xapian shardsXapian\n");
+               return warn("$desc has no Xapian shards\n");
 
        if (delete $self->{xdb}) { # XXX: do we need this?
                # clobber existing {xdb} if amending
@@ -176,6 +177,13 @@ sub each_eml { # callback for MboxReader->mboxrd
        $each_smsg->($smsg, undef, $eml);
 }
 
+# PublicInbox::OnDestroy callback
+sub kill_reap {
+       my ($pid) = @_;
+       kill('KILL', $pid); # spawn() blocks other signals
+       waitpid($pid, 0);
+}
+
 sub query_remote_mboxrd {
        my ($self, $lei, $uris) = @_;
        local $0 = "$0 query_remote_mboxrd";
@@ -184,9 +192,22 @@ sub query_remote_mboxrd {
        my ($opt, $env) = @$lei{qw(opt env)};
        my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
        push(@qform, t => 1) if $opt->{thread};
-       my @cmd = (qw(curl -sSf -d), '');
+       my @cmd = ($self->{curl}, qw(-sSf -d), '');
        my $verbose = $opt->{verbose};
-       push @cmd, '-v' if $verbose;
+       my $reap;
+       my $cerr = File::Temp->new(TEMPLATE => 'curl.err-XXXX', TMPDIR => 1);
+       fcntl($cerr, F_SETFL, O_APPEND|O_RDWR) or warn "set O_APPEND: $!";
+       my $rdr = { 2 => $cerr };
+       my $coff = 0;
+       if ($verbose) {
+               # spawn a process to force line-buffering, otherwise curl
+               # will write 1 character at-a-time and parallel outputs
+               # mmmaaayyy llloookkk llliiikkkeee ttthhhiiisss
+               push @cmd, '-v';
+               my $o = { 1 => $lei->{2}, 2 => $lei->{2} };
+               my $pid = spawn(['tail', '-f', $cerr->filename], undef, $o);
+               $reap = PublicInbox::OnDestroy->new(\&kill_reap, $pid);
+       }
        for my $o ($lei->curl_opt) {
                $o =~ s/\|[a-z0-9]\b//i; # remove single char short option
                if ($o =~ s/=[is]@\z//) {
@@ -207,27 +228,35 @@ sub query_remote_mboxrd {
                my $cmd = [ @cmd, $uri->as_string ];
                if ($tor eq 'auto' && substr($uri->host, -6) eq '.onion' &&
                                (($env->{LD_PRELOAD}//'') !~ /torsocks/)) {
-                       unshift @$cmd, 'torsocks';
+                       unshift @$cmd, which('torsocks');
                } elsif (PublicInbox::Config::git_bool($tor)) {
-                       unshift @$cmd, 'torsocks';
+                       unshift @$cmd, which('torsocks');
                }
+
+               # continue anyways if torsocks is missing; a proxy may be
+               # specified via CLI, curlrc, environment variable, or even
+               # firewall rule
+               shift(@$cmd) if !$cmd->[0];
+
                $lei->err("# @$cmd") if $verbose;
                $? = 0;
-               my $fh = popen_rd($cmd, $env, { 2 => $lei->{2} });
+               my $fh = popen_rd($cmd, $env, $rdr);
                $fh = IO::Uncompress::Gunzip->new($fh);
                eval {
                        PublicInbox::MboxReader->mboxrd($fh, \&each_eml, $self,
                                                        $lei, $each_smsg);
                };
                return $lei->fail("E: @$cmd: $@") if $@;
-               if (($? >> 8) == 22) { # HTTP 404 from curl(1)
-                       $uri->query_form(q => $lei->{mset_opt}->{qstr});
-                       $lei->err('# no results from '.$uri->as_string);
-               } elsif ($?) {
-                       $uri->query_form(q => $lei->{mset_opt}->{qstr});
-                       $lei->err('E: '.$uri->as_string);
-                       $lei->child_error($?);
-               }
+               next unless $?;
+               seek($cerr, $coff, SEEK_SET) or warn "seek(curl stderr): $!\n";
+               my $e = do { local $/; <$cerr> } //
+                               die "read(curl stderr): $!\n";
+               $coff += length($e);
+               next if (($? >> 8) == 22 && $e =~ /\b404\b/);
+               $lei->child_error($?);
+               $uri->query_form(q => $lei->{mset_opt}->{qstr});
+               # --verbose already showed the error via tail(1)
+               $lei->err("E: $uri \$?=$?\n", $verbose ? () : $e);
        }
        undef $each_smsg;
        $lei->{ovv}->ovv_atexit_child($lei);
@@ -388,13 +417,22 @@ sub ipc_atfork_prepare {
        $self->SUPER::ipc_atfork_prepare; # PublicInbox::IPC
 }
 
+sub add_uri {
+       my ($self, $uri) = @_;
+       if (my $curl = $self->{curl} //= which('curl') // 0) {
+               push @{$self->{remotes}}, $uri;
+       } else {
+               warn "curl missing, ignoring $uri\n";
+       }
+}
+
 sub prepare_external {
        my ($self, $loc, $boost) = @_; # n.b. already ordered by boost
        if (ref $loc) { # already a URI, or PublicInbox::Inbox-like object
-               return push(@{$self->{remotes}}, $loc) if $loc->can('scheme');
+               return add_uri($self, $loc) if $loc->can('scheme');
        } elsif ($loc =~ m!\Ahttps?://!) {
                require URI;
-               return push(@{$self->{remotes}}, URI->new($loc));
+               return add_uri($self, URI->new($loc));
        } elsif (-f "$loc/ei.lock") {
                require PublicInbox::ExtSearch;
                $loc = PublicInbox::ExtSearch->new($loc);