www_stream: note existence of IMAP and NNTP URLs

[public-inbox.git] / lib / PublicInbox / LeiXSearch.pm

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index cac7fb7d5902c9aa51cbd40b395721fe4768a9f3..556ffd58ff7c969f536ef179cee01414a92fa4de 100644 (file)
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -32,7 +32,7 @@ sub attach_external {
        my ($self, $ibxish) = @_; # ibxish = ExtSearch or Inbox
        my $desc = $ibxish->{inboxdir} // $ibxish->{topdir};
        my $srch = $ibxish->search or
-               return warn("$desc not indexed for Xapian\n");
+               return warn("$desc not indexed for Xapian ($@ $!)\n");
        my @shards = $srch->xdb_shards_flat or
                return warn("$desc has no Xapian shards\n");
 
@@ -161,6 +161,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
        my ($srch, $over) = ($ibxish->search, $ibxish->over);
        my $dir = $ibxish->{inboxdir} // $ibxish->{topdir};
        return warn("$dir not indexed by Xapian\n") unless ($srch && $over);
+       bless $srch, 'PublicInbox::LeiSearch'; # for ->qparse_new
        my $mo = { %{$lei->{mset_opt}} }; # copy
        my $mset;
        my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
@@ -266,12 +267,15 @@ sub _smsg_fill ($$) {
 sub each_remote_eml { # callback for MboxReader->mboxrd
        my ($eml, $self, $lei, $each_smsg) = @_;
        my $xoids = $lei->{ale}->xoids_for($eml, 1);
+       my $smsg = bless {}, 'PublicInbox::Smsg';
        if ($self->{import_sto} && !$xoids) {
-               $self->{import_sto}->ipc_do('add_eml', $eml);
+               my $res = $self->{import_sto}->ipc_do('add_eml', $eml);
+               if (ref($res) eq ref($smsg)) { # totally new message
+                       $smsg = $res;
+                       $smsg->{kw} = []; # short-circuit xsmsg_vmd
+               }
        }
-       my $smsg = bless {}, 'PublicInbox::Smsg';
-       $smsg->{blob} = $xoids ? (keys(%$xoids))[0]
-                               : git_sha(1, $eml)->hexdigest;
+       $smsg->{blob} //= $xoids ? (keys(%$xoids))[0] : $lei->git_blob_id($eml);
        _smsg_fill($smsg, $eml);
        wait_startq($lei);
        if ($lei->{-progress}) {
@@ -293,7 +297,9 @@ sub query_remote_mboxrd {
        local $SIG{TERM} = sub { exit(0) }; # for DESTROY (File::Temp, $reap)
        my $lei = $self->{lei};
        my $opt = $lei->{opt};
-       my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
+       my $qstr = $lei->{mset_opt}->{qstr};
+       $qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
+       my @qform = (q => $qstr, x => 'm');
        push(@qform, t => 1) if $opt->{threads};
        my $verbose = $opt->{verbose};
        my ($reap_tail, $reap_curl);
@@ -371,8 +377,8 @@ sub query_done { # EOF callback for main daemon
        if ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) {
                warn "BUG: {sto} missing with --mail-sync";
        }
-       my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
-       $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
+       $lei->sto_done_request if $lei->{sto};
+       my $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
        $lei->{ovv}->ovv_end($lei);
        my $start_mua;
        if ($l2m) { # close() calls LeiToMail reap_compress
@@ -395,9 +401,13 @@ Error closing $lei->{ovv}->{dst}: $!
        if ($lei->{-progress}) {
                my $tot = $lei->{-mset_total} // 0;
                my $nr = $lei->{-nr_write} // 0;
-               $lei->qerr($l2m ?
-                       "# $nr written to $lei->{ovv}->{dst} ($tot matches)" :
-                       "# $tot matches");
+               if ($l2m) {
+                       my $m = "# $nr written to " .
+                               "$lei->{ovv}->{dst} ($tot matches)";
+                       $nr ? $lei->qfin($m) : $lei->qerr($m);
+               } else {
+                       $lei->qerr("# $tot matches");
+               }
        }
        $lei->start_mua if $start_mua;
        $lei->dclose;
@@ -504,7 +514,7 @@ sub do_query {
                my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
                if ($l2m->{-wq_nr_workers} > 1 &&
                                $l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
-                       # setup two barriers to coordinate dedupe_nr
+                       # setup two barriers to coordinate ->has_entries
                        # between l2m workers
                        pipe(my ($a_r, $a_w)) or die "pipe: $!";
                        fcntl($a_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
@@ -562,8 +572,11 @@ sub prepare_external {
                die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
                require PublicInbox::Inbox; # v2, v1
                $loc = bless { inboxdir => $loc }, 'PublicInbox::Inbox';
+       } elsif (!-e $loc) {
+               warn "W: $loc gone, perhaps run: lei forget-external $loc\n";
+               return;
        } else {
-               warn "W: ignoring $loc, unable to determine type\n";
+               warn "W: $loc ignored, unable to determine external type\n";
                return;
        }
        push @{$self->{locals}}, $loc;