lei: normalize whitespace in remote queries

[public-inbox.git] / lib / PublicInbox / LeiXSearch.pm

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index f7c1e5599e6b14b5fd84a454ee55dffc15e1f336..9f7f3885a3a6fce2a120bb23416d109495c324ad 100644 (file)
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -9,7 +9,6 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::LeiSearch PublicInbox::IPC);
 use PublicInbox::DS qw(now);
-use PublicInbox::PktOp qw(pkt_do);
 use File::Temp 0.19 (); # 0.19 for ->newdir
 use File::Spec ();
 use PublicInbox::Search qw(xap_terms);
@@ -33,7 +32,7 @@ sub attach_external {
        my ($self, $ibxish) = @_; # ibxish = ExtSearch or Inbox
        my $desc = $ibxish->{inboxdir} // $ibxish->{topdir};
        my $srch = $ibxish->search or
-               return warn("$desc not indexed for Xapian\n");
+               return warn("$desc not indexed for Xapian ($@ $!)\n");
        my @shards = $srch->xdb_shards_flat or
                return warn("$desc has no Xapian shards\n");
 
@@ -72,11 +71,11 @@ sub _mitem_kw { # retry_reopen callback
        my $doc = $mitem->get_document;
        my $kw = xap_terms('K', $doc);
        $kw->{flagged} = 1 if $flagged;
-       my $L = xap_terms('L', $doc);
+       my @L = xap_terms('L', $doc);
        # we keep the empty {kw} array here to prevent expensive work in
        # ->xsmsg_vmd, _unbless_smsg will clobber it iff it's empty
        $smsg->{kw} = [ sort keys %$kw ];
-       $smsg->{L} = [ sort keys %$L ] if scalar(keys %$L);
+       $smsg->{L} = \@L if scalar(@L);
 }
 
 sub mitem_kw ($$$;$) {
@@ -142,7 +141,7 @@ sub mset_progress {
        my $lei = shift;
        return if $lei->{early_mua} || !$lei->{-progress};
        if ($lei->{pkt_op_p}) {
-               pkt_do($lei->{pkt_op_p}, 'mset_progress', @_);
+               $lei->{pkt_op_p}->pkt_do('mset_progress', @_);
        } else { # single lei-daemon consumer
                my ($desc, $mset_size, $mset_total_est) = @_;
                $lei->{-mset_total} += $mset_size if $mset_total_est ne '?';
@@ -162,6 +161,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
        my ($srch, $over) = ($ibxish->search, $ibxish->over);
        my $dir = $ibxish->{inboxdir} // $ibxish->{topdir};
        return warn("$dir not indexed by Xapian\n") unless ($srch && $over);
+       bless $srch, 'PublicInbox::LeiSearch'; # for ->qparse_new
        my $mo = { %{$lei->{mset_opt}} }; # copy
        my $mset;
        my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
@@ -267,12 +267,15 @@ sub _smsg_fill ($$) {
 sub each_remote_eml { # callback for MboxReader->mboxrd
        my ($eml, $self, $lei, $each_smsg) = @_;
        my $xoids = $lei->{ale}->xoids_for($eml, 1);
+       my $smsg = bless {}, 'PublicInbox::Smsg';
        if ($self->{import_sto} && !$xoids) {
-               $self->{import_sto}->ipc_do('add_eml', $eml);
+               my $res = $self->{import_sto}->ipc_do('add_eml', $eml);
+               if (ref($res) eq ref($smsg)) { # totally new message
+                       $smsg = $res;
+                       $smsg->{kw} = []; # short-circuit xsmsg_vmd
+               }
        }
-       my $smsg = bless {}, 'PublicInbox::Smsg';
-       $smsg->{blob} = $xoids ? (keys(%$xoids))[0]
-                               : git_sha(1, $eml)->hexdigest;
+       $smsg->{blob} //= $xoids ? (keys(%$xoids))[0] : $lei->git_blob_id($eml);
        _smsg_fill($smsg, $eml);
        wait_startq($lei);
        if ($lei->{-progress}) {
@@ -294,7 +297,9 @@ sub query_remote_mboxrd {
        local $SIG{TERM} = sub { exit(0) }; # for DESTROY (File::Temp, $reap)
        my $lei = $self->{lei};
        my $opt = $lei->{opt};
-       my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
+       my $qstr = $lei->{mset_opt}->{qstr};
+       $qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
+       my @qform = (q => $qstr, x => 'm');
        push(@qform, t => 1) if $opt->{threads};
        my $verbose = $opt->{verbose};
        my ($reap_tail, $reap_curl);
@@ -372,8 +377,8 @@ sub query_done { # EOF callback for main daemon
        if ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) {
                warn "BUG: {sto} missing with --mail-sync";
        }
-       my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
-       $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
+       $lei->sto_done_request if $lei->{sto};
+       my $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
        $lei->{ovv}->ovv_end($lei);
        my $start_mua;
        if ($l2m) { # close() calls LeiToMail reap_compress
@@ -396,9 +401,13 @@ Error closing $lei->{ovv}->{dst}: $!
        if ($lei->{-progress}) {
                my $tot = $lei->{-mset_total} // 0;
                my $nr = $lei->{-nr_write} // 0;
-               $lei->qerr($l2m ?
-                       "# $nr written to $lei->{ovv}->{dst} ($tot matches)" :
-                       "# $tot matches");
+               if ($l2m) {
+                       my $m = "# $nr written to " .
+                               "$lei->{ovv}->{dst} ($tot matches)";
+                       $nr ? $lei->qfin($m) : $lei->qerr($m);
+               } else {
+                       $lei->qerr("# $tot matches");
+               }
        }
        $lei->start_mua if $start_mua;
        $lei->dclose;
@@ -505,7 +514,7 @@ sub do_query {
                my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
                if ($l2m->{-wq_nr_workers} > 1 &&
                                $l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
-                       # setup two barriers to coordinate dedupe_nr
+                       # setup two barriers to coordinate ->has_entries
                        # between l2m workers
                        pipe(my ($a_r, $a_w)) or die "pipe: $!";
                        fcntl($a_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
@@ -533,7 +542,7 @@ sub do_query {
                start_query($self);
        }
        $lei->event_step_init; # wait for shutdowns
-       $op_c->op_wait_event($ops);
+       $lei->wait_wq_events($op_c, $ops);
 }
 
 sub add_uri {