X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiXSearch.pm;h=9f7f3885a3a6fce2a120bb23416d109495c324ad;hb=57fed2e4b78ed394;hp=f7c1e5599e6b14b5fd84a454ee55dffc15e1f336;hpb=528d69bbbda3cd26d9cc317eff92d21d980b2056;p=public-inbox.git

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index f7c1e559..9f7f3885 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -9,7 +9,6 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::LeiSearch PublicInbox::IPC);
 use PublicInbox::DS qw(now);
-use PublicInbox::PktOp qw(pkt_do);
 use File::Temp 0.19 (); # 0.19 for ->newdir
 use File::Spec ();
 use PublicInbox::Search qw(xap_terms);
@@ -33,7 +32,7 @@ sub attach_external {
 	my ($self, $ibxish) = @_; # ibxish = ExtSearch or Inbox
 	my $desc = $ibxish->{inboxdir} // $ibxish->{topdir};
 	my $srch = $ibxish->search or
-		return warn("$desc not indexed for Xapian\n");
+		return warn("$desc not indexed for Xapian ($@ $!)\n");
 	my @shards = $srch->xdb_shards_flat or
 		return warn("$desc has no Xapian shards\n");
 
@@ -72,11 +71,11 @@ sub _mitem_kw { # retry_reopen callback
 	my $doc = $mitem->get_document;
 	my $kw = xap_terms('K', $doc);
 	$kw->{flagged} = 1 if $flagged;
-	my $L = xap_terms('L', $doc);
+	my @L = xap_terms('L', $doc);
 	# we keep the empty {kw} array here to prevent expensive work in
 	# ->xsmsg_vmd, _unbless_smsg will clobber it iff it's empty
 	$smsg->{kw} = [ sort keys %$kw ];
-	$smsg->{L} = [ sort keys %$L ] if scalar(keys %$L);
+	$smsg->{L} = \@L if scalar(@L);
 }
 
 sub mitem_kw ($$$;$) {
@@ -142,7 +141,7 @@ sub mset_progress {
 	my $lei = shift;
 	return if $lei->{early_mua} || !$lei->{-progress};
 	if ($lei->{pkt_op_p}) {
-		pkt_do($lei->{pkt_op_p}, 'mset_progress', @_);
+		$lei->{pkt_op_p}->pkt_do('mset_progress', @_);
 	} else { # single lei-daemon consumer
 		my ($desc, $mset_size, $mset_total_est) = @_;
 		$lei->{-mset_total} += $mset_size if $mset_total_est ne '?';
@@ -162,6 +161,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
 	my ($srch, $over) = ($ibxish->search, $ibxish->over);
 	my $dir = $ibxish->{inboxdir} // $ibxish->{topdir};
 	return warn("$dir not indexed by Xapian\n") unless ($srch && $over);
+	bless $srch, 'PublicInbox::LeiSearch'; # for ->qparse_new
 	my $mo = { %{$lei->{mset_opt}} }; # copy
 	my $mset;
 	my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
@@ -267,12 +267,15 @@ sub _smsg_fill ($$) {
 sub each_remote_eml { # callback for MboxReader->mboxrd
 	my ($eml, $self, $lei, $each_smsg) = @_;
 	my $xoids = $lei->{ale}->xoids_for($eml, 1);
+	my $smsg = bless {}, 'PublicInbox::Smsg';
 	if ($self->{import_sto} && !$xoids) {
-		$self->{import_sto}->ipc_do('add_eml', $eml);
+		my $res = $self->{import_sto}->ipc_do('add_eml', $eml);
+		if (ref($res) eq ref($smsg)) { # totally new message
+			$smsg = $res;
+			$smsg->{kw} = []; # short-circuit xsmsg_vmd
+		}
 	}
-	my $smsg = bless {}, 'PublicInbox::Smsg';
-	$smsg->{blob} = $xoids ? (keys(%$xoids))[0]
-				: git_sha(1, $eml)->hexdigest;
+	$smsg->{blob} //= $xoids ? (keys(%$xoids))[0] : $lei->git_blob_id($eml);
 	_smsg_fill($smsg, $eml);
 	wait_startq($lei);
 	if ($lei->{-progress}) {
@@ -294,7 +297,9 @@ sub query_remote_mboxrd {
 	local $SIG{TERM} = sub { exit(0) }; # for DESTROY (File::Temp, $reap)
 	my $lei = $self->{lei};
 	my $opt = $lei->{opt};
-	my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
+	my $qstr = $lei->{mset_opt}->{qstr};
+	$qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
+	my @qform = (q => $qstr, x => 'm');
 	push(@qform, t => 1) if $opt->{threads};
 	my $verbose = $opt->{verbose};
 	my ($reap_tail, $reap_curl);
@@ -372,8 +377,8 @@ sub query_done { # EOF callback for main daemon
 	if ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) {
 		warn "BUG: {sto} missing with --mail-sync";
 	}
-	my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
-	$wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
+	$lei->sto_done_request if $lei->{sto};
+	my $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
 	$lei->{ovv}->ovv_end($lei);
 	my $start_mua;
 	if ($l2m) { # close() calls LeiToMail reap_compress
@@ -396,9 +401,13 @@ Error closing $lei->{ovv}->{dst}: $!
 	if ($lei->{-progress}) {
 		my $tot = $lei->{-mset_total} // 0;
 		my $nr = $lei->{-nr_write} // 0;
-		$lei->qerr($l2m ?
-			"# $nr written to $lei->{ovv}->{dst} ($tot matches)" :
-			"# $tot matches");
+		if ($l2m) {
+			my $m = "# $nr written to " .
+				"$lei->{ovv}->{dst} ($tot matches)";
+			$nr ? $lei->qfin($m) : $lei->qerr($m);
+		} else {
+			$lei->qerr("# $tot matches");
+		}
 	}
 	$lei->start_mua if $start_mua;
 	$lei->dclose;
@@ -505,7 +514,7 @@ sub do_query {
 		my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
 		if ($l2m->{-wq_nr_workers} > 1 &&
 				$l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
-			# setup two barriers to coordinate dedupe_nr
+			# setup two barriers to coordinate ->has_entries
 			# between l2m workers
 			pipe(my ($a_r, $a_w)) or die "pipe: $!";
 			fcntl($a_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
@@ -533,7 +542,7 @@ sub do_query {
 		start_query($self);
 	}
 	$lei->event_step_init; # wait for shutdowns
-	$op_c->op_wait_event($ops);
+	$lei->wait_wq_events($op_c, $ops);
 }
 
 sub add_uri {