X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiXSearch.pm;h=1266b3b3396f9466b5eb26759f7e857baaf7ffb7;hb=7d2e572aca7297ea2015d2b6e7c71b672521ec82;hp=2d399653591d1323749ff615929903ffcaa2bd6c;hpb=f0731b9b653640afd8078aed994814b3049f230a;p=public-inbox.git

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 2d399653..1266b3b3 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -63,9 +63,16 @@ sub locals { @{$_[0]->{locals} // []} }
 
 sub remotes { @{$_[0]->{remotes} // []} }
 
-# called by PublicInbox::Search::xdb
+# called by PublicInbox::Search::xdb (usually via ->mset)
 sub xdb_shards_flat { @{$_[0]->{shards_flat} // []} }
 
+sub mitem_kw ($$;$) {
+	my ($smsg, $mitem, $flagged) = @_;
+	my $kw = xap_terms('K', $mitem->get_document);
+	$kw->{flagged} = 1 if $flagged;
+	$smsg->{kw} = [ sort keys %$kw ];
+}
+
 # like over->get_art
 sub smsg_for {
 	my ($self, $mitem) = @_;
@@ -76,10 +83,7 @@ sub smsg_for {
 	my $num = int(($docid - 1) / $nshard) + 1;
 	my $ibx = $self->{shard2ibx}->[$shard];
 	my $smsg = $ibx->over->get_art($num);
-	if (ref($ibx->can('msg_keywords'))) {
-		my $kw = xap_terms('K', $mitem->get_document);
-		$smsg->{kw} = [ sort keys %$kw ];
-	}
+	mitem_kw($smsg, $mitem) if $ibx->can('msg_keywords');
 	$smsg->{docid} = $docid;
 	$smsg;
 }
@@ -88,11 +92,16 @@ sub recent {
 	my ($self, $qstr, $opt) = @_;
 	$opt //= {};
 	$opt->{relevance} //= -2;
-	$self->mset($qstr //= 'bytes:1..', $opt);
+	$self->mset($qstr //= 'z:1..', $opt);
 }
 
 sub over {}
 
+sub overs_all { # for xids_for
+	my ($self) = @_;
+	grep(defined, map { $_->over } locals($self))
+}
+
 sub _mset_more ($$) {
 	my ($mset, $mo) = @_;
 	my $size = $mset->size;
@@ -143,6 +152,8 @@ sub query_thread_mset { # for --threads
 	my $mo = { %{$lei->{mset_opt}} };
 	my $mset;
 	my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $ibxish);
+	my $can_kw = !!$ibxish->can('msg_keywords');
+	my $fl = $lei->{opt}->{threads} > 1 ? [ 'flagged' ] : undef;
 	do {
 		$mset = $srch->mset($mo->{qstr}, $mo);
 		mset_progress($lei, $desc, $mset->size,
@@ -156,6 +167,13 @@ sub query_thread_mset { # for --threads
 				my $smsg = $over->get_art($n) or next;
 				wait_startq($lei);
 				my $mitem = delete $n2item{$smsg->{num}};
+				if ($mitem) {
+					if ($can_kw) {
+						mitem_kw($smsg, $mitem, $fl);
+					} elsif ($fl) {
+						$smsg->{kw} = $fl;
+					}
+				}
 				$each_smsg->($smsg, $mitem);
 			}
 			@{$ctx->{xids}} = ();
@@ -191,7 +209,9 @@ sub query_mset { # non-parallel for non-"--threads" users
 
 sub each_remote_eml { # callback for MboxReader->mboxrd
 	my ($eml, $self, $lei, $each_smsg) = @_;
-	$lei->{sto}->ipc_do('set_eml', $eml) if $lei->{sto}; # --import-remote
+	if (my $sto = $self->{import_sto}) {
+		$sto->ipc_do('add_eml_maybe', $eml);
+	}
 	my $smsg = bless {}, 'PublicInbox::Smsg';
 	$smsg->populate($eml);
 	$smsg->parse_references($eml, mids($eml));
@@ -236,6 +256,7 @@ sub query_remote_mboxrd {
 	my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
 	push @$curl, '-s', '-d', '';
 	my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
+	$self->{import_sto} = $lei->{sto} if $lei->{opt}->{'import-remote'};
 	for my $uri (@$uris) {
 		$lei->{-current_url} = $uri->as_string;
 		$lei->{-nr_remote_eml} = 0;
@@ -276,27 +297,7 @@ sub query_remote_mboxrd {
 	$lei->{ovv}->ovv_atexit_child($lei);
 }
 
-# called by LeiOverview::each_smsg_cb
-sub git { $_[0]->{git_tmp} // die 'BUG: caller did not set {git_tmp}' }
-
-sub git_tmp ($) {
-	my ($self) = @_;
-	my (%seen, @dirs);
-	my $tmp = File::Temp->newdir("lei_xsearch_git.$$-XXXX", TMPDIR => 1);
-	for my $ibxish (locals($self)) {
-		my $d = File::Spec->canonpath($ibxish->git->{git_dir});
-		$seen{$d} //= push @dirs, "$d/objects\n"
-	}
-	my $git_dir = $tmp->dirname;
-	PublicInbox::Import::init_bare($git_dir);
-	my $f = "$git_dir/objects/info/alternates";
-	open my $alt, '>', $f or die "open($f): $!";
-	print $alt @dirs or die "print $f: $!";
-	close $alt or die "close $f: $!";
-	my $git = PublicInbox::Git->new($git_dir);
-	$git->{-tmp} = $tmp;
-	$git;
-}
+sub git { $_[0]->{git} // die 'BUG: git uninitialized' }
 
 sub xsearch_done_wait { # dwaitpid callback
 	my ($arg, $pid) = @_;
@@ -325,6 +326,7 @@ Error closing $lei->{ovv}->{dst}: $!
 			$l2m->poke_dst;
 			$lei->poke_mua;
 		} else { # mbox users
+			delete $l2m->{mbl}; # drop dotlock
 			$lei->start_mua;
 		}
 	}
@@ -335,7 +337,7 @@ Error closing $lei->{ovv}->{dst}: $!
 
 sub do_post_augment {
 	my ($lei) = @_;
-	my $l2m = $lei->{l2m} or die 'BUG: unexpected do_post_augment';
+	my $l2m = $lei->{l2m} or return; # client disconnected
 	my $err;
 	eval { $l2m->post_augment($lei) };
 	$err = $@;
@@ -354,7 +356,7 @@ sub do_post_augment {
 
 sub incr_post_augment { # called whenever an l2m shard finishes augment
 	my ($lei) = @_;
-	my $l2m = $lei->{l2m} or die 'BUG: unexpected incr_post_augment';
+	my $l2m = $lei->{l2m} or return; # client disconnected
 	return if ++$lei->{nr_post_augment} != $l2m->{-wq_nr_workers};
 	do_post_augment($lei);
 }
@@ -402,6 +404,11 @@ sub ipc_atfork_child {
 	$self->SUPER::ipc_atfork_child;
 }
 
+sub delete_pkt_op { # OnDestroy callback
+	my $unclosed_after_die = delete($_[0])->{pkt_op_p} or return;
+	close $unclosed_after_die;
+}
+
 sub do_query {
 	my ($self, $lei) = @_;
 	my $l2m = $lei->{l2m};
@@ -417,6 +424,7 @@ sub do_query {
 		'incr_start_query' => [ \&incr_start_query, $self, $l2m ],
 	};
 	$lei->{auth}->op_merge($ops, $l2m) if $l2m && $lei->{auth};
+	my $od = PublicInbox::OnDestroy->new($$, \&delete_pkt_op, $lei);
 	($lei->{pkt_op_c}, $lei->{pkt_op_p}) = PublicInbox::PktOp->pair($ops);
 	$lei->{1}->autoflush(1);
 	$lei->start_pager if delete $lei->{need_pager};
@@ -432,11 +440,6 @@ sub do_query {
 		# 1031: F_SETPIPE_SZ
 		fcntl($lei->{startq}, 1031, 4096) if $^O eq 'linux';
 	}
-	if (!$lei->{opt}->{threads} && locals($self)) { # for query_mset
-		# lei->{git_tmp} is set for wq_wait_old so we don't
-		# delete until all lei2mail + lei_xsearch workers are reaped
-		$lei->{git_tmp} = $self->{git_tmp} = git_tmp($self);
-	}
 	$self->wq_workers_start('lei_xsearch', undef,
 				$lei->oldset, { lei => $lei });
 	my $op = delete $lei->{pkt_op_c};
@@ -474,8 +477,10 @@ sub prepare_external {
 		return add_uri($self, URI->new($loc));
 	} elsif (-f "$loc/ei.lock") {
 		require PublicInbox::ExtSearch;
+		die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
 		$loc = PublicInbox::ExtSearch->new($loc);
 	} elsif (-f "$loc/inbox.lock" || -d "$loc/public-inbox") {
+		die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
 		require PublicInbox::Inbox; # v2, v1
 		$loc = bless { inboxdir => $loc }, 'PublicInbox::Inbox';
 	} else {