X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiXSearch.pm;h=f64b2c62d307887fb451df5d97b6e114c6f13b33;hb=43c43f785aa53607a0dd050989da5d7fd0dcfff4;hp=dcc4880614a8b727f7bc14f322836cf33d71eaf5;hpb=06349427654334deb14b21769a66e03a3e646684;p=public-inbox.git

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index dcc48806..f64b2c62 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -18,6 +18,7 @@ use PublicInbox::MID qw(mids);
 use PublicInbox::Smsg;
 use PublicInbox::Eml;
 use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
+use PublicInbox::ContentHash qw(git_sha);
 
 sub new {
 	my ($class) = @_;
@@ -68,9 +69,13 @@ sub xdb_shards_flat { @{$_[0]->{shards_flat} // []} }
 
 sub mitem_kw ($$;$) {
 	my ($smsg, $mitem, $flagged) = @_;
-	my $kw = xap_terms('K', $mitem->get_document);
+	my $kw = xap_terms('K', my $doc = $mitem->get_document);
 	$kw->{flagged} = 1 if $flagged;
+	# we keep the empty {kw} array here to prevent expensive work in
+	# ->xsmsg_vmd, _unbless_smsg will clobber it iff it's empty
 	$smsg->{kw} = [ sort keys %$kw ];
+	my $L = xap_terms('L', $doc);
+	$smsg->{L} = [ sort keys %$L ] if scalar(keys %$L);
 }
 
 # like over->get_art
@@ -83,8 +88,10 @@ sub smsg_for {
 	my $num = int(($docid - 1) / $nshard) + 1;
 	my $ibx = $self->{shard2ibx}->[$shard];
 	my $smsg = $ibx->over->get_art($num);
-	mitem_kw($smsg, $mitem) if $ibx->can('msg_keywords');
-	$smsg->{docid} = $docid;
+	return if $smsg->{bytes} == 0; # external message
+	if ($ibx->can('msg_keywords')) {
+		mitem_kw($smsg, $mitem);
+	}
 	$smsg;
 }
 
@@ -92,7 +99,7 @@ sub recent {
 	my ($self, $qstr, $opt) = @_;
 	$opt //= {};
 	$opt->{relevance} //= -2;
-	$self->mset($qstr //= 'bytes:1..', $opt);
+	$self->mset($qstr //= 'z:1..', $opt);
 }
 
 sub over {}
@@ -148,11 +155,12 @@ sub query_thread_mset { # for --threads
 	my $mset;
 	my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $ibxish);
 	my $can_kw = !!$ibxish->can('msg_keywords');
-	my $fl = $lei->{opt}->{threads} > 1;
+	my $fl = $lei->{opt}->{threads} > 1 ? 1 : undef;
 	do {
 		$mset = $srch->mset($mo->{qstr}, $mo);
 		mset_progress($lei, $desc, $mset->size,
 				$mset->get_matches_estimated);
+		wait_startq($lei); # wait for keyword updates
 		my $ids = $srch->mset_to_artnums($mset, $mo);
 		my $ctx = { ids => $ids };
 		my $i = 0;
@@ -160,13 +168,14 @@ sub query_thread_mset { # for --threads
 		while ($over->expand_thread($ctx)) {
 			for my $n (@{$ctx->{xids}}) {
 				my $smsg = $over->get_art($n) or next;
-				wait_startq($lei);
 				my $mitem = delete $n2item{$smsg->{num}};
+				next if $smsg->{bytes} == 0;
 				if ($mitem) {
 					if ($can_kw) {
 						mitem_kw($smsg, $mitem, $fl);
-					} else {
-						$smsg->{kw} = [ 'flagged' ];
+					} elsif ($fl) {
+						# call ->xsmsg_vmd, later
+						$smsg->{lei_q_tt_flagged} = 1;
 					}
 				}
 				$each_smsg->($smsg, $mitem);
@@ -192,9 +201,9 @@ sub query_mset { # non-parallel for non-"--threads" users
 		$mset = $self->mset($mo->{qstr}, $mo);
 		mset_progress($lei, 'xsearch', $mset->size,
 				$mset->size, $mset->get_matches_estimated);
+		wait_startq($lei); # wait for keyword updates
 		for my $mitem ($mset->items) {
 			my $smsg = smsg_for($self, $mitem) or next;
-			wait_startq($lei);
 			$each_smsg->($smsg, $mitem);
 		}
 	} while (_mset_more($mset, $mo));
@@ -204,8 +213,13 @@ sub query_mset { # non-parallel for non-"--threads" users
 
 sub each_remote_eml { # callback for MboxReader->mboxrd
 	my ($eml, $self, $lei, $each_smsg) = @_;
-	$lei->{sto}->ipc_do('add_eml', $eml) if $lei->{sto}; # --import-remote
+	my $xoids = $lei->{ale}->xoids_for($eml, 1);
+	if ($self->{import_sto} && !$xoids) {
+		$self->{import_sto}->ipc_do('add_eml', $eml);
+	}
 	my $smsg = bless {}, 'PublicInbox::Smsg';
+	$smsg->{blob} = $xoids ? (keys(%$xoids))[0]
+				: git_sha(1, $eml)->hexdigest;
 	$smsg->populate($eml);
 	$smsg->parse_references($eml, mids($eml));
 	$smsg->{$_} //= '' for qw(from to cc ds subject references mid);
@@ -249,6 +263,7 @@ sub query_remote_mboxrd {
 	my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
 	push @$curl, '-s', '-d', '';
 	my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
+	$self->{import_sto} = $lei->{sto} if $lei->{opt}->{'import-remote'};
 	for my $uri (@$uris) {
 		$lei->{-current_url} = $uri->as_string;
 		$lei->{-nr_remote_eml} = 0;
@@ -289,27 +304,7 @@ sub query_remote_mboxrd {
 	$lei->{ovv}->ovv_atexit_child($lei);
 }
 
-# called by LeiOverview::each_smsg_cb
-sub git { $_[0]->{git_tmp} // die 'BUG: caller did not set {git_tmp}' }
-
-sub git_tmp ($) {
-	my ($self) = @_;
-	my (%seen, @dirs);
-	my $tmp = File::Temp->newdir("lei_xsearch_git.$$-XXXX", TMPDIR => 1);
-	for my $ibxish (locals($self)) {
-		my $d = File::Spec->canonpath($ibxish->git->{git_dir});
-		$seen{$d} //= push @dirs, "$d/objects\n"
-	}
-	my $git_dir = $tmp->dirname;
-	PublicInbox::Import::init_bare($git_dir);
-	my $f = "$git_dir/objects/info/alternates";
-	open my $alt, '>', $f or die "open($f): $!";
-	print $alt @dirs or die "print $f: $!";
-	close $alt or die "close $f: $!";
-	my $git = PublicInbox::Git->new($git_dir);
-	$git->{-tmp} = $tmp;
-	$git;
-}
+sub git { $_[0]->{git} // die 'BUG: git uninitialized' }
 
 sub xsearch_done_wait { # dwaitpid callback
 	my ($arg, $pid) = @_;
@@ -411,7 +406,7 @@ sub incr_start_query { # called whenever an l2m shard starts do_post_auth
 
 sub ipc_atfork_child {
 	my ($self) = @_;
-	$self->{lei}->lei_atfork_child;
+	$self->{lei}->_lei_atfork_child;
 	$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 	$self->SUPER::ipc_atfork_child;
 }
@@ -431,7 +426,7 @@ sub do_query {
 		'incr_start_query' => [ \&incr_start_query, $self, $l2m ],
 	};
 	$lei->{auth}->op_merge($ops, $l2m) if $l2m && $lei->{auth};
-	($lei->{pkt_op_c}, $lei->{pkt_op_p}) = PublicInbox::PktOp->pair($ops);
+	my $end = $lei->pkt_op_pair($ops);
 	$lei->{1}->autoflush(1);
 	$lei->start_pager if delete $lei->{need_pager};
 	$lei->{ovv}->ovv_begin($lei);
@@ -446,15 +441,11 @@ sub do_query {
 		# 1031: F_SETPIPE_SZ
 		fcntl($lei->{startq}, 1031, 4096) if $^O eq 'linux';
 	}
-	if (!$lei->{opt}->{threads} && locals($self)) { # for query_mset
-		# lei->{git_tmp} is set for wq_wait_old so we don't
-		# delete until all lei2mail + lei_xsearch workers are reaped
-		$lei->{git_tmp} = $self->{git_tmp} = git_tmp($self);
-	}
 	$self->wq_workers_start('lei_xsearch', undef,
 				$lei->oldset, { lei => $lei });
 	my $op = delete $lei->{pkt_op_c};
 	delete $lei->{pkt_op_p};
+	@$end = ();
 	$self->{threads} = $lei->{opt}->{threads};
 	if ($l2m) {
 		$l2m->net_merge_complete unless $lei->{auth};
@@ -488,8 +479,10 @@ sub prepare_external {
 		return add_uri($self, URI->new($loc));
 	} elsif (-f "$loc/ei.lock") {
 		require PublicInbox::ExtSearch;
+		die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
 		$loc = PublicInbox::ExtSearch->new($loc);
 	} elsif (-f "$loc/inbox.lock" || -d "$loc/public-inbox") {
+		die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
 		require PublicInbox::Inbox; # v2, v1
 		$loc = bless { inboxdir => $loc }, 'PublicInbox::Inbox';
 	} else {