X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSearchIdx.pm;h=c36fc6c7dee5307ea803ec17f17ac8af7eb10dc8;hb=59cc88bb5bc5ce3e63133d0b9d04c55c1959e6b4;hp=be46b2b972acd8954094930ba5ac16defd37f980;hpb=f344d64066f85dd6737daeb42c94902e1bbfda78;p=public-inbox.git

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index be46b2b9..c36fc6c7 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -69,8 +69,8 @@ sub new {
 	if ($version == 1) {
 		$self->{lock_path} = "$inboxdir/ssoma.lock";
 		my $dir = $self->xdir;
-		$self->{over} = PublicInbox::OverIdx->new("$dir/over.sqlite3");
-		$self->{over}->{-no_fsync} = 1 if $ibx->{-no_fsync};
+		$self->{oidx} = PublicInbox::OverIdx->new("$dir/over.sqlite3");
+		$self->{oidx}->{-no_fsync} = 1 if $ibx->{-no_fsync};
 	} elsif ($version == 2) {
 		defined $shard or die "shard is required for v2\n";
 		# shard is a number
@@ -131,14 +131,13 @@ sub idx_acquire {
 				($is_shard && need_xapian($self)))) {
 			File::Path::mkpath($dir);
 			nodatacow_dir($dir);
+			$self->{-set_has_threadid_once} = 1;
 		}
 	}
 	return unless defined $flag;
 	$flag |= $DB_NO_SYNC if $self->{ibx}->{-no_fsync};
 	my $xdb = eval { ($X->{WritableDatabase})->new($dir, $flag) };
-	if ($@) {
-		die "Failed opening $dir: ", $@;
-	}
+	croak "Failed opening $dir: $@" if $@;
 	$self->{xdb} = $xdb;
 }
 
@@ -339,7 +338,7 @@ sub index_ids ($$$$) {
 	$doc->add_boolean_term('Q' . $_) for @$mids;
 	for my $l ($hdr->header_raw('List-Id')) {
 		$l =~ /<([^>]+)>/ or next;
-		my $lid = $1;
+		my $lid = lc $1;
 		$doc->add_boolean_term('G' . $lid);
 		index_text($self, $lid, 1, 'XL'); # probabilistic
 	}
@@ -356,6 +355,7 @@ sub add_xapian ($$$$) {
 	add_val($doc, PublicInbox::Search::DT(), $dt);
 	add_val($doc, PublicInbox::Search::BYTES(), $smsg->{bytes});
 	add_val($doc, PublicInbox::Search::UID(), $smsg->{num});
+	add_val($doc, PublicInbox::Search::THREADID, $smsg->{tid});
 
 	my $tg = term_generator($self);
 	$tg->set_document($doc);
@@ -419,8 +419,8 @@ sub add_message {
 		# of the fields which exist in over.sqlite3.  We may stop
 		# storing doc_data in Xapian sometime after we get multi-inbox
 		# search working.
-		if (my $over = $self->{over}) { # v1 only
-			$over->add_overview($mime, $smsg);
+		if (my $oidx = $self->{oidx}) { # v1 only
+			$oidx->add_overview($mime, $smsg);
 		}
 		if (need_xapian($self)) {
 			add_xapian($self, $mime, $smsg, $mids);
@@ -457,7 +457,7 @@ sub xdb_remove {
 
 sub remove_by_oid {
 	my ($self, $oid, $num) = @_;
-	die "BUG: remove_by_oid is v2-only\n" if $self->{over};
+	die "BUG: remove_by_oid is v2-only\n" if $self->{oidx};
 	$self->begin_txn_lazy;
 	xdb_remove($self, $oid, $num) if need_xapian($self);
 }
@@ -479,13 +479,9 @@ sub unindex_eml {
 	my $nr = 0;
 	my %tmp;
 	for my $mid (@$mids) {
-		my @removed = eval { $self->{over}->remove_oid($oid, $mid) };
-		if ($@) {
-			warn "E: failed to remove <$mid> from overview: $@\n";
-		} else {
-			$nr += scalar @removed;
-			$tmp{$_}++ for @removed;
-		}
+		my @removed = $self->{oidx}->remove_oid($oid, $mid);
+		$nr += scalar @removed;
+		$tmp{$_}++ for @removed;
 	}
 	if (!$nr) {
 		$mids = join('> <', @$mids);
@@ -507,9 +503,9 @@ sub index_mm {
 	my $mids = mids($mime);
 	my $mm = $self->{mm};
 	if ($sync->{reindex}) {
-		my $over = $self->{over};
+		my $oidx = $self->{oidx};
 		for my $mid (@$mids) {
-			my ($num, undef) = $over->num_mid0_for_oid($oid, $mid);
+			my ($num, undef) = $oidx->num_mid0_for_oid($oid, $mid);
 			return $num if defined $num;
 		}
 		$mm->num_for($mids->[0]) // $mm->mid_insert($mids->[0]);
@@ -589,13 +585,21 @@ sub v1_checkpoint ($$;$) {
 
 	$self->{mm}->{dbh}->commit;
 	if ($newest && need_xapian($self)) {
-		my $cur = $self->{xdb}->get_metadata('last_commit');
+		my $xdb = $self->{xdb};
+		my $cur = $xdb->get_metadata('last_commit');
 		if (need_update($self, $cur, $newest)) {
-			$self->{xdb}->set_metadata('last_commit', $newest);
+			$xdb->set_metadata('last_commit', $newest);
+		}
+
+		# let SearchView know a full --reindex was done so it can
+		# generate ->has_threadid-dependent links
+		if ($sync->{reindex} && !ref($sync->{reindex})) {
+			my $n = $xdb->get_metadata('has_threadid');
+			$xdb->set_metadata('has_threadid', '1') if $n ne '1';
 		}
 	}
 
-	$self->{over}->rethread_done($sync->{-opt}) if $newest; # all done
+	$self->{oidx}->rethread_done($sync->{-opt}) if $newest; # all done
 	commit_txn_lazy($self);
 	$self->{ibx}->git->cleanup;
 	my $nr = ${$sync->{nr}};
@@ -765,7 +769,7 @@ sub _index_sync {
 	my $pr = $opt->{-progress};
 	my $sync = { reindex => $opt->{reindex}, -opt => $opt };
 	my $xdb = $self->begin_txn_lazy;
-	$self->{over}->rethread_prepare($opt);
+	$self->{oidx}->rethread_prepare($opt);
 	my $mm = _msgmap_init($self);
 	if ($sync->{reindex}) {
 		my $last = $mm->last_commit;
@@ -793,44 +797,10 @@ sub DESTROY {
 	$_[0]->{lockfh} = undef;
 }
 
-# remote_* subs are only used by SearchIdxPart
-sub remote_commit {
-	my ($self) = @_;
-	if (my $w = $self->{w}) {
-		print $w "commit\n" or die "failed to write commit: $!";
-	} else {
-		$self->commit_txn_lazy;
-	}
-}
-
-sub remote_close {
-	my ($self) = @_;
-	if (my $w = delete $self->{w}) {
-		my $pid = delete $self->{pid} or die "no process to wait on\n";
-		print $w "close\n" or die "failed to write to pid:$pid: $!\n";
-		close $w or die "failed to close pipe for pid:$pid: $!\n";
-		waitpid($pid, 0) == $pid or die "remote process did not finish";
-		$? == 0 or die ref($self)." pid:$pid exited with: $?";
-	} else {
-		die "transaction in progress $self\n" if $self->{txn};
-		idx_release($self) if $self->{xdb};
-	}
-}
-
-sub remote_remove {
-	my ($self, $oid, $num) = @_;
-	if (my $w = $self->{w}) {
-		# triggers remove_by_oid in a shard
-		print $w "D $oid $num\n" or die "failed to write remove $!";
-	} else {
-		$self->remove_by_oid($oid, $num);
-	}
-}
-
 sub _begin_txn {
 	my ($self) = @_;
 	my $xdb = $self->{xdb} || idx_acquire($self);
-	$self->{over}->begin_lazy if $self->{over};
+	$self->{oidx}->begin_lazy if $self->{oidx};
 	$xdb->begin_transaction if $xdb;
 	$self->{txn} = 1;
 	$xdb;
@@ -849,6 +819,9 @@ sub set_metadata_once {
 	return if $self->{shard}; # only continue if undef or 0, not >0
 	my $xdb = $self->{xdb};
 
+	if (delete($self->{-set_has_threadid_once})) {
+		$xdb->set_metadata('has_threadid', '1');
+	}
 	if (delete($self->{-set_indexlevel_once})) {
 		my $level = $xdb->get_metadata('indexlevel');
 		if (!$level || $level ne 'medium') {
@@ -867,7 +840,7 @@ sub _commit_txn {
 		set_metadata_once($self);
 		$xdb->commit_transaction;
 	}
-	$self->{over}->commit_lazy if $self->{over};
+	$self->{oidx}->commit_lazy if $self->{oidx};
 }
 
 sub commit_txn_lazy {