X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSearchIdx.pm;h=0b1dc219ba45922334ebc9a4ebdf3722b860cbe7;hb=2a49233ab00a366251974824658a20bf68e519da;hp=06bce70a4218ba82e3d7e420131e6b58a30c6d16;hpb=2b6580afae0c0449fa536b62c8fbad4408337a3f;p=public-inbox.git

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 06bce70a..0b1dc219 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -12,7 +12,7 @@ use warnings;
 use base qw(PublicInbox::Search PublicInbox::Lock);
 use PublicInbox::MIME;
 use PublicInbox::InboxWritable;
-use PublicInbox::MID qw/mid_clean id_compress mid_mime mids references/;
+use PublicInbox::MID qw/mid_clean id_compress mid_mime mids/;
 use PublicInbox::MsgIter;
 use Carp qw(croak);
 use POSIX qw(strftime);
@@ -273,18 +273,12 @@ sub add_message {
 		my $smsg = PublicInbox::SearchMsg->new($mime);
 		my $doc = $smsg->{doc};
 		my $subj = $smsg->subject;
-		my $xpath;
-		if ($subj ne '') {
-			$xpath = $self->subject_path($subj);
-			$xpath = id_compress($xpath);
-		}
-
-		$smsg->{lines} = $mime->body_raw =~ tr!\n!\n!;
-		defined $bytes or $bytes = length($mime->as_string);
-		$smsg->{bytes} = $bytes;
 		add_val($doc, PublicInbox::Search::TS(), $smsg->ts);
-		my $yyyymmdd = strftime('%Y%m%d', gmtime($smsg->ds));
-		add_val($doc, PublicInbox::Search::YYYYMMDD, $yyyymmdd);
+		my @ds = gmtime($smsg->ds);
+		my $yyyymmdd = strftime('%Y%m%d', @ds);
+		add_val($doc, PublicInbox::Search::YYYYMMDD(), $yyyymmdd);
+		my $dt = strftime('%Y%m%d%H%M%S', @ds);
+		add_val($doc, PublicInbox::Search::DT(), $dt);
 
 		my $tg = $self->term_generator;
 
@@ -333,12 +327,12 @@ sub add_message {
 			index_body($tg, \@orig, $doc) if @orig;
 		});
 
-		# populates smsg->references for smsg->to_doc_data
-		my $refs = parse_references($smsg, $mid0, $mids);
-		my $data = $smsg->to_doc_data($oid, $mid0);
 		foreach my $mid (@$mids) {
 			$tg->index_text($mid, 1, 'XM');
 		}
+		$smsg->{to} = $smsg->{cc} = '';
+		PublicInbox::OverIdx::parse_references($smsg, $mid0, $mids);
+		my $data = $smsg->to_doc_data($oid, $mid0);
 		$doc->set_data($data);
 		if (my $altid = $self->{-altid}) {
 			foreach my $alt (@$altid) {
@@ -351,15 +345,11 @@ sub add_message {
 			}
 		}
 
-		$self->delete_article($num) if defined $num; # for reindexing
-
-		utf8::encode($data);
-		$data = compress($data);
-		my @vals = ($smsg->ts, $num, $mids, $refs, $xpath, $data);
-		$self->{over}->add_over(\@vals);
+		if (my $over = $self->{over}) {
+			$over->add_overview($mime, $bytes, $num, $oid, $mid0);
+		}
 		$doc->add_boolean_term('Q' . $_) foreach @$mids;
-		$doc->add_boolean_term('XNUM' . $num) if defined $num;
-		$doc_id = $self->{xdb}->add_document($doc);
+		$self->{xdb}->replace_document($doc_id = $num, $doc);
 	};
 
 	if ($@) {
@@ -396,11 +386,13 @@ sub remove_message {
 	my $db = $self->{xdb};
 	my $called;
 	$mid = mid_clean($mid);
+	my $over = $self->{over};
 
 	eval {
 		batch_do($self, 'Q' . $mid, sub {
 			my ($ids) = @_;
 			$db->delete_document($_) for @$ids;
+			$over->delete_articles($ids) if $over;
 			$called = 1;
 		});
 	};
@@ -411,21 +403,13 @@ sub remove_message {
 	}
 }
 
-sub delete_article {
-	my ($self, $num) = @_;
-	my $ndel = 0;
-	batch_do($self, 'XNUM' . $num, sub {
-		my ($ids) = @_;
-		$ndel += scalar @$ids;
-		$self->{xdb}->delete_document($_) for @$ids;
-	});
-}
-
 # MID is a hint in V2
 sub remove_by_oid {
 	my ($self, $oid, $mid) = @_;
 	my $db = $self->{xdb};
 
+	$self->{over}->remove_oid($oid, $mid) if $self->{over};
+
 	# XXX careful, we cannot use batch_do here since we conditionally
 	# delete documents based on other factors, so we cannot call
 	# find_doc_ids twice.
@@ -435,7 +419,6 @@ sub remove_by_oid {
 	# there is only ONE element in @delete unless we
 	# have bugs in our v2writable deduplication check
 	my @delete;
-	my @over_del;
 	for (; $head != $tail; $head->inc) {
 		my $docid = $head->get_docid;
 		my $doc = $db->get_document($docid);
@@ -443,11 +426,9 @@ sub remove_by_oid {
 		$smsg->load_expand;
 		if ($smsg->{blob} eq $oid) {
 			push(@delete, $docid);
-			push(@over_del, $smsg->num);
 		}
 	}
 	$db->delete_document($_) foreach @delete;
-	$self->{over}->remove_oid($oid, $mid);
 	scalar(@delete);
 }
 
@@ -463,29 +444,6 @@ sub term_generator { # write-only
 	$self->{term_generator} = $tg;
 }
 
-sub parse_references ($$$) {
-	my ($smsg, $mid0, $mids) = @_;
-	my $mime = $smsg->{mime};
-	my $hdr = $mime->header_obj;
-	my $refs = references($hdr);
-	push(@$refs, @$mids) if scalar(@$mids) > 1;
-	return $refs if scalar(@$refs) == 0;
-
-	# prevent circular references here:
-	my %seen = ( $mid0 => 1 );
-	my @keep;
-	foreach my $ref (@$refs) {
-		if (length($ref) > PublicInbox::MID::MAX_MID_SIZE) {
-			warn "References: <$ref> too long, ignoring\n";
-			next;
-		}
-		next if $seen{$ref}++;
-		push @keep, $ref;
-	}
-	$smsg->{references} = '<'.join('> <', @keep).'>' if @keep;
-	\@keep;
-}
-
 sub index_git_blob_id {
 	my ($doc, $pfx, $objid) = @_;
 
@@ -613,7 +571,7 @@ sub _git_log {
 				--raw -r --no-abbrev/, $range);
 }
 
-# indexes all unindexed messages
+# indexes all unindexed messages (v1 only)
 sub _index_sync {
 	my ($self, $opts) = @_;
 	my $tip = $opts->{ref} || 'HEAD';
@@ -744,7 +702,7 @@ sub begin_txn_lazy {
 	my ($self) = @_;
 	return if $self->{txn};
 	my $xdb = $self->{xdb} || $self->_xdb_acquire;
-	$self->{over}->begin_lazy;
+	$self->{over}->begin_lazy if $self->{over};
 	$xdb->begin_transaction;
 	$self->{txn} = 1;
 	$xdb;
@@ -754,7 +712,7 @@ sub commit_txn_lazy {
 	my ($self) = @_;
 	delete $self->{txn} or return;
 	$self->{xdb}->commit_transaction;
-	$self->{over}->commit_lazy;
+	$self->{over}->commit_lazy if $self->{over};
 }
 
 sub worker_done {