searchidx: add NNTP article number as a searchable term

[public-inbox.git] / lib / PublicInbox / SearchIdxSkeleton.pm

diff --git a/lib/PublicInbox/SearchIdxSkeleton.pm b/lib/PublicInbox/SearchIdxSkeleton.pm
index 0016f89a6c59ab081f73e424d0d521ef118af3c5..4066b5915470872b2fc766e45650534b62560469 100644 (file)
--- a/lib/PublicInbox/SearchIdxSkeleton.pm
+++ b/lib/PublicInbox/SearchIdxSkeleton.pm
@@ -42,7 +42,6 @@ sub new {
 sub skeleton_worker_loop {
        my ($self, $r) = @_;
        $0 = 'pi-v2-skeleton';
-       my $msg;
        my $xdb = $self->_xdb_acquire;
        $xdb->begin_transaction;
        my $txn = 1;
@@ -54,9 +53,12 @@ sub skeleton_worker_loop {
                        $self->_xdb_release;
                        $xdb = $txn = undef;
                } else {
-                       read($r, $msg, $line) or die "read failed: $!\n";
+                       my $len = int($line);
+                       my $n = read($r, my $msg, $len) or die "read: $!\n";
+                       $n == $len or die "short read: $n != $len\n";
                        $msg = thaw($msg); # should raise on error
                        defined $msg or die "failed to thaw buffer\n";
+                       $xdb ||= $self->_xdb_acquire;
                        if (!$txn) {
                                $xdb->begin_transaction;
                                $txn = 1;
@@ -65,6 +67,8 @@ sub skeleton_worker_loop {
                        warn "failed to index message <$msg->[-1]>: $@\n" if $@;
                }
        }
+       die "xdb not released\n" if $xdb;
+       die "in transaction\n" if $txn;
 }
 
 # called by a partition worker
@@ -76,6 +80,8 @@ sub index_skeleton {
        $str = length($str) . "\n" . $str;
 
        # multiple processes write to the same pipe, so use flock
+       # We can't avoid this lock for <=PIPE_BUF writes, either,
+       # because those atomic writes can break up >PIPE_BUF ones
        $self->_lock_acquire;
        print $w $str or $err = $!;
        $self->_lock_release;
@@ -88,34 +94,20 @@ sub index_skeleton_real ($$) {
        my ($self, $values) = @_;
        my $doc_data = pop @$values;
        my $xpath = pop @$values;
-       my $mid = pop @$values;
+       my $mids = pop @$values;
        my $ts = $values->[PublicInbox::Search::TS];
-       my $smsg = $self->lookup_message($mid);
-       my ($old_tid, $doc_id);
-       if ($smsg) {
-               # convert a ghost to a regular message
-               # it will also clobber any existing regular message
-               $doc_id = $smsg->{doc_id};
-               $old_tid = $smsg->thread_id;
-       } else {
-               $smsg = PublicInbox::SearchMsg->new(undef);
-               $smsg->{mid} = $mid;
-       }
+       my $smsg = PublicInbox::SearchMsg->new(undef);
        my $doc = $smsg->{doc};
-       $doc->add_term('XPATH' . $xpath) if defined $xpath;
-       $doc->add_term('XMID' . $mid);
+       foreach my $mid (@$mids) {
+               $doc->add_term('Q' . $mid);
+       }
        PublicInbox::SearchIdx::add_values($doc, $values);
        $doc->set_data($doc_data);
        $smsg->{ts} = $ts;
        $smsg->load_from_data($doc_data);
+       my $num = $values->[PublicInbox::Search::NUM];
        my @refs = ($smsg->references =~ /<([^>]+)>/g);
-       $self->link_message($smsg, \@refs, $old_tid);
-       my $db = $self->{xdb};
-       if (defined $doc_id) {
-               $db->replace_document($doc_id, $doc);
-       } else {
-               $doc_id = $db->add_document($doc);
-       }
+       $self->link_and_save($doc, $mids, \@refs, $num, $xpath);
 }
 
 1;