nntp: fix NEWNEWS command

[public-inbox.git] / lib / PublicInbox / Search.pm

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index b20b2ccd3e39f3b0ede3425c1b46c55f03654674..4dc274723d6ca8e34ca19821256308d47ea7058d 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -56,9 +56,10 @@ my %bool_pfx_internal = (
 );
 
 my %bool_pfx_external = (
-       mid => 'XMID', # Message-ID (full/exact)
+       mid => 'Q', # Message-ID (full/exact), this is mostly uniQue
 );
 
+my $non_quoted_body = 'XNQ XDFN XDFA XDFB XDFHH XDFCTX XDFPRE XDFPOST';
 my %prob_prefix = (
        # for mairix compatibility
        s => 'S',
@@ -69,12 +70,12 @@ my %prob_prefix = (
        c => 'XCC',
        tcf => 'XTO XCC A',
        a => 'XTO XCC A',
-       b => 'XNQ XQUOT',
-       bs => 'XNQ XQUOT S',
+       b => $non_quoted_body . ' XQUOT',
+       bs => $non_quoted_body . ' XQUOT S',
        n => 'XFN',
 
        q => 'XQUOT',
-       nq => 'XNQ',
+       nq => $non_quoted_body,
        dfn => 'XDFN',
        dfa => 'XDFA',
        dfb => 'XDFB',
@@ -85,7 +86,7 @@ my %prob_prefix = (
        dfblob => 'XDFPRE XDFPOST',
 
        # default:
-       '' => 'XM S A XNQ XQUOT XFN',
+       '' => 'XM S A XQUOT XFN ' . $non_quoted_body,
 );
 
 # not documenting m: and mid: for now, the using the URLs works w/o Xapian
@@ -158,16 +159,21 @@ sub new {
                                $xdb = $sub;
                        }
                }
-               warn "v2 repo with $parts found in $dir\n";
                $self->{xdb} = $xdb;
-               $self->{skel} = Search::Xapian::Database->new("$dir/all");
+               $self->{skel} = Search::Xapian::Database->new("$dir/skel");
        } else {
                $self->{xdb} = Search::Xapian::Database->new($self->xdir);
        }
        $self;
 }
 
-sub reopen { $_[0]->{xdb}->reopen }
+sub reopen {
+       my ($self) = @_;
+       $self->{xdb}->reopen;
+       if (my $skel = $self->{skel}) {
+               $skel->reopen;
+       }
+}
 
 # read-only
 sub query {
@@ -185,7 +191,7 @@ sub query {
 
 sub get_thread {
        my ($self, $mid, $opts) = @_;
-       my $smsg = eval { $self->lookup_skeleton($mid) };
+       my $smsg = retry_reopen($self, sub { lookup_skeleton($self, $mid) });
 
        return { total => 0, msgs => [] } unless $smsg;
        my $qtid = Search::Xapian::Query->new('G' . $smsg->thread_id);
@@ -201,7 +207,7 @@ sub get_thread {
        # always sort threads by timestamp, this makes life easier
        # for the threading algorithm (in SearchThread.pm)
        $opts->{asc} = 1;
-
+       $opts->{enquire} = enquire_skel($self);
        _do_enquire($self, $qtid, $opts);
 }
 
@@ -216,6 +222,7 @@ sub retry_reopen {
                if (ref($@) eq 'Search::Xapian::DatabaseModifiedError') {
                        reopen($self);
                } else {
+                       warn "ref: ", ref($@), "\n";
                        die;
                }
        }
@@ -228,7 +235,7 @@ sub _do_enquire {
 
 sub _enquire_once {
        my ($self, $query, $opts) = @_;
-       my $enquire = $self->enquire;
+       my $enquire = $opts->{enquire} || enquire($self);
        if (defined $query) {
                $query = Search::Xapian::Query->new(OP_AND,$query,$mail_query);
        } else {
@@ -310,18 +317,38 @@ sub num_range_processor {
 sub query_xover {
        my ($self, $beg, $end, $offset) = @_;
        my $qp = Search::Xapian::QueryParser->new;
-       $qp->set_database($self->{xdb});
+       $qp->set_database($self->{skel} || $self->{xdb});
        $qp->add_valuerangeprocessor($self->num_range_processor);
        my $query = $qp->parse_query("$beg..$end", QP_FLAGS);
 
-       _do_enquire($self, $query, {num => 1, limit => 200, offset => $offset});
+       my $opts = {
+               enquire => enquire_skel($self),
+               num => 1,
+               limit => 200,
+               offset => $offset,
+       };
+       _do_enquire($self, $query, $opts);
+}
+
+sub query_ts {
+       my ($self, $ts, $opts) = @_;
+       my $qp = $self->{qp_ts} ||= eval {
+               my $q = Search::Xapian::QueryParser->new;
+               $q->set_database($self->{skel} || $self->{xdb});
+               $q->add_valuerangeprocessor(
+                       Search::Xapian::NumberValueRangeProcessor->new(TS));
+               $q
+       };
+       my $query = $qp->parse_query($ts, QP_FLAGS);
+       $opts->{enquire} = enquire_skel($self);
+       _do_enquire($self, $query, $opts);
 }
 
 sub lookup_skeleton {
        my ($self, $mid) = @_;
        my $skel = $self->{skel} or return lookup_message($self, $mid);
        $mid = mid_clean($mid);
-       my $term = 'XMID' . $mid;
+       my $term = 'Q' . $mid;
        my $smsg;
        my $beg = $skel->postlist_begin($term);
        if ($beg != $skel->postlist_end($term)) {
@@ -340,7 +367,7 @@ sub lookup_message {
        my ($self, $mid) = @_;
        $mid = mid_clean($mid);
 
-       my $doc_id = $self->find_first_doc_id('XMID' . $mid);
+       my $doc_id = $self->find_first_doc_id('Q' . $mid);
        my $smsg;
        if (defined $doc_id) {
                # raises on error:
@@ -359,6 +386,42 @@ sub lookup_mail { # no ghosts!
        });
 }
 
+sub lookup_article {
+       my ($self, $num) = @_;
+       my $term = 'XNUM'.$num;
+       my $smsg;
+       eval {
+               retry_reopen($self, sub {
+                       my $db = $self->{skel} || $self->{xdb};
+                       my $head = $db->postlist_begin($term);
+                       return if $head == $db->postlist_end($term);
+                       my $doc_id = $head->get_docid;
+                       return unless defined $doc_id;
+                       # raises on error:
+                       my $doc = $db->get_document($doc_id);
+                       $smsg = PublicInbox::SearchMsg->wrap($doc);
+                       $smsg->load_expand;
+                       $smsg->{doc_id} = $doc_id;
+               });
+       };
+       $smsg;
+}
+
+sub each_smsg_by_mid {
+       my ($self, $mid, $cb) = @_;
+       my $xdb = $self->{xdb};
+       # XXX retry_reopen isn't necessary for V2Writable, but the PSGI
+       # interface will need it...
+       my ($head, $tail) = $self->find_doc_ids('Q' . $mid);
+       for (; $head->nequal($tail); $head->inc) {
+               my $doc_id = $head->get_docid;
+               my $doc = $xdb->get_document($doc_id);
+               my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid);
+               $smsg->{doc_id} = $doc_id;
+               $cb->($smsg) or return;
+       }
+}
+
 sub find_unique_doc_id {
        my ($self, $termval) = @_;
 
@@ -416,6 +479,15 @@ sub enquire {
        $self->{enquire} ||= Search::Xapian::Enquire->new($self->{xdb});
 }
 
+sub enquire_skel {
+       my ($self) = @_;
+       if (my $skel = $self->{skel}) {
+               $self->{enquire_skel} ||= Search::Xapian::Enquire->new($skel);
+       } else {
+               enquire($self);
+       }
+}
+
 sub help {
        my ($self) = @_;
        $self->qp; # parse altids