search: retry document loading from Xapian

[public-inbox.git] / lib / PublicInbox / Search.pm

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index c8e297f42634a842c3b9b3ba033d21cd71353d9e..24cb26670754b2f4a21b62e7d22c3cdc76098a5a 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -38,7 +38,8 @@ use constant {
        # 9 - disable Message-ID compression (SHA-1)
        # 10 - optimize doc for NNTP overviews
        # 11 - merge threads when vivifying ghosts
-       SCHEMA_VERSION => 11,
+       # 12 - change YYYYMMDD value column to numeric
+       SCHEMA_VERSION => 12,
 
        # n.b. FLAG_PURE_NOT is expensive not suitable for a public website
        # as it could become a denial-of-service vector
@@ -66,39 +67,40 @@ my %prob_prefix = (
        tc => 'XTO XCC',
        c => 'XCC',
        tcf => 'XTO XCC A',
+       a => 'XTO XCC A',
        b => 'XNQ XQUOT',
        bs => 'XNQ XQUOT S',
+       n => 'XFN',
 
-       # n.b.: leaving out "a:" alias for "tcf:" even though
-       # mairix supports it.  It is only mentioned in passing in mairix(1)
-       # and the extra two letters are not significantly longer.
        q => 'XQUOT',
        nq => 'XNQ',
 
        # default:
-       '' => 'XMID S A XNQ XQUOT',
+       '' => 'XMID S A XNQ XQUOT XFN',
 );
 
 # not documenting m: and mid: for now, the using the URLs works w/o Xapian
 our @HELP = (
-       's:' => <<EOF,
-match within Subject only  e.g. s:"a quick brown fox"
-This is a probabilistic search with support for stemming
-and wildcards '*'
-EOF
+       's:' => 'match within Subject  e.g. s:"a quick brown fox"',
        'd:' => <<EOF,
 date range as YYYYMMDD  e.g. d:19931002..20101002
 Open-ended ranges such as d:19931002.. and d:..20101002
-are also supported.
+are also supported
 EOF
+       'b:' => 'match within message body, including text attachments',
+       'nq:' => 'match non-quoted text within message body',
+       'quot:' => 'match quoted text within message body',
+       'n:' => 'match filename of attachment(s)',
+       't:' => 'match within the To header',
+       'c:' => 'match within the Cc header',
+       'f:' => 'match within the From header',
+       'a:' => 'match within the To, Cc, and From headers',
+       'tc:' => 'match within the To and Cc headers',
+       'bs:' => 'match within the Subject and body',
 );
-# TODO: (from mairix, some of these are maybe)
-# b (body), f (From:), c (Cc:), n (attachment), t (To:)
-# tc (To:+Cc:), bs (body + Subject), tcf (To: +Cc: +From:)
-#
-# Non-mairix:
+chomp @HELP;
+# TODO:
 # df (filenames from diff)
-# nq (non-quoted body)
 # da (diff a/ removed lines)
 # db (diff b/ added lines)
 
@@ -156,25 +158,35 @@ sub get_thread {
        }
        $opts ||= {};
        $opts->{limit} ||= 1000;
+
+       # always sort threads by timestamp, this makes life easier
+       # for the threading algorithm (in SearchThread.pm)
+       $opts->{asc} = 1;
+
        _do_enquire($self, $qtid, $opts);
 }
 
-sub _do_enquire {
-       my ($self, $query, $opts) = @_;
+sub retry_reopen {
+       my ($self, $cb) = @_;
        my $ret;
        for (1..10) {
-               eval { $ret = _enquire_once($self, $query, $opts) };
+               eval { $ret = $cb->() };
                return $ret unless $@;
                # Exception: The revision being read has been discarded -
                # you should call Xapian::Database::reopen()
-               if (index($@, 'Xapian::Database::reopen') >= 0) {
+               if (ref($@) eq 'Search::Xapian::DatabaseModifiedError') {
                        reopen($self);
                } else {
-                       die $@;
+                       die;
                }
        }
 }
 
+sub _do_enquire {
+       my ($self, $query, $opts) = @_;
+       retry_reopen($self, sub { _enquire_once($self, $query, $opts) });
+}
+
 sub _enquire_once {
        my ($self, $query, $opts) = @_;
        my $enquire = $self->enquire;
@@ -221,7 +233,7 @@ sub qp {
        $qp->set_stemmer($self->stemmer);
        $qp->set_stemming_strategy(STEM_SOME);
        $qp->add_valuerangeprocessor(
-               Search::Xapian::StringValueRangeProcessor->new(YYYYMMDD, 'd:'));
+               Search::Xapian::NumberValueRangeProcessor->new(YYYYMMDD, 'd:'));
 
        while (my ($name, $prefix) = each %bool_pfx_external) {
                $qp->add_boolean_prefix($name, $prefix);
@@ -236,11 +248,12 @@ sub qp {
                        /\Aserial:(\w+):/ or next;
                        my $pfx = $1;
                        push @$user_pfx, "$pfx:", <<EOF;
-alternate serial number  e.g. $pfx:12345
+alternate serial number  e.g. $pfx:12345 (boolean)
 EOF
                        # gmane => XGMANE
                        $qp->add_boolean_prefix($pfx, 'X'.uc($pfx));
                }
+               chomp @$user_pfx;
        }
 
        while (my ($name, $prefix) = each %prob_prefix) {