miscsearch: index UIDVALIDITY, use as startup cache

[public-inbox.git] / lib / PublicInbox / Search.pm

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 5a57657ff4da5b1332afe8db5ed39b35c06f7d4e..05c679c9a6b3f4386c55d1e52411bc6a985f4002 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -6,7 +6,7 @@
 package PublicInbox::Search;
 use strict;
 use parent qw(Exporter);
-our @EXPORT_OK = qw(mdocid);
+our @EXPORT_OK = qw(retry_reopen int_val);
 use List::Util qw(max);
 
 # values for searching, changing the numeric value breaks
@@ -54,11 +54,11 @@ use constant {
 
 use PublicInbox::Smsg;
 use PublicInbox::Over;
-my $QP_FLAGS;
-our %X = map { $_ => 0 } qw(BoolWeight Database Enquire QueryParser Stem);
+our $QP_FLAGS;
+our %X = map { $_ => 0 } qw(BoolWeight Database Enquire QueryParser Stem Query);
 our $Xap; # 'Search::Xapian' or 'Xapian'
-my $NVRP; # '$Xap::'.('NumberValueRangeProcessor' or 'NumberRangeProcessor')
-my $ENQ_ASCENDING;
+our $NVRP; # '$Xap::'.('NumberValueRangeProcessor' or 'NumberRangeProcessor')
+our $ENQ_ASCENDING;
 
 sub load_xapian () {
        return 1 if defined $Xap;
@@ -90,6 +90,8 @@ sub load_xapian () {
                $ENQ_ASCENDING = $x eq 'Xapian' ?
                                1 : Search::Xapian::ENQ_ASCENDING();
 
+               *sortable_serialise = $x.'::sortable_serialise';
+               *sortable_unserialise = $x.'::sortable_unserialise';
                # n.b. FLAG_PURE_NOT is expensive not suitable for a public
                # website as it could become a denial-of-service vector
                # FLAG_PHRASE also seems to cause performance problems chert
@@ -190,17 +192,16 @@ sub xdir ($;$) {
        }
 }
 
-sub _xdb_sharded {
-       my ($self, $xpfx) = @_;
-       opendir(my $dh, $xpfx) or return; # not initialized yet
+sub xdb_sharded {
+       my ($self) = @_;
+       opendir(my $dh, $self->{xpfx}) or return; # not initialized yet
 
        # We need numeric sorting so shard[0] is first for reading
        # Xapian metadata, if needed
-       my $last = max(grep(/\A[0-9]+\z/, readdir($dh)));
-       return if !defined($last);
+       my $last = max(grep(/\A[0-9]+\z/, readdir($dh))) // return;
        my (@xdb, $slow_phrase);
        for (0..$last) {
-               my $shard_dir = "$xpfx/$_";
+               my $shard_dir = "$self->{xpfx}/$_";
                if (-d $shard_dir && -r _) {
                        push @xdb, $X{Database}->new($shard_dir);
                        $slow_phrase ||= -f "$shard_dir/iamchert";
@@ -221,7 +222,7 @@ sub _xdb {
        my $dir = xdir($self, 1);
        $self->{qp_flags} //= $QP_FLAGS;
        if ($self->{ibx_ver} >= 2) {
-               _xdb_sharded($self, $dir);
+               xdb_sharded($self);
        } else {
                $self->{qp_flags} |= FLAG_PHRASE() if !-f "$dir/iamchert";
                $X{Database}->new($dir);
@@ -291,15 +292,15 @@ sub mset {
 }
 
 sub retry_reopen {
-       my ($self, $cb, $arg) = @_;
+       my ($self, $cb, @arg) = @_;
        for my $i (1..10) {
                if (wantarray) {
                        my @ret;
-                       eval { @ret = $cb->($arg) };
+                       eval { @ret = $cb->($self, @arg) };
                        return @ret unless $@;
                } else {
                        my $ret;
-                       eval { $ret = $cb->($arg) };
+                       eval { $ret = $cb->($self, @arg) };
                        return $ret unless $@;
                }
                # Exception: The revision being read has been discarded -
@@ -319,7 +320,7 @@ sub retry_reopen {
 
 sub _do_enquire {
        my ($self, $query, $opts) = @_;
-       retry_reopen($self, \&_enquire_once, [ $self, $query, $opts ]);
+       retry_reopen($self, \&_enquire_once, $query, $opts);
 }
 
 # returns true if all docs have the THREADID value
@@ -329,8 +330,17 @@ sub has_threadid ($) {
 }
 
 sub _enquire_once { # retry_reopen callback
-       my ($self, $query, $opts) = @{$_[0]};
+       my ($self, $query, $opts) = @_;
        my $xdb = xdb($self);
+       if (defined(my $eidx_key = $opts->{eidx_key})) {
+               $query = $X{Query}->new(OP_FILTER(), $query, 'O'.$eidx_key);
+       }
+       if (defined(my $uid_range = $opts->{uid_range})) {
+               my $range = $X{Query}->new(OP_VALUE_RANGE(), UID,
+                                       sortable_serialise($uid_range->[0]),
+                                       sortable_serialise($uid_range->[1]));
+               $query = $X{Query}->new(OP_FILTER(), $query, $range);
+       }
        my $enquire = $X{Enquire}->new($xdb);
        $enquire->set_query($query);
        $opts ||= {};
@@ -427,4 +437,10 @@ sub help {
        \@ret;
 }
 
+sub int_val ($$) {
+       my ($doc, $col) = @_;
+       my $val = $doc->get_value($col) or return; # undefined is '' in Xapian
+       sortable_unserialise($val) + 0; # PV => IV conversion
+}
+
 1;