X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FSearch.pm;h=ba2392557bc5abfc42147483a65bb1f7dc10ef2a;hb=525555d14118f92f86be54c683f797089c52a78d;hp=546884a9a1fc4867f80a97f95494f4330470f70b;hpb=1561be76574111a53c9d7c284fd03f68d6a5208b;p=public-inbox.git diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index 546884a9..ba239255 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -6,7 +6,8 @@ package PublicInbox::Search; use strict; use parent qw(Exporter); -our @EXPORT_OK = qw(mdocid); +our @EXPORT_OK = qw(retry_reopen); +use List::Util qw(max); # values for searching, changing the numeric value breaks # compatibility with old indices (so don't change them it) @@ -53,11 +54,11 @@ use constant { use PublicInbox::Smsg; use PublicInbox::Over; -my $QP_FLAGS; -our %X = map { $_ => 0 } qw(BoolWeight Database Enquire QueryParser Stem); +our $QP_FLAGS; +our %X = map { $_ => 0 } qw(BoolWeight Database Enquire QueryParser Stem Query); our $Xap; # 'Search::Xapian' or 'Xapian' -my $NVRP; # '$Xap::'.('NumberValueRangeProcessor' or 'NumberRangeProcessor') -my $ENQ_ASCENDING; +our $NVRP; # '$Xap::'.('NumberValueRangeProcessor' or 'NumberRangeProcessor') +our $ENQ_ASCENDING; sub load_xapian () { return 1 if defined $Xap; @@ -89,9 +90,6 @@ sub load_xapian () { $ENQ_ASCENDING = $x eq 'Xapian' ? 1 : Search::Xapian::ENQ_ASCENDING(); - # for Smsg: - *PublicInbox::Smsg::sortable_unserialise = - $Xap.'::sortable_unserialise'; # n.b. FLAG_PURE_NOT is expensive not suitable for a public # website as it could become a denial-of-service vector # FLAG_PHRASE also seems to cause performance problems chert @@ -192,36 +190,42 @@ sub xdir ($;$) { } } -sub _xdb ($) { +sub xdb_sharded { + my ($self) = @_; + opendir(my $dh, $self->{xpfx}) or return; # not initialized yet + + # We need numeric sorting so shard[0] is first for reading + # Xapian metadata, if needed + my $last = max(grep(/\A[0-9]+\z/, readdir($dh))); + return if !defined($last); + my (@xdb, $slow_phrase); + for (0..$last) { + my $shard_dir = "$self->{xpfx}/$_"; + if (-d $shard_dir && -r _) { + push @xdb, $X{Database}->new($shard_dir); + $slow_phrase ||= -f "$shard_dir/iamchert"; + } else { # gaps from missing epochs throw off mdocid() + warn "E: $shard_dir missing or unreadable\n"; + return; + } + } + $self->{qp_flags} |= FLAG_PHRASE() if !$slow_phrase; + $self->{nshard} = scalar(@xdb); + my $xdb = shift @xdb; + $xdb->add_database($_) for @xdb; + $xdb; +} + +sub _xdb { my ($self) = @_; my $dir = xdir($self, 1); - my ($xdb, $slow_phrase); - my $qpf = \($self->{qp_flags} ||= $QP_FLAGS); + $self->{qp_flags} //= $QP_FLAGS; if ($self->{ibx_ver} >= 2) { - my @xdb; - opendir(my $dh, $dir) or return; # not initialized yet - - # We need numeric sorting so shard[0] is first for reading - # Xapian metadata, if needed - for (sort { $a <=> $b } grep(/\A[0-9]+\z/, readdir($dh))) { - my $shard_dir = "$dir/$_"; - if (-d $shard_dir && -r _) { - push @xdb, $X{Database}->new($shard_dir); - $slow_phrase ||= -f "$shard_dir/iamchert"; - } else { # gaps from missing epochs throw off mdocid() - warn "E: $shard_dir missing or unreadable\n"; - return; - } - } - $self->{nshard} = scalar(@xdb); - $xdb = shift @xdb; - $xdb->add_database($_) for @xdb; + xdb_sharded($self); } else { - $slow_phrase = -f "$dir/iamchert"; - $xdb = $X{Database}->new($dir); + $self->{qp_flags} |= FLAG_PHRASE() if !-f "$dir/iamchert"; + $X{Database}->new($dir); } - $$qpf |= FLAG_PHRASE() unless $slow_phrase; - $xdb; } # v2 Xapian docids don't conflict, so they're identical to @@ -241,9 +245,9 @@ sub mset_to_artnums { sub xdb ($) { my ($self) = @_; - $self->{xdb} ||= do { + $self->{xdb} //= do { load_xapian(); - _xdb($self); + $self->_xdb; }; } @@ -265,8 +269,6 @@ sub new { ibx_ver => $ibx->version, }, $class; xpfx_init($self); - my $dir = xdir($self, 1); - $self->{over_ro} = PublicInbox::Over->new("$dir/over.sqlite3"); $self; } @@ -279,7 +281,7 @@ sub reopen { } # read-only -sub query { +sub mset { my ($self, $query_string, $opts) = @_; $opts ||= {}; my $qp = $self->{qp} //= qparse_new($self); @@ -289,15 +291,15 @@ sub query { } sub retry_reopen { - my ($self, $cb, $arg) = @_; + my ($self, $cb, @arg) = @_; for my $i (1..10) { if (wantarray) { my @ret; - eval { @ret = $cb->($arg) }; + eval { @ret = $cb->($self, @arg) }; return @ret unless $@; } else { my $ret; - eval { $ret = $cb->($arg) }; + eval { $ret = $cb->($self, @arg) }; return $ret unless $@; } # Exception: The revision being read has been discarded - @@ -317,7 +319,7 @@ sub retry_reopen { sub _do_enquire { my ($self, $query, $opts) = @_; - retry_reopen($self, \&_enquire_once, [ $self, $query, $opts ]); + retry_reopen($self, \&_enquire_once, $query, $opts); } # returns true if all docs have the THREADID value @@ -327,8 +329,11 @@ sub has_threadid ($) { } sub _enquire_once { # retry_reopen callback - my ($self, $query, $opts) = @{$_[0]}; + my ($self, $query, $opts) = @_; my $xdb = xdb($self); + if (defined(my $eidx_key = $opts->{eidx_key})) { + $query = $X{Query}->new(OP_FILTER(), $query, 'O'.$eidx_key); + } my $enquire = $X{Enquire}->new($xdb); $enquire->set_query($query); $opts ||= {}; @@ -346,17 +351,17 @@ sub _enquire_once { # retry_reopen callback if ($opts->{thread} && has_threadid($self)) { $enquire->set_collapse_key(THREADID); } + $enquire->get_mset($opts->{offset} || 0, $opts->{limit} || 50); +} - my $offset = $opts->{offset} || 0; - my $limit = $opts->{limit} || 50; - my $mset = $enquire->get_mset($offset, $limit); - return $mset if $opts->{mset}; +sub mset_to_smsg { + my ($self, $ibx, $mset) = @_; my $nshard = $self->{nshard} // 1; my $i = 0; my %order = map { mdocid($nshard, $_) => ++$i } $mset->items; my @msgs = sort { $order{$a->{num}} <=> $order{$b->{num}} - } @{$self->{over_ro}->get_all(keys %order)}; + } @{$ibx->over->get_all(keys %order)}; wantarray ? ($mset->get_matches_estimated, \@msgs) : \@msgs; }