search: add note about SCHEMA_VERSION 15

[public-inbox.git] / lib / PublicInbox / Search.pm

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index eae10d8ef852d3ccc9f4d03f1f2716726725e3c4..88c8dc70e17c3bedf3883f07555894b6c54eb7c3 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2015-2019 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # based on notmuch, but with no concept of folders, files or flags
 #
@@ -24,8 +24,8 @@ sub load_xapian () {
 
                # n.b. FLAG_PURE_NOT is expensive not suitable for a public
                # website as it could become a denial-of-service vector
-               # FLAG_PHRASE also seems to cause performance problems
-               # sometimes.
+               # FLAG_PHRASE also seems to cause performance problems chert
+               # (and probably earlier Xapian DBs).  glass seems fine...
                # TODO: make this an option, maybe?
                # or make indexlevel=medium as default
                FLAG_PHRASE()|FLAG_BOOLEAN()|FLAG_LOVEHATE()|FLAG_WILDCARD();
@@ -54,6 +54,9 @@ use constant {
        # 13 - fix threading for empty References/In-Reply-To
        #      (commit 83425ef12e4b65cdcecd11ddcb38175d4a91d5a0)
        # 14 - fix ghost root vivification
+       # 15 - see public-inbox-v2-format(5)
+       #      further bumps likely unnecessary, we'll suggest in-place
+       #      "--reindex" use for further fixes and tweaks
        SCHEMA_VERSION => 15,
 };
 
@@ -126,52 +129,56 @@ chomp @HELP;
 sub xdir ($;$) {
        my ($self, $rdonly) = @_;
        if ($self->{version} == 1) {
-               "$self->{mainrepo}/public-inbox/xapian" . SCHEMA_VERSION;
+               "$self->{inboxdir}/public-inbox/xapian" . SCHEMA_VERSION;
        } else {
-               my $dir = "$self->{mainrepo}/xap" . SCHEMA_VERSION;
+               my $dir = "$self->{inboxdir}/xap" . SCHEMA_VERSION;
                return $dir if $rdonly;
 
-               my $part = $self->{partition};
-               defined $part or die "partition not given";
-               $dir .= "/$part";
+               my $shard = $self->{shard};
+               defined $shard or die "shard not given";
+               $dir .= "/$shard";
        }
 }
 
+sub _xdb ($) {
+       my ($self) = @_;
+       my $dir = xdir($self, 1);
+       my ($xdb, $slow_phrase);
+       my $qpf = \($self->{qp_flags} ||= $QP_FLAGS);
+       if ($self->{version} >= 2) {
+               foreach my $shard (<$dir/*>) {
+                       -d $shard && $shard =~ m!/[0-9]+\z! or next;
+                       my $sub = Search::Xapian::Database->new($shard);
+                       if ($xdb) {
+                               $xdb->add_database($sub);
+                       } else {
+                               $xdb = $sub;
+                       }
+                       $slow_phrase ||= -f "$shard/iamchert";
+               }
+       } else {
+               $slow_phrase = -f "$dir/iamchert";
+               $xdb = Search::Xapian::Database->new($dir);
+       }
+       $$qpf |= FLAG_PHRASE() unless $slow_phrase;
+       $xdb;
+}
+
 sub xdb ($) {
        my ($self) = @_;
        $self->{xdb} ||= do {
                load_xapian();
-               my $dir = xdir($self, 1);
-               if ($self->{version} >= 2) {
-                       my $xdb;
-                       foreach my $part (<$dir/*>) {
-                               -d $part && $part =~ m!/\d+\z! or next;
-                               my $sub = Search::Xapian::Database->new($part);
-                               if ($xdb) {
-                                       $xdb->add_database($sub);
-                               } else {
-                                       $xdb = $sub;
-                               }
-                       }
-                       $xdb;
-               } else {
-                       Search::Xapian::Database->new($dir);
-               }
+               _xdb($self);
        };
 }
 
 sub new {
-       my ($class, $mainrepo, $altid) = @_;
-       my $version = 1;
-       my $ibx = $mainrepo;
-       if (ref $ibx) {
-               $version = $ibx->{version} || 1;
-               $mainrepo = $ibx->{mainrepo};
-       }
+       my ($class, $ibx) = @_;
+       ref $ibx or die "BUG: expected PublicInbox::Inbox object: $ibx";
        my $self = bless {
-               mainrepo => $mainrepo,
-               altid => $altid,
-               version => $version,
+               inboxdir => $ibx->{inboxdir},
+               altid => $ibx->{altid},
+               version => $ibx->{version} // 1,
        }, $class;
        my $dir = xdir($self, 1);
        $self->{over_ro} = PublicInbox::Over->new("$dir/over.sqlite3");
@@ -194,7 +201,8 @@ sub query {
                $self->{over_ro}->recent($opts);
        } else {
                my $qp = qp($self);
-               my $query = $qp->parse_query($query_string, $QP_FLAGS);
+               my $qp_flags = $self->{qp_flags};
+               my $query = $qp->parse_query($query_string, $qp_flags);
                $opts->{relevance} = 1 unless exists $opts->{relevance};
                _do_enquire($self, $query, $opts);
        }
@@ -218,7 +226,9 @@ sub retry_reopen {
                        warn "reopen try #$i on $@\n";
                        reopen($self);
                } else {
-                       warn "ref: ", ref($@), "\n";
+                       # let caller decide how to spew, because ExtMsg queries
+                       # get wonky and trigger:
+                       # "something terrible happened at .../Xapian/Enquire.pm"
                        die;
                }
        }