rename SearchIdxThread to SearchIdxSkeleton

[public-inbox.git] / lib / PublicInbox / Search.pm

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 3ec96ca93a35fc966166c3c08ebb872cf3fb27d1..6b14942a1045cda131e8b66347e72b7a0db2e0a7 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -56,7 +56,7 @@ my %bool_pfx_internal = (
 );
 
 my %bool_pfx_external = (
-       mid => 'XMID', # uniQue id (Message-ID)
+       mid => 'XMID', # Message-ID (full/exact)
 );
 
 my %prob_prefix = (
@@ -120,18 +120,60 @@ chomp @HELP;
 my $mail_query = Search::Xapian::Query->new('T' . 'mail');
 
 sub xdir {
-       my (undef, $git_dir) = @_;
-       "$git_dir/public-inbox/xapian" . SCHEMA_VERSION;
+       my ($self) = @_;
+       if ($self->{version} == 1) {
+               "$self->{mainrepo}/public-inbox/xapian" . SCHEMA_VERSION;
+       } else {
+               my $dir = "$self->{mainrepo}/xap" . SCHEMA_VERSION;
+               my $part = $self->{partition};
+               defined $part or die "partition not given";
+               $dir .= "/$part";
+       }
 }
 
 sub new {
-       my ($class, $git_dir, $altid) = @_;
-       my $dir = $class->xdir($git_dir);
-       my $db = Search::Xapian::Database->new($dir);
-       bless { xdb => $db, git_dir => $git_dir, altid => $altid }, $class;
+       my ($class, $mainrepo, $altid) = @_;
+       my $version = 1;
+       my $ibx = $mainrepo;
+       if (ref $ibx) {
+               $version = $ibx->{version} || 1;
+               $mainrepo = $ibx->{mainrepo};
+       }
+       my $self = bless {
+               mainrepo => $mainrepo,
+               altid => $altid,
+               version => $version,
+       }, $class;
+       if ($version >= 2) {
+               my $dir = "$self->{mainrepo}/xap" . SCHEMA_VERSION;
+               my $xdb;
+               my $parts = 0;
+               foreach my $part (<$dir/*>) {
+                       -d $part && $part =~ m!/\d+\z! or next;
+                       $parts++;
+                       my $sub = Search::Xapian::Database->new($part);
+                       if ($xdb) {
+                               $xdb->add_database($sub);
+                       } else {
+                               $xdb = $sub;
+                       }
+               }
+               warn "v2 repo with $parts found in $dir\n";
+               $self->{xdb} = $xdb;
+               $self->{skel} = Search::Xapian::Database->new("$dir/skel");
+       } else {
+               $self->{xdb} = Search::Xapian::Database->new($self->xdir);
+       }
+       $self;
 }
 
-sub reopen { $_[0]->{xdb}->reopen }
+sub reopen {
+       my ($self) = @_;
+       $self->{xdb}->reopen;
+       if (my $skel = $self->{skel}) {
+               $skel->reopen;
+       }
+}
 
 # read-only
 sub query {
@@ -149,7 +191,7 @@ sub query {
 
 sub get_thread {
        my ($self, $mid, $opts) = @_;
-       my $smsg = eval { $self->lookup_message($mid) };
+       my $smsg = retry_reopen($self, sub { lookup_skeleton($self, $mid) });
 
        return { total => 0, msgs => [] } unless $smsg;
        my $qtid = Search::Xapian::Query->new('G' . $smsg->thread_id);
@@ -165,7 +207,7 @@ sub get_thread {
        # always sort threads by timestamp, this makes life easier
        # for the threading algorithm (in SearchThread.pm)
        $opts->{asc} = 1;
-
+       $opts->{enquire} = enquire_skel($self);
        _do_enquire($self, $qtid, $opts);
 }
 
@@ -180,6 +222,7 @@ sub retry_reopen {
                if (ref($@) eq 'Search::Xapian::DatabaseModifiedError') {
                        reopen($self);
                } else {
+                       warn "ref: ", ref($@), "\n";
                        die;
                }
        }
@@ -192,7 +235,7 @@ sub _do_enquire {
 
 sub _enquire_once {
        my ($self, $query, $opts) = @_;
-       my $enquire = $self->enquire;
+       my $enquire = $opts->{enquire} || enquire($self);
        if (defined $query) {
                $query = Search::Xapian::Query->new(OP_AND,$query,$mail_query);
        } else {
@@ -281,11 +324,30 @@ sub query_xover {
        _do_enquire($self, $query, {num => 1, limit => 200, offset => $offset});
 }
 
+sub lookup_skeleton {
+       my ($self, $mid) = @_;
+       my $skel = $self->{skel} or return lookup_message($self, $mid);
+       $mid = mid_clean($mid);
+       my $term = 'XMID' . $mid;
+       my $smsg;
+       my $beg = $skel->postlist_begin($term);
+       if ($beg != $skel->postlist_end($term)) {
+               my $doc_id = $beg->get_docid;
+               if (defined $doc_id) {
+                       # raises on error:
+                       my $doc = $skel->get_document($doc_id);
+                       $smsg = PublicInbox::SearchMsg->wrap($doc, $mid);
+                       $smsg->{doc_id} = $doc_id;
+               }
+       }
+       $smsg;
+}
+
 sub lookup_message {
        my ($self, $mid) = @_;
        $mid = mid_clean($mid);
 
-       my $doc_id = $self->find_unique_doc_id('XMID' . $mid);
+       my $doc_id = $self->find_first_doc_id('XMID' . $mid);
        my $smsg;
        if (defined $doc_id) {
                # raises on error:
@@ -327,6 +389,16 @@ sub find_doc_ids {
        ($db->postlist_begin($termval), $db->postlist_end($termval));
 }
 
+sub find_first_doc_id {
+       my ($self, $termval) = @_;
+
+       my ($begin, $end) = $self->find_doc_ids($termval);
+
+       return undef if $begin->equal($end); # not found
+
+       $begin->get_docid;
+}
+
 # normalize subjects so they are suitable as pathnames for URLs
 # XXX: consider for removal
 sub subject_path {
@@ -351,6 +423,15 @@ sub enquire {
        $self->{enquire} ||= Search::Xapian::Enquire->new($self->{xdb});
 }
 
+sub enquire_skel {
+       my ($self) = @_;
+       if (my $skel = $self->{skel}) {
+               $self->{enquire_skel} ||= Search::Xapian::Enquire->new($skel);
+       } else {
+               enquire($self);
+       }
+}
+
 sub help {
        my ($self) = @_;
        $self->qp; # parse altids