Merge remote-tracking branch 'origin/master' into lorelei

[public-inbox.git] / lib / PublicInbox / MiscSearch.pm

diff --git a/lib/PublicInbox/MiscSearch.pm b/lib/PublicInbox/MiscSearch.pm
index 8beb8349fea839a4a132298ec9001a6c1758fb52..6683d5644379129fdb976fa56c187a247c99c2f8 100644 (file)
--- a/lib/PublicInbox/MiscSearch.pm
+++ b/lib/PublicInbox/MiscSearch.pm
@@ -5,10 +5,12 @@
 package PublicInbox::MiscSearch;
 use strict;
 use v5.10.1;
-use PublicInbox::Search qw(retry_reopen);
+use PublicInbox::Search qw(retry_reopen int_val);
+my $json;
 
 # Xapian value columns:
 our $MODIFIED = 0;
+our $UIDVALIDITY = 1; # (created time)
 
 # avoid conflicting with message Search::prob_prefix for UI/UX reasons
 my %PROB_PREFIX = (
@@ -23,6 +25,8 @@ my %PROB_PREFIX = (
 
 sub new {
        my ($class, $dir) = @_;
+       PublicInbox::Search::load_xapian();
+       $json //= PublicInbox::Config::json();
        bless {
                xdb => $PublicInbox::Search::X{Database}->new($dir)
        }, $class;
@@ -50,7 +54,7 @@ sub mi_qp_new ($) {
 }
 
 sub misc_enquire_once { # retry_reopen callback
-       my ($self, $qr, $opt) = @{$_[0]};
+       my ($self, $qr, $opt) = @_;
        my $eq = $PublicInbox::Search::X{Enquire}->new($self->{xdb});
        $eq->set_query($qr);
         my $desc = !$opt->{asc};
@@ -69,11 +73,119 @@ sub misc_enquire_once { # retry_reopen callback
 sub mset {
        my ($self, $qs, $opt) = @_;
        $opt ||= {};
+       reopen($self);
        my $qp = $self->{qp} //= mi_qp_new($self);
        $qs = 'type:inbox' if $qs eq '';
        my $qr = $qp->parse_query($qs, $PublicInbox::Search::QP_FLAGS);
        $opt->{relevance} = 1 unless exists $opt->{relevance};
-       retry_reopen($self, \&misc_enquire_once, [ $self, $qr, $opt ]);
+       retry_reopen($self, \&misc_enquire_once, $qr, $opt);
 }
 
+sub ibx_matches_once { # retry_reopen callback
+       my ($self, $qr, $by_newsgroup) = @_;
+       # double in case no newsgroups are configured:
+       my $limit = scalar(keys %$by_newsgroup) * 2;
+       my $opt = { limit => $limit, offset => 0, relevance => -1 };
+       my $ret = {}; # newsgroup => $ibx of matches
+       while (1) {
+               my $mset = misc_enquire_once($self, $qr, $opt);
+               for my $mi ($mset->items) {
+                       my $doc = $mi->get_document;
+                       my $end = $doc->termlist_end;
+                       my $cur = $doc->termlist_begin;
+                       $cur->skip_to('Q');
+                       if ($cur != $end) {
+                               my $ng = $cur->get_termname; # eidx_key
+                               $ng =~ s/\AQ// or warn "BUG: no `Q': $ng";
+                               if (my $ibx = $by_newsgroup->{$ng}) {
+                                       $ret->{$ng} = $ibx;
+                               }
+                       } else {
+                               warn <<EOF;
+W: docid=${\$mi->get_docid} has no `Q' (eidx_key) term
+EOF
+                       }
+               }
+               my $nr = $mset->size;
+               return $ret if $nr < $limit;
+               $opt->{offset} += $nr;
+       }
+}
+
+# returns a newsgroup => PublicInbox::Inbox mapping
+sub newsgroup_matches {
+       my ($self, $qs, $pi_cfg) = @_;
+       my $qp = $self->{qp} //= mi_qp_new($self);
+       $qs .= ' type:inbox';
+       my $qr = $qp->parse_query($qs, $PublicInbox::Search::QP_FLAGS);
+       retry_reopen($self, \&ibx_matches_once, $qr, $pi_cfg->{-by_newsgroup});
+}
+
+sub ibx_data_once {
+       my ($self, $ibx) = @_;
+       my $xdb = $self->{xdb};
+       my $term = 'Q'.$ibx->eidx_key; # may be {inboxdir}, so private
+       my $head = $xdb->postlist_begin($term);
+       my $tail = $xdb->postlist_end($term);
+       if ($head != $tail) {
+               my $doc = $xdb->get_document($head->get_docid);
+               $ibx->{uidvalidity} //= int_val($doc, $UIDVALIDITY);
+               $ibx->{-modified} = int_val($doc, $MODIFIED);
+               $doc->get_data;
+       } else {
+               undef;
+       }
+}
+
+sub inbox_data {
+       my ($self, $ibx) = @_;
+       retry_reopen($self, \&ibx_data_once, $ibx);
+}
+
+sub ibx_cache_load {
+       my ($doc, $cache) = @_;
+       my $end = $doc->termlist_end;
+       my $cur = $doc->termlist_begin;
+       $cur->skip_to('Q');
+       return if $cur == $end;
+       my $eidx_key = $cur->get_termname;
+       $eidx_key =~ s/\AQ// or return; # expired
+       my $ce = $cache->{$eidx_key} = {};
+       $ce->{uidvalidity} = int_val($doc, $UIDVALIDITY);
+       $ce->{-modified} = int_val($doc, $MODIFIED);
+       $ce->{description} = do {
+               # extract description from manifest.js.gz epoch description
+               my $d;
+               my $data = $json->decode($doc->get_data);
+               for (values %$data) {
+                       $d = $_->{description} // next;
+                       $d =~ s/ \[epoch [0-9]+\]\z// or next;
+                       last;
+               }
+               $d;
+       }
+}
+
+sub _nntpd_cache_load { # retry_reopen callback
+       my ($self) = @_;
+       my $opt = { limit => $self->{xdb}->get_doccount * 10, relevance => -1 };
+       my $mset = mset($self, 'type:newsgroup type:inbox', $opt);
+       my $cache = {};
+       for my $it ($mset->items) {
+               ibx_cache_load($it->get_document, $cache);
+       }
+       $cache
+}
+
+# returns { newsgroup => $cache_entry } mapping, $cache_entry contains
+# anything which may trigger seeks at startup, currently: description,
+# -modified, and uidvalidity.
+sub nntpd_cache_load {
+       my ($self) = @_;
+       retry_reopen($self, \&_nntpd_cache_load);
+}
+
+no warnings 'once';
+*reopen = \&PublicInbox::Search::reopen;
+
 1;