search: make xap_terms easier-to-use and use it more

This allows us to simplify callers throughout, and exceptions are
can no longer be silently hidden.  MiscSearch now uses xap_terms
for looking up eidx_key terms for a code reduction.

We also simplify LeiStore->_msg_kw for runtime use by moving the
MsetIterator handling into t/lei_store.t test case.

diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index d0963e928c6500402f8560f6e89515ce957d3f95..06ea62995a2b089f1bc10922fa40a8d1be640fbc 100644 (file)
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -19,16 +19,13 @@ sub num2docid ($$) {
 }
 
 sub _msg_kw { # retry_reopen callback
-       my ($self, $num) = @_; # num_or_mitem
-       my $xdb = $self->xdb; # set {nshard};
-       my $docid = ref($num) ? $num->get_docid : num2docid($self, $num);
-       my $kw = xap_terms('K', $xdb, $docid);
-       warn "E: #$docid ($num): $@\n" if $@;
-       wantarray ? sort(keys(%$kw)) : $kw;
+       my ($self, $num) = @_;
+       my $xdb = $self->xdb; # set {nshard} for num2docid;
+       xap_terms('K', $xdb, num2docid($self, $num));
 }
 
-sub msg_keywords {
-       my ($self, $num) = @_; # num_or_mitem
+sub msg_keywords { # array or hashref
+       my ($self, $num) = @_;
        $self->retry_reopen(\&_msg_kw, $num);
 }
 
@@ -138,7 +135,8 @@ sub kw_changed {
                $docids //= [];
                @$docids = sort { $a <=> $b } values %$xoids;
        }
-       my $cur_kw = msg_keywords($self, $docids->[0]);
+       my $cur_kw = eval { msg_keywords($self, $docids->[0]) };
+       die "E: #$docids->[0] keyword lookup failure: $@\n" if $@;
 
        # RFC 5550 sec 5.9 on the $Forwarded keyword states:
        # "Once set, the flag SHOULD NOT be cleared"

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index beb955bb6de599a5ce675276888f17f6f0c83495..cac7fb7d5902c9aa51cbd40b395721fe4768a9f3 100644 (file)
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -71,11 +71,11 @@ sub _mitem_kw { # retry_reopen callback
        my $doc = $mitem->get_document;
        my $kw = xap_terms('K', $doc);
        $kw->{flagged} = 1 if $flagged;
-       my $L = xap_terms('L', $doc);
+       my @L = xap_terms('L', $doc);
        # we keep the empty {kw} array here to prevent expensive work in
        # ->xsmsg_vmd, _unbless_smsg will clobber it iff it's empty
        $smsg->{kw} = [ sort keys %$kw ];
-       $smsg->{L} = [ sort keys %$L ] if scalar(keys %$L);
+       $smsg->{L} = \@L if scalar(@L);
 }
 
 sub mitem_kw ($$$;$) {

diff --git a/lib/PublicInbox/MiscSearch.pm b/lib/PublicInbox/MiscSearch.pm
index ead9a2781bb235d6606b0e1bd6b196b4c43dc941..4e01045392886135fbb9d2f7aa1bebb90016be5c 100644 (file)
--- a/lib/PublicInbox/MiscSearch.pm
+++ b/lib/PublicInbox/MiscSearch.pm
@@ -5,7 +5,7 @@
 package PublicInbox::MiscSearch;
 use strict;
 use v5.10.1;
-use PublicInbox::Search qw(retry_reopen int_val);
+use PublicInbox::Search qw(retry_reopen int_val xap_terms);
 my $json;
 
 # Xapian value columns:
@@ -90,15 +90,10 @@ sub ibx_matches_once { # retry_reopen callback
        while (1) {
                my $mset = misc_enquire_once($self, $qr, $opt);
                for my $mi ($mset->items) {
-                       my $doc = $mi->get_document;
-                       my $end = $doc->termlist_end;
-                       my $cur = $doc->termlist_begin;
-                       $cur->skip_to('Q');
-                       if ($cur != $end) {
-                               my $ng = $cur->get_termname; # eidx_key
-                               $ng =~ s/\AQ// or warn "BUG: no `Q': $ng";
-                               if (my $ibx = $by_newsgroup->{$ng}) {
-                                       $ret->{$ng} = $ibx;
+                       my ($eidx_key) = xap_terms('Q', $mi->get_document);
+                       if (defined($eidx_key)) {
+                               if (my $ibx = $by_newsgroup->{$eidx_key}) {
+                                       $ret->{$eidx_key} = $ibx;
                                }
                        } else {
                                warn <<EOF;
@@ -144,12 +139,8 @@ sub inbox_data {
 
 sub ibx_cache_load {
        my ($doc, $cache) = @_;
-       my $end = $doc->termlist_end;
-       my $cur = $doc->termlist_begin;
-       $cur->skip_to('Q');
-       return if $cur == $end;
-       my $eidx_key = $cur->get_termname;
-       $eidx_key =~ s/\AQ// or return; # expired
+       my ($eidx_key) = xap_terms('Q', $doc);
+       return unless defined($eidx_key); # expired
        my $ce = $cache->{$eidx_key} = {};
        $ce->{uidvalidity} = int_val($doc, $UIDVALIDITY);
        $ce->{-modified} = int_val($doc, $MODIFIED);

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 59a5a3b006755967699f1731e19039d61574d81d..7e19e616a2180e28fe7470698b3d555360ee4c93 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -557,19 +557,15 @@ sub get_pct ($) { # mset item
 sub xap_terms ($$;@) {
        my ($pfx, $xdb_or_doc, @docid) = @_; # @docid may be empty ()
        my %ret;
-       eval {
-               my $end = $xdb_or_doc->termlist_end(@docid);
-               my $cur = $xdb_or_doc->termlist_begin(@docid);
-               for (; $cur != $end; $cur++) {
-                       $cur->skip_to($pfx);
-                       last if $cur == $end;
-                       my $tn = $cur->get_termname;
-                       if (index($tn, $pfx) == 0) {
-                               $ret{substr($tn, length($pfx))} = undef;
-                       }
-               }
-       };
-       \%ret;
+       my $end = $xdb_or_doc->termlist_end(@docid);
+       my $cur = $xdb_or_doc->termlist_begin(@docid);
+       for (; $cur != $end; $cur++) {
+               $cur->skip_to($pfx);
+               last if $cur == $end;
+               my $tn = $cur->get_termname;
+               $ret{substr($tn, length($pfx))} = undef if !index($tn, $pfx);
+       }
+       wantarray ? sort(keys(%ret)) : \%ret;
 }
 
 1;

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index f553eda69b42d3ec8dfe23142812afe15d4f4104..65764cc81a81b167a7e5a59561d4e1084401531e 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -435,8 +435,9 @@ sub add_xapian ($$$$) {
        if (my $old = $merge_vmd ? _get_doc($self, $smsg->{num}) : undef) {
                my @x = @VMD_MAP;
                while (my ($field, $pfx) = splice(@x, 0, 2)) {
-                       my $vals = xap_terms($pfx, $old);
-                       $doc->add_boolean_term($pfx.$_) for keys %$vals;
+                       for my $term (xap_terms($pfx, $old)) {
+                               $doc->add_boolean_term($pfx.$term);
+                       }
                }
        }
        $self->{xdb}->replace_document($smsg->{num}, $doc);

diff --git a/t/lei_store.t b/t/lei_store.t
index db94f6daa0cbfe5b3a1c71275cd774b6b7089172..73b5c74db0e3468cffe29dadd0054136a950c641 100644 (file)
--- a/t/lei_store.t
+++ b/t/lei_store.t
@@ -31,7 +31,8 @@ $sto->done;
        is($mset->size, 1, 'search works');
        is_deeply($es->mset_to_artnums($mset), [ $msgs->[0]->{num} ],
                'mset_to_artnums');
-       my @kw = $es->msg_keywords(($mset->items)[0]);
+       my $mi = ($mset->items)[0];
+       my @kw = PublicInbox::Search::xap_terms('K', $mi->get_document);
        is_deeply(\@kw, [], 'no flags');
 }