config: lazy-load coderepos, support extindex

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index adced0762b41487d5bdaa69517db157901bf3076..772f5a644a5b0f580e531c78d3c4971d6915d2bf 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -22,6 +22,7 @@ use PublicInbox::OverIdx;
 use PublicInbox::Spawn qw(spawn nodatacow_dir);
 use PublicInbox::Git qw(git_unquote);
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
+use PublicInbox::Address;
 our @EXPORT_OK = qw(log2stack is_ancestor check_size prepare_stack
        index_text term_generator add_val is_bad_blob);
 my $X = \%PublicInbox::Search::X;
@@ -105,8 +106,11 @@ sub load_xapian_writable () {
        $DB_CREATE_OR_OPEN = eval($xap.'::DB_CREATE_OR_OPEN()');
        $DB_OPEN = eval($xap.'::DB_OPEN()');
        my $ver = (eval($xap.'::major_version()') << 16) |
-               (eval($xap.'::minor_version()') << 8);
+               (eval($xap.'::minor_version()') << 8) |
+               eval($xap.'::revision()');
        $DB_NO_SYNC = 0x4 if $ver >= 0x10400;
+       # Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks
+       $X->{CLOEXEC_UNSET} = 1 if $ver >= 0x010215 && $ver <= 0x010218;
        1;
 }
 
@@ -155,22 +159,44 @@ sub term_generator ($) { # write-only
        }
 }
 
+sub index_phrase ($$$$) {
+       my ($self, $text, $wdf_inc, $prefix) = @_;
+
+       my $tg = term_generator($self);
+       $tg->index_text($text, $wdf_inc, $prefix);
+       $tg->increase_termpos;
+}
+
 sub index_text ($$$$) {
        my ($self, $text, $wdf_inc, $prefix) = @_;
-       my $tg = term_generator($self); # man Search::Xapian::TermGenerator
 
        if ($self->{indexlevel} eq 'full') {
-               $tg->index_text($text, $wdf_inc, $prefix);
-               $tg->increase_termpos;
+               index_phrase($self, $text, $wdf_inc, $prefix);
        } else {
+               my $tg = term_generator($self);
                $tg->index_text_without_positions($text, $wdf_inc, $prefix);
        }
 }
 
 sub index_headers ($$) {
        my ($self, $smsg) = @_;
-       my @x = (from => 'A', # Author
-               subject => 'S', to => 'XTO', cc => 'XCC');
+       my @x = (from => 'A', to => 'XTO', cc => 'XCC'); # A: Author
+       while (my ($field, $pfx) = splice(@x, 0, 2)) {
+               my $val = $smsg->{$field};
+               next if $val eq '';
+               # include "(comments)" after the address, too, so not using
+               # PublicInbox::Address::names or pairs
+               index_text($self, $val, 1, $pfx);
+
+               # we need positional info for email addresses since they
+               # can be considered phrases
+               if ($self->{indexlevel} eq 'medium') {
+                       for my $addr (PublicInbox::Address::emails($val)) {
+                               index_phrase($self, $addr, 1, $pfx);
+                       }
+               }
+       }
+       @x = (subject => 'S');
        while (my ($field, $pfx) = splice(@x, 0, 2)) {
                my $val = $smsg->{$field};
                index_text($self, $val, 1, $pfx) if $val ne '';
@@ -183,7 +209,11 @@ sub index_diff_inc ($$$$) {
                index_text($self, join("\n", @$xnq), 1, 'XNQ');
                @$xnq = ();
        }
-       index_text($self, $text, 1, $pfx);
+       if ($pfx eq 'XDFN') {
+               index_phrase($self, $text, 1, $pfx);
+       } else {
+               index_text($self, $text, 1, $pfx);
+       }
 }
 
 sub index_old_diff_fn {
@@ -289,7 +319,7 @@ sub index_xapian { # msg_iter callback
        my $ct = $part->content_type || 'text/plain';
        my $fn = $part->filename;
        if (defined $fn && $fn ne '') {
-               index_text($self, $fn, 1, 'XFN');
+               index_phrase($self, $fn, 1, 'XFN');
        }
        if ($part->{is_submsg}) {
                my $mids = mids_for_index($part);
@@ -327,20 +357,20 @@ sub index_list_id ($$$) {
                $l =~ /<([^>]+)>/ or next;
                my $lid = lc $1;
                $doc->add_boolean_term('G' . $lid);
-               index_text($self, $lid, 1, 'XL'); # probabilistic
+               index_phrase($self, $lid, 1, 'XL'); # probabilistic
        }
 }
 
 sub index_ids ($$$$) {
        my ($self, $doc, $hdr, $mids) = @_;
        for my $mid (@$mids) {
-               index_text($self, $mid, 1, 'XM');
+               index_phrase($self, $mid, 1, 'XM');
 
                # because too many Message-IDs are prefixed with
                # "Pine.LNX."...
                if ($mid =~ /\w{12,}/) {
                        my @long = ($mid =~ /(\w{3,}+)/g);
-                       index_text($self, join(' ', @long), 1, 'XM');
+                       index_phrase($self, join(' ', @long), 1, 'XM');
                }
        }
        $doc->add_boolean_term('Q' . $_) for @$mids;
@@ -377,7 +407,7 @@ sub eml2doc ($$$;$) {
        if (!$self->{-skip_docdata}) {
                # WWW doesn't need {to} or {cc}, only NNTP
                $smsg->{to} = $smsg->{cc} = '';
-               PublicInbox::OverIdx::parse_references($smsg, $eml, $mids);
+               $smsg->parse_references($eml, $mids);
                my $data = $smsg->to_doc_data;
                $doc->set_data($data);
        }
@@ -539,18 +569,6 @@ sub remove_keywords {
        $self->{xdb}->replace_document($docid, $doc) if $replace;
 }
 
-sub smsg_from_doc ($) {
-       my ($doc) = @_;
-       my $data = $doc->get_data or return;
-       my $smsg = bless {}, 'PublicInbox::Smsg';
-       $smsg->{ts} = int_val($doc, PublicInbox::Search::TS());
-       my $dt = int_val($doc, PublicInbox::Search::DT());
-       my ($yyyy, $mon, $dd, $hh, $mm, $ss) = unpack('A4A2A2A2A2A2', $dt);
-       $smsg->{ds} = timegm($ss, $mm, $hh, $dd, $mon - 1, $yyyy);
-       $smsg->load_from_data($data);
-       $smsg;
-}
-
 sub xdb_remove {
        my ($self, @docids) = @_;
        $self->begin_txn_lazy;