nntp: smsg_range_i: favor ->{$field} lookups when possible

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 4bdd69f540b58e769499e5abd72f8349293712ed..f10a9104e789fe82beae8e8a7b98d3ec9932f6c9 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -22,11 +22,9 @@ use PublicInbox::Git qw(git_unquote);
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
 my $X = \%PublicInbox::Search::X;
 my ($DB_CREATE_OR_OPEN, $DB_OPEN);
-use constant {
-       BATCH_BYTES => defined($ENV{XAPIAN_FLUSH_THRESHOLD}) ?
-                       0x7fffffff : 1_000_000,
-       DEBUG => !!$ENV{DEBUG},
-};
+our $BATCH_BYTES = defined($ENV{XAPIAN_FLUSH_THRESHOLD}) ?
+                       0x7fffffff : 1_000_000;
+use constant DEBUG => !!$ENV{DEBUG};
 
 my $xapianlevels = qr/\A(?:full|medium)\z/;
 
@@ -158,16 +156,14 @@ sub index_text ($$$$) {
        }
 }
 
-sub index_users ($$) {
+sub index_headers ($$) {
        my ($self, $smsg) = @_;
-
-       my $from = $smsg->from;
-       my $to = $smsg->to;
-       my $cc = $smsg->cc;
-
-       index_text($self, $from, 1, 'A'); # A - author
-       index_text($self, $to, 1, 'XTO') if $to ne '';
-       index_text($self, $cc, 1, 'XCC') if $cc ne '';
+       my @x = (from => 'A', # Author
+               subject => 'S', to => 'XTO', cc => 'XCC');
+       while (my ($field, $pfx) = splice(@x, 0, 2)) {
+               my $val = $smsg->{$field};
+               index_text($self, $val, 1, $pfx) if $val ne '';
+       }
 }
 
 sub index_diff_inc ($$$$) {
@@ -284,6 +280,13 @@ sub index_xapian { # msg_iter callback
        if (defined $fn && $fn ne '') {
                index_text($self, $fn, 1, 'XFN');
        }
+       if ($part->{is_submsg}) {
+               my $mids = mids_for_index($part);
+               index_ids($self, $doc, $part, $mids);
+               my $smsg = bless {}, 'PublicInbox::Smsg';
+               $smsg->populate($part);
+               index_headers($self, $smsg);
+       }
 
        my ($s, undef) = msg_part_text($part, $ct);
        defined $s or return;
@@ -307,12 +310,31 @@ sub index_xapian { # msg_iter callback
        }
 }
 
+sub index_ids ($$$$) {
+       my ($self, $doc, $hdr, $mids) = @_;
+       for my $mid (@$mids) {
+               index_text($self, $mid, 1, 'XM');
+
+               # because too many Message-IDs are prefixed with
+               # "Pine.LNX."...
+               if ($mid =~ /\w{12,}/) {
+                       my @long = ($mid =~ /(\w{3,}+)/g);
+                       index_text($self, join(' ', @long), 1, 'XM');
+               }
+       }
+       $doc->add_boolean_term('Q' . $_) for @$mids;
+       for my $l ($hdr->header_raw('List-Id')) {
+               $l =~ /<([^>]+)>/ or next;
+               my $lid = $1;
+               $doc->add_boolean_term('G' . $lid);
+               index_text($self, $lid, 1, 'XL'); # probabilistic
+       }
+}
+
 sub add_xapian ($$$$) {
        my ($self, $mime, $smsg, $mids) = @_;
-       $smsg->{mime} = $mime; # XXX dangerous
        my $hdr = $mime->header_obj;
        my $doc = $X->{Document}->new;
-       my $subj = $smsg->subject;
        add_val($doc, PublicInbox::Search::TS(), $smsg->{ts});
        my @ds = gmtime($smsg->{ds});
        my $yyyymmdd = strftime('%Y%m%d', @ds);
@@ -321,22 +343,11 @@ sub add_xapian ($$$$) {
        add_val($doc, PublicInbox::Search::DT(), $dt);
 
        my $tg = term_generator($self);
-
        $tg->set_document($doc);
-       index_text($self, $subj, 1, 'S') if $subj;
-       index_users($self, $smsg);
+       index_headers($self, $smsg);
 
        msg_iter($mime, \&index_xapian, [ $self, $doc ]);
-       foreach my $mid (@$mids) {
-               index_text($self, $mid, 1, 'XM');
-
-               # because too many Message-IDs are prefixed with
-               # "Pine.LNX."...
-               if ($mid =~ /\w{12,}/) {
-                       my @long = ($mid =~ /(\w{3,}+)/g);
-                       index_text($self, join(' ', @long), 1, 'XM');
-               }
-       }
+       index_ids($self, $doc, $hdr, $mids);
        $smsg->{to} = $smsg->{cc} = ''; # WWW doesn't need these, only NNTP
        PublicInbox::OverIdx::parse_references($smsg, $hdr, $mids);
        my $data = $smsg->to_doc_data;
@@ -351,13 +362,6 @@ sub add_xapian ($$$$) {
                        }
                }
        }
-       $doc->add_boolean_term('Q' . $_) foreach @$mids;
-       for my $l ($hdr->header_raw('List-Id')) {
-               $l =~ /<([^>]+)>/ or next;
-               my $lid = $1;
-               $doc->add_boolean_term('G' . $lid);
-               index_text($self, $lid, 1, 'XL'); # probabilistic
-       }
        $self->{xdb}->replace_document($smsg->{num}, $doc);
 }
 
@@ -383,8 +387,7 @@ sub add_message {
        };
 
        # v1 and tests only:
-       $smsg->{ds} //= msg_datestamp($hdr, $self->{autime});
-       $smsg->{ts} //= msg_timestamp($hdr, $self->{cotime});
+       $smsg->populate($hdr, $self);
 
        eval {
                # order matters, overview stores every possible piece of
@@ -481,7 +484,7 @@ sub remove_by_oid {
        for (; $head != $tail; $head++) {
                my $docid = $head->get_docid;
                my $doc = $db->get_document($docid);
-               my $smsg = PublicInbox::Smsg->wrap($mid);
+               my $smsg = bless { mid => $mid }, 'PublicInbox::Smsg';
                $smsg->load_expand($doc);
                if ($smsg->{blob} eq $oid) {
                        push(@delete, $docid);
@@ -574,7 +577,7 @@ sub batch_adjust ($$$$$) {
        my ($max, $bytes, $batch_cb, $latest, $nr) = @_;
        $$max -= $bytes;
        if ($$max <= 0) {
-               $$max = BATCH_BYTES;
+               $$max = $BATCH_BYTES;
                $batch_cb->($nr, $latest);
        }
 }
@@ -599,7 +602,7 @@ sub read_log {
        my $git = $self->{git};
        my $latest;
        my $bytes;
-       my $max = BATCH_BYTES;
+       my $max = $BATCH_BYTES;
        local $/ = "\n";
        my %D;
        my $line;
@@ -640,6 +643,7 @@ sub read_log {
                my $mime = do_cat_mail($git, $blob, \$bytes);
                $del_cb->($self, $mime);
        }
+       delete @$self{qw(autime cotime)};
        $batch_cb->($nr, $latest, $newest);
 }