search: index_sync allows specifying alternate HEAD

[public-inbox.git] / lib / PublicInbox / Search.pm

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 6a05ce7a4e72e5ff2303caf04bdb1c6eb7354869..d9e5fd1f13820575dca626c948e54585a92de682 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -6,13 +6,13 @@ use strict;
 use warnings;
 use PublicInbox::SearchMsg;
 use Search::Xapian qw/:standard/;
-require PublicInbox::View;
 use Email::MIME;
 use PublicInbox::MID qw/mid_clean mid_compressed/;
 
 # This is English-only, everything else is non-standard and may be confused as
 # a prefix common in patch emails
 our $REPLY_RE = qr/^re:\s+/i;
+our $LANG = 'english';
 
 use constant {
        TS => 0,
@@ -21,8 +21,9 @@ use constant {
        # 1 - subject_path is lower-cased
        # 2 - subject_path is mid_compressed in the index, only
        # 3 - message-ID is compressed if it includes '%' (hack!)
-       SCHEMA_VERSION => 3,
-       LANG => 'english',
+       # 4 - change "Re: " normalization, avoid circular Reference ghosts
+       # 5 - subject_path drops trailing '.'
+       SCHEMA_VERSION => 5,
        QP_FLAGS => FLAG_PHRASE|FLAG_BOOLEAN|FLAG_LOVEHATE|FLAG_WILDCARD,
 };
 
@@ -87,7 +88,6 @@ sub add_message {
        my $mid = mid_compressed($mid_orig);
        my $was_ghost = 0;
        my $ct_msg = $mime->header('Content-Type') || 'text/plain';
-       my $enc_msg = PublicInbox::View::enc_for($ct_msg);
 
        eval {
                my $smsg = $self->lookup_message($mid);
@@ -150,9 +150,11 @@ sub add_message {
                        # account for filter bugs...
                        $ct =~ m!\btext/plain\b!i or return;
 
-                       my $enc = PublicInbox::View::enc_for($ct, $enc_msg);
                        my (@orig, @quot);
-                       foreach my $l (split(/\n/, $enc->decode($part->body))) {
+                       my $body = $part->body;
+                       $part->body_set('');
+                       my @lines = split(/\n/, $body);
+                       while (defined(my $l = shift @lines)) {
                                if ($l =~ /^\s*>/) {
                                        push @quot, $l;
                                } else {
@@ -161,10 +163,12 @@ sub add_message {
                        }
                        if (@quot) {
                                $tg->index_text(join("\n", @quot), 0);
+                               @quot = ();
                                $tg->increase_termpos;
                        }
                        if (@orig) {
                                $tg->index_text(join("\n", @orig));
+                               @orig = ();
                                $tg->increase_termpos;
                        }
                });
@@ -211,7 +215,6 @@ sub query {
        my ($self, $query_string, $opts) = @_;
        my $query = $self->qp->parse_query($query_string, QP_FLAGS);
 
-       $query = Search::Xapian::Query->new(OP_AND, $mail_query, $query);
        $self->do_enquire($query, $opts);
 }
 
@@ -252,6 +255,7 @@ sub do_enquire {
        my ($self, $query, $opts) = @_;
        my $enquire = $self->enquire;
 
+       $query = Search::Xapian::Query->new(OP_AND, $query, $mail_query);
        $enquire->set_query($query);
        $enquire->set_sort_by_relevance_then_value(TS, 0);
        $opts ||= {};
@@ -266,7 +270,7 @@ sub do_enquire {
 }
 
 # read-write
-sub stemmer { Search::Xapian::Stem->new(LANG) }
+sub stemmer { Search::Xapian::Stem->new($LANG) }
 
 # read-only
 sub qp {
@@ -491,14 +495,21 @@ sub merge_threads {
 # normalize subjects so they are suitable as pathnames for URLs
 sub subject_path {
        my $subj = pop;
-
-       $subj =~ s/\A\s+//;
-       $subj =~ s/\s+\z//;
-       $subj =~ s/$REPLY_RE//igo; # remove reply prefix
+       $subj = subject_normalized($subj);
        $subj =~ s![^a-zA-Z0-9_\.~/\-]+!_!g;
        lc($subj);
 }
 
+sub subject_normalized {
+       my $subj = pop;
+       $subj =~ s/\A\s+//s; # no leading space
+       $subj =~ s/\s+\z//s; # no trailing space
+       $subj =~ s/\s+/ /gs; # no redundant spaces
+       $subj =~ s/\.+\z//; # no trailing '.'
+       $subj =~ s/$REPLY_RE//igo; # remove reply prefix
+       $subj;
+}
+
 sub do_cat_mail {
        my ($git, $blob) = @_;
        my $mime = eval {
@@ -530,20 +541,21 @@ sub enquire {
 
 # indexes all unindexed messages
 sub index_sync {
-       my ($self) = @_;
+       my ($self, $head) = @_;
        require PublicInbox::GitCatFile;
        my $db = $self->{xdb};
        my $hex = '[a-f0-9]';
        my $h40 = $hex .'{40}';
        my $addmsg = qr!^:000000 100644 \S+ ($h40) A\t${hex}{2}/${hex}{38}$!;
        my $delmsg = qr!^:100644 000000 ($h40) \S+ D\t${hex}{2}/${hex}{38}$!;
+       $head ||= 'HEAD';
 
        $db->begin_transaction;
        eval {
                my $git = PublicInbox::GitCatFile->new($self->{git_dir});
 
                my $latest = $db->get_metadata('last_commit');
-               my $range = length $latest ? "$latest..HEAD" : 'HEAD';
+               my $range = length $latest ? "$latest..$head" : $head;
                $latest = undef;
 
                # get indexed messages