search: bump schema version to 5 for subject_path

[public-inbox.git] / lib / PublicInbox / Search.pm

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 617c267b789534ad10a31d5da4c747197fbc41bf..abd9db4c4289345200079845a888a94993b42fba 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -6,10 +6,14 @@ use strict;
 use warnings;
 use PublicInbox::SearchMsg;
 use Search::Xapian qw/:standard/;
-require PublicInbox::View;
 use Email::MIME;
 use PublicInbox::MID qw/mid_clean mid_compressed/;
 
+# This is English-only, everything else is non-standard and may be confused as
+# a prefix common in patch emails
+our $REPLY_RE = qr/^re:\s+/i;
+our $LANG = 'english';
+
 use constant {
        TS => 0,
        # SCHEMA_VERSION history
@@ -17,8 +21,9 @@ use constant {
        # 1 - subject_path is lower-cased
        # 2 - subject_path is mid_compressed in the index, only
        # 3 - message-ID is compressed if it includes '%' (hack!)
-       SCHEMA_VERSION => 3,
-       LANG => 'english',
+       # 4 - change "Re: " normalization, avoid circular Reference ghosts
+       # 5 - subject_path drops trailing '.'
+       SCHEMA_VERSION => 5,
        QP_FLAGS => FLAG_PHRASE|FLAG_BOOLEAN|FLAG_LOVEHATE|FLAG_WILDCARD,
 };
 
@@ -79,11 +84,10 @@ sub add_message {
        my $db = $self->{xdb};
 
        my $doc_id;
-       my $mid = mid_clean($mime->header_obj->header_raw('Message-ID'));
-       $mid = mid_compressed($mid);
+       my $mid_orig = mid_clean($mime->header_obj->header_raw('Message-ID'));
+       my $mid = mid_compressed($mid_orig);
        my $was_ghost = 0;
        my $ct_msg = $mime->header('Content-Type') || 'text/plain';
-       my $enc_msg = PublicInbox::View::enc_for($ct_msg);
 
        eval {
                my $smsg = $self->lookup_message($mid);
@@ -146,9 +150,11 @@ sub add_message {
                        # account for filter bugs...
                        $ct =~ m!\btext/plain\b!i or return;
 
-                       my $enc = PublicInbox::View::enc_for($ct, $enc_msg);
                        my (@orig, @quot);
-                       foreach my $l (split(/\n/, $enc->decode($part->body))) {
+                       my $body = $part->body;
+                       $part->body_set('');
+                       my @lines = split(/\n/, $body);
+                       while (defined(my $l = shift @lines)) {
                                if ($l =~ /^\s*>/) {
                                        push @quot, $l;
                                } else {
@@ -157,10 +163,12 @@ sub add_message {
                        }
                        if (@quot) {
                                $tg->index_text(join("\n", @quot), 0);
+                               @quot = ();
                                $tg->increase_termpos;
                        }
                        if (@orig) {
                                $tg->index_text(join("\n", @orig));
+                               @orig = ();
                                $tg->increase_termpos;
                        }
                });
@@ -176,7 +184,7 @@ sub add_message {
        };
 
        if ($@) {
-               warn "failed to index message <$mid>: $@\n";
+               warn "failed to index message <$mid_orig>: $@\n";
                return undef;
        }
        $doc_id;
@@ -184,11 +192,11 @@ sub add_message {
 
 # returns deleted doc_id on success, undef on missing
 sub remove_message {
-       my ($self, $mid) = @_;
+       my ($self, $mid_orig) = @_;
        my $db = $self->{xdb};
        my $doc_id;
-       $mid = mid_clean($mid);
-       $mid = mid_compressed($mid);
+       $mid_orig = mid_clean($mid_orig);
+       my $mid = mid_compressed($mid_orig);
 
        eval {
                $doc_id = $self->find_unique_doc_id('mid', $mid);
@@ -196,7 +204,7 @@ sub remove_message {
        };
 
        if ($@) {
-               warn "failed to remove message <$mid>: $@\n";
+               warn "failed to remove message <$mid_orig>: $@\n";
                return undef;
        }
        $doc_id;
@@ -207,7 +215,6 @@ sub query {
        my ($self, $query_string, $opts) = @_;
        my $query = $self->qp->parse_query($query_string, QP_FLAGS);
 
-       $query = Search::Xapian::Query->new(OP_AND, $mail_query, $query);
        $self->do_enquire($query, $opts);
 }
 
@@ -248,6 +255,7 @@ sub do_enquire {
        my ($self, $query, $opts) = @_;
        my $enquire = $self->enquire;
 
+       $query = Search::Xapian::Query->new(OP_AND, $query, $mail_query);
        $enquire->set_query($query);
        $enquire->set_sort_by_relevance_then_value(TS, 0);
        $opts ||= {};
@@ -262,7 +270,7 @@ sub do_enquire {
 }
 
 # read-write
-sub stemmer { Search::Xapian::Stem->new(LANG) }
+sub stemmer { Search::Xapian::Stem->new($LANG) }
 
 # read-only
 sub qp {
@@ -347,16 +355,33 @@ sub link_message_to_parents {
                if ($irt =~ /<([^>]+)>/) {
                        $irt = $1;
                }
-               push @refs, $irt;
+
+               # maybe some crazies will try to make a circular reference:
+               if ($irt eq $mid) {
+                       $irt = undef;
+               } else {
+                       push @refs, $irt;
+               }
        }
 
        my $tid;
        if (@refs) {
-               @refs = map { mid_compressed($_) } @refs;
-               my %uniq;
-               @refs = grep { !$uniq{$_}++ } @refs; # uniq
-
-               $doc->add_term(xpfx('inreplyto') . $refs[-1]);
+               my @crefs = map { mid_compressed($_) } @refs;
+               my %uniq = ($mid => 1);
+
+               # prevent circular references via References: here:
+               @refs = ();
+               foreach my $ref (@crefs) {
+                       next if $uniq{$ref};
+                       $uniq{$ref} = 1;
+                       push @refs, $ref;
+               }
+               $irt = undef if (defined $irt && !$uniq{$irt});
+       }
+       if (@refs) {
+               if (defined $irt) {
+                       $doc->add_term(xpfx('inreplyto') . $irt);
+               }
 
                my $ref_pfx = xpfx('references');
 
@@ -470,14 +495,21 @@ sub merge_threads {
 # normalize subjects so they are suitable as pathnames for URLs
 sub subject_path {
        my $subj = pop;
-
-       $subj =~ s/\A\s+//;
-       $subj =~ s/\s+\z//;
-       $subj =~ s/^(?:re|aw):\s*//i; # remove reply prefix (aw: German)
+       $subj = subject_normalized($subj);
        $subj =~ s![^a-zA-Z0-9_\.~/\-]+!_!g;
        lc($subj);
 }
 
+sub subject_normalized {
+       my $subj = pop;
+       $subj =~ s/\A\s+//s; # no leading space
+       $subj =~ s/\s+\z//s; # no trailing space
+       $subj =~ s/\s+/ /gs; # no redundant spaces
+       $subj =~ s/\.+\z//; # no trailing '.'
+       $subj =~ s/$REPLY_RE//igo; # remove reply prefix
+       $subj;
+}
+
 sub do_cat_mail {
        my ($git, $blob) = @_;
        my $mime = eval {