searchidx: use cached local $@ copy

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 86be9ed49811722bd64f667e002a41c18bdf300b..fd0d320e8685fe34c37b5f07162d3e81da2f0c06 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -4,13 +4,13 @@
 #
 # Indexes mail with Xapian and our (SQLite-based) ::Msgmap for use
 # with the web and NNTP interfaces.  This index maintains thread
-# relationships for use by Mail::Thread.  This writes to the search
-# index.
+# relationships for use by PublicInbox::SearchThread.
+# This writes to the search index.
 package PublicInbox::SearchIdx;
 use strict;
 use warnings;
 use Fcntl qw(:flock :DEFAULT);
-use Email::MIME;
+use PublicInbox::MIME;
 use Email::MIME::ContentType;
 $Email::MIME::ContentType::STRICT_PARAMS = 0;
 use base qw(PublicInbox::Search);
@@ -148,7 +148,6 @@ sub add_message {
 
        my ($doc_id, $old_tid);
        my $mid = mid_clean(mid_mime($mime));
-       my $ct_msg = $mime->header('Content-Type') || 'text/plain';
 
        eval {
                die 'Message-ID too long' if length($mid) > MAX_MID_SIZE;
@@ -156,7 +155,7 @@ sub add_message {
                if ($smsg) {
                        # convert a ghost to a regular message
                        # it will also clobber any existing regular message
-                       $doc_id = $smsg->doc_id;
+                       $doc_id = $smsg->{doc_id};
                        $old_tid = $smsg->thread_id;
                }
                $smsg = PublicInbox::SearchMsg->new($mime);
@@ -181,14 +180,29 @@ sub add_message {
 
                msg_iter($mime, sub {
                        my ($part, $depth, @idx) = @{$_[0]};
-                       my $ct = $part->content_type || $ct_msg;
+                       my $ct = $part->content_type || 'text/plain';
+                       my $fn = $part->filename;
+                       if (defined $fn && $fn ne '') {
+                               $tg->index_text($fn, 1, 'XFN');
+                       }
 
-                       # account for filter bugs...
-                       $ct =~ m!\btext/plain\b!i or return;
+                       return if $ct =~ m!\btext/x?html\b!i;
+
+                       my $s = eval { $part->body_str };
+                       if ($@) {
+                               if ($ct =~ m!\btext/plain\b!i) {
+                                       # Try to assume UTF-8 because Alpine
+                                       # seems to do wacky things and set
+                                       # charset=X-UNKNOWN
+                                       $part->charset_set('UTF-8');
+                                       $s = eval { $part->body_str };
+                                       $s = $part->body if $@;
+                               }
+                       }
+                       defined $s or return;
 
                        my (@orig, @quot);
                        my $body = $part->body;
-                       $part->body_set('');
                        my @lines = split(/\n/, $body);
                        while (defined(my $l = shift @lines)) {
                                if ($l =~ /^>/) {
@@ -275,15 +289,14 @@ sub link_message {
        my ($self, $smsg, $old_tid) = @_;
        my $doc = $smsg->{doc};
        my $mid = $smsg->mid;
-       my $mime = $smsg->mime;
+       my $mime = $smsg->{mime};
        my $hdr = $mime->header_obj;
-       my $refs = $hdr->header_raw('References');
-       my @refs = $refs ? ($refs =~ /<([^>]+)>/g) : ();
-       if (my $irt = $hdr->header_raw('In-Reply-To')) {
-               # last References should be $irt
-               # we will de-dupe later
-               push @refs, mid_clean($irt);
-       }
+
+       # last References should be IRT, but some mail clients do things
+       # out of order, so trust IRT over References iff IRT exists
+       my @refs = ($hdr->header_raw('References'),
+                       $hdr->header_raw('In-Reply-To'));
+       @refs = ((join(' ', @refs)) =~ /<([^>]+)>/g);
 
        my $tid;
        if (@refs) {
@@ -301,6 +314,7 @@ sub link_message {
                        push @refs, $ref;
                }
        }
+
        if (@refs) {
                $smsg->{references} = '<'.join('> <', @refs).'>';
 
@@ -316,7 +330,7 @@ sub link_message {
                        merge_threads($self, $tid, $ptid);
                }
        } else {
-               $tid = $self->next_thread_id;
+               $tid = defined $old_tid ? $old_tid : $self->next_thread_id;
        }
        $doc->add_term(xpfx('thread') . $tid);
 }
@@ -372,7 +386,7 @@ sub do_cat_mail {
                my $str = $git->cat_file($blob, $sizeref);
                # fixup bugs from import:
                $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
-               Email::MIME->new($str);
+               PublicInbox::MIME->new($str);
        };
        $@ ? undef : $mime;
 }
@@ -606,7 +620,7 @@ sub with_umask {
        my $rv = eval { $cb->() };
        my $err = $@;
        umask $old;
-       die $err if $@;
+       die $err if $err;
        $rv;
 }