introduce PublicInbox::MIME wrapper class

[public-inbox.git] / lib / PublicInbox / SearchIdx.pm

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 86be9ed49811722bd64f667e002a41c18bdf300b..d63dd7c742b4c21ef9420d810954df3e8332f45e 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -4,13 +4,13 @@
 #
 # Indexes mail with Xapian and our (SQLite-based) ::Msgmap for use
 # with the web and NNTP interfaces.  This index maintains thread
-# relationships for use by Mail::Thread.  This writes to the search
-# index.
+# relationships for use by PublicInbox::SearchThread.
+# This writes to the search index.
 package PublicInbox::SearchIdx;
 use strict;
 use warnings;
 use Fcntl qw(:flock :DEFAULT);
-use Email::MIME;
+use PublicInbox::MIME;
 use Email::MIME::ContentType;
 $Email::MIME::ContentType::STRICT_PARAMS = 0;
 use base qw(PublicInbox::Search);
@@ -148,7 +148,6 @@ sub add_message {
 
        my ($doc_id, $old_tid);
        my $mid = mid_clean(mid_mime($mime));
-       my $ct_msg = $mime->header('Content-Type') || 'text/plain';
 
        eval {
                die 'Message-ID too long' if length($mid) > MAX_MID_SIZE;
@@ -156,7 +155,7 @@ sub add_message {
                if ($smsg) {
                        # convert a ghost to a regular message
                        # it will also clobber any existing regular message
-                       $doc_id = $smsg->doc_id;
+                       $doc_id = $smsg->{doc_id};
                        $old_tid = $smsg->thread_id;
                }
                $smsg = PublicInbox::SearchMsg->new($mime);
@@ -181,14 +180,29 @@ sub add_message {
 
                msg_iter($mime, sub {
                        my ($part, $depth, @idx) = @{$_[0]};
-                       my $ct = $part->content_type || $ct_msg;
+                       my $ct = $part->content_type || 'text/plain';
+                       my $fn = $part->filename;
+                       if (defined $fn && $fn ne '') {
+                               $tg->index_text($fn, 1, 'XFN');
+                       }
 
-                       # account for filter bugs...
-                       $ct =~ m!\btext/plain\b!i or return;
+                       return if $ct =~ m!\btext/x?html\b!i;
+
+                       my $s = eval { $part->body_str };
+                       if ($@) {
+                               if ($ct =~ m!\btext/plain\b!i) {
+                                       # Try to assume UTF-8 because Alpine
+                                       # seems to do wacky things and set
+                                       # charset=X-UNKNOWN
+                                       $part->charset_set('UTF-8');
+                                       $s = eval { $part->body_str };
+                                       $s = $part->body if $@;
+                               }
+                       }
+                       defined $s or return;
 
                        my (@orig, @quot);
                        my $body = $part->body;
-                       $part->body_set('');
                        my @lines = split(/\n/, $body);
                        while (defined(my $l = shift @lines)) {
                                if ($l =~ /^>/) {
@@ -275,14 +289,14 @@ sub link_message {
        my ($self, $smsg, $old_tid) = @_;
        my $doc = $smsg->{doc};
        my $mid = $smsg->mid;
-       my $mime = $smsg->mime;
+       my $mime = $smsg->{mime};
        my $hdr = $mime->header_obj;
        my $refs = $hdr->header_raw('References');
        my @refs = $refs ? ($refs =~ /<([^>]+)>/g) : ();
-       if (my $irt = $hdr->header_raw('In-Reply-To')) {
-               # last References should be $irt
-               # we will de-dupe later
-               push @refs, mid_clean($irt);
+       my $irt = $hdr->header_raw('In-Reply-To');
+       if (defined $irt) {
+               $irt = mid_clean($irt);
+               $irt = undef if $mid eq $irt;
        }
 
        my $tid;
@@ -291,6 +305,15 @@ sub link_message {
                my @orig_refs = @refs;
                @refs = ();
 
+               if (defined $irt) {
+                       # to check MAX_MID_SIZE
+                       push @orig_refs, $irt;
+
+                       # below, we will ensure IRT (if specified)
+                       # is the last References
+                       $uniq{$irt} = 1;
+               }
+
                # prevent circular references via References: here:
                foreach my $ref (@orig_refs) {
                        if (length($ref) > MAX_MID_SIZE) {
@@ -301,6 +324,11 @@ sub link_message {
                        push @refs, $ref;
                }
        }
+
+       # last References should be IRT, but some mail clients do things
+       # out of order, so trust IRT over References iff IRT exists
+       push @refs, $irt if defined $irt;
+
        if (@refs) {
                $smsg->{references} = '<'.join('> <', @refs).'>';
 
@@ -372,7 +400,7 @@ sub do_cat_mail {
                my $str = $git->cat_file($blob, $sizeref);
                # fixup bugs from import:
                $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
-               Email::MIME->new($str);
+               PublicInbox::MIME->new($str);
        };
        $@ ? undef : $mime;
 }