search: match the behavior of WWW for indexing text

The basic rule is that if it is displayable via our WWW
interface, it should be indexable text for Xapian search.

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 0e2d225e52e945ecab134aeff3fc5af462b2822f..fb68f4b12a66878dbe53108194fbea4396e6ce2c 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -148,7 +148,6 @@ sub add_message {
 
        my ($doc_id, $old_tid);
        my $mid = mid_clean(mid_mime($mime));
-       my $ct_msg = $mime->header('Content-Type') || 'text/plain';
 
        eval {
                die 'Message-ID too long' if length($mid) > MAX_MID_SIZE;
@@ -181,10 +180,22 @@ sub add_message {
 
                msg_iter($mime, sub {
                        my ($part, $depth, @idx) = @{$_[0]};
-                       my $ct = $part->content_type || $ct_msg;
-
-                       # account for filter bugs...
-                       $ct =~ m!\btext/plain\b!i or return;
+                       my $ct = $part->content_type || 'text/plain';
+
+                       return if $ct =~ m!\btext/x?html\b!i;
+
+                       my $s = eval { $part->body_str };
+                       if ($@) {
+                               if ($ct =~ m!\btext/plain\b!i) {
+                                       # Try to assume UTF-8 because Alpine
+                                       # seems to do wacky things and set
+                                       # charset=X-UNKNOWN
+                                       $part->charset_set('UTF-8');
+                                       $s = eval { $part->body_str };
+                                       $s = $part->body if $@;
+                               }
+                       }
+                       defined $s or return;
 
                        my (@orig, @quot);
                        my $body = $part->body;