msgiter: attempt to decode all text/* bodies

[public-inbox.git] / lib / PublicInbox / MsgIter.pm

diff --git a/lib/PublicInbox/MsgIter.pm b/lib/PublicInbox/MsgIter.pm
index ef0d209f6b391efc45ab843c3c8154a5cbd66557..d9df32abc37c7fc7da68c0e09ed2f942ddbb4827 100644 (file)
--- a/lib/PublicInbox/MsgIter.pm
+++ b/lib/PublicInbox/MsgIter.pm
@@ -1,22 +1,13 @@
-# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# Copyright (C) 2016-2019 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-#
+
+# read-only utilities for Email::MIME
 package PublicInbox::MsgIter;
 use strict;
 use warnings;
 use base qw(Exporter);
-our @EXPORT = qw(msg_iter);
-use Email::MIME;
-use Scalar::Util qw(readonly);
-
-# Workaround Email::MIME versions without
-# commit dcef9be66c49ae89c7a5027a789bbbac544499ce
-# ("removing all trailing newlines was too much")
-# This is necessary for Debian jessie
-my $bad = 1.923;
-my $good = 1.935;
-my $ver = $Email::MIME::VERSION;
-my $extra_nl = 1 if ($ver >= $bad && $ver < $good);
+our @EXPORT = qw(msg_iter msg_part_text);
+use PublicInbox::MIME;
 
 # Like Email::MIME::walk_parts, but this is:
 # * non-recursive
@@ -36,16 +27,6 @@ sub msg_iter ($$) {
                                @sub = map { [ $_, $depth, @idx, ++$i ] } @sub;
                                @parts = (@sub, @parts);
                        } else {
-                               if ($extra_nl) {
-                                       my $lf = $part->{mycrlf};
-                                       my $bref = $part->{body};
-                                       if (readonly($$bref)) {
-                                               my $s = $$bref . $lf;
-                                               $part->{body} = \$s;
-                                       } else {
-                                               $$bref .= $lf;
-                                       }
-                               }
                                $cb->($p);
                        }
                }
@@ -54,4 +35,34 @@ sub msg_iter ($$) {
        }
 }
 
+sub msg_part_text ($$) {
+       my ($part, $ct) = @_;
+
+       my $s = eval { $part->body_str };
+       my $err = $@;
+
+       # text/plain is the default, multipart/mixed happened a few
+       # times when it should not have been:
+       #   <87llgalspt.fsf@free.fr>
+       #   <200308111450.h7BEoOu20077@mail.osdl.org>
+       if ($err && ($ct =~ m!\btext/\b!i ||
+                       $ct =~ m!\bmultipart/mixed\b!i)) {
+               my $cte = $part->header_raw('Content-Transfer-Encoding');
+               if (defined($cte) && $cte =~ /\b7bit\b/i) {
+                       $s = $part->body;
+                       $err = undef if $s =~ /\A[[:ascii:]]+\z/s;
+               } else {
+                       # Try to assume UTF-8 because Alpine seems to
+                       # do wacky things and set charset=X-UNKNOWN
+                       $part->charset_set('UTF-8');
+                       $s = eval { $part->body_str };
+               }
+
+               # If forcing charset=UTF-8 failed,
+               # caller will warn further down...
+               $s = $part->body if $@;
+       }
+       ($s, $err);
+}
+
 1;