X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FMsgIter.pm;h=6c18d2bf3102c56e070d8a1ff8dbce44b47e847d;hb=95bdac7f09c69036efed537a4d03d5bdd2ae4eb6;hp=ef0d209f6b391efc45ab843c3c8154a5cbd66557;hpb=c1de840e23416eea63739b21995b07b2e8a1e9ac;p=public-inbox.git

diff --git a/lib/PublicInbox/MsgIter.pm b/lib/PublicInbox/MsgIter.pm
index ef0d209f..6c18d2bf 100644
--- a/lib/PublicInbox/MsgIter.pm
+++ b/lib/PublicInbox/MsgIter.pm
@@ -1,30 +1,22 @@
-# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# Copyright (C) 2016-2020 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-#
+
+# read-only utilities for Email::MIME
 package PublicInbox::MsgIter;
 use strict;
 use warnings;
 use base qw(Exporter);
-our @EXPORT = qw(msg_iter);
-use Email::MIME;
-use Scalar::Util qw(readonly);
-
-# Workaround Email::MIME versions without
-# commit dcef9be66c49ae89c7a5027a789bbbac544499ce
-# ("removing all trailing newlines was too much")
-# This is necessary for Debian jessie
-my $bad = 1.923;
-my $good = 1.935;
-my $ver = $Email::MIME::VERSION;
-my $extra_nl = 1 if ($ver >= $bad && $ver < $good);
+our @EXPORT = qw(msg_iter msg_part_text);
+use PublicInbox::MIME;
 
 # Like Email::MIME::walk_parts, but this is:
 # * non-recursive
 # * passes depth and indices to the iterator callback
-sub msg_iter ($$) {
-	my ($mime, $cb) = @_;
+sub msg_iter ($$;$$) {
+	my ($mime, $cb, $cb_arg, $do_undef) = @_;
 	my @parts = $mime->subparts;
 	if (@parts) {
+		$mime = $_[0] = undef if $do_undef; # saves some memory
 		my $i = 0;
 		@parts = map { [ $_, 1, ++$i ] } @parts;
 		while (my $p = shift @parts) {
@@ -36,22 +28,47 @@ sub msg_iter ($$) {
 				@sub = map { [ $_, $depth, @idx, ++$i ] } @sub;
 				@parts = (@sub, @parts);
 			} else {
-				if ($extra_nl) {
-					my $lf = $part->{mycrlf};
-					my $bref = $part->{body};
-					if (readonly($$bref)) {
-						my $s = $$bref . $lf;
-						$part->{body} = \$s;
-					} else {
-						$$bref .= $lf;
-					}
-				}
-				$cb->($p);
+				$cb->($p, $cb_arg);
 			}
 		}
 	} else {
-		$cb->([$mime, 0, 0]);
+		$cb->([$mime, 0, 0], $cb_arg);
+	}
+}
+
+sub msg_part_text ($$) {
+	my ($part, $ct) = @_;
+
+	# TODO: we may offer a separate sub for people who need to index
+	# HTML-only mail, but the majority of HTML mail is multipart/alternative
+	# with a text part which we don't have to waste cycles decoding
+	return if $ct =~ m!\btext/x?html\b!;
+
+	my $s = eval { $part->body_str };
+	my $err = $@;
+
+	# text/plain is the default, multipart/mixed happened a few
+	# times when it should not have been:
+	#   <87llgalspt.fsf@free.fr>
+	#   <200308111450.h7BEoOu20077@mail.osdl.org>
+	if ($err && ($ct =~ m!\btext/\b!i ||
+			$ct =~ m!\bmultipart/mixed\b!i)) {
+		my $cte = $part->header_raw('Content-Transfer-Encoding');
+		if (defined($cte) && $cte =~ /\b7bit\b/i) {
+			$s = $part->body;
+			$err = undef if $s =~ /\A[[:ascii:]]+\z/s;
+		} else {
+			# Try to assume UTF-8 because Alpine seems to
+			# do wacky things and set charset=X-UNKNOWN
+			$part->charset_set('UTF-8');
+			$s = eval { $part->body_str };
+		}
+
+		# If forcing charset=UTF-8 failed,
+		# caller will warn further down...
+		$s = $part->body if $@;
 	}
+	($s, $err);
 }
 
 1;