MANIFEST | 1 + lib/PublicInbox/ContentId.pm | 9 +-------- lib/PublicInbox/MsgIter.pm | 25 ++++++++++++++++++++++++- lib/PublicInbox/SearchIdx.pm | 14 +------------- lib/PublicInbox/View.pm | 25 +++---------------------- t/psgi_multipart_not.t | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/MANIFEST b/MANIFEST index dc851676e3a4e2eed2252d5de420a2e6292f5c0a..f25a580fc4c41c7d81bd41a433b4c77d9b2ddca7 100644 --- a/MANIFEST +++ b/MANIFEST @@ -189,6 +189,7 @@ t/precheck.t t/psgi_attach.t t/psgi_bad_mids.t t/psgi_mount.t +t/psgi_multipart_not.t t/psgi_search.t t/psgi_text.t t/psgi_v2.t diff --git a/lib/PublicInbox/ContentId.pm b/lib/PublicInbox/ContentId.pm index b1d27eb8e48a8e835c0d088f77e84f3a7e38dd88..dd3155befefbd2d628683e6099847e72a34aec5a 100644 --- a/lib/PublicInbox/ContentId.pm +++ b/lib/PublicInbox/ContentId.pm @@ -75,14 +75,7 @@ $dig->add("d\0$d\0"); } $dig->add("b\0"); my $ct = $part->content_type || 'text/plain'; - my $s = eval { $part->body_str }; - if ($@ && $ct =~ m!\btext/plain\b!i) { - # Try to assume UTF-8 because Alpine - # seems to do wacky things and set - # charset=X-UNKNOWN - $part->charset_set('UTF-8'); - $s = eval { $part->body_str }; - } + my ($s, undef) = msg_part_text($part, $ct); if (defined $s) { $s =~ s/\r\n/\n/gs; $s =~ s/\s*\z//s; diff --git a/lib/PublicInbox/MsgIter.pm b/lib/PublicInbox/MsgIter.pm index a795f6175b93003118e64b229e10197f94b5c847..9e2d797fecc240d4f191ba8f3fe7fd84b410148e 100644 --- a/lib/PublicInbox/MsgIter.pm +++ b/lib/PublicInbox/MsgIter.pm @@ -5,7 +5,7 @@ package PublicInbox::MsgIter; use strict; use warnings; use base qw(Exporter); -our @EXPORT = qw(msg_iter); +our @EXPORT = qw(msg_iter msg_part_text); use PublicInbox::MIME; # Like Email::MIME::walk_parts, but this is: @@ -32,6 +32,29 @@ } } else { $cb->([$mime, 0, 0]); } +} + +sub msg_part_text ($$) { + my ($part, $ct) = @_; + + my $s = eval { $part->body_str }; + my $err = $@; + + # text/plain is the default, multipart/mixed happened a few + # times when it should not have been: + # <87llgalspt.fsf@free.fr> + # <200308111450.h7BEoOu20077@mail.osdl.org> + if ($ct =~ m!\btext/plain\b!i || $ct =~ m!\bmultipart/mixed\b!i) { + # Try to assume UTF-8 because Alpine seems to + # do wacky things and set charset=X-UNKNOWN + $part->charset_set('UTF-8'); + $s = eval { $part->body_str }; + + # If forcing charset=UTF-8 failed, + # caller will warn further down... + $s = $part->body if $@; + } + ($s, $err); } 1; diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index ca832ad39d9d2d8520f673d7311127ac66008c3a..76f3f33a4de5b9fa6925101c20d95ac870da9529 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -305,19 +305,7 @@ if (defined $fn && $fn ne '') { $self->index_text($fn, 1, 'XFN'); } - return if $ct =~ m!\btext/x?html\b!i; - - my $s = eval { $part->body_str }; - if ($@) { - if ($ct =~ m!\btext/plain\b!i) { - # Try to assume UTF-8 because Alpine - # seems to do wacky things and set - # charset=X-UNKNOWN - $part->charset_set('UTF-8'); - $s = eval { $part->body_str }; - $s = $part->body if $@; - } - } + my ($s, undef) = msg_part_text($part, $ct); defined $s or return; my (@orig, @quot); diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm index 86acd824cd0d6871d82a164550987288eadedced..bb49c035d976db7a65b83924a594e4b16c407b99 100644 --- a/lib/PublicInbox/View.pm +++ b/lib/PublicInbox/View.pm @@ -543,33 +543,14 @@ # $p - from msg_iter: [ Email::MIME, depth, @idx ] my ($part, $depth) = @$p; # attachment @idx is unused my $ct = $part->content_type || 'text/plain'; my $fn = $part->filename; + my ($s, $err) = msg_part_text($part, $ct); - if ($ct =~ m!\btext/x?html\b!i) { - return attach_link($upfx, $ct, $p, $fn); - } - - my $s = eval { $part->body_str }; - - # badly-encoded message? tell the world about it! - my $err = $@; - if ($err) { - if ($ct =~ m!\btext/plain\b!i) { - # Try to assume UTF-8 because Alpine seems to - # do wacky things and set charset=X-UNKNOWN - $part->charset_set('UTF-8'); - $s = eval { $part->body_str }; - - # If forcing charset=UTF-8 failed, - # attach_link will warn further down... - $s = $part->body if $@; - } else { - return attach_link($upfx, $ct, $p, $fn); - } - } + return attach_link($upfx, $ct, $p, $fn) unless defined $s; my @lines = split(/^/m, $s); $s = ''; if (defined($fn) || $depth > 0 || $err) { + # badly-encoded message with $err? tell the world about it! $s .= attach_link($upfx, $ct, $p, $fn, $err); $s .= "\n"; } diff --git a/t/psgi_multipart_not.t b/t/psgi_multipart_not.t new file mode 100644 index 0000000000000000000000000000000000000000..4c9fa57ab74d2e24a406f8646846827bcc00f53d --- /dev/null +++ b/t/psgi_multipart_not.t @@ -0,0 +1,65 @@ +# Copyright (C) 2018 all contributors +# License: AGPL-3.0+ +use strict; +use warnings; +use Test::More; +use File::Temp qw/tempdir/; +use Email::MIME; +use PublicInbox::Config; +use PublicInbox::WWW; +my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common + Plack::Test URI::Escape Plack::Builder Plack::Test); +foreach my $mod (@mods) { + eval "require $mod"; + plan skip_all => "$mod missing for psgi_multipart_not.t" if $@; +} +use_ok($_) for @mods; +use_ok 'PublicInbox::V2Writable'; +my $repo = tempdir('pi-psgi-multipart-not.XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $ibx = PublicInbox::Inbox->new({ + mainrepo => $repo, + name => 'multipart-not', + version => 2, + -primary_address => 'test@example.com', +}); +my $im = PublicInbox::V2Writable->new($ibx, 1); +$im->{parallel} = 0; + +my $mime = PublicInbox::MIME->new(<<'EOF'); +Message-Id: <200308111450.h7BEoOu20077@mail.osdl.org> +To: linux-kernel@vger.kernel.org +Subject: [OSDL] linux-2.6.0-test3 reaim results +Mime-Version: 1.0 +Content-Type: multipart/mixed ; + boundary="==_Exmh_120757360" +Date: Mon, 11 Aug 2003 07:50:24 -0700 +From: exmh user + +Freed^Wmultipart ain't what it used to be +EOF + +ok($im->add($mime), 'added broken multipart message'); +$im->done; + +my $cfgpfx = "publicinbox.v2test"; +my $cfg = { + "$cfgpfx.address" => $ibx->{-primary_address}, + "$cfgpfx.mainrepo" => $repo, +}; +my $config = PublicInbox::Config->new($cfg); +my $www = PublicInbox::WWW->new($config); + +my ($res, $raw); +test_psgi(sub { $www->call(@_) }, sub { + my ($cb) = @_; + for my $u ('/v2test/?q=%22ain\'t what it used to be%22&x=t', + '/v2test/new.atom', '/v2test/new.html') { + $res = $cb->(GET($u)); + $raw = $res->content; + ok(index($raw, 'Freed^Wmultipart') >= 0, $u); + ok(index($raw, 'Warning: decoded text') >= 0, $u.' warns'); + } +}); + +done_testing(); +1;