msg_iter: workaround broken Email::MIME versions

Email::MIME >= 1.923 and < 1.935 would drop too many newlines
in attachments.  This would lead to ugly text files without
a proper trailing newline if using quoted-printable, 7bit, or
8bit.  Attachments encoded with base64 were not affected.

These versions of Email::MIME are widely available in Debian 8
(Jessie) and even Ubuntu LTS distros so we will need to support
this workaround for a while.

diff --git a/lib/PublicInbox/MsgIter.pm b/lib/PublicInbox/MsgIter.pm
index d0dd82fb3e1dc1cd676bf91380041b20e1a0de5a..e0127ab99de9478c78121eac03049de7fbd09f3b 100644 (file)
--- a/lib/PublicInbox/MsgIter.pm
+++ b/lib/PublicInbox/MsgIter.pm
@@ -6,6 +6,16 @@ use strict;
 use warnings;
 use base qw(Exporter);
 our @EXPORT = qw(msg_iter);
+use Email::MIME;
+
+# Workaround Email::MIME versions without
+# commit dcef9be66c49ae89c7a5027a789bbbac544499ce
+# ("removing all trailing newlines was too much")
+# This is necessary for Debian jessie
+my $bad = 1.923;
+my $good = 1.935;
+my $ver = $Email::MIME::VERSION;
+my $extra_nl = 1 if ($ver >= $bad && $ver < $good);
 
 # Like Email::MIME::walk_parts, but this is:
 # * non-recursive
@@ -25,6 +35,9 @@ sub msg_iter ($$) {
                                @sub = map { [ $_, $depth, @idx, ++$i ] } @sub;
                                @parts = (@sub, @parts);
                        } else {
+                               if ($extra_nl) {
+                                       ${$part->{body}} .= $part->{mycrlf};
+                               }
                                $cb->($p);
                        }
                }

diff --git a/t/msg_iter.t b/t/msg_iter.t
index cc58b93fd6ffff9d8f3b0e81975a71533e6ae6f8..7ade6e41424f7b71a0b99234a86c9fd738c10f57 100644 (file)
--- a/t/msg_iter.t
+++ b/t/msg_iter.t
@@ -7,14 +7,16 @@ use Email::MIME;
 use_ok('PublicInbox::MsgIter');
 
 {
-       my $parts = [ Email::MIME->create(body => 'a'),
-                       Email::MIME->create(body => 'b') ];
+       my $parts = [ Email::MIME->create(body => "a\n"),
+                       Email::MIME->create(body => "b\n") ];
        my $mime = Email::MIME->create(parts => $parts,
                                header_str => [ From => 'root@localhost' ]);
        my @parts;
        msg_iter($mime, sub {
                my ($part, $level, @ex) = @{$_[0]};
-               push @parts, [ $part->body_str, $level, @ex ];
+               my $s = $part->body_str;
+               $s =~ s/\s+//s;
+               push @parts, [ $s, $level, @ex ];
        });
        is_deeply(\@parts, [ [ qw(a 1 1) ], [ qw(b 1 2) ] ], 'order is fine');
 }
@@ -30,9 +32,11 @@ use_ok('PublicInbox::MsgIter');
        my @parts;
        msg_iter($mime, sub {
                my ($part, $level, @ex) = @{$_[0]};
-               push @parts, [ $part->body_str, $level, @ex ];
+               my $s = $part->body_str;
+               $s =~ s/\s+//s;
+               push @parts, [ $s, $level, @ex ];
        });
-       is_deeply(\@parts, [ [ qw(a 2 1 1)], [qw(b 2 1 2)], [qw(sig 1 2)] ],
+       is_deeply(\@parts, [ [qw(a 2 1 1)], [qw(b 2 1 2)], [qw(sig 1 2)] ],
                'nested part shows up properly');
 }
 

diff --git a/t/psgi_attach.t b/t/psgi_attach.t
new file mode 100644 (file)
index 0000000..535cd21
--- /dev/null
+++ b/t/psgi_attach.t
@@ -0,0 +1,96 @@
+# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use Email::MIME;
+use File::Temp qw/tempdir/;
+my $tmpdir = tempdir('psgi-attach-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my $maindir = "$tmpdir/main.git";
+my $addr = 'test-public@example.com';
+my $cfgpfx = "publicinbox.test";
+my @mods = qw(HTTP::Request::Common Plack::Request Plack::Test URI::Escape);
+foreach my $mod (@mods) {
+       eval "require $mod";
+       plan skip_all => "$mod missing for plack.t" if $@;
+}
+use_ok $_ foreach @mods;
+use PublicInbox::Import;
+use PublicInbox::Git;
+use PublicInbox::Config;
+use PublicInbox::WWW;
+use_ok 'PublicInbox::WwwAttach';
+use Plack::Builder;
+my $config = PublicInbox::Config->new({
+       "$cfgpfx.address" => $addr,
+       "$cfgpfx.mainrepo" => $maindir,
+});
+is(0, system(qw(git init -q --bare), $maindir), "git init (main)");
+my $git = PublicInbox::Git->new($maindir);
+my $im = PublicInbox::Import->new($git, 'test', $addr);
+
+{
+       open my $fh, '<', '/dev/urandom' or die "unable to open urandom: $!\n";
+       sysread($fh, my $buf, 8);
+       is(8, length($buf), 'read some random data');
+       my $qp = "abcdef=g\n==blah\n";
+       my $b64 = 'b64'.$buf."\n";
+       my $txt = "plain\ntext\npass\nthrough\n";
+       my $parts = [
+               Email::MIME->create(
+                       attributes => {
+                               filename => 'queue-pee',
+                               content_type => 'text/plain',
+                               encoding => 'quoted-printable'
+                       },
+                       body => $qp),
+               Email::MIME->create(
+                       attributes => {
+                               filename => 'bayce-sixty-four',
+                               content_type => 'appication/octet-stream',
+                               encoding => 'base64',
+                       },
+                       body => $b64),
+               Email::MIME->create(
+                       attributes => {
+                               filename => 'noop',
+                               content_type => 'text/plain',
+                       },
+                       body => $txt),
+       ];
+       my $mime = Email::MIME->create(
+               parts => $parts,
+               header_str => [ From => 'root@z', 'Message-Id' => '<Z@B>',
+                       Subject => 'hi']
+       );
+       $mime = $mime->as_string;
+       $mime =~ s/\r\n/\n/g; # normalize to LF only
+       $mime = Email::MIME->new($mime);
+       $im->add($mime);
+       $im->done;
+
+       my $www = PublicInbox::WWW->new($config);
+       test_psgi(sub { $www->call(@_) }, sub {
+               my ($cb) = @_;
+               my $res;
+
+               $res = $cb->(GET('/test/Z%40B/1-a.txt'));
+               my $qp_res = $res->content;
+               ok(length($qp_res) >= length($qp), 'QP length is close');
+               like($qp_res, qr/\n\z/s, 'trailing newline exists');
+               # is(index($qp_res, $qp), 0, 'QP trailing newline is there');
+               $qp_res =~ s/\r\n/\n/g;
+               is(index($qp_res, $qp), 0, 'QP trailing newline is there');
+
+               $res = $cb->(GET('/test/Z%40B/2-a.txt'));
+               is(quotemeta($res->content), quotemeta($b64),
+                       'Base64 matches exactly');
+
+               $res = $cb->(GET('/test/Z%40B/3-a.txt'));
+               my $txt_res = $res->content;
+               ok(length($txt_res) >= length($txt), 'plain text almost matches');
+               like($txt_res, qr/\n\z/s, 'trailing newline exists in text');
+               is(index($txt_res, $txt), 0, 'plain text not truncated');
+       });
+}
+done_testing();