-# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# Lazy MIME parser, it still slurps the full message but keeps short
$MIME_DEC{quotedprint} = $MIME_DEC{'quoted-printable'} = $MIME_DEC{qp};
$MIME_ENC{$_} = \&identity_codec for qw(7bit 8bit binary);
-my %DECODE_ADDRESS = map { $_ => 1 } qw(From To Cc Sender Reply-To);
+my %DECODE_ADDRESS = map {
+ ($_ => 1, "Resent-$_" => 1)
+} qw(From To Cc Sender Reply-To Bcc);
my %DECODE_FULL = (
Subject => 1,
'Content-Description' => 1,
our %STR_TYPE = (text => 1);
our %STR_SUBTYPE = (plain => 1, html => 1);
+# message/* subtypes we descend into
+our %MESSAGE_DESCEND = (
+ news => 1, # RFC 1849 (obsolete, but archives are forever)
+ rfc822 => 1, # RFC 2046
+ rfc2822 => 1, # gmime handles this (but not rfc5322)
+ global => 1, # RFC 6532
+);
+
my %re_memo;
sub re_memo ($) {
my ($k) = @_;
sub new_sub {
my (undef, $ref) = @_;
# special case for messages like <85k5su9k59.fsf_-_@lola.goethe.zz>
- $$ref =~ /\A(\r?\n)/s or goto &new;
+ $$ref =~ /\A(\r?\n)/s or return new(undef, $ref);
my $hdr = substr($$ref, 0, $+[0], ''); # sv_chop on $$ref
bless { hdr => \$hdr, crlf => $1, bdy => $ref }, __PACKAGE__;
}
}
# returns a queue of sub-parts iff it's worth descending into
-# TODO: descend into message/rfc822 parts (Email::MIME didn't)
sub mp_descend ($$) {
my ($self, $nr) = @_; # or $once for top-level
- my $bnd = ct($self)->{attributes}->{boundary} // return; # single-part
+ my $ct = ct($self);
+ my $type = lc($ct->{type});
+ if ($type eq 'message' && $MESSAGE_DESCEND{lc($ct->{subtype})}) {
+ my $nxt = new(undef, body_raw($self));
+ $self->{-call_cb} = $nxt->{is_submsg} = 1;
+ return [ $nxt ];
+ }
+ return if $type ne 'multipart';
+ my $bnd = $ct->{attributes}->{boundary} // return; # single-part
return if $bnd eq '' || length($bnd) >= $mime_boundary_length_limit;
$bnd = quotemeta($bnd);
+ # this is a multipart message that didn't get descended into in
+ # public-inbox <= 1.5.0, so ensure we call the user callback for
+ # this part to not break PSGI downloads.
+ $self->{-call_cb} = $self->{is_submsg};
+
# "multipart" messages can exist w/o a body
my $bdy = ($nr ? delete($self->{bdy}) : \(body_raw($self))) or return;
# compatibility with Email::MIME
$parts[-1] =~ s/\n\r?\n\z/\n/s if $epilogue_missing;
- @parts = grep /[^ \t\r\n]/s, @parts; # ignore empty parts
+ # ignore empty parts
+ @parts = map { new_sub(undef, \$_) } grep /[^ \t\r\n]/s, @parts;
# Keep "From: someone..." from preamble in old,
# buggy versions of git-send-email, otherwise drop it
# There's also a case where quoted text showed up in the
# preamble
# <20060515162817.65F0F1BBAE@citi.umich.edu>
- unshift(@parts, $pre) if $pre =~ /:/s;
+ unshift(@parts, new_sub(undef, \$pre)) if index($pre, ':') >= 0;
return \@parts;
}
# "multipart", but no boundary found, treat as single part
# $cb - user-supplied callback sub
# $arg - user-supplied arg (think pthread_create)
# $once - unref body scalar during iteration
+# $all - used by IMAP server, only
sub each_part {
- my ($self, $cb, $arg, $once) = @_;
+ my ($self, $cb, $arg, $once, $all) = @_;
my $p = mp_descend($self, $once // 0) or
- return $cb->([$self, 0, 0], $arg);
+ return $cb->([$self, 0, 1], $arg);
+
+ $cb->([$self, 0, 0], $arg) if ($all || $self->{-call_cb}); # rare
+
$p = [ $p, 0 ];
my @s; # our virtual stack
my $nr = 0;
my (undef, @idx) = @$p;
@idx = (join('.', @idx));
my $depth = ($idx[0] =~ tr/././) + 1;
- my $sub = new_sub(undef, \(shift @{$p->[0]}));
+ my $sub = shift @{$p->[0]};
if ($depth < $mime_nesting_limit &&
(my $nxt = mp_descend($sub, $nr))) {
push(@s, $p) if scalar @{$p->[0]};
$p = [ $nxt, @idx, 0 ];
+ ($all || $sub->{-call_cb}) and
+ $cb->([$sub, $depth, @idx], $arg);
} else { # a leaf node
$cb->([$sub, $depth, @idx], $arg);
}
if ($$bdy =~ /^--\Q$bnd\E--[ \t]*\r?\n(.+)\z/sm) {
$self->{epilogue} = $1;
}
- map { new_sub(undef, \$_) } @$parts;
+ @$parts;
}
sub parts_set {
sub body_str_set {
my ($self, $body_str) = @_;
my $charset = ct($self)->{attributes}->{charset} or
- Carp::confess('body_str was given, but no charset is defined');
+ croak('body_str was given, but no charset is defined');
body_set($self, \(encode($charset, $body_str, Encode::FB_CROAK)));
}
my ($self, $name, @vals) = @_;
for (@vals) {
next unless /[^\x20-\x7e]/;
- utf8::encode($_); # to octets
# 39: int((75 - length("Subject: =?UTF-8?B?".'?=') ) / 4) * 3;
- s/(.{1,39})/'=?UTF-8?B?'.encode_base64($1, '').'?='/ges;
+ s/(.{1,39})/
+ my $x = $1;
+ utf8::encode($x); # to octets
+ '=?UTF-8?B?'.encode_base64($x, '').'?='
+ /xges;
}
header_set($self, $name, @vals);
}
-sub mhdr_decode ($) { eval { $MIME_Header->decode($_[0]) } // $_[0] }
+sub mhdr_decode ($) {
+ eval { $MIME_Header->decode($_[0], Encode::FB_DEFAULT) } // $_[0];
+}
sub filename {
my $dis = header_raw($_[0], 'Content-Disposition');
if ($STR_TYPE{$ct->{type}} && $STR_SUBTYPE{$ct->{subtype}}) {
return body($self);
}
- Carp::confess("can't get body as a string for ",
+ croak("can't get body as a string for ",
join("\n\t", header_raw($self, 'Content-Type')));
}
decode($charset, body($self), Encode::FB_CROAK);
sub crlf { $_[0]->{crlf} // "\n" }
+# warnings to ignore when handling spam mailboxes and maybe other places
+sub warn_ignore {
+ my $s = "@_";
+ # Email::Address::XS warnings
+ $s =~ /^Argument contains empty /
+ || $s =~ /^Element at index [0-9]+.*? contains /
+ # PublicInbox::MsgTime
+ || $s =~ /^bogus TZ offset: .+?, ignoring and assuming \+0000/
+ || $s =~ /^bad Date: .+? in /
+ # Encode::Unicode::UTF7
+ || $s =~ /^Bad UTF7 data escape at /
+}
+
+# this expects to be RHS in this assignment: "local $SIG{__WARN__} = ..."
+sub warn_ignore_cb {
+ my $cb = $SIG{__WARN__} // \&CORE::warn;
+ sub { $cb->(@_) unless warn_ignore(@_) }
+}
+
sub willneed { re_memo($_) for @_ }
willneed(qw(From To Cc Date Subject Content-Type In-Reply-To References