X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FEml.pm;h=ef401141c135db358861e83b7271336d9df12f20;hb=8b44e99ec009508d7e050ee44d34a1cf0f111dd5;hp=4508bd84c6b1b3457fa7b9247331b94694eabc51;hpb=b2f2d28e20f7bf9e7234d3134ec91cd9549001b1;p=public-inbox.git

diff --git a/lib/PublicInbox/Eml.pm b/lib/PublicInbox/Eml.pm
index 4508bd84..ef401141 100644
--- a/lib/PublicInbox/Eml.pm
+++ b/lib/PublicInbox/Eml.pm
@@ -38,9 +38,12 @@ my $MIME_Header = find_encoding('MIME-Header');
 use PublicInbox::EmlContentFoo qw(parse_content_type parse_content_disposition);
 $PublicInbox::EmlContentFoo::STRICT_PARAMS = 0;
 
-our $MAXPARTS = 1000; # same as SpamAssassin
-our $MAXDEPTH = 20; # seems enough, Perl sucks, here
-our $MAXBOUNDLEN = 2048; # same as postfix
+our $mime_parts_limit = 1000; # same as SpamAssassin (not in postfix AFAIK)
+
+# the rest of the limit names are taken from postfix:
+our $mime_nesting_limit = 20; # seems enough, Perl sucks, here
+our $mime_boundary_length_limit = 2048; # same as postfix
+our $header_size_limit = 102400; # same as postfix
 
 my %MIME_ENC = (qp => \&enc_qp, base64 => \&encode_base64);
 my %MIME_DEC = (qp => \&dec_qp, base64 => \&decode_base64);
@@ -68,19 +71,47 @@ sub re_memo ($) {
 					/ismx
 }
 
+sub hdr_truncate ($) {
+	my $len = length($_[0]);
+	substr($_[0], $header_size_limit, $len) = '';
+	my $end = rindex($_[0], "\n");
+	if ($end >= 0) {
+		++$end;
+		substr($_[0], $end, $len) = '';
+		warn "header of $len bytes truncated to $end bytes\n";
+	} else {
+		$_[0] = '';
+		warn <<EOF
+header of $len bytes without `\\n' within $header_size_limit ignored
+EOF
+	}
+}
+
 # compatible with our uses of Email::MIME
 sub new {
 	my $ref = ref($_[1]) ? $_[1] : \(my $cpy = $_[1]);
-	if ($$ref =~ /(?:\r?\n(\r?\n))/gs) { # likely
-		# This can modify $$ref in-place and to avoid memcpy/memmove
-		# on a potentially large $$ref.  It does need to make a
-		# copy for $hdr, though.  Idea stolen from Email::Simple
-		my $hdr = substr($$ref, 0, pos($$ref), ''); # sv_chop on $$ref
+	# substr() can modify the first arg in-place and to avoid
+	# memcpy/memmove on a potentially large scalar.  It does need
+	# to make a copy for $hdr, though.  Idea stolen from Email::Simple.
+
+	# We also prefer index() on common LFLF emails since it's faster
+	# and re scan can bump RSS by length($$ref) on big strings
+	if (index($$ref, "\r\n") < 0 && (my $pos = index($$ref, "\n\n")) >= 0) {
+		# likely on *nix
+		my $hdr = substr($$ref, 0, $pos + 2, ''); # sv_chop on $$ref
+		chop($hdr); # lower SvCUR
+		hdr_truncate($hdr) if length($hdr) > $header_size_limit;
+		bless { hdr => \$hdr, crlf => "\n", bdy => $ref }, __PACKAGE__;
+	} elsif ($$ref =~ /\r?\n(\r?\n)/s) {
+		my $hdr = substr($$ref, 0, $+[0], ''); # sv_chop on $$ref
 		substr($hdr, -(length($1))) = ''; # lower SvCUR
+		hdr_truncate($hdr) if length($hdr) > $header_size_limit;
 		bless { hdr => \$hdr, crlf => $1, bdy => $ref }, __PACKAGE__;
 	} elsif ($$ref =~ /^[a-z0-9-]+[ \t]*:/ims && $$ref =~ /(\r?\n)\z/s) {
 		# body is optional :P
-		bless { hdr => \($$ref), crlf => $1 }, __PACKAGE__;
+		my $hdr = substr($$ref, 0, $header_size_limit + 1);
+		hdr_truncate($hdr) if length($hdr) > $header_size_limit;
+		bless { hdr => \$hdr, crlf => $1 }, __PACKAGE__;
 	} else { # nothing useful
 		my $hdr = $$ref = '';
 		bless { hdr => \$hdr, crlf => "\n" }, __PACKAGE__;
@@ -90,8 +121,8 @@ sub new {
 sub new_sub {
 	my (undef, $ref) = @_;
 	# special case for messages like <85k5su9k59.fsf_-_@lola.goethe.zz>
-	$$ref =~ /\A(?:(\r?\n))/gs or goto &new;
-	my $hdr = substr($$ref, 0, pos($$ref), ''); # sv_chop on $$ref
+	$$ref =~ /\A(\r?\n)/s or goto &new;
+	my $hdr = substr($$ref, 0, $+[0], ''); # sv_chop on $$ref
 	bless { hdr => \$hdr, crlf => $1, bdy => $ref }, __PACKAGE__;
 }
 
@@ -122,7 +153,7 @@ sub ct ($) {
 sub mp_descend ($$) {
 	my ($self, $nr) = @_; # or $once for top-level
 	my $bnd = ct($self)->{attributes}->{boundary} // return; # single-part
-	return if $bnd eq '' || length($bnd) >= $MAXBOUNDLEN;
+	return if $bnd eq '' || length($bnd) >= $mime_boundary_length_limit;
 	$bnd = quotemeta($bnd);
 
 	# "multipart" messages can exist w/o a body
@@ -132,8 +163,8 @@ sub mp_descend ($$) {
 	# *sigh* just the regexp match alone seems to bump RSS by
 	# length($$bdy) on a ~30M string:
 	my $epilogue_missing;
-	if ($$bdy =~ /((?:\r?\n)?^--$bnd--[ \t]*\r?$)/gsm) {
-		substr($$bdy, pos($$bdy) - length($1)) = '';
+	if ($$bdy =~ /(?:\r?\n)?^--$bnd--[ \t]*\r?$/sm) {
+		substr($$bdy, $-[0]) = '';
 	} else {
 		$epilogue_missing = 1;
 	}
@@ -150,7 +181,7 @@ sub mp_descend ($$) {
 				# + 3 since we don't want the last part
 				# processed to include any other excluded
 				# parts ($nr starts at 1, and I suck at math)
-				$MAXPARTS + 3 - $nr);
+				$mime_parts_limit + 3 - $nr);
 
 	if (@parts) { # the usual path if we got this far:
 		undef $bdy; # release memory ASAP if $nr > 0
@@ -189,13 +220,15 @@ sub each_part {
 	$p = [ $p, 0 ];
 	my @s; # our virtual stack
 	my $nr = 0;
-	while ((scalar(@{$p->[0]}) || ($p = pop @s)) && ++$nr <= $MAXPARTS) {
+	while ((scalar(@{$p->[0]}) || ($p = pop @s)) &&
+			++$nr <= $mime_parts_limit) {
 		++$p->[-1]; # bump index
 		my (undef, @idx) = @$p;
 		@idx = (join('.', @idx));
 		my $depth = ($idx[0] =~ tr/././) + 1;
 		my $sub = new_sub(undef, \(shift @{$p->[0]}));
-		if ($depth < $MAXDEPTH && (my $nxt = mp_descend($sub, $nr))) {
+		if ($depth < $mime_nesting_limit &&
+				(my $nxt = mp_descend($sub, $nr))) {
 			push(@s, $p) if scalar @{$p->[0]};
 			$p = [ $nxt, @idx, 0 ];
 		} else { # a leaf node