lib/PublicInbox/MsgIter.pm | 10 ++++++++++ lib/PublicInbox/SearchIdx.pm | 2 +- lib/PublicInbox/View.pm | 2 +- lib/PublicInbox/ViewDiff.pm | 11 +++++++++++ t/msg_iter.t | 30 ++++++++++++++++++++++++++++++ diff --git a/lib/PublicInbox/MsgIter.pm b/lib/PublicInbox/MsgIter.pm index 6c18d2bf3102c56e070d8a1ff8dbce44b47e847d..fa25564a5db1042f6623ae838270920414fd12f2 100644 --- a/lib/PublicInbox/MsgIter.pm +++ b/lib/PublicInbox/MsgIter.pm @@ -71,4 +71,14 @@ } ($s, $err); } +# returns an array of quoted or unquoted sections +sub split_quotes { + # Quiet "Complex regular subexpression recursion limit" warning + # in case an inconsiderate sender quotes 32K of text at once. + # The warning from Perl is harmless for us since our callers can + # tolerate less-than-ideal matches which work within Perl limits. + no warnings 'regexp'; + split(/((?:^>[^\n]*\n)+)/sm, shift); +} + 1; diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index fe00df53c268ffbc77bd4872fa6fb7ab5b4918e0..89d8bc2b282dbf64bc71d87b1d1e1fb66b4fef24 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -302,7 +302,7 @@ my ($s, undef) = msg_part_text($part, $ct); defined $s or return; # split off quoted and unquoted blocks: - my @sections = split(/((?:^>[^\n]*\n)+)/sm, $s); + my @sections = PublicInbox::MsgIter::split_quotes($s); $part = $s = undef; index_body($self, $_, /\A>/ ? 0 : $doc) for @sections; } diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm index c42654b68204c67f697278bedfba504850b8c889..70c10604e246ec23314d66580a01bd8b54ac7560 100644 --- a/lib/PublicInbox/View.pm +++ b/lib/PublicInbox/View.pm @@ -576,7 +576,7 @@ # some editors don't put trailing newlines at the end: $s .= "\n" unless $s =~ /\n\z/s; # split off quoted and unquoted blocks: - my @sections = split(/((?:^>[^\n]*\n)+)/sm, $s); + my @sections = PublicInbox::MsgIter::split_quotes($s); $s = ''; my $rv = $ctx->{obuf}; if (defined($fn) || $depth > 0 || $err) { diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm index d22c80b97b4871a00dba47f3e8b055bd1659ab06..5d391a13e0e0b869cac074761e7f716207b643cc 100644 --- a/lib/PublicInbox/ViewDiff.pm +++ b/lib/PublicInbox/ViewDiff.pm @@ -202,6 +202,17 @@ $top[0] =~ $IS_OID) { $dctx = diff_header($dst, \$x, $ctx, \@top); } elsif ($dctx) { my $after = ''; + + # Quiet "Complex regular subexpression recursion limit" + # warning. Perl will truncate matches upon hitting + # that limit, giving us more (and shorter) scalars than + # would be ideal, but otherwise it's harmless. + # + # We could replace the `+' metacharacter with `{1,100}' + # to limit the matches ourselves to 100, but we can + # let Perl do it for us, quietly. + no warnings 'regexp'; + for my $s (split(/((?:(?:^\+[^\n]*\n)+)| (?:(?:^-[^\n]*\n)+)| (?:^@@ [^\n]+\n))/xsm, $x)) { diff --git a/t/msg_iter.t b/t/msg_iter.t index e33bfc69df1bcc2d0b00d16424e3f70993d9d419..d303564fd0eaa321308fc8993fb8a782a5234f2f 100644 --- a/t/msg_iter.t +++ b/t/msg_iter.t @@ -78,5 +78,35 @@ like(ascii_html($raw), qr/• bullet point/s, 'got bullet point when X-UNKNOWN assumes UTF-8'); } +{ # API not finalized + my @warn; + local $SIG{__WARN__} = sub { push @warn, [ @_ ] }; + my $attr = "So and so wrote:\n"; + my $q = "> hello world\n" x 10; + my $nq = "hello world\n" x 10; + my @sections = PublicInbox::MsgIter::split_quotes($attr . $q . $nq); + is($sections[0], $attr, 'attribution matches'); + is($sections[1], $q, 'quoted section matches'); + is($sections[2], $nq, 'non-quoted section matches'); + is(scalar(@sections), 3, 'only three sections for short message'); + is_deeply(\@warn, [], 'no warnings'); + + $q x= 3300; + $nq x= 3300; + @sections = PublicInbox::MsgIter::split_quotes($attr . $q . $nq); + is_deeply(\@warn, [], 'no warnings on giant message'); + is(join('', @sections), $attr . $q . $nq, 'result matches expected'); + is(shift(@sections), $attr, 'attribution is first section'); + my @check = ('', ''); + while (defined(my $l = shift @sections)) { + next if $l eq ''; + like($l, qr/\n\z/s, 'section ends with newline'); + my $idx = ($l =~ /\A>/) ? 0 : 1; + $check[$idx] .= $l; + } + is($check[0], $q, 'long quoted section matches'); + is($check[1], $nq, 'long quoted section matches'); +} + done_testing(); 1;