quiet "Complex regular subexpression recursion limit" warnings

These seem mostly harmless since Perl will just truncate the
match and start a new one on a newline boundary in our case.
The only downside is we'd end up with redundant <span> tags in
HTML.

Limiting the number of line matched ourselves with `{1,$NUM}'
doesn't seem prudent since lines vary in length, so we continue
to defer the job of limiting matches to the Perl regexp engine.

I've noticed this warning in practice on 100K+ line patches to
locale data.

diff --git a/lib/PublicInbox/MsgIter.pm b/lib/PublicInbox/MsgIter.pm
index 6c18d2bf3102c56e070d8a1ff8dbce44b47e847d..fa25564a5db1042f6623ae838270920414fd12f2 100644 (file)
--- a/lib/PublicInbox/MsgIter.pm
+++ b/lib/PublicInbox/MsgIter.pm
@@ -71,4 +71,14 @@ sub msg_part_text ($$) {
        ($s, $err);
 }
 
+# returns an array of quoted or unquoted sections
+sub split_quotes {
+       # Quiet "Complex regular subexpression recursion limit" warning
+       # in case an inconsiderate sender quotes 32K of text at once.
+       # The warning from Perl is harmless for us since our callers can
+       # tolerate less-than-ideal matches which work within Perl limits.
+       no warnings 'regexp';
+       split(/((?:^>[^\n]*\n)+)/sm, shift);
+}
+
 1;

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index fe00df53c268ffbc77bd4872fa6fb7ab5b4918e0..89d8bc2b282dbf64bc71d87b1d1e1fb66b4fef24 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -302,7 +302,7 @@ sub index_xapian { # msg_iter callback
        defined $s or return;
 
        # split off quoted and unquoted blocks:
-       my @sections = split(/((?:^>[^\n]*\n)+)/sm, $s);
+       my @sections = PublicInbox::MsgIter::split_quotes($s);
        $part = $s = undef;
        index_body($self, $_, /\A>/ ? 0 : $doc) for @sections;
 }

diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index c42654b68204c67f697278bedfba504850b8c889..70c10604e246ec23314d66580a01bd8b54ac7560 100644 (file)
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -576,7 +576,7 @@ sub add_text_body { # callback for msg_iter
        $s .= "\n" unless $s =~ /\n\z/s;
 
        # split off quoted and unquoted blocks:
-       my @sections = split(/((?:^>[^\n]*\n)+)/sm, $s);
+       my @sections = PublicInbox::MsgIter::split_quotes($s);
        $s = '';
        my $rv = $ctx->{obuf};
        if (defined($fn) || $depth > 0 || $err) {

diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index d22c80b97b4871a00dba47f3e8b055bd1659ab06..5d391a13e0e0b869cac074761e7f716207b643cc 100644 (file)
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -202,6 +202,17 @@ sub flush_diff ($$$) {
                        $dctx = diff_header($dst, \$x, $ctx, \@top);
                } elsif ($dctx) {
                        my $after = '';
+
+                       # Quiet "Complex regular subexpression recursion limit"
+                       # warning.  Perl will truncate matches upon hitting
+                       # that limit, giving us more (and shorter) scalars than
+                       # would be ideal, but otherwise it's harmless.
+                       #
+                       # We could replace the `+' metacharacter with `{1,100}'
+                       # to limit the matches ourselves to 100, but we can
+                       # let Perl do it for us, quietly.
+                       no warnings 'regexp';
+
                        for my $s (split(/((?:(?:^\+[^\n]*\n)+)|
                                        (?:(?:^-[^\n]*\n)+)|
                                        (?:^@@ [^\n]+\n))/xsm, $x)) {

diff --git a/t/msg_iter.t b/t/msg_iter.t
index e33bfc69df1bcc2d0b00d16424e3f70993d9d419..d303564fd0eaa321308fc8993fb8a782a5234f2f 100644 (file)
--- a/t/msg_iter.t
+++ b/t/msg_iter.t
@@ -78,5 +78,35 @@ use_ok('PublicInbox::MsgIter');
                'got bullet point when X-UNKNOWN assumes UTF-8');
 }
 
+{ # API not finalized
+       my @warn;
+       local $SIG{__WARN__} = sub { push @warn, [ @_ ] };
+       my $attr = "So and so wrote:\n";
+       my $q = "> hello world\n" x 10;
+       my $nq = "hello world\n" x 10;
+       my @sections = PublicInbox::MsgIter::split_quotes($attr . $q . $nq);
+       is($sections[0], $attr, 'attribution matches');
+       is($sections[1], $q, 'quoted section matches');
+       is($sections[2], $nq, 'non-quoted section matches');
+       is(scalar(@sections), 3, 'only three sections for short message');
+       is_deeply(\@warn, [], 'no warnings');
+
+       $q x= 3300;
+       $nq x= 3300;
+       @sections = PublicInbox::MsgIter::split_quotes($attr . $q . $nq);
+       is_deeply(\@warn, [], 'no warnings on giant message');
+       is(join('', @sections), $attr . $q . $nq, 'result matches expected');
+       is(shift(@sections), $attr, 'attribution is first section');
+       my @check = ('', '');
+       while (defined(my $l = shift @sections)) {
+               next if $l eq '';
+               like($l, qr/\n\z/s, 'section ends with newline');
+               my $idx = ($l =~ /\A>/) ? 0 : 1;
+               $check[$idx] .= $l;
+       }
+       is($check[0], $q, 'long quoted section matches');
+       is($check[1], $nq, 'long quoted section matches');
+}
+
 done_testing();
 1;