msg_iter: make ->each_part method for PublicInbox::MIME

[public-inbox.git] / lib / PublicInbox / MsgIter.pm

diff --git a/lib/PublicInbox/MsgIter.pm b/lib/PublicInbox/MsgIter.pm
index eb94d621d73cdbcde4fee70d93a52fe3ba4d8f1d..cd5a5d99564e9d05c100d521677ca7d62c9deb4f 100644 (file)
--- a/lib/PublicInbox/MsgIter.pm
+++ b/lib/PublicInbox/MsgIter.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2016-2020 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # read-only utilities for Email::MIME
@@ -7,15 +7,16 @@ use strict;
 use warnings;
 use base qw(Exporter);
 our @EXPORT = qw(msg_iter msg_part_text);
-use PublicInbox::MIME;
 
+# This becomes PublicInbox::MIME->each_part:
 # Like Email::MIME::walk_parts, but this is:
 # * non-recursive
 # * passes depth and indices to the iterator callback
-sub msg_iter ($$) {
-       my ($mime, $cb) = @_;
+sub em_each_part ($$;$$) {
+       my ($mime, $cb, $cb_arg, $do_undef) = @_;
        my @parts = $mime->subparts;
        if (@parts) {
+               $mime = $_[0] = undef if $do_undef; # saves some memory
                my $i = 0;
                @parts = map { [ $_, 1, ++$i ] } @parts;
                while (my $p = shift @parts) {
@@ -27,17 +28,33 @@ sub msg_iter ($$) {
                                @sub = map { [ $_, $depth, @idx, ++$i ] } @sub;
                                @parts = (@sub, @parts);
                        } else {
-                               $cb->($p);
+                               $cb->($p, $cb_arg);
                        }
                }
        } else {
-               $cb->([$mime, 0, 0]);
+               $cb->([$mime, 0, 0], $cb_arg);
+       }
+}
+
+# Use this when we may accept Email::MIME from user scripts
+# (not just PublicInbox::MIME)
+sub msg_iter ($$;$$) { # $_[0] = PublicInbox::MIME/Email::MIME-like obj
+       my (undef, $cb, $cb_arg, $once) = @_;
+       if (my $ep = $_[0]->can('each_part')) { # PublicInbox::{MIME,*}
+               $ep->($_[0], $cb, $cb_arg, $once);
+       } else { # for compatibility with existing Email::MIME users:
+               em_each_part($_[0], $cb, $cb_arg, $once);
        }
 }
 
 sub msg_part_text ($$) {
        my ($part, $ct) = @_;
 
+       # TODO: we may offer a separate sub for people who need to index
+       # HTML-only mail, but the majority of HTML mail is multipart/alternative
+       # with a text part which we don't have to waste cycles decoding
+       return if $ct =~ m!\btext/x?html\b!;
+
        my $s = eval { $part->body_str };
        my $err = $@;
 
@@ -45,11 +62,18 @@ sub msg_part_text ($$) {
        # times when it should not have been:
        #   <87llgalspt.fsf@free.fr>
        #   <200308111450.h7BEoOu20077@mail.osdl.org>
-       if ($ct =~ m!\btext/plain\b!i || $ct =~ m!\bmultipart/mixed\b!i) {
-               # Try to assume UTF-8 because Alpine seems to
-               # do wacky things and set charset=X-UNKNOWN
-               $part->charset_set('UTF-8');
-               $s = eval { $part->body_str };
+       if ($err && ($ct =~ m!\btext/\b!i ||
+                       $ct =~ m!\bmultipart/mixed\b!i)) {
+               my $cte = $part->header_raw('Content-Transfer-Encoding');
+               if (defined($cte) && $cte =~ /\b7bit\b/i) {
+                       $s = $part->body;
+                       $err = undef if $s =~ /\A[[:ascii:]]+\z/s;
+               } else {
+                       # Try to assume UTF-8 because Alpine seems to
+                       # do wacky things and set charset=X-UNKNOWN
+                       $part->charset_set('UTF-8');
+                       $s = eval { $part->body_str };
+               }
 
                # If forcing charset=UTF-8 failed,
                # caller will warn further down...
@@ -58,4 +82,14 @@ sub msg_part_text ($$) {
        ($s, $err);
 }
 
+# returns an array of quoted or unquoted sections
+sub split_quotes {
+       # Quiet "Complex regular subexpression recursion limit" warning
+       # in case an inconsiderate sender quotes 32K of text at once.
+       # The warning from Perl is harmless for us since our callers can
+       # tolerate less-than-ideal matches which work within Perl limits.
+       no warnings 'regexp';
+       split(/((?:^>[^\n]*\n)+)/sm, shift);
+}
+
 1;