filter: loosen regexp on type matches

[public-inbox.git] / lib / PublicInbox / Filter.pm

diff --git a/lib/PublicInbox/Filter.pm b/lib/PublicInbox/Filter.pm
index a1587682a3f3a8d4dd866db535b15a0623382252..bcc57c4527602d74e000f5edd670807c93d90307 100644 (file)
--- a/lib/PublicInbox/Filter.pm
+++ b/lib/PublicInbox/Filter.pm
@@ -12,6 +12,7 @@ use Email::MIME::ContentType qw/parse_content_type/;
 use Email::Filter;
 use IPC::Run;
 our $VERSION = '0.0.1';
+use constant NO_HTML => '*** We only accept plain-text email, no HTML ***';
 
 # start with the same defaults as mailman
 our $BAD_EXT = qr/\.(?:exe|bat|cmd|com|pif|scr|vbs|cpl)\z/i;
@@ -21,7 +22,7 @@ our $MIME_TEXT_ANY = qr!\btext/[a-z0-9\+\._-]+\b!i;
 # this is highly opinionated delivery
 # returns 0 only if there is nothing to deliver
 sub run {
-       my ($class, $mime) = @_;
+       my ($class, $mime, $filter) = @_;
 
        my $content_type = $mime->header('Content-Type') || 'text/plain';
 
@@ -38,6 +39,7 @@ sub run {
        if ($content_type =~ m!\btext/plain\b!i) {
                return 1; # yay, nothing to do
        } elsif ($content_type =~ $MIME_HTML) {
+               $filter->reject(NO_HTML) if $filter;
                # HTML-only, non-multipart
                my $body = $mime->body;
                my $ct_parsed = parse_content_type($content_type);
@@ -45,7 +47,7 @@ sub run {
                replace_body($mime, $body);
                return 1;
        } elsif ($content_type =~ m!\bmultipart/!i) {
-               return strip_multipart($mime, $content_type);
+               return strip_multipart($mime, $content_type, $filter);
        } else {
                replace_body($mime, "$content_type message scrubbed");
                return 0;
@@ -97,6 +99,7 @@ sub dump_html {
                push @cmd, "-assume_charset=$charset";
        }
        if (IPC::Run::run(\@cmd, $body, \$out, \$err)) {
+               $out =~ s/\r\n/\n/sg;
                $$body = $out;
        } else {
                # give them an ugly version:
@@ -108,9 +111,9 @@ sub dump_html {
 # this is to correct user errors and not expected to cover all corner cases
 # if users don't want to hit this, they should be sending text/plain messages
 # unfortunately, too many people send HTML mail and we'll attempt to convert
-# it to something safer, smaller and harder-to-track.
+# it to something safer, smaller and harder-to-spy-on-users-with.
 sub strip_multipart {
-       my ($mime, $content_type) = @_;
+       my ($mime, $content_type, $filter) = @_;
 
        my (@html, @keep);
        my $rejected = 0;
@@ -128,10 +131,11 @@ sub strip_multipart {
                        return;
                }
 
-               my $part_type = $part->content_type;
+               my $part_type = $part->content_type || '';
                if ($part_type =~ m!\btext/plain\b!i) {
                        push @keep, $part;
                } elsif ($part_type =~ $MIME_HTML) {
+                       $filter->reject(NO_HTML) if $filter;
                        push @html, $part;
                } elsif ($part_type =~ $MIME_TEXT_ANY) {
                        # Give other text attachments the benefit of the doubt,
@@ -139,7 +143,8 @@ sub strip_multipart {
                        # help with.
 
                        push @keep, $part;
-               } elsif ($part_type =~ m!\Aapplication/octet-stream\z!i) {
+               } elsif ($part_type eq '' ||
+                        $part_type =~ m!\bapplication/octet-stream\b!i) {
                        # unfortunately, some mailers don't set correct types,
                        # let messages of unknown type through but do not
                        # change the sender-specified type
@@ -148,7 +153,7 @@ sub strip_multipart {
                        } else {
                                $rejected++;
                        }
-               } elsif ($part_type =~ m!\Aapplication/pgp-signature\z!i) {
+               } elsif ($part_type =~ m!\bapplication/pgp-signature\b!i) {
                        # PGP signatures are not huge, we may keep them.
                        # They can only be valid if it's the last element,
                        # so we keep them iff the message is unmodified:
@@ -201,6 +206,10 @@ sub collapse {
        my ($mime, $part) = @_;
        $mime->header_set('Content-Type', $part->content_type);
        $mime->body_set($part->body_raw);
+       my $cte = $part->header('Content-Transfer-Encoding');
+       if (defined($cte) && length($cte)) {
+               $mime->header_set('Content-Transfer-Encoding', $cte);
+       }
        mark_changed($mime);
        return 1;
 }