1 # Copyright (C) 2013-2015 all contributors <meta@public-inbox.org>
2 # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
4 # Used to filter incoming mail for -mda and importers
5 # This only exposes one function: run
6 # Note: the settings here are highly opinionated. Obviously, this is
7 # Free Software (AGPLv3), so you may change it if you host yourself.
8 package PublicInbox::Filter;
12 use Email::MIME::ContentType qw/parse_content_type/;
15 our $VERSION = '0.0.1';
16 use constant NO_HTML => '*** We only accept plain-text email, no HTML ***';
17 use constant TEXT_ONLY => '*** We only accept plain-text email ***';
19 # start with the same defaults as mailman
20 our $BAD_EXT = qr/\.(exe|bat|cmd|com|pif|scr|vbs|cpl|zip)\s*\z/i;
21 our $MIME_HTML = qr!\btext/x?html\b!i;
22 our $MIME_TEXT_ANY = qr!\btext/[a-z0-9\+\._-]+\b!i;
24 # this is highly opinionated delivery
25 # returns 0 only if there is nothing to deliver
27 my ($class, $mime, $filter) = @_;
29 my $content_type = $mime->header('Content-Type') || 'text/plain';
31 # kill potentially bad/confusing headers
32 # Note: ssoma already does this, but since we mangle the message,
33 # we should do this before it gets to ssoma.
34 # We also kill Mail-{Followup,Reply}-To headers due to
35 # the nature of public-inbox having no real subscribers.
36 foreach my $d (qw(status lines content-length
37 mail-followup-to mail-reply-to)) {
38 $mime->header_set($d);
41 if ($content_type =~ m!\btext/plain\b!i) {
42 return 1; # yay, nothing to do
43 } elsif ($content_type =~ $MIME_HTML) {
44 $filter->reject(NO_HTML) if $filter;
45 # HTML-only, non-multipart
46 my $body = $mime->body;
47 my $ct_parsed = parse_content_type($content_type);
48 dump_html(\$body, $ct_parsed->{attributes}->{charset});
49 replace_body($mime, $body);
51 } elsif ($content_type =~ m!\bmultipart/!i) {
52 return strip_multipart($mime, $content_type, $filter);
54 $filter->reject(TEXT_ONLY) if $filter;
55 replace_body($mime, "$content_type message scrubbed");
61 my ($mime, $part, $type) = ($_[0], $_[1], $_[3]);
62 # don't copy $_[2], that's the body (it may be huge)
64 # Email::MIME insists on setting Date:, so just set it consistently
65 # to avoid conflicts to avoid git merge conflicts in a split brain
67 unless (defined $part->header('Date')) {
68 my $date = $mime->header('Date') ||
69 'Thu, 01 Jan 1970 00:00:00 +0000';
70 $part->header_set('Date', $date);
73 $part->charset_set(undef);
74 $part->name_set(undef);
75 $part->filename_set(undef);
76 $part->format_set(undef);
77 $part->encoding_set('8bit');
78 $part->disposition_set(undef);
79 $part->content_type_set($type);
80 $part->body_set($_[2]);
83 # converts one part of a multipart message to text
84 sub html_part_to_text {
85 my ($mime, $part) = @_;
86 my $body = $part->body;
87 my $ct_parsed = parse_content_type($part->content_type);
88 dump_html(\$body, $ct_parsed->{attributes}->{charset});
89 replace_part($mime, $part, $body, 'text/plain');
92 # modifies $_[0] in place
94 my ($body, $charset) = @_;
95 $charset ||= 'US-ASCII';
96 my @cmd = qw(lynx -stdin -stderr -dump);
100 # be careful about remote command injection!
101 if ($charset =~ /\A([A-Za-z0-9\-]+)\z/) {
102 push @cmd, "-assume_charset=$charset";
104 if (IPC::Run::run(\@cmd, $body, \$out, \$err)) {
105 $out =~ s/\r\n/\n/sg;
108 # give them an ugly version:
109 $$body = "public-inbox HTML conversion failed: $err\n" .
114 # this is to correct old archives during import.
115 sub strip_multipart {
116 my ($mime, $content_type, $filter) = @_;
122 # scan through all parts once
123 $mime->walk_parts(sub {
125 return if $part->subparts; # walk_parts already recurses
127 # some extensions are just bad, reject them outright
128 my $fn = $part->filename;
129 if (defined($fn) && $fn =~ $BAD_EXT) {
130 $filter->reject("Bad file type: $1") if $filter;
135 my $part_type = $part->content_type || '';
136 if ($part_type =~ m!\btext/plain\b!i) {
138 } elsif ($part_type =~ $MIME_HTML) {
139 $filter->reject(NO_HTML) if $filter;
141 } elsif ($part_type =~ $MIME_TEXT_ANY) {
142 # Give other text attachments the benefit of the doubt,
143 # here? Could be source code or script the user wants
147 } elsif ($part_type eq '' ||
148 $part_type =~ m!\bapplication/octet-stream\b!i) {
149 # unfortunately, some mailers don't set correct types,
150 # let messages of unknown type through but do not
151 # change the sender-specified type
152 if (recheck_type_ok($part)) {
155 $filter->reject("Bad attachment: $part_type ".
160 } elsif ($part_type =~ m!\bapplication/pgp-signature\b!i) {
161 # PGP signatures are not huge, we may keep them.
162 # They can only be valid if it's the last element,
163 # so we keep them iff the message is unmodified:
164 if ($rejected == 0 && !@html) {
168 $filter->reject("unacceptable mime-type: $part_type ".
171 # reject everything else, including non-PGP signatures
176 if ($content_type =~ m!\bmultipart/alternative\b!i) {
177 if (scalar @keep == 1) {
178 return collapse($mime, $keep[0]);
180 } else { # convert HTML parts to plain text
181 foreach my $part (@html) {
182 html_part_to_text($mime, $part);
188 @keep = (Email::MIME->create(
190 content_type => 'text/plain',
191 charset => 'US-ASCII',
194 body_str => 'all attachments scrubbed by '. __PACKAGE__
198 if (scalar(@html) || $rejected) {
199 $mime->parts_set(\@keep);
200 $mime->body_set($mime->body_raw);
209 $mime->header_set('X-Content-Filtered-By', __PACKAGE__ ." $VERSION");
213 my ($mime, $part) = @_;
214 $mime->header_set('Content-Type', $part->content_type);
215 $mime->body_set($part->body_raw);
216 my $cte = $part->header('Content-Transfer-Encoding');
217 if (defined($cte) && $cte ne '') {
218 $mime->header_set('Content-Transfer-Encoding', $cte);
226 $mime->body_set($_[1]);
227 $mime->header_set('Content-Type', 'text/plain');
228 if ($mime->header('Content-Transfer-Encoding')) {
229 $mime->header_set('Content-Transfer-Encoding', undef);
234 # Check for display-able text, no messed up binaries
235 # Note: we can not rewrite the message with the detected mime type
236 sub recheck_type_ok {
239 ((length($s) < 0x10000) && ($s =~ /\A([[:print:]\s]+)\z/s));