1 # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
2 # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
4 # This only exposes one function: run
5 # Note: the settings here are highly opinionated. Obviously, this is
6 # Free Software (AGPLv3), so you may change it if you host yourself.
7 package PublicInbox::Filter;
11 use Email::MIME::ContentType qw/parse_content_type/;
14 our $VERSION = '0.0.1';
16 # start with the same defaults as mailman
17 our $BAD_EXT = qr/\.(?:exe|bat|cmd|com|pif|scr|vbs|cpl)\z/i;
19 # this is highly opinionated delivery
20 # returns 0 only if there is nothing to deliver
22 my ($class, $simple) = @_;
24 my $content_type = $simple->header("Content-Type") || "text/plain";
26 # kill potentially bad/confusing headers
27 # Note: ssoma already does this, but since we mangle the message,
28 # we should do this before it gets to ssoma.
29 # We also kill Mail-{Followup,Reply}-To and Reply-To headers due to
30 # the nature of public-inbox having no real subscribers.
31 foreach my $d (qw(status lines content-length
32 mail-followup-to mail-reply-to reply-to)) {
33 $simple->header_set($d);
36 if ($content_type =~ m!\btext/plain\b!i) {
37 return 1; # yay, nothing to do
38 } elsif ($content_type =~ m!\btext/html\b!i) {
39 # HTML-only, non-multipart
40 my $body = $simple->body;
41 my $ct_parsed = parse_content_type($content_type);
42 dump_html($body, $ct_parsed->{attributes}->{charset});
43 replace_body($simple, $body);
45 } elsif ($content_type =~ m!\bmultipart/!i) {
46 return strip_multipart($simple, $content_type);
48 replace_body($simple, "$content_type message scrubbed");
54 my ($simple, $part, $type) = ($_[0], $_[1], $_[3]);
55 # don't copy $_[2], that's the body (it may be huge)
57 # Email::MIME insists on setting Date:, so just set it consistently
58 # to avoid conflicts to avoid git merge conflicts in a split brain
60 unless (defined $part->header('Date')) {
61 my $date = $simple->header('Date') ||
62 'Thu, 01 Jan 1970 00:00:00 +0000';
63 $part->header_set('Date', $date);
66 $part->charset_set(undef);
67 $part->name_set(undef);
68 $part->filename_set(undef);
69 $part->format_set(undef);
70 $part->encoding_set('8bit');
71 $part->disposition_set(undef);
72 $part->content_type_set($type);
73 $part->body_set($_[2]);
76 # converts one part of a multipart message to text
77 sub html_part_to_text {
78 my ($simple, $part) = @_;
79 my $body = $part->body;
80 my $ct_parsed = parse_content_type($part->content_type);
81 dump_html($body, $ct_parsed->{attributes}->{charset});
82 replace_part($simple, $part, $body, 'text/plain');
85 # modifies $_[0] in place
87 my $charset = $_[1] || 'US-ASCII';
88 my $cmd = "lynx -stdin -dump";
90 # be careful about remote command injection!
91 if ($charset =~ /\A[A-Za-z0-9\-]+\z/) {
92 $cmd .= " -assume_charset=$charset";
95 my $pid = open2(my $out, my $in, $cmd);
105 # this is to correct user errors and not expected to cover all corner cases
106 # if users don't want to hit this, they should be sending text/plain messages
107 # unfortunately, too many people send HTML mail and we'll attempt to convert
108 # it to something safer, smaller and harder-to-track.
109 sub strip_multipart {
110 my ($simple, $content_type) = @_;
111 my $mime = Email::MIME->new($simple->as_string);
117 # scan through all parts once
118 $mime->walk_parts(sub {
120 return if $part->subparts; # walk_parts already recurses
122 # some extensions are just bad, reject them outright
123 my $fn = $part->filename;
124 if (defined($fn) && $fn =~ $BAD_EXT) {
129 my $part_type = $part->content_type;
130 if ($part_type =~ m!\btext/plain\b!i) {
132 } elsif ($part_type =~ m!\btext/html\b!i) {
134 } elsif ($part_type =~ m!\btext/[a-z0-9\+\._-]+\b!i) {
135 # Give other text attachments the benefit of the doubt,
136 # here? Could be source code or script the user wants
141 # reject everything else
143 # Yes, we drop GPG/PGP signatures because:
144 # * hardly anybody bothers to verify signatures
145 # * we strip/convert HTML parts, which could invalidate
147 # * they increase the size of messages greatly
148 # (especially short ones)
149 # * they do not compress well
151 # Instead, rely on soft verification measures:
152 # * content of the message is most important
153 # * we encourage Cc: all replies, so replies go to
154 # the original sender
155 # * Received, User-Agent, and similar headers
156 # (this is also to encourage using self-hosted mail
157 # servers (using 100% Free Software, of course :)
159 # Furthermore, identity theft is uncommon in Free/Open
160 # Source, even in communities where signatures are rare.
165 if ($content_type =~ m!\bmultipart/alternative\b!i) {
166 if (scalar @keep == 1) {
167 return collapse($simple, $keep[0]);
169 } else { # convert HTML parts to plain text
170 foreach my $part (@html) {
171 html_part_to_text($simple, $part);
177 @keep = (Email::MIME->create(
179 content_type => 'text/plain',
180 charset => 'US-ASCII',
183 body_str => 'all attachments scrubbed by '. __PACKAGE__
187 if (scalar(@html) || $rejected) {
188 $mime->parts_set(\@keep);
189 $simple->body_set($mime->body_raw);
190 mark_changed($simple);
198 $simple->header_set("X-Content-Filtered-By", __PACKAGE__ ." $VERSION");
202 my ($simple, $part) = @_;
203 $simple->header_set("Content-Type", $part->content_type);
204 $simple->body_set($part->body_raw);
205 mark_changed($simple);
211 $simple->body_set($_[1]);
212 $simple->header_set("Content-Type", "text/plain");
213 if ($simple->header("Content-Transfer-Encoding")) {
214 $simple->header_set("Content-Transfer-Encoding", undef);
216 mark_changed($simple);