watch_maildir: spam removal support

We can support spam removal by watching a special "spam"
Maildir, too.  We can run public-inbox-learn as a separate
step, and that command will be improved to support
auto-learning, too.

diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index f1a21b9b42d6cdaa9d7702305ba481c0410347a2..cb64f899fb27e277b8a6032100f18683c35ff5e3 100644 (file)
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -9,11 +9,24 @@ $Email::MIME::ContentType::STRICT_PARAMS = 0; # user input is imperfect
 use PublicInbox::Git;
 use PublicInbox::Import;
 use PublicInbox::MDA;
+use PublicInbox::Spawn qw(spawn);
 
 sub new {
        my ($class, $config) = @_;
        my (%mdmap, @mdir);
-       foreach my $k (keys %$config) {
+       my $k = 'publicinboxlearn.watchspam';
+       if (my $spamdir = $config->{$k}) {
+               if ($spamdir =~ s/\Amaildir://) {
+                       $spamdir =~ s!/+\z!!;
+                       # skip "new", no MUA has seen it, yet.
+                       my $cur = "$spamdir/cur";
+                       push @mdir, $cur;
+                       $mdmap{$cur} = 'watchspam';
+               } else {
+                       warn "unsupported $k=$spamdir\n";
+               }
+       }
+       foreach $k (keys %$config) {
                $k =~ /\Apublicinbox\.([^\.]+)\.watch\z/ or next;
                my $name = $1;
                my $watch = $config->{$k};
@@ -27,8 +40,9 @@ sub new {
                        my $new = "$watch/new";
                        my $cur = "$watch/cur";
                        push @mdir, $new, $cur;
-                       $mdmap{$new} = $inbox;
-                       $mdmap{$cur} = $inbox;
+                       die "$new already in use\n" if $mdmap{$new};
+                       die "$cur already in use\n" if $mdmap{$cur};
+                       $mdmap{$new} = $mdmap{$cur} = $inbox;
                } else {
                        warn "watch unsupported: $k=$watch\n";
                }
@@ -55,6 +69,42 @@ sub _try_fsn_paths {
        _done_for_now($self);
 }
 
+sub _check_spam {
+       my ($self, $path) = @_;
+       my $mime = _path_to_mime($path) or return;
+       _force_mid($mime);
+       foreach my $inbox (values %{$self->{mdmap}}) {
+               next unless ref $inbox;
+               my $im = _importer_for($self, $inbox);
+               $im->remove($mime);
+               if (my $scrub = _scrubber_for($inbox)) {
+                       my $scrubbed = $scrub->scrub($mime) or next;
+                       $im->remove($scrubbed);
+               }
+       }
+}
+
+# used to hash the relevant portions of a message when there are conflicts
+sub _hash_mime2 {
+       my ($mime) = @_;
+       require Digest::SHA;
+       my $dig = Digest::SHA->new('SHA-1');
+       $dig->add($mime->header_obj->header_raw('Subject'));
+       $dig->add($mime->body_raw);
+       $dig->hexdigest;
+}
+
+sub _force_mid {
+       my ($mime) = @_;
+       # probably a bad idea, but we inject a Message-Id if
+       # one is missing, here..
+       my $mid = $mime->header_obj->header_raw('Message-Id');
+       if (!defined $mid || $mid =~ /\A\s*\z/) {
+               $mid = '<' . _hash_mime2($mime) . '@generated>';
+               $mime->header_set('Message-Id', $mid);
+       }
+}
+
 sub _try_path {
        my ($self, $path) = @_;
        if ($path !~ $self->{mdre}) {
@@ -66,44 +116,22 @@ sub _try_path {
                warn "unmappable dir: $1\n";
                return;
        }
-       my $im = $inbox->{-import} ||= eval {
-               my $git = $inbox->git;
-               my $name = $inbox->{name};
-               my $addr = $inbox->{-primary_address};
-               PublicInbox::Import->new($git, $name, $addr);
-       };
-       $self->{importers}->{"$im"} = $im;
-       my $mime;
-       if (open my $fh, '<', $path) {
-               local $/;
-               my $str = <$fh>;
-               $str or return;
-               $mime = Email::MIME->new(\$str);
-       } elsif ($!{ENOENT}) {
-               return;
-       } else {
-               warn "failed to open $path: $!\n";
-               return;
+       if (!ref($inbox) && $inbox eq 'watchspam') {
+               return _check_spam($self, $path);
        }
-
+       my $im = _importer_for($self, $inbox);
+       my $mime = _path_to_mime($path) or return;
        $mime->header_set($_) foreach @PublicInbox::MDA::BAD_HEADERS;
        my $wm = $inbox->{-watchheader};
        if ($wm) {
                my $v = $mime->header_obj->header_raw($wm->[0]);
                return unless ($v && $v =~ $wm->[1]);
        }
-       my $f = $inbox->{filter};
-       if ($f && $f =~ /::/) {
-               eval "require $f";
-               if ($@) {
-                       warn $@;
-               } else {
-                       $f = $f->new;
-                       $mime = $f->scrub($mime);
-               }
+       if (my $scrub = _scrubber_for($inbox)) {
+               $mime = $scrub->scrub($mime) or return;
        }
-       $mime or return;
-       my $mid = $mime->header_obj->header_raw('Message-Id');
+
+       _force_mid($mime);
        $im->add($mime);
 }
 
@@ -140,4 +168,44 @@ sub scan {
        _done_for_now($self);
 }
 
+sub _path_to_mime {
+       my ($path) = @_;
+       if (open my $fh, '<', $path) {
+               local $/;
+               my $str = <$fh>;
+               $str or return;
+               return Email::MIME->new(\$str);
+       } elsif ($!{ENOENT}) {
+               return;
+       } else {
+               warn "failed to open $path: $!\n";
+               return;
+       }
+}
+
+sub _importer_for {
+       my ($self, $inbox) = @_;
+       my $im = $inbox->{-import} ||= eval {
+               my $git = $inbox->git;
+               my $name = $inbox->{name};
+               my $addr = $inbox->{-primary_address};
+               PublicInbox::Import->new($git, $name, $addr);
+       };
+       $self->{importers}->{"$im"} = $im;
+}
+
+sub _scrubber_for {
+       my ($inbox) = @_;
+       my $f = $inbox->{filter};
+       if ($f && $f =~ /::/) {
+               eval "require $f";
+               if ($@) {
+                       warn $@;
+               } else {
+                       return $f->new;
+               }
+       }
+       undef;
+}
+
 1;

diff --git a/t/watch_maildir.t b/t/watch_maildir.t
index 6564a8665830f2881d88c83ca6861fc4e98ad3a2..8a2c934a36f05623d373392c648142c8fd630afd 100644 (file)
--- a/t/watch_maildir.t
+++ b/t/watch_maildir.t
@@ -8,6 +8,7 @@ use PublicInbox::Config;
 my $tmpdir = tempdir('watch_maildir-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $git_dir = "$tmpdir/test.git";
 my $maildir = "$tmpdir/md";
+my $spamdir = "$tmpdir/spam";
 use_ok 'PublicInbox::WatchMaildir';
 use_ok 'PublicInbox::Emergency';
 my $cfgpfx = "publicinbox.test";
@@ -21,14 +22,17 @@ Subject: spam
 Message-Id: <a\@b.com>
 Date: Sat, 18 Jun 2016 00:00:00 +0000
 
-msg
+something
 EOF
 PublicInbox::Emergency->new($maildir)->prepare(\$msg);
+my $sem = PublicInbox::Emergency->new($spamdir); # create dirs
 
 my $config = PublicInbox::Config->new({
        "$cfgpfx.address" => $addr,
        "$cfgpfx.mainrepo" => $git_dir,
        "$cfgpfx.watch" => "maildir:$maildir",
+       "$cfgpfx.filter" => 'PublicInbox::Filter::Vger',
+       "publicinboxlearn.watchspam" => "maildir:$spamdir",
 });
 
 PublicInbox::WatchMaildir->new($config)->scan;
@@ -36,4 +40,44 @@ my $git = PublicInbox::Git->new($git_dir);
 my @list = $git->qx(qw(rev-list refs/heads/master));
 is(scalar @list, 1, 'one revision in rev-list');
 
+my $write_spam = sub {
+       is(scalar glob("$spamdir/new/*"), undef, 'no spam existing');
+       $sem->prepare(\$msg);
+       $sem->commit;
+       my @new = glob("$spamdir/new/*");
+       is(scalar @new, 1);
+       my @p = split(m!/+!, $new[0]);
+       ok(link($new[0], "$spamdir/cur/".$p[-1]));
+       is(unlink($new[0]), 1);
+};
+$write_spam->();
+is(unlink(glob("$maildir/new/*")), 1, 'unlinked old spam');
+PublicInbox::WatchMaildir->new($config)->scan;
+@list = $git->qx(qw(rev-list refs/heads/master));
+is(scalar @list, 2, 'two revisions in rev-list');
+@list = $git->qx(qw(ls-tree -r --name-only refs/heads/master));
+is(scalar @list, 0, 'tree is empty');
+
+# check with scrubbing
+{
+       $msg .= qq(--
+To unsubscribe from this list: send the line "unsubscribe git" in
+the body of a message to majordomo\@vger.kernel.org
+More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
+       PublicInbox::Emergency->new($maildir)->prepare(\$msg);
+       PublicInbox::WatchMaildir->new($config)->scan;
+       @list = $git->qx(qw(ls-tree -r --name-only refs/heads/master));
+       is(scalar @list, 1, 'tree has one file');
+       my $mref = $git->cat_file('HEAD:'.$list[0]);
+       like($$mref, qr/something\n\z/s, 'message scrubbed on import');
+
+       is(unlink(glob("$maildir/new/*")), 1, 'unlinked spam');
+       $write_spam->();
+       PublicInbox::WatchMaildir->new($config)->scan;
+       @list = $git->qx(qw(ls-tree -r --name-only refs/heads/master));
+       is(scalar @list, 0, 'tree is empty');
+       @list = $git->qx(qw(rev-list refs/heads/master));
+       is(scalar @list, 4, 'four revisions in rev-list');
+}
+
 done_testing;