watch_maildir: implement optional spam checking

Mailing lists I watch and mirror may not have the best spam
filtering, and an extra layer should not hurt.

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 5ffc26ef61760de872fd6a27b87ca1d89fc72500..27f36a7e486b84337e1e4e4586ab9ec5251e4c65 100644 (file)
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -140,7 +140,7 @@ sub remove {
 
 # returns undef on duplicate
 sub add {
-       my ($self, $mime) = @_; # mime = Email::MIME
+       my ($self, $mime, $check_cb) = @_; # mime = Email::MIME
 
        my $from = $mime->header('From');
        my ($email) = ($from =~ /([^<\s]+\@[^>\s]+)/g);
@@ -170,6 +170,10 @@ sub add {
 
        # kill potentially confusing/misleading headers
        $mime->header_set($_) for qw(bytes lines content-length status);
+       if ($check_cb) {
+               $mime = $check_cb->($mime) or return;
+       }
+
        $mime = $mime->as_string;
        my $blob = $self->{mark}++;
        print $w "blob\nmark :$blob\ndata ", length($mime), "\n" or wfail;

diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index c1fe81ecfe5a5a39f891bf231763ec83f6c4f489..72bd3d085dc6a5fcbf8fe2c01308caf1e948ea4c 100644 (file)
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -13,7 +13,9 @@ use PublicInbox::Spawn qw(spawn);
 
 sub new {
        my ($class, $config) = @_;
-       my (%mdmap, @mdir);
+       my (%mdmap, @mdir, $spamc);
+
+       # XXX is "publicinboxlearn" really a good namespace for this?
        my $k = 'publicinboxlearn.watchspam';
        if (my $spamdir = $config->{$k}) {
                if ($spamdir =~ s/\Amaildir://) {
@@ -26,6 +28,21 @@ sub new {
                        warn "unsupported $k=$spamdir\n";
                }
        }
+
+       $k = 'publicinboxwatch.spamcheck';
+       my $spamcheck = $config->{$k};
+       if ($spamcheck) {
+               if ($spamcheck eq 'spamc') {
+                       $spamcheck = 'PublicInbox::Spamcheck::Spamc';
+               }
+               if ($spamcheck =~ /::/) {
+                       eval "require $spamcheck";
+                       $spamcheck = _spamcheck_cb($spamcheck->new);
+               } else {
+                       warn "unsupported $k=$spamcheck\n";
+                       $spamcheck = undef;
+               }
+       }
        foreach $k (keys %$config) {
                $k =~ /\Apublicinbox\.([^\.]+)\.watch\z/ or next;
                my $name = $1;
@@ -52,6 +69,7 @@ sub new {
        my $mdre = join('|', map { quotemeta($_) } @mdir);
        $mdre = qr!\A($mdre)/!;
        bless {
+               spamcheck => $spamcheck,
                mdmap => \%mdmap,
                mdir => \@mdir,
                mdre => $mdre,
@@ -136,7 +154,7 @@ sub _try_path {
        }
 
        _force_mid($mime);
-       $im->add($mime);
+       $im->add($mime, $self->{spamcheck});
 }
 
 sub watch {
@@ -208,4 +226,16 @@ sub _scrubber_for {
        undef;
 }
 
+sub _spamcheck_cb {
+       my ($sc) = @_;
+       sub {
+               my ($mime) = @_;
+               my $tmp = '';
+               if ($sc->spamcheck($mime, \$tmp)) {
+                       return Email::MIME->new(\$tmp);
+               }
+               undef;
+       }
+}
+
 1;

diff --git a/t/import.t b/t/import.t
index 09c003624df5c3f12839a625b5a38a66594a75b9..73f92adbbb6e6e3381c0880b63d4c9d26bf8be1e 100644 (file)
--- a/t/import.t
+++ b/t/import.t
@@ -30,7 +30,7 @@ is(scalar @revs, 1, 'one revision created');
 
 $mime->header_set('Message-ID', '<b@example.com>');
 $mime->header_set('Subject', 'msg2');
-like($im->add($mime), qr/\A:\d+\z/, 'added 2nd message');
+like($im->add($mime, sub { $mime }), qr/\A:\d+\z/, 'added 2nd message');
 $im->done;
 @revs = $git->qx(qw(rev-list HEAD));
 is(scalar @revs, 2, '2 revisions exist');
@@ -61,5 +61,9 @@ is($mark, 'MISMATCH', 'mark == MISMATCH on mismatch');
 is($msg->header('Message-ID'), '<a@example.com>', 'Message-ID matches');
 isnt($msg->header('Subject'), $mime->header('Subject'), 'subject mismatch');
 
+$mime->header_set('Message-Id', '<failcheck@example.com>');
+is($im->add($mime, sub { undef }), undef, 'check callback fails');
+is($im->remove($mime), undef, 'message not added, so not removed');
+
 $im->done;
 done_testing();

diff --git a/t/watch_maildir.t b/t/watch_maildir.t
index be1a31285bbefd295722b3b9457f8ebf9256db16..213896385d7e59622059dbf8fc88db7d7bdac67d 100644 (file)
--- a/t/watch_maildir.t
+++ b/t/watch_maildir.t
@@ -3,6 +3,7 @@
 use Test::More;
 use File::Temp qw/tempdir/;
 use Email::MIME;
+use Cwd;
 use PublicInbox::Config;
 my @mods = qw(Filesys::Notify::Simple);
 foreach my $mod (@mods) {
@@ -86,4 +87,37 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
        is(scalar @list, 4, 'four revisions in rev-list');
 }
 
+{
+       my $fail_bin = getcwd()."/t/fail-bin";
+       ok(-x "$fail_bin/spamc", "mock spamc exists");
+       my $fail_path = "$fail_bin:$ENV{PATH}"; # for spamc ham mock
+       local $ENV{PATH} = $fail_path;
+       PublicInbox::Emergency->new($maildir)->prepare(\$msg);
+       $config->{'publicinboxwatch.spamcheck'} = 'spamc';
+       PublicInbox::WatchMaildir->new($config)->scan;
+       @list = $git->qx(qw(ls-tree -r --name-only refs/heads/master));
+       is(scalar @list, 0, 'tree has no files spamc checked');
+       is(unlink(glob("$maildir/new/*")), 1);
+}
+
+{
+       my $main_bin = getcwd()."/t/main-bin";
+       ok(-x "$main_bin/spamc", "mock spamc exists");
+       my $main_path = "$main_bin:$ENV{PATH}"; # for spamc ham mock
+       local $ENV{PATH} = $main_path;
+       PublicInbox::Emergency->new($maildir)->prepare(\$msg);
+       $config->{'publicinboxwatch.spamcheck'} = 'spamc';
+       @list = $git->qx(qw(ls-tree -r --name-only refs/heads/master));
+       PublicInbox::WatchMaildir->new($config)->scan;
+       @list = $git->qx(qw(ls-tree -r --name-only refs/heads/master));
+       is(scalar @list, 1, 'tree has one file after spamc checked');
+
+       # XXX: workaround some weird caching/memoization in cat-file,
+       # shouldn't be an issue in real-world use, though...
+       $git = PublicInbox::Git->new($git_dir);
+
+       my $mref = $git->cat_file('refs/heads/master:'.$list[0]);
+       like($$mref, qr/something\n\z/s, 'message scrubbed on import');
+}
+
 done_testing;