watchmaildir: scan all matching headers

[public-inbox.git] / lib / PublicInbox / WatchMaildir.pm

diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index 8d8e817ad767d9097b438c84ae336ea503e12a4b..02e4b32f254a8933e81e60b2001b8cc5ebce63c8 100644 (file)
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2016-2020 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # ref: https://cr.yp.to/proto/maildir.html
@@ -7,15 +7,11 @@ package PublicInbox::WatchMaildir;
 use strict;
 use warnings;
 use PublicInbox::MIME;
-use PublicInbox::Git;
-use PublicInbox::Import;
-use PublicInbox::MDA;
-use PublicInbox::Spawn qw(spawn);
 use PublicInbox::InboxWritable;
-use File::Temp qw//;
-use PublicInbox::Filter::Base;
+use File::Temp 0.19 (); # 0.19 for ->newdir
+use PublicInbox::Filter::Base qw(REJECT);
 use PublicInbox::Spamcheck;
-*REJECT = *PublicInbox::Filter::Base::REJECT;
+*mime_from_path = \&PublicInbox::InboxWritable::mime_from_path;
 
 sub new {
        my ($class, $config) = @_;
@@ -27,9 +23,10 @@ sub new {
        # indefinitely...
        foreach my $pfx (qw(publicinboxwatch publicinboxlearn)) {
                my $k = "$pfx.watchspam";
-               if (my $dir = $config->{$k}) {
-                       if ($dir =~ s/\Amaildir://) {
-                               $dir =~ s!/+\z!!;
+               defined(my $dirs = $config->{$k}) or next;
+               $dirs = [ $dirs ] if !ref($dirs);
+               for my $dir (@$dirs) {
+                       if (is_maildir($dir)) {
                                # skip "new", no MUA has seen it, yet.
                                my $cur = "$dir/cur";
                                my $old = $mdmap{$cur};
@@ -60,11 +57,22 @@ sub new {
                my $ibx = $_[0] = PublicInbox::InboxWritable->new($_[0]);
 
                my $watch = $ibx->{watch} or return;
-               if ($watch =~ s/\Amaildir://) {
-                       $watch =~ s!/+\z!!;
-                       if (my $wm = $ibx->{watchheader}) {
-                               my ($k, $v) = split(/:/, $wm, 2);
-                               $ibx->{-watchheader} = [ $k, qr/\Q$v\E/ ];
+               if (is_maildir($watch)) {
+                       my $watch_hdrs = [];
+                       if (my $whs = $ibx->{watchheader}) {
+                               for (@$whs) {
+                                       my ($k, $v) = split(/:/, $_, 2);
+                                       push @$watch_hdrs, [ $k, qr/\Q$v\E/ ];
+                               }
+                       }
+                       if (my $list_ids = $ibx->{listid}) {
+                               for (@$list_ids) {
+                                       my $re = qr/<[ \t]*\Q$_\E[ \t]*>/;
+                                       push @$watch_hdrs, ['List-Id', $re ];
+                               }
+                       }
+                       if (scalar @$watch_hdrs) {
+                               $ibx->{-watchheaders} = $watch_hdrs;
                        }
                        my $new = "$watch/new";
                        my $cur = "$watch/cur";
@@ -117,13 +125,13 @@ sub _remove_spam {
        my ($self, $path) = @_;
        # path must be marked as (S)een
        $path =~ /:2,[A-R]*S[T-Za-z]*\z/ or return;
-       my $mime = _path_to_mime($path) or return;
+       my $mime = mime_from_path($path) or return;
        $self->{config}->each_inbox(sub {
                my ($ibx) = @_;
                eval {
                        my $im = _importer_for($self, $ibx);
                        $im->remove($mime, 'spam');
-                       if (my $scrub = $ibx->filter) {
+                       if (my $scrub = $ibx->filter($im)) {
                                my $scrubbed = $scrub->scrub($mime, 1);
                                $scrubbed or return;
                                $scrubbed == REJECT() and return;
@@ -152,16 +160,28 @@ sub _try_path {
        if (!ref($inboxes) && $inboxes eq 'watchspam') {
                return _remove_spam($self, $path);
        }
+
+       my $warn_cb = $SIG{__WARN__} || sub { print STDERR @_ };
+       local $SIG{__WARN__} = sub {
+               $warn_cb->("path: $path\n");
+               $warn_cb->(@_);
+       };
        foreach my $ibx (@$inboxes) {
-               my $mime = _path_to_mime($path) or next;
+               my $mime = mime_from_path($path) or next;
                my $im = _importer_for($self, $ibx);
 
-               my $wm = $ibx->{-watchheader};
-               if ($wm) {
-                       my $v = $mime->header_obj->header_raw($wm->[0]);
-                       next unless ($v && $v =~ $wm->[1]);
+               # any header match means it's eligible for the inbox:
+               if (my $watch_hdrs = $ibx->{-watchheaders}) {
+                       my $ok;
+                       my $hdr = $mime->header_obj;
+                       for my $wh (@$watch_hdrs) {
+                               my @v = $hdr->header_raw($wh->[0]);
+                               $ok = grep(/$wh->[1]/, @v) and last;
+                       }
+                       next unless $ok;
                }
-               if (my $scrub = $ibx->filter) {
+
+               if (my $scrub = $ibx->filter($im)) {
                        my $ret = $scrub->scrub($mime) or next;
                        $ret == REJECT() and next;
                        $mime = $ret;
@@ -182,7 +202,8 @@ sub watch {
 
        # lazy load here, we may support watching via IMAP IDLE
        # in the future...
-       require Filesys::Notify::Simple;
+       eval { require Filesys::Notify::Simple } or
+               die "Filesys::Notify::Simple is currently required for $0\n";
        my $fsn = Filesys::Notify::Simple->new([@{$self->{mdir}}, $scandir]);
        $fsn->wait($cb) until $self->{quit};
 }
@@ -238,21 +259,6 @@ sub scan {
        trigger_scan($self, 'cont') if keys %$opendirs;
 }
 
-sub _path_to_mime {
-       my ($path) = @_;
-       if (open my $fh, '<', $path) {
-               local $/;
-               my $str = <$fh>;
-               $str or return;
-               return PublicInbox::MIME->new(\$str);
-       } elsif ($!{ENOENT}) {
-               return;
-       } else {
-               warn "failed to open $path: $!\n";
-               return;
-       }
-}
-
 sub _importer_for {
        my ($self, $ibx) = @_;
        my $importers = $self->{importers};
@@ -278,4 +284,11 @@ sub _spamcheck_cb {
        }
 }
 
+sub is_maildir {
+       $_[0] =~ s!\Amaildir:!! or return;
+       $_[0] =~ tr!/!/!s;
+       $_[0] =~ s!/\z!!;
+       $_[0];
+}
+
 1;