mda, watch: wire up List-ID header support

This also adds watchheader tests for -watch, which we never
had before :x

diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod
index 8d545f7afa247339653b72ceb13183aa749fc70d..6a9739f71433c85ac8bba39ea8e0a0a11b0de5a0 100644 (file)
--- a/Documentation/public-inbox-config.pod
+++ b/Documentation/public-inbox-config.pod
@@ -85,6 +85,24 @@ the given header.  Multiple values are not currently supported.
 
 Default: none; only for L<public-inbox-watch(1)> users
 
 
 Default: none; only for L<public-inbox-watch(1)> users
 

+=item publicinbox.<name>.listid
+
+The L<rfc2919|https://tools.ietf.org/html/rfc2919> header without
+angle brackets for L<public-inbox-mda(1)> deliveries and
+L<public-inbox-watch(1)>.
+
+For public-inbox-watch users, this is a shortcut for specifying
+C<publicinbox.$NAME.watchheader=List-Id:<foo.example.com>>
+
+For public-inbox-mda users, this may be used to avoid recipient
+matching via C<ORIGINAL_RECIPIENT> environment variable.
+
+This may be specified multiple times for merging multiple mailing
+lists into a single public-inbox, only one C<List-Id> header
+needs to match.
+
+Default: none
+

 =item publicinbox.<name>.nntpmirror
 
 This may be the full NNTP URL of an independently-run mirror.
 =item publicinbox.<name>.nntpmirror
 
 This may be the full NNTP URL of an independently-run mirror.

diff --git a/Documentation/public-inbox-mda.pod b/Documentation/public-inbox-mda.pod
index 64ec690cc568a3ccc71f085b81c4bef94221b427..921b7a152fb440a27f623f3a2255bec84e39961c 100644 (file)
--- a/Documentation/public-inbox-mda.pod
+++ b/Documentation/public-inbox-mda.pod
@@ -25,6 +25,9 @@ L<public-inbox-config(5)/publicinboxmda.spamcheck>
 The original recipient email address, set by the MTA.  Postfix
 sets it by default, untested on other MTAs.
 
 The original recipient email address, set by the MTA.  Postfix
 sets it by default, untested on other MTAs.
 

+This does not have to be set if relying on C<publicinbox.$NAME.listid>
+directives configured in L<public-inbox-config(5)>.
+

 =item PI_CONFIG
 
 Per-user config file parseable by L<git-config(1)>.
 =item PI_CONFIG
 
 Per-user config file parseable by L<git-config(1)>.

diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index f63140c8ed32724a20554cb9bbb014cefdfd709b..08b1aab43ad1e8a4f6b06e6def310a4ecfb29eb0 100644 (file)
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -59,9 +59,19 @@ sub new {
 
                my $watch = $ibx->{watch} or return;
                if (is_maildir($watch)) {
 
                my $watch = $ibx->{watch} or return;
                if (is_maildir($watch)) {

-                       if (my $wm = $ibx->{watchheader}) {
-                               my ($k, $v) = split(/:/, $wm, 2);
-                               $ibx->{-watchheader} = [ $k, qr/\Q$v\E/ ];
+                       my $watch_hdrs = [];
+                       if (my $wh = $ibx->{watchheader}) {
+                               my ($k, $v) = split(/:/, $wh, 2);
+                               push @$watch_hdrs, [ $k, qr/\Q$v\E/ ];
+                       }
+                       if (my $list_ids = $ibx->{listid}) {
+                               for (@$list_ids) {
+                                       my $re = qr/<[ \t]*\Q$_\E[ \t]*>/;
+                                       push @$watch_hdrs, ['List-Id', $re ];
+                               }
+                       }
+                       if (scalar @$watch_hdrs) {
+                               $ibx->{-watchheaders} = $watch_hdrs;

                        }
                        my $new = "$watch/new";
                        my $cur = "$watch/cur";
                        }
                        my $new = "$watch/new";
                        my $cur = "$watch/cur";


@@ -159,10 +169,17 @@ sub _try_path {
                my $mime = _path_to_mime($path) or next;
                my $im = _importer_for($self, $ibx);
 
                my $mime = _path_to_mime($path) or next;
                my $im = _importer_for($self, $ibx);
 

-               my $wm = $ibx->{-watchheader};
-               if ($wm) {
-                       my $v = $mime->header_obj->header_raw($wm->[0]);
-                       next unless ($v && $v =~ $wm->[1]);
+               # any header match means it's eligible for the inbox:
+               if (my $watch_hdrs = $ibx->{-watchheaders}) {
+                       my $ok;
+                       my $hdr = $mime->header_obj;
+                       for my $wh (@$watch_hdrs) {
+                               my $v = $hdr->header_raw($wh->[0]);
+                               next unless defined($v) && $v =~ $wh->[1];
+                               $ok = 1;
+                               last;
+                       }
+                       next unless $ok;

                }
 
                if (my $scrub = $ibx->filter($im)) {
                }
 
                if (my $scrub = $ibx->filter($im)) {

diff --git a/script/public-inbox-mda b/script/public-inbox-mda
index 4e6e04e2174ebd9ab64d815df691c68294bae99f..2655a6c5ed064aad67b97f0c575a3212cde68344 100755 (executable)
--- a/script/public-inbox-mda
+++ b/script/public-inbox-mda
@@ -36,10 +36,21 @@ my $config = PublicInbox::Config->new;
 my $key = 'publicinboxmda.spamcheck';
 my $default = 'PublicInbox::Spamcheck::Spamc';
 my $spamc = PublicInbox::Spamcheck::get($config, $key, $default);
 my $key = 'publicinboxmda.spamcheck';
 my $default = 'PublicInbox::Spamcheck::Spamc';
 my $spamc = PublicInbox::Spamcheck::get($config, $key, $default);

+my $dst;

 my $recipient = $ENV{ORIGINAL_RECIPIENT};
 my $recipient = $ENV{ORIGINAL_RECIPIENT};

-defined $recipient or die "ORIGINAL_RECIPIENT not defined in ENV\n";
-my $dst = $config->lookup($recipient); # first check
-defined $dst or do_exit(67); # EX_NOUSER 5.1.1 user unknown
+if (defined $recipient) {
+       $dst = $config->lookup($recipient); # first check
+}
+if (!defined $dst) {
+       my $list_id = $simple->header('List-Id');
+       if (defined $list_id && $list_id =~ /<[ \t]*(.+)?[ \t]*>/) {
+               $dst = $config->lookup_list_id($1);
+       }
+       if (!defined $dst && !defined $recipient) {
+               die "ORIGINAL_RECIPIENT not defined in ENV\n";
+       }
+       defined $dst or do_exit(67); # EX_NOUSER 5.1.1 user unknown
+}

 $dst->{mainrepo} or do_exit(67);
 $dst = PublicInbox::InboxWritable->new($dst);
 
 $dst->{mainrepo} or do_exit(67);
 $dst = PublicInbox::InboxWritable->new($dst);
 

diff --git a/t/mda.t b/t/mda.t
index 5621b7d6b4e01d8449894b54ea2959cf1731db5a..3cab590b78f9a9d2728c213706bc229e3ba81aad 100644 (file)
--- a/t/mda.t
+++ b/t/mda.t
@@ -267,6 +267,34 @@ EOF
        }
 }
 
        }
 }
 

+# List-ID based delivery
+{
+       local $ENV{PI_EMERGENCY} = $faildir;
+       local $ENV{HOME} = $home;
+       local $ENV{ORIGINAL_RECIPIENT} = undef;
+       local $ENV{PATH} = $main_path;
+       my $list_id = 'foo.example.com';
+       my $mid = 'list-id-delivery@example.com';
+       my $simple = Email::Simple->new(<<EOF);
+From: user <user\@example.com>
+To: You <you\@example.com>
+Cc: $addr
+Message-ID: <$mid>
+List-Id: <$list_id>
+Subject: this message will be trained as spam
+Date: Thu, 01 Jan 1970 00:00:00 +0000
+
+EOF
+       system(qw(git config --file), $pi_config, "$cfgpfx.listid", $list_id);
+       $? == 0 or die "failed to set listid $?";
+       my $in = $simple->as_string;
+       IPC::Run::run([$mda], \$in);
+       is($?, 0, 'mda OK with List-Id match');
+       my $path = mid2path($mid);
+       my $msg = `git --git-dir=$maindir cat-file blob HEAD:$path`;
+       like($msg, qr/\Q$list_id\E/, 'delivered message w/ List-ID matches');
+}
+

 done_testing();
 
 sub fail_bad_header {
 done_testing();
 
 sub fail_bad_header {

diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t
index 0a5a8017ebf067aa273283776c0ce005040b7a92..99551ceb7d51191c8dac2cab0482ed50e013bac2 100644 (file)
--- a/t/watch_maildir_v2.t
+++ b/t/watch_maildir_v2.t
@@ -171,4 +171,35 @@ EOF
        is($both, $$msg, 'got original message back from v2');
 }
 
        is($both, $$msg, 'got original message back from v2');
 }
 

+{
+       my $want = <<'EOF';
+From: <u@example.com>
+List-Id: <i.want.you.to.want.me>
+Message-ID: <do.want@example.com>
+EOF
+       my $do_not_want = <<'EOF';
+From: <u@example.com>
+List-Id: <do.not.want>
+X-Mailing-List: no@example.com
+Message-ID: <do.not.want@example.com>
+EOF
+       my $cfg = $orig."$cfgpfx.listid=i.want.you.to.want.me\n";
+       PublicInbox::Emergency->new($maildir)->prepare(\$want);
+       PublicInbox::Emergency->new($maildir)->prepare(\$do_not_want);
+       my $config = PublicInbox::Config->new(\$cfg);
+       PublicInbox::WatchMaildir->new($config)->scan('full');
+       $ibx = $config->lookup_name('test');
+       my $num = $ibx->mm->num_for('do.want@example.com');
+       ok(defined $num, 'List-ID matched for watch');
+       $num = $ibx->mm->num_for('do.not.want@example.com');
+       is($num, undef, 'unaccepted List-ID matched for watch');
+
+       $cfg = $orig."$cfgpfx.watchheader=X-Mailing-List:no\@example.com\n";
+       $config = PublicInbox::Config->new(\$cfg);
+       PublicInbox::WatchMaildir->new($config)->scan('full');
+       $ibx = $config->lookup_name('test');
+       $num = $ibx->mm->num_for('do.not.want@example.com');
+       ok(defined $num, 'X-Mailing-List matched');
+}
+

 done_testing;
 done_testing;