X-Git-Url: http://www.git.stargrave.org/?p=public-inbox.git;a=blobdiff_plain;f=lib%2FPublicInbox%2FWatchMaildir.pm;h=8a8c1262737034f6415bec853d87c7b2898b2cd3;hp=424186bd35f5f36ea5d02cc18cfdc8561ec55551;hb=55b707d788ce13696e4411389583e720ea6dab01;hpb=44788dcad1863988c4535dd94ddd751bf9164cd2 diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm index 424186bd..8a8c1262 100644 --- a/lib/PublicInbox/WatchMaildir.pm +++ b/lib/PublicInbox/WatchMaildir.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2016 all contributors +# Copyright (C) 2016-2019 all contributors # License: AGPL-3.0+ # # ref: https://cr.yp.to/proto/maildir.html @@ -7,30 +7,40 @@ package PublicInbox::WatchMaildir; use strict; use warnings; use PublicInbox::MIME; -use Email::MIME::ContentType; -$Email::MIME::ContentType::STRICT_PARAMS = 0; # user input is imperfect -use PublicInbox::Git; -use PublicInbox::Import; -use PublicInbox::MDA; use PublicInbox::Spawn qw(spawn); -use File::Temp qw//; +use PublicInbox::InboxWritable; +use File::Temp 0.19 (); # 0.19 for ->newdir +use PublicInbox::Filter::Base qw(REJECT); +use PublicInbox::Spamcheck; +*maildir_path_load = *PublicInbox::InboxWritable::maildir_path_load; sub new { my ($class, $config) = @_; - my (%mdmap, @mdir, $spamc, $spamdir); + my (%mdmap, @mdir, $spamc); + my %uniq; # "publicinboxwatch" is the documented namespace # "publicinboxlearn" is legacy but may be supported # indefinitely... foreach my $pfx (qw(publicinboxwatch publicinboxlearn)) { my $k = "$pfx.watchspam"; - if (my $dir = $config->{$k}) { - if ($dir =~ s/\Amaildir://) { - $dir =~ s!/+\z!!; + defined(my $dirs = $config->{$k}) or next; + $dirs = [ $dirs ] if !ref($dirs); + for my $dir (@$dirs) { + if (is_maildir($dir)) { # skip "new", no MUA has seen it, yet. my $cur = "$dir/cur"; - $spamdir = $cur; + my $old = $mdmap{$cur}; + if (ref($old)) { + foreach my $ibx (@$old) { + warn <<""; +"$cur already watched for `$ibx->{name}' + + } + die; + } push @mdir, $cur; + $uniq{$cur}++; $mdmap{$cur} = 'watchspam'; } else { warn "unsupported $k=$dir\n"; @@ -39,47 +49,47 @@ sub new { } my $k = 'publicinboxwatch.spamcheck'; - my $spamcheck = $config->{$k}; - if ($spamcheck) { - if ($spamcheck eq 'spamc') { - $spamcheck = 'PublicInbox::Spamcheck::Spamc'; - } - if ($spamcheck =~ /::/) { - eval "require $spamcheck"; - $spamcheck = _spamcheck_cb($spamcheck->new); - } else { - warn "unsupported $k=$spamcheck\n"; - $spamcheck = undef; - } - } - foreach $k (keys %$config) { - $k =~ /\Apublicinbox\.([^\.]+)\.watch\z/ or next; - my $name = $1; - my $watch = $config->{$k}; - if ($watch =~ s/\Amaildir://) { - $watch =~ s!/+\z!!; - my $inbox = $config->lookup_name($name); - if (my $wm = $inbox->{watchheader}) { - my ($k, $v) = split(/:/, $wm, 2); - $inbox->{-watchheader} = [ $k, qr/\Q$v\E/ ]; + my $default = undef; + my $spamcheck = PublicInbox::Spamcheck::get($config, $k, $default); + $spamcheck = _spamcheck_cb($spamcheck) if $spamcheck; + + $config->each_inbox(sub { + # need to make all inboxes writable for spam removal: + my $ibx = $_[0] = PublicInbox::InboxWritable->new($_[0]); + + my $watch = $ibx->{watch} or return; + if (is_maildir($watch)) { + my $watch_hdrs = []; + if (my $wh = $ibx->{watchheader}) { + my ($k, $v) = split(/:/, $wh, 2); + push @$watch_hdrs, [ $k, qr/\Q$v\E/ ]; + } + if (my $list_ids = $ibx->{listid}) { + for (@$list_ids) { + my $re = qr/<[ \t]*\Q$_\E[ \t]*>/; + push @$watch_hdrs, ['List-Id', $re ]; + } + } + if (scalar @$watch_hdrs) { + $ibx->{-watchheaders} = $watch_hdrs; } my $new = "$watch/new"; my $cur = "$watch/cur"; - push @mdir, $new, $cur; - die "$new already in use\n" if $mdmap{$new}; - die "$cur already in use\n" if $mdmap{$cur}; - $mdmap{$new} = $mdmap{$cur} = $inbox; + push @mdir, $new unless $uniq{$new}++; + push @mdir, $cur unless $uniq{$cur}++; + + push @{$mdmap{$new} ||= []}, $ibx; + push @{$mdmap{$cur} ||= []}, $ibx; } else { warn "watch unsupported: $k=$watch\n"; } - } + }); return unless @mdir; my $mdre = join('|', map { quotemeta($_) } @mdir); $mdre = qr!\A($mdre)/!; bless { spamcheck => $spamcheck, - spamdir => $spamdir, mdmap => \%mdmap, mdir => \@mdir, mdre => $mdre, @@ -93,18 +103,6 @@ sub _done_for_now { my ($self) = @_; my $importers = $self->{importers}; foreach my $im (values %$importers) { - $im->done if $im->{nchg}; - } - - my $opendirs = $self->{opendirs}; - - # spamdir scanning means every importer remains open - my $spamdir = $self->{spamdir}; - return if defined($spamdir) && $opendirs->{$spamdir}; - - foreach my $im (values %$importers) { - # not done if we're scanning - next if $opendirs->{$im->{git}->{git_dir}}; $im->done; } } @@ -126,16 +124,16 @@ sub _remove_spam { my ($self, $path) = @_; # path must be marked as (S)een $path =~ /:2,[A-R]*S[T-Za-z]*\z/ or return; - my $mime = _path_to_mime($path) or return; - _force_mid($mime); + my $mime = maildir_path_load($path) or return; $self->{config}->each_inbox(sub { my ($ibx) = @_; eval { my $im = _importer_for($self, $ibx); $im->remove($mime, 'spam'); - if (my $scrub = _scrubber_for($ibx)) { - my $scrubbed = $scrub->scrub($mime) or return; - $scrubbed == 100 and return; + if (my $scrub = $ibx->filter($im)) { + my $scrubbed = $scrub->scrub($mime, 1); + $scrubbed or return; + $scrubbed == REJECT() and return; $im->remove($scrubbed, 'spam'); } }; @@ -146,64 +144,51 @@ sub _remove_spam { }) } -# used to hash the relevant portions of a message when there are conflicts -sub _hash_mime2 { - my ($mime) = @_; - require Digest::SHA; - my $dig = Digest::SHA->new('SHA-1'); - $dig->add($mime->header_obj->header_raw('Subject')); - $dig->add($mime->body_raw); - $dig->hexdigest; -} - -sub _force_mid { - my ($mime) = @_; - # probably a bad idea, but we inject a Message-Id if - # one is missing, here.. - my $mid = $mime->header_obj->header_raw('Message-Id'); - if (!defined $mid || $mid =~ /\A\s*\z/) { - $mid = '<' . _hash_mime2($mime) . '@generated>'; - $mime->header_set('Message-Id', $mid); - } -} - sub _try_path { my ($self, $path) = @_; - my @p = split(m!/+!, $path); - return if $p[-1] !~ /\A[a-zA-Z0-9][\w:,=\.]+\z/; - if ($p[-1] =~ /:2,([A-Z]+)\z/i) { - my $flags = $1; - return if $flags =~ /[DT]/; # no [D]rafts or [T]rashed mail - } - return unless -f $path; + return unless PublicInbox::InboxWritable::is_maildir_path($path); if ($path !~ $self->{mdre}) { warn "unrecognized path: $path\n"; return; } - my $inbox = $self->{mdmap}->{$1}; - unless ($inbox) { + my $inboxes = $self->{mdmap}->{$1}; + unless ($inboxes) { warn "unmappable dir: $1\n"; return; } - if (!ref($inbox) && $inbox eq 'watchspam') { + if (!ref($inboxes) && $inboxes eq 'watchspam') { return _remove_spam($self, $path); } - my $im = _importer_for($self, $inbox); - my $mime = _path_to_mime($path) or return; - $mime->header_set($_) foreach @PublicInbox::MDA::BAD_HEADERS; - my $wm = $inbox->{-watchheader}; - if ($wm) { - my $v = $mime->header_obj->header_raw($wm->[0]); - return unless ($v && $v =~ $wm->[1]); - } - if (my $scrub = _scrubber_for($inbox)) { - my $ret = $scrub->scrub($mime) or return; - $ret == 100 and return; - $mime = $ret; - } - _force_mid($mime); - $im->add($mime, $self->{spamcheck}); + my $warn_cb = $SIG{__WARN__} || sub { print STDERR @_ }; + local $SIG{__WARN__} = sub { + $warn_cb->("path: $path\n"); + $warn_cb->(@_); + }; + foreach my $ibx (@$inboxes) { + my $mime = maildir_path_load($path) or next; + my $im = _importer_for($self, $ibx); + + # any header match means it's eligible for the inbox: + if (my $watch_hdrs = $ibx->{-watchheaders}) { + my $ok; + my $hdr = $mime->header_obj; + for my $wh (@$watch_hdrs) { + my $v = $hdr->header_raw($wh->[0]); + next unless defined($v) && $v =~ $wh->[1]; + $ok = 1; + last; + } + next unless $ok; + } + + if (my $scrub = $ibx->filter($im)) { + my $ret = $scrub->scrub($mime) or next; + $ret == REJECT() and next; + $mime = $ret; + } + $im->add($mime, $self->{spamcheck}); + } } sub quit { trigger_scan($_[0], 'quit') } @@ -274,57 +259,16 @@ sub scan { trigger_scan($self, 'cont') if keys %$opendirs; } -sub _path_to_mime { - my ($path) = @_; - if (open my $fh, '<', $path) { - local $/; - my $str = <$fh>; - $str or return; - return PublicInbox::MIME->new(\$str); - } elsif ($!{ENOENT}) { - return; - } else { - warn "failed to open $path: $!\n"; - return; - } -} - sub _importer_for { - my ($self, $inbox) = @_; - my $im = $inbox->{-import} ||= eval { - my $git = $inbox->git; - my $name = $inbox->{name}; - my $addr = $inbox->{-primary_address}; - PublicInbox::Import->new($git, $name, $addr, $inbox); - }; - + my ($self, $ibx) = @_; my $importers = $self->{importers}; + my $im = $importers->{"$ibx"} ||= $ibx->importer(0); if (scalar(keys(%$importers)) > 2) { - delete $importers->{"$im"}; + delete $importers->{"$ibx"}; _done_for_now($self); } - $importers->{"$im"} = $im; -} - -sub _scrubber_for { - my ($inbox) = @_; - my $f = $inbox->{filter}; - if ($f && $f =~ /::/) { - my @args = (-inbox => $inbox); - # basic line splitting, only - # Perhaps we can have proper quote splitting one day... - ($f, @args) = split(/\s+/, $f) if $f =~ /\s+/; - - eval "require $f"; - if ($@) { - warn $@; - } else { - # e.g: PublicInbox::Filter::Vger->new(@args) - return $f->new(@args); - } - } - undef; + $importers->{"$ibx"} = $im; } sub _spamcheck_cb { @@ -340,4 +284,11 @@ sub _spamcheck_cb { } } +sub is_maildir { + $_[0] =~ s!\Amaildir:!! or return; + $_[0] =~ tr!/!/!s; + $_[0] =~ s!/\z!!; + $_[0]; +} + 1;