watch: avoid unnecessary spawning on spam removals

This should further mitigate lock contention problems
when -watch is configured to watch on a Maildir for spam
while performing a large NNTP import.

There is now a small risk a message won't get removed because if
it's in the current (uncommitted) fast-import batch, but
unlikely given the batch size is now only 10 messages.

If a that small window is hit, flipping the \Seen flag
(e.g. marking it unread, and then read again) will trigger
another removal attempt via IMAP or Maildir.

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 700b40262e36d6eae460d12639106b6c2529062c..ee5ca2ea11b4f383e514ecdf117ac1dabe2c14c1 100644 (file)
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -461,6 +461,9 @@ sub init_bare {
        }
 }
 
        }
 }
 

+# true if locked and active
+sub active { !!$_[0]->{out} }
+

 sub done {
        my ($self) = @_;
        my $w = delete $self->{out} or return;
 sub done {
        my ($self) = @_;
        my $w = delete $self->{out} or return;

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index f2288904638e98e147af30bc68a1678a5dcda7f7..553dd839f39a9a56227e8a092d55e791ca54b2e0 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -655,6 +655,9 @@ sub checkpoint ($;$) {
 # public
 sub barrier { checkpoint($_[0], 1) };
 
 # public
 sub barrier { checkpoint($_[0], 1) };
 

+# true if locked and active
+sub active { !!$_[0]->{im} }
+

 # public
 sub done {
        my ($self) = @_;
 # public
 sub done {
        my ($self) = @_;

diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 5f78613961db1058506b8193c4ea7bd55c0522f3..0bb92d0a9055703233b7af5cafe53bdfa0232e2f 100644 (file)
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -134,15 +134,34 @@ sub _done_for_now {
 sub remove_eml_i { # each_inbox callback
        my ($ibx, $arg) = @_;
        my ($self, $eml, $loc) = @$arg;
 sub remove_eml_i { # each_inbox callback
        my ($ibx, $arg) = @_;
        my ($self, $eml, $loc) = @$arg;

+

        eval {
        eval {

-               my $im = _importer_for($self, $ibx);
-               $im->remove($eml, 'spam');
-               if (my $scrub = $ibx->filter($im)) {
-                       my $scrubbed = $scrub->scrub($eml, 1);
-                       if ($scrubbed && $scrubbed != REJECT) {
-                               $im->remove($scrubbed, 'spam');
+               # try to avoid taking a lock or unnecessary spawning
+               my $im = $self->{importers}->{"$ibx"};
+               my $scrubbed;
+               if ((!$im || !$im->active) && $ibx->over) {
+                       if (content_exists($ibx, $eml)) {
+                               # continue
+                       } elsif (my $scrub = $ibx->filter($im)) {
+                               $scrubbed = $scrub->scrub($eml, 1);
+                               if ($scrubbed && $scrubbed != REJECT &&
+                                         !content_exists($ibx, $scrubbed)) {
+                                       return;
+                               }
+                       } else {
+                               return;

                        }
                }
                        }
                }

+
+               $im //= _importer_for($self, $ibx); # may spawn fast-import
+               $im->remove($eml, 'spam');
+               $scrubbed //= do {
+                       my $scrub = $ibx->filter($im);
+                       $scrub ? $scrub->scrub($eml, 1) : undef;
+               };
+               if ($scrubbed && $scrubbed != REJECT) {
+                       $im->remove($scrubbed, 'spam');
+               }

        };
        if ($@) {
                warn "error removing spam at: $loc from $ibx->{name}: $@\n";
        };
        if ($@) {
                warn "error removing spam at: $loc from $ibx->{name}: $@\n";