v2writable: allow disabling parallelization

[public-inbox.git] / lib / PublicInbox / WatchMaildir.pm

diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index 8e10e43f056c677d4deb6e116a71692a55eed704..e28e602ad026c3bb6b5790f8d33c12b137049762 100644 (file)
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # ref: https://cr.yp.to/proto/maildir.html
@@ -7,8 +7,6 @@ package PublicInbox::WatchMaildir;
 use strict;
 use warnings;
 use PublicInbox::MIME;
-use Email::MIME::ContentType;
-$Email::MIME::ContentType::STRICT_PARAMS = 0; # user input is imperfect
 use PublicInbox::Git;
 use PublicInbox::Import;
 use PublicInbox::MDA;
@@ -93,18 +91,6 @@ sub _done_for_now {
        my ($self) = @_;
        my $importers = $self->{importers};
        foreach my $im (values %$importers) {
-               $im->done if $im->{nchg};
-       }
-
-       my $opendirs = $self->{opendirs};
-
-       # spamdir scanning means every importer remains open
-       my $spamdir = $self->{spamdir};
-       return if defined($spamdir) && $opendirs->{$spamdir};
-
-       foreach my $im (values %$importers) {
-               # not done if we're scanning
-               next if $opendirs->{$im->{git}->{git_dir}};
                $im->done;
        }
 }
@@ -127,16 +113,15 @@ sub _remove_spam {
        # path must be marked as (S)een
        $path =~ /:2,[A-R]*S[T-Za-z]*\z/ or return;
        my $mime = _path_to_mime($path) or return;
-       _force_mid($mime);
        $self->{config}->each_inbox(sub {
                my ($ibx) = @_;
                eval {
                        my $im = _importer_for($self, $ibx);
-                       $im->remove($mime);
+                       $im->remove($mime, 'spam');
                        if (my $scrub = _scrubber_for($ibx)) {
                                my $scrubbed = $scrub->scrub($mime) or return;
                                $scrubbed == 100 and return;
-                               $im->remove($scrubbed);
+                               $im->remove($scrubbed, 'spam');
                        }
                };
                if ($@) {
@@ -146,31 +131,10 @@ sub _remove_spam {
        })
 }
 
-# used to hash the relevant portions of a message when there are conflicts
-sub _hash_mime2 {
-       my ($mime) = @_;
-       require Digest::SHA;
-       my $dig = Digest::SHA->new('SHA-1');
-       $dig->add($mime->header_obj->header_raw('Subject'));
-       $dig->add($mime->body_raw);
-       $dig->hexdigest;
-}
-
-sub _force_mid {
-       my ($mime) = @_;
-       # probably a bad idea, but we inject a Message-Id if
-       # one is missing, here..
-       my $mid = $mime->header_obj->header_raw('Message-Id');
-       if (!defined $mid || $mid =~ /\A\s*\z/) {
-               $mid = '<' . _hash_mime2($mime) . '@generated>';
-               $mime->header_set('Message-Id', $mid);
-       }
-}
-
 sub _try_path {
        my ($self, $path) = @_;
        my @p = split(m!/+!, $path);
-       return if $p[-1] !~ /\A[a-zA-Z0-9][\w:,=\.]+\z/;
+       return if $p[-1] !~ /\A[a-zA-Z0-9][\-\w:,=\.]+\z/;
        if ($p[-1] =~ /:2,([A-Z]+)\z/i) {
                my $flags = $1;
                return if $flags =~ /[DT]/; # no [D]rafts or [T]rashed mail
@@ -202,7 +166,6 @@ sub _try_path {
                $mime = $ret;
        }
 
-       _force_mid($mime);
        $im->add($mime, $self->{spamcheck});
 }
 
@@ -225,7 +188,7 @@ sub watch {
 
 sub trigger_scan {
        my ($self, $base) = @_;
-       my $dir = $self->{scandir} or die "not watch-ing, yet\n";
+       my $dir = $self->{scandir} or return;
        open my $fh, '>', "$dir/$base" or die "open $dir/$base failed: $!\n";
        close $fh or die "close $dir/$base failed: $!\n";
 }
@@ -235,6 +198,7 @@ sub scan {
        if ($path =~ /quit\z/) {
                %{$self->{opendirs}} = ();
                _done_for_now($self);
+               delete $self->{scandir};
                $self->{quit} = 1;
                return;
        }
@@ -291,10 +255,21 @@ sub _path_to_mime {
 sub _importer_for {
        my ($self, $inbox) = @_;
        my $im = $inbox->{-import} ||= eval {
-               my $git = $inbox->git;
-               my $name = $inbox->{name};
-               my $addr = $inbox->{-primary_address};
-               PublicInbox::Import->new($git, $name, $addr, $inbox);
+               my $v = $inbox->{version} || 1;
+               if ($v == 2) {
+                       eval { require PublicInbox::V2Writable };
+                       die "v2 not supported: $@\n" if $@;
+                       my $v2w = PublicInbox::V2Writable->new($inbox);
+                       $v2w->{parallel} = 0;
+                       $v2w;
+               } elsif ($v == 1) {
+                       my $git = $inbox->git;
+                       my $name = $inbox->{name};
+                       my $addr = $inbox->{-primary_address};
+                       PublicInbox::Import->new($git, $name, $addr, $inbox);
+               } else {
+                       die "unsupported inbox version: $v\n";
+               }
        };
 
        my $importers = $self->{importers};