lei import|convert: support mbox locking on reads

In case somebody is writing non-atomically, ensure we
take read locks when opening mbox files for reading.

v2: squash: load MboxLock even for .eml files

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b5bdda215008ebd95ba0bb49418e865112f1f44e..0da244994a1c3a437b7a44a09b77fa2abd24d2f0 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -172,12 +172,12 @@ our %CMD = ( # sorted in order of importance/use:
 'import' => [ 'LOCATION...|--stdin',
        'one-time import/update from URL or filesystem',
        qw(stdin| offset=i recursive|r exclude=s include|I=s
-       in-format|F=s kw|keywords|flags! C=s@),
+       lock=s@ in-format|F=s kw|keywords|flags! C=s@),
        ],
 'convert' => [ 'LOCATION...|--stdin',
        'one-time conversion from URL or filesystem to another format',
        qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s quiet|q
-       kw|keywords|flags! C=s@),
+       lock=s@ kw|keywords|flags! C=s@),
        ],
 'config' => [ '[...]', sub {
                'git-config(1) wrapper for '._config_path($_[0]);
@@ -218,6 +218,9 @@ my %OPTDESC = (
 'help|h' => 'show this built-in help',
 'C=s@' => [ 'DIR', 'chdir to specify to directory' ],
 'quiet|q' => 'be quiet',
+'lock=s@' => [ 'METHOD|dotlock|fcntl|flock|none',
+       'mbox(5) locking method(s) to use (default: fcntl,dotlock)' ],
+
 'globoff|g' => "do not match locations using '*?' wildcards ".
                "and\xa0'[]'\x{a0}ranges",
 'verbose|v+' => 'be more verbose',
@@ -407,11 +410,13 @@ sub check_input_format ($;$) {
                my $err = $files ? "regular file(s):\n@$files" : '--stdin';
                return fail($self, "--$opt_key unset for $err");
        }
+       require PublicInbox::MboxLock if $files;
        return 1 if $fmt eq 'eml';
        # XXX: should this handle {gz,bz2,xz}? that's currently in LeiToMail
        require PublicInbox::MboxReader;
-       PublicInbox::MboxReader->can($fmt) ||
-                               fail($self, "--$opt_key=$fmt unrecognized");
+       PublicInbox::MboxReader->can($fmt) or
+               return fail($self, "--$opt_key=$fmt unrecognized");
+       1;
 }
 
 sub out ($;@) {

diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 45d42c9c76040376908756c0b6b64f7d5a711635..4c0bbd882c1c8bfedf0970a9e5e3c65198b2f8ec 100644 (file)
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -62,9 +62,11 @@ sub do_convert { # via wq_do
                        $ifmt = lc $1;
                }
                if (-f $input) {
-                       open my $fh, '<', $input or
-                                       return $lei->fail("open $input: $!");
-                       convert_fh($self, $ifmt, $fh, $input);
+                       my $m = $lei->{opt}->{'lock'} //
+                                       ($ifmt eq 'eml' ? ['none'] :
+                                       PublicInbox::MboxLock->defaults);
+                       my $mbl = PublicInbox::MboxLock->acq($input, 0, $m);
+                       convert_fh($self, $ifmt, $mbl->{fh}, $input);
                } elsif (-d _) {
                        PublicInbox::MdirReader::maildir_each_eml($input,
                                                        \&mdir_cb, $self);
@@ -109,6 +111,7 @@ sub call { # the main "lei convert" method
 
                        }
                        if (-f $input_path) {
+                               require PublicInbox::MboxLock;
                                require PublicInbox::MboxReader;
                                PublicInbox::MboxReader->can($ifmt) or return
                                        $lei->fail("$ifmt not supported");

diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 7f247b640937bf4387ac353112ade62bff197976..c2c98030074282e0329767069d41dcd340102054 100644 (file)
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -80,10 +80,11 @@ sub call { # the main "lei import" method
                        my $ifmt = lc $1;
                        if (($fmt // $ifmt) ne $ifmt) {
                                return $lei->fail(<<"");
---format=$fmt and `$ifmt:' conflict
+--in-format=$fmt and `$ifmt:' conflict
 
                        }
                        if (-f $input_path) {
+                               require PublicInbox::MboxLock;
                                require PublicInbox::MboxReader;
                                PublicInbox::MboxReader->can($ifmt) or return
                                        $lei->fail("$ifmt not supported");
@@ -142,7 +143,7 @@ error reading $input: $!
                        $cb->(undef, $fh, \&_import_eml, $lei->{sto}, $set_kw);
                }
        };
-       $lei->child_error(1 << 8, "<stdin>: $@") if $@;
+       $lei->child_error(1 << 8, "$input: $@") if $@;
 }
 
 sub _import_maildir { # maildir_each_file cb
@@ -171,10 +172,10 @@ sub import_path_url {
                $ifmt = lc $1;
        }
        if (-f $input) {
-               open my $fh, '<', $input or return $lei->child_error(1 << 8, <<"");
-unable to open $input: $!
-
-               _import_fh($lei, $fh, $input, $ifmt);
+               my $m = $lei->{opt}->{'lock'} // ($ifmt eq 'eml' ? ['none'] :
+                               PublicInbox::MboxLock->defaults);
+               my $mbl = PublicInbox::MboxLock->acq($input, 0, $m);
+               _import_fh($lei, $mbl->{fh}, $input, $ifmt);
        } elsif (-d _ && (-d "$input/cur" || -d "$input/new")) {
                return $lei->fail(<<EOM) if $ifmt && $ifmt ne 'maildir';
 $input appears to a be a maildir, not $ifmt