lei q: import flags when clobbering/augmenting Maildirs

[public-inbox.git] / lib / PublicInbox / LeiToMail.pm

diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index df813064c992593939dbe46deb0dc8a49ae97066..3420b06e3af36cbbbf8c6b5fdeb431c9da25d004 100644 (file)
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -11,7 +11,6 @@ use PublicInbox::Lock;
 use PublicInbox::ProcessPipe;
 use PublicInbox::Spawn qw(which spawn popen_rd);
 use PublicInbox::LeiDedupe;
-use PublicInbox::OnDestroy;
 use PublicInbox::Git;
 use PublicInbox::GitAsyncCat;
 use PublicInbox::PktOp qw(pkt_do);
@@ -20,7 +19,6 @@ use IO::Handle; # ->autoflush
 use Fcntl qw(SEEK_SET SEEK_END O_CREAT O_EXCL O_WRONLY);
 use Errno qw(EEXIST ESPIPE ENOENT EPIPE);
 use Digest::SHA qw(sha256_hex);
-my ($maildir_each_file);
 
 # struggles with short-lived repos, Gcf2Client makes little sense with lei;
 # but we may use in-process libgit2 in the future.
@@ -219,7 +217,7 @@ sub _post_augment_mbox { # open a compressor process
        my $cmd = zsfx2cmd($zsfx, undef, $lei);
        my ($r, $w) = @{delete $lei->{zpipe}};
        my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2} };
-       my $pid = spawn($cmd, $lei->{env}, $rdr);
+       my $pid = spawn($cmd, undef, $rdr);
        my $pp = gensym;
        my $dup = bless { "pid.$pid" => $cmd }, ref($lei);
        $dup->{$_} = $lei->{$_} for qw(2 sock);
@@ -232,7 +230,7 @@ sub _post_augment_mbox { # open a compressor process
 sub decompress_src ($$$) {
        my ($in, $zsfx, $lei) = @_;
        my $cmd = zsfx2cmd($zsfx, 1, $lei);
-       popen_rd($cmd, $lei->{env}, { 0 => $in, 2 => $lei->{2} });
+       popen_rd($cmd, undef, { 0 => $in, 2 => $lei->{2} });
 }
 
 sub dup_src ($) {
@@ -269,8 +267,8 @@ sub _mbox_write_cb ($$) {
        }
 }
 
-sub _augment_file { # maildir_each_file cb
-       my ($f, $lei, $mod, $shard) = @_;
+sub _augment_or_unlink { # maildir_each_eml cb
+       my ($f, $kw, $eml, $lei, $lse, $mod, $shard, $unlink) = @_;
        if ($mod) {
                # can't get dirent.d_ino w/ pure Perl, so we extract the OID
                # if it looks like one:
@@ -278,9 +276,16 @@ sub _augment_file { # maildir_each_file cb
                                $1 : sha256_hex($f);
                my $recno = hex(substr($hex, 0, 8));
                return if ($recno % $mod) != $shard;
+               if ($lse) {
+                       my $x = $lse->kw_changed($eml, $kw);
+                       if ($x) {
+                               $lei->{sto}->ipc_do('set_eml', $eml, @$kw);
+                       } elsif (!defined($x)) {
+                               # TODO: xkw
+                       }
+               }
        }
-       my $eml = PublicInbox::InboxWritable::eml_from_path($f) or return;
-       _augment($eml, $lei);
+       $unlink ? unlink($f) : _augment($eml, $lei);
 }
 
 # maildir_each_file callback, \&CORE::unlink doesn't work with it
@@ -376,12 +381,7 @@ sub new {
        my $dst = $lei->{ovv}->{dst};
        my $self = bless {}, $cls;
        if ($fmt eq 'maildir') {
-               $maildir_each_file //= do {
-                       require PublicInbox::MdirReader;
-                       PublicInbox::MdirReader->can('maildir_each_file');
-               };
-               $lei->{opt}->{augment} and
-                       require PublicInbox::InboxWritable; # eml_from_path
+               require PublicInbox::MdirReader;
                $self->{base_type} = 'maildir';
                -e $dst && !-d _ and die
                                "$dst exists and is not a directory\n";
@@ -427,19 +427,31 @@ sub _pre_augment_maildir {
 sub _do_augment_maildir {
        my ($self, $lei) = @_;
        my $dst = $lei->{ovv}->{dst};
+       my $lse = $lei->{sto}->search if $lei->{opt}->{'import-augment'};
+       my ($mod, $shard) = @{$self->{shard_info} // []};
        if ($lei->{opt}->{augment}) {
                my $dedupe = $lei->{dedupe};
                if ($dedupe && $dedupe->prepare_dedupe) {
-                       my ($mod, $shard) = @{$self->{shard_info} // []};
-                       $maildir_each_file->($dst, \&_augment_file,
-                                               $lei, $mod, $shard);
+                       PublicInbox::MdirReader::maildir_each_eml($dst,
+                                               \&_augment_or_unlink,
+                                               $lei, $lse, $mod, $shard);
                        $dedupe->pause_dedupe;
                }
-       } else { # clobber existing Maildir
-               $maildir_each_file->($dst, \&_unlink);
+       } elsif ($lse) {
+               PublicInbox::MdirReader::maildir_each_eml($dst,
+                                       \&_augment_or_unlink,
+                                       $lei, $lse, $mod, $shard, 1);
+       } else {# clobber existing Maildir
+               PublicInbox::MdirReader::maildir_each_file($dst, \&_unlink);
        }
 }
 
+sub _post_augment_maildir {
+       my ($self, $lei) = @_;
+       $lei->{opt}->{'import-augment'} or return;
+       my $wait = $lei->{sto}->ipc_do('checkpoint', 1);
+}
+
 sub _augment_imap { # PublicInbox::NetReader::imap_each cb
        my ($url, $uid, $kw, $eml, $lei) = @_;
        _augment($eml, $lei);
@@ -464,11 +476,19 @@ sub _pre_augment_mbox {
        my ($self, $lei) = @_;
        my $dst = $lei->{ovv}->{dst};
        if ($dst ne '/dev/stdout') {
-               my $mode = -p $dst ? '>' : '+>>';
-               if (-f _ && !$lei->{opt}->{augment} and !unlink($dst)) {
-                       $! == ENOENT or die "unlink($dst): $!";
+               my $out;
+               if (-p $dst) {
+                       open $out, '>', $dst or die "open($dst): $!";
+               } elsif (-f _ || !-e _) {
+                       require PublicInbox::MboxLock;
+                       my $m = $lei->{opt}->{'lock'} //
+                                       PublicInbox::MboxLock->defaults;
+                       $self->{mbl} = PublicInbox::MboxLock->acq($dst, 1, $m);
+                       $out = $self->{mbl}->{fh};
+                       if (!$lei->{opt}->{augment} and !truncate($out, 0)) {
+                               die "truncate($dst): $!";
+                       }
                }
-               open my $out, $mode, $dst or die "open($dst): $!";
                $lei->{old_1} = $lei->{1}; # keep for spawning MUA
                $lei->{1} = $out;
        }