lei q: support --import-augment for IMAP

[public-inbox.git] / lib / PublicInbox / LeiToMail.pm

diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 31b8aba8ef073b10f9953f3b6645524099f9182b..b3228a59e31f4219e0a5b8cf83f9ac8608bd91e5 100644 (file)
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -267,8 +267,19 @@ sub _mbox_write_cb ($$) {
        }
 }
 
-sub _augment_file { # maildir_each_eml cb
-       my ($f, undef, $eml, $lei, $mod, $shard) = @_;
+sub update_kw_maybe ($$$$) {
+       my ($lei, $lse, $eml, $kw) = @_;
+       return unless $lse;
+       my $x = $lse->kw_changed($eml, $kw);
+       if ($x) {
+               $lei->{sto}->ipc_do('set_eml', $eml, @$kw);
+       } elsif (!defined($x)) {
+               # TODO: xkw
+       }
+}
+
+sub _augment_or_unlink { # maildir_each_eml cb
+       my ($f, $kw, $eml, $lei, $lse, $mod, $shard, $unlink) = @_;
        if ($mod) {
                # can't get dirent.d_ino w/ pure Perl, so we extract the OID
                # if it looks like one:
@@ -276,8 +287,9 @@ sub _augment_file { # maildir_each_eml cb
                                $1 : sha256_hex($f);
                my $recno = hex(substr($hex, 0, 8));
                return if ($recno % $mod) != $shard;
+               update_kw_maybe($lei, $lse, $eml, $kw);
        }
-       _augment($eml, $lei);
+       $unlink ? unlink($f) : _augment($eml, $lei);
 }
 
 # maildir_each_file callback, \&CORE::unlink doesn't work with it
@@ -419,34 +431,51 @@ sub _pre_augment_maildir {
 sub _do_augment_maildir {
        my ($self, $lei) = @_;
        my $dst = $lei->{ovv}->{dst};
+       my $lse = $lei->{sto}->search if $lei->{opt}->{'import-augment'};
+       my ($mod, $shard) = @{$self->{shard_info} // []};
        if ($lei->{opt}->{augment}) {
                my $dedupe = $lei->{dedupe};
                if ($dedupe && $dedupe->prepare_dedupe) {
-                       my ($mod, $shard) = @{$self->{shard_info} // []};
                        PublicInbox::MdirReader::maildir_each_eml($dst,
-                                               \&_augment_file,
-                                               $lei, $mod, $shard);
+                                               \&_augment_or_unlink,
+                                               $lei, $lse, $mod, $shard);
                        $dedupe->pause_dedupe;
                }
-       } else { # clobber existing Maildir
+       } elsif ($lse) {
+               PublicInbox::MdirReader::maildir_each_eml($dst,
+                                       \&_augment_or_unlink,
+                                       $lei, $lse, $mod, $shard, 1);
+       } else {# clobber existing Maildir
                PublicInbox::MdirReader::maildir_each_file($dst, \&_unlink);
        }
 }
 
-sub _augment_imap { # PublicInbox::NetReader::imap_each cb
-       my ($url, $uid, $kw, $eml, $lei) = @_;
-       _augment($eml, $lei);
+sub _imap_augment_or_delete { # PublicInbox::NetReader::imap_each cb
+       my ($url, $uid, $kw, $eml, $lei, $lse, $delete_mic) = @_;
+       update_kw_maybe($lei, $lse, $eml, $kw);
+       if ($delete_mic) {
+               $lei->{net}->imap_delete_1($url, $uid, $delete_mic);
+       } else {
+               _augment($eml, $lei);
+       }
 }
 
 sub _do_augment_imap {
        my ($self, $lei) = @_;
        my $net = $lei->{net};
+       my $lse = $lei->{sto}->search if $lei->{opt}->{'import-augment'};
        if ($lei->{opt}->{augment}) {
                my $dedupe = $lei->{dedupe};
                if ($dedupe && $dedupe->prepare_dedupe) {
-                       $net->imap_each($self->{uri}, \&_augment_imap, $lei);
+                       $net->imap_each($self->{uri}, \&_imap_augment_or_delete,
+                                       $lei, $lse);
                        $dedupe->pause_dedupe;
                }
+       } elsif ($lse) {
+               my $delete_mic;
+               $net->imap_each($self->{uri}, \&_imap_augment_or_delete,
+                                       $lei, $lse, \$delete_mic);
+               $delete_mic->expunge if $delete_mic;
        } elsif (!$self->{-wq_worker_nr}) { # undef or 0
                # clobber existing IMAP folder
                $net->imap_delete_all($self->{uri});
@@ -520,6 +549,8 @@ sub do_augment { # slow, runs in wq worker
 # fast (spawn compressor or mkdir), runs in same process as pre_augment
 sub post_augment {
        my ($self, $lei, @args) = @_;
+       my $wait = $lei->{opt}->{'import-augment'} ?
+                       $lei->{sto}->ipc_do('checkpoint', 1) : 0;
        # _post_augment_mbox
        my $m = $self->can("_post_augment_$self->{base_type}") or return;
        $m->($self, $lei, @args);