MANIFEST | 1 + lib/PublicInbox/LEI.pm | 2 ++ lib/PublicInbox/LeiPruneMailSync.pm | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/PublicInbox/NetReader.pm | 19 +++++++++++++++++++ lib/PublicInbox/NetWriter.pm | 21 ++------------------- diff --git a/MANIFEST b/MANIFEST index 7bdbf25200cdc0b894aeac35e7ab150d5e77d1ff..3d4c6cbd91668b9da3916d220484214402ff4a67 100644 --- a/MANIFEST +++ b/MANIFEST @@ -222,6 +222,7 @@ lib/PublicInbox/LeiMirror.pm lib/PublicInbox/LeiOverview.pm lib/PublicInbox/LeiP2q.pm lib/PublicInbox/LeiPmdir.pm +lib/PublicInbox/LeiPruneMailSync.pm lib/PublicInbox/LeiQuery.pm lib/PublicInbox/LeiRediff.pm lib/PublicInbox/LeiRemote.pm diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 77fc5b8fa278983dea3a332fc3277894ac5ff7d7..265b7047fdf63541fb0bd263071efc445e674c26 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -245,6 +245,8 @@ lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!), qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ], 'forget-mail-sync' => [ 'LOCATION...', 'forget sync information for a mail folder', @c_opt ], +'prune-mail-sync' => [ 'LOCATION...|--all', + 'prune dangling sync data for a mail folder', 'all:s', @c_opt ], 'export-kw' => [ 'LOCATION...|--all', 'one-time export of keywords of sync sources', qw(all:s mode=s), @c_opt ], diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiPruneMailSync.pm new file mode 100644 index 0000000000000000000000000000000000000000..79f3325df68855127d6eb0b77e31a16b138024d1 --- /dev/null +++ b/lib/PublicInbox/LeiPruneMailSync.pm @@ -0,0 +1,97 @@ +# Copyright (C) 2021 all contributors +# License: AGPL-3.0+ + +# "lei prune-mail-sync" drops dangling sync information +package PublicInbox::LeiPruneMailSync; +use strict; +use v5.10.1; +use parent qw(PublicInbox::IPC PublicInbox::LeiInput); +use PublicInbox::LeiExportKw; +use PublicInbox::InboxWritable qw(eml_from_path); + +sub eml_match ($$) { + my ($eml, $oidbin) = @_; + $oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest; +} + +sub prune_mdir { # lms->each_src callback + my ($oidbin, $id, $self, $mdir) = @_; + my @try = $$id =~ /:2,[a-zA-Z]*\z/ ? qw(cur new) : qw(new cur); + for my $d (@try) { + my $src = "$mdir/$d/$$id"; + if ($self->{verify}) { + my $eml = eml_from_path($src) or next; + return if eml_match($eml, $oidbin); + } elsif (-f $src) { + return; + } + } + # both tries failed + $self->{lei}->qerr("# maildir:$mdir $$id gone"); + $self->{lei}->{sto}->ipc_do('lms_clear_src', "maildir:$mdir", $id); +} + +sub prune_imap { # lms->each_src callback + my ($oidbin, $uid, $self, $uids, $url) = @_; + return if exists $uids->{$uid}; + $self->{lei}->qerr("# $url $uid gone"); + $self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid); +} + +sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url + my ($self, $input, @args) = @_; + my $lms = $self->{-lms_ro} //= $self->{lse}->lms; + if ($input =~ /\Amaildir:(.+)/i) { + my $mdir = $1; + $lms->each_src($input, \&prune_mdir, $self, $mdir); + } elsif ($input =~ m!\Aimaps?://!i) { + my $uri = PublicInbox::URIimap->new($input); + my $mic = $self->{lei}->{net}->mic_for_folder($uri); + my $uids = $mic->search('UID 1:*'); + $uids = +{ map { $_ => undef } @$uids }; + $lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri); + } else { die "BUG: $input not supported" } + my $wait = $self->{lei}->{sto}->ipc_do('done'); +} + +sub lei_prune_mail_sync { + my ($lei, @folders) = @_; + my $sto = $lei->_lei_store or return $lei->fail(<search; + my $lms = $lse->lms or return $lei->fail(<{opt}->{all})) { + $lms->group2folders($lei, $all, \@folders) or return; + } else { + my $err = $lms->arg2folder($lei, \@folders); + $lei->qerr(@{$err->{qerr}}) if $err->{qerr}; + return $lei->fail($err->{fail}) if $err->{fail}; + } + delete $lms->{dbh}; + $sto->write_prepare($lei); + my $self = bless { lse => $lse }, __PACKAGE__; + $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs + $self->prepare_inputs($lei, \@folders) or return; + my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1; + undef $lms; # for fork + my $ops = {}; + $sto->write_prepare($lei); + $lei->{auth}->op_merge($ops, $self) if $lei->{auth}; + $self->{-wq_nr_workers} = $j // 1; # locked + (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); + $lei->{wq1} = $self; + $lei->{-err_type} = 'non-fatal'; + net_merge_all_done($self) unless $lei->{auth}; + $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth} +} + +no warnings 'once'; +*_complete_prune_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw; +*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child; +*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all; +*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done; + +1; diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm index 058f431398a6a8bcf5579e668db30ef6c1660bd5..2795a9d4d85e4bbc21c7669a13825db1d6870ba8 100644 --- a/lib/PublicInbox/NetReader.pm +++ b/lib/PublicInbox/NetReader.pm @@ -747,4 +747,23 @@ } sub new { bless {}, shift }; +# updates $uri with UIDVALIDITY +sub mic_for_folder { + my ($self, $uri) = @_; + my $mic = $self->mic_get($uri) or die "E: not connected: $@"; + my $m = $self->isa('PublicInbox::NetWriter') ? 'select' : 'examine'; + $mic->$m($uri->mailbox) or return; + my $uidval; + for ($mic->Results) { + /^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next; + $uidval = $1; + last; + } + $uidval //= $mic->uidvalidity($uri->mailbox) or + die "E: failed to get uidvalidity from <$uri>: $@"; + $uri->uidvalidity($uidval); + $mic; +} + + 1; diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm index 8ec7f85c876b568cf6102cc9dbdf705203bb2d03..82288e6bc6391106c0d2123ee7bc1e03c24af489 100644 --- a/lib/PublicInbox/NetWriter.pm +++ b/lib/PublicInbox/NetWriter.pm @@ -26,26 +26,9 @@ $mic->append_string($folder, $$bref, $f, $smsg->internaldate) or die "APPEND $folder: $@"; } -# updates $uri with UIDVALIDITY -sub mic_for_folder { - my ($self, $uri) = @_; - my $mic = $self->mic_get($uri) or die "E: not connected: $@"; - $mic->select($uri->mailbox) or return; - my $uidval; - for ($mic->Results) { - /^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next; - $uidval = $1; - last; - } - $uidval //= $mic->uidvalidity($uri->mailbox) or - die "E: failed to get uidvalidity from <$uri>: $@"; - $uri->uidvalidity($uidval); - $mic; -} - sub imap_delete_all { my ($self, $uri) = @_; - my $mic = mic_for_folder($self, $uri) or return; + my $mic = $self->mic_for_folder($uri) or return; my $sec = $self->can('uri_section')->($uri); local $0 = $uri->mailbox." $sec"; if ($mic->delete_message('1:*')) { @@ -55,7 +38,7 @@ } sub imap_delete_1 { my ($self, $uri, $uid, $delete_mic) = @_; - $$delete_mic //= mic_for_folder($self, $uri) or return; + $$delete_mic //= $self->mic_for_folder($uri) or return; $$delete_mic->delete_message($uid); }