MANIFEST | 3 ++- lib/PublicInbox/LEI.pm | 3 ++- lib/PublicInbox/LeiPruneMailSync.pm => lib/PublicInbox/LeiRefreshMailSync.pm | 54 ++++++++++++++++++++++++++++++++++------------------- lib/PublicInbox/LeiStore.pm | 5 +++++ t/lei-export-kw.t | 1 - t/lei-refresh-mail-sync.t | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/MANIFEST b/MANIFEST index 640eabd1f0f8d0f2be42e6d9269a6ef9ad2a0f8a..9f11f2f9ec010f775ae6bc44d23ab45ca045b685 100644 --- a/MANIFEST +++ b/MANIFEST @@ -235,9 +235,9 @@ lib/PublicInbox/LeiNoteEvent.pm lib/PublicInbox/LeiOverview.pm lib/PublicInbox/LeiP2q.pm lib/PublicInbox/LeiPmdir.pm -lib/PublicInbox/LeiPruneMailSync.pm lib/PublicInbox/LeiQuery.pm lib/PublicInbox/LeiRediff.pm +lib/PublicInbox/LeiRefreshMailSync.pm lib/PublicInbox/LeiRemote.pm lib/PublicInbox/LeiRm.pm lib/PublicInbox/LeiRmWatch.pm @@ -450,6 +450,7 @@ t/lei-q-kw.t t/lei-q-remote-import.t t/lei-q-save.t t/lei-q-thread.t +t/lei-refresh-mail-sync.t t/lei-sigpipe.t t/lei-tag.t t/lei-up.t diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index ec10323155cd8cbeeea258b7acb732f454fd72cf..9794497b6e3e03a4612a139d7f37f335bb673b49 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -263,7 +263,7 @@ lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!), @net_opt, @c_opt ], 'forget-mail-sync' => [ 'LOCATION...', 'forget sync information for a mail folder', @c_opt ], -'prune-mail-sync' => [ 'LOCATION...|--all', +'refresh-mail-sync' => [ 'LOCATION...|--all', 'prune dangling sync data for a mail folder', 'all:s', @c_opt ], 'export-kw' => [ 'LOCATION...|--all', 'one-time export of keywords of sync sources', @@ -616,6 +616,7 @@ $ops->{'|'} = [ \&sigpipe_handler, $lei ]; $ops->{x_it} = [ \&x_it, $lei ]; $ops->{child_error} = [ \&child_error, $lei ]; $ops->{incr} = [ \&incr, $lei ]; + $ops->{sto_done_request} = [ \&sto_done_request, $lei, $lei->{sock} ]; $ops; } diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm rename from lib/PublicInbox/LeiPruneMailSync.pm rename to lib/PublicInbox/LeiRefreshMailSync.pm index 3678bd048dbc781f55047c998c3f3ba26215f945..3c083965514296779b4a1c5503af29c5cb30995a 100644 --- a/lib/PublicInbox/LeiPruneMailSync.pm +++ b/lib/PublicInbox/LeiRefreshMailSync.pm @@ -1,16 +1,20 @@ # Copyright (C) 2021 all contributors # License: AGPL-3.0+ -# "lei prune-mail-sync" drops dangling sync information -package PublicInbox::LeiPruneMailSync; +# "lei refresh-mail-sync" drops dangling sync information +# and attempts to detect moved files +package PublicInbox::LeiRefreshMailSync; use strict; use v5.10.1; use parent qw(PublicInbox::IPC PublicInbox::LeiInput); use PublicInbox::LeiExportKw; use PublicInbox::InboxWritable qw(eml_from_path); +use PublicInbox::ContentHash qw(git_sha); +use PublicInbox::Import; sub eml_match ($$) { my ($eml, $oidbin) = @_; + $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS; $oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest; } @@ -20,7 +24,7 @@ my @try = $$id =~ /:2,[a-zA-Z]*\z/ ? qw(cur new) : qw(new cur); for my $d (@try) { my $src = "$mdir/$d/$$id"; if ($self->{verify}) { - my $eml = eml_from_path($src) or next; + my $eml = eml_from_path($src) // next; return if eml_match($eml, $oidbin); } elsif (-f $src) { return; @@ -38,12 +42,27 @@ $self->{lei}->qerr("# $url $uid gone"); $self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid); } +# detects missed file moves +sub pmdir_cb { # called via LeiPmdir->each_mdir_fn + my ($self, $f, $fl) = @_; + my ($folder, $bn) = ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) or + die "BUG: $f was not from a Maildir?"; + substr($folder, 0, 0) = 'maildir:'; # add prefix + my $lms = $self->{-lms_ro} //= $self->{lei}->lms; + return if defined($lms->name_oidbin($folder, $bn)); + my $eml = eml_from_path($f) // return; + my $oidbin = $self->{lei}->git_oid($eml)->digest; + $self->{lei}->{sto}->ipc_do('lms_set_src', $oidbin, $folder, \$bn); +} + sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url my ($self, $input, @args) = @_; my $lms = $self->{-lms_ro} //= $self->{lei}->lms; if ($input =~ /\Amaildir:(.+)/i) { - my $mdir = $1; - $lms->each_src($input, \&prune_mdir, $self, $mdir); + $lms->each_src($input, \&prune_mdir, $self, my $mdir = $1); + $self->{lse} //= $self->{lei}->{sto}->search; + # call pmdir_cb (via maildir_each_file -> each_mdir_fn) + PublicInbox::LeiInput::input_path_url($self, $input); } elsif ($input =~ m!\Aimaps?://!i) { my $uri = PublicInbox::URIimap->new($input); my $mic = $self->{lei}->{net}->mic_for_folder($uri); @@ -51,34 +70,31 @@ my $uids = $mic->search('UID 1:*'); $uids = +{ map { $_ => undef } @$uids }; $lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri); } else { die "BUG: $input not supported" } - my $wait = $self->{lei}->{sto}->ipc_do('done'); + $self->{lei}->{pkt_op_p}->pkt_do('sto_done_request'); } -sub lei_prune_mail_sync { +sub lei_refresh_mail_sync { my ($lei, @folders) = @_; my $sto = $lei->_lei_store or return $lei->fail(<lms) { - if (defined(my $all = $lei->{opt}->{all})) { - $lms->group2folders($lei, $all, \@folders) or return; - } else { - my $err = $lms->arg2folder($lei, \@folders); - $lei->qerr(@{$err->{qerr}}) if $err->{qerr}; - return $lei->fail($err->{fail}) if $err->{fail}; - } - } else { - return $lei->fail(<lms or return $lei->fail(<{opt}->{all})) { + $lms->group2folders($lei, $all, \@folders) or return; + } else { + my $err = $lms->arg2folder($lei, \@folders); + $lei->qerr(@{$err->{qerr}}) if $err->{qerr}; + return $lei->fail($err->{fail}) if $err->{fail}; } + undef $lms; # must be done before fork $sto->write_prepare($lei); my $self = bless { missing_ok => 1 }, __PACKAGE__; $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs $self->prepare_inputs($lei, \@folders) or return; my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1; my $ops = {}; - $sto->write_prepare($lei); $lei->{auth}->op_merge($ops, $self) if $lei->{auth}; $self->{-wq_nr_workers} = $j // 1; # locked (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); @@ -89,7 +105,7 @@ $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth} } no warnings 'once'; -*_complete_prune_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw; +*_complete_refresh_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw; *ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child; *net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done; diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index e8bcb04e972bcf31945f13800ac04be2bb3bf3c9..32f55abd2ad7384e52e2481b4ed96a5cb004d80a 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -293,6 +293,11 @@ my ($self, $oidhex, $folder, $id) = @_; _lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id); } +sub lms_set_src { + my ($self, $oidbin, $folder, $id) = @_; + _lms_rw($self)->set_src($oidbin, $folder, $id); +} + sub _remove_if_local { # git->cat_async arg my ($bref, $oidhex, $type, $size, $self) = @_; $self->{im}->remove($bref) if $bref; diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t index 9531949a1627f648fd871240ab3e55f31abf6d58..1fe940bb6d894977edb062fda3ca03dae3bffc5e 100644 --- a/t/lei-export-kw.t +++ b/t/lei-export-kw.t @@ -6,7 +6,6 @@ use File::Copy qw(cp); use File::Path qw(make_path); require_mods(qw(lei -imapd Mail::IMAPClient)); my ($tmpdir, $for_destroy) = tmpdir; -my ($ro_home, $cfg_path) = setup_public_inboxes; my $expect = eml_load('t/data/0001.patch'); test_lei({ tmpdir => $tmpdir }, sub { my $home = $ENV{HOME}; diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t new file mode 100644 index 0000000000000000000000000000000000000000..ff558277f8cc003d751c0f47a8aae4ce9e67f9c7 --- /dev/null +++ b/t/lei-refresh-mail-sync.t @@ -0,0 +1,67 @@ +#!perl -w +# Copyright (C) all contributors +# License: AGPL-3.0+ +use strict; use v5.10.1; use PublicInbox::TestCommon; +require_mods(qw(lei)); + +my $stop_daemon = sub { # needed since we don't have inotify + lei_ok qw(daemon-pid); + chomp(my $pid = $lei_out); + $pid > 0 or xbail "bad pid: $pid"; + kill('TERM', $pid) or xbail "kill: $!"; + for (0..10) { + tick; + kill(0, $pid) or last; + } + kill(0, $pid) and xbail "daemon still running (PID:$pid)"; +}; + +test_lei({ daemon_only => 1 }, sub { + my $d = "$ENV{HOME}/d"; + my ($ro_home, $cfg_path) = setup_public_inboxes; + lei_ok qw(daemon-pid); + lei_ok qw(add-external), "$ro_home/t2"; + lei_ok qw(q mid:testmessage@example.com -o), "Maildir:$d"; + my (@o) = glob("$d/*/*"); + scalar(@o) == 1 or xbail('multiple results', \@o); + my ($bn0) = ($o[0] =~ m!/([^/]+)\z!); + + my $oid = '9bf1002c49eb075df47247b74d69bcd555e23422'; + lei_ok 'inspect', "blob:$oid"; + my $before = json_utf8->decode($lei_out); + my $exp0 = { 'mail-sync' => { "maildir:$d" => [ $bn0 ] } }; + is_deeply($before, $exp0, 'inspect shows expected'); + + $stop_daemon->(); + my $dst = $o[0]; + $dst =~ s/:2,.*\z// and $dst =~ s!/cur/!/new/! and + rename($o[0], $dst) or xbail "rename($o[0] => $dst): $!"; + + lei_ok 'inspect', "blob:$oid"; + is_deeply(json_utf8->decode($lei_out), + $before, 'inspect unchanged immediately after restart'); + lei_ok 'refresh-mail-sync', '--all'; + lei_ok 'inspect', "blob:$oid"; + my ($bn1) = ($dst =~ m!/([^/]+)\z!); + my $exp1 = { 'mail-sync' => { "maildir:$d" => [ $bn1 ] } }; + is_deeply(json_utf8->decode($lei_out), $exp1, + 'refresh-mail-sync updated location'); + + $stop_daemon->(); + rename($dst, "$d/unwatched") or xbail "rename $dst out-of-the-way $!"; + + lei_ok 'refresh-mail-sync', $d; + lei_ok 'inspect', "blob:$oid"; + is($lei_out, '{}', 'no known locations after "removal"'); + lei_ok 'refresh-mail-sync', "Maildir:$d"; + + $stop_daemon->(); + rename("$d/unwatched", $dst) or xbail "rename $dst back"; + + lei_ok 'refresh-mail-sync', "Maildir:$d"; + lei_ok 'inspect', "blob:$oid"; + is_deeply(json_utf8->decode($lei_out), $exp1, + 'replaced file noted again'); +}); + +done_testing;