# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# "lei prune-mail-sync" drops dangling sync information
-package PublicInbox::LeiPruneMailSync;
+# "lei refresh-mail-sync" drops dangling sync information
+# and attempts to detect moved files
+package PublicInbox::LeiRefreshMailSync;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiExportKw;
use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::Import;
sub eml_match ($$) {
my ($eml, $oidbin) = @_;
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
$oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest;
}
for my $d (@try) {
my $src = "$mdir/$d/$$id";
if ($self->{verify}) {
- my $eml = eml_from_path($src) or next;
+ my $eml = eml_from_path($src) // next;
return if eml_match($eml, $oidbin);
} elsif (-f $src) {
return;
$self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid);
}
+# detects missed file moves
+sub pmdir_cb { # called via LeiPmdir->each_mdir_fn
+ my ($self, $f, $fl) = @_;
+ my ($folder, $bn) = ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) or
+ die "BUG: $f was not from a Maildir?";
+ substr($folder, 0, 0) = 'maildir:'; # add prefix
+ my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
+ return if defined($lms->name_oidbin($folder, $bn));
+ my $eml = eml_from_path($f) // return;
+ my $oidbin = $self->{lei}->git_oid($eml)->digest;
+ $self->{lei}->{sto}->ipc_do('lms_set_src', $oidbin, $folder, \$bn);
+}
+
sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
my ($self, $input, @args) = @_;
my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
if ($input =~ /\Amaildir:(.+)/i) {
- my $mdir = $1;
- $lms->each_src($input, \&prune_mdir, $self, $mdir);
+ $lms->each_src($input, \&prune_mdir, $self, my $mdir = $1);
+ $self->{lse} //= $self->{lei}->{sto}->search;
+ # call pmdir_cb (via maildir_each_file -> each_mdir_fn)
+ PublicInbox::LeiInput::input_path_url($self, $input);
} elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
my $mic = $self->{lei}->{net}->mic_for_folder($uri);
$uids = +{ map { $_ => undef } @$uids };
$lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri);
} else { die "BUG: $input not supported" }
- my $wait = $self->{lei}->{sto}->ipc_do('done');
+ $self->{lei}->{pkt_op_p}->pkt_do('sto_done_request');
}
-sub lei_prune_mail_sync {
+sub lei_refresh_mail_sync {
my ($lei, @folders) = @_;
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
lei/store uninitialized, see lei-import(1)
EOM
- if (my $lms = $lei->lms) {
- if (defined(my $all = $lei->{opt}->{all})) {
- $lms->group2folders($lei, $all, \@folders) or return;
- } else {
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
- }
- } else {
- return $lei->fail(<<EOM);
+ my $lms = $lei->lms or return $lei->fail(<<EOM);
lei mail_sync.sqlite3 uninitialized, see lei-import(1)
EOM
+ if (defined(my $all = $lei->{opt}->{all})) {
+ $lms->group2folders($lei, $all, \@folders) or return;
+ } else {
+ my $err = $lms->arg2folder($lei, \@folders);
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ return $lei->fail($err->{fail}) if $err->{fail};
}
+ undef $lms; # must be done before fork
$sto->write_prepare($lei);
my $self = bless { missing_ok => 1 }, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
my $ops = {};
- $sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
}
no warnings 'once';
-*_complete_prune_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+*_complete_refresh_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
--- /dev/null
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+require_mods(qw(lei));
+
+my $stop_daemon = sub { # needed since we don't have inotify
+ lei_ok qw(daemon-pid);
+ chomp(my $pid = $lei_out);
+ $pid > 0 or xbail "bad pid: $pid";
+ kill('TERM', $pid) or xbail "kill: $!";
+ for (0..10) {
+ tick;
+ kill(0, $pid) or last;
+ }
+ kill(0, $pid) and xbail "daemon still running (PID:$pid)";
+};
+
+test_lei({ daemon_only => 1 }, sub {
+ my $d = "$ENV{HOME}/d";
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
+ lei_ok qw(daemon-pid);
+ lei_ok qw(add-external), "$ro_home/t2";
+ lei_ok qw(q mid:testmessage@example.com -o), "Maildir:$d";
+ my (@o) = glob("$d/*/*");
+ scalar(@o) == 1 or xbail('multiple results', \@o);
+ my ($bn0) = ($o[0] =~ m!/([^/]+)\z!);
+
+ my $oid = '9bf1002c49eb075df47247b74d69bcd555e23422';
+ lei_ok 'inspect', "blob:$oid";
+ my $before = json_utf8->decode($lei_out);
+ my $exp0 = { 'mail-sync' => { "maildir:$d" => [ $bn0 ] } };
+ is_deeply($before, $exp0, 'inspect shows expected');
+
+ $stop_daemon->();
+ my $dst = $o[0];
+ $dst =~ s/:2,.*\z// and $dst =~ s!/cur/!/new/! and
+ rename($o[0], $dst) or xbail "rename($o[0] => $dst): $!";
+
+ lei_ok 'inspect', "blob:$oid";
+ is_deeply(json_utf8->decode($lei_out),
+ $before, 'inspect unchanged immediately after restart');
+ lei_ok 'refresh-mail-sync', '--all';
+ lei_ok 'inspect', "blob:$oid";
+ my ($bn1) = ($dst =~ m!/([^/]+)\z!);
+ my $exp1 = { 'mail-sync' => { "maildir:$d" => [ $bn1 ] } };
+ is_deeply(json_utf8->decode($lei_out), $exp1,
+ 'refresh-mail-sync updated location');
+
+ $stop_daemon->();
+ rename($dst, "$d/unwatched") or xbail "rename $dst out-of-the-way $!";
+
+ lei_ok 'refresh-mail-sync', $d;
+ lei_ok 'inspect', "blob:$oid";
+ is($lei_out, '{}', 'no known locations after "removal"');
+ lei_ok 'refresh-mail-sync', "Maildir:$d";
+
+ $stop_daemon->();
+ rename("$d/unwatched", $dst) or xbail "rename $dst back";
+
+ lei_ok 'refresh-mail-sync', "Maildir:$d";
+ lei_ok 'inspect', "blob:$oid";
+ is_deeply(json_utf8->decode($lei_out), $exp1,
+ 'replaced file noted again');
+});
+
+done_testing;