X-Git-Url: http://www.git.stargrave.org/?a=blobdiff_plain;f=lib%2FPublicInbox%2FLeiExportKw.pm;h=d2396fa7d697bb7b8b9ddf092ff27de65230f350;hb=23af251dd607c4e75ab1e68063f2c885c48cc035;hp=fabc01f8851272609ab38377d7b5d697149c1b1d;hpb=30bf4d7a2a8c1cf933ce8c6187559518423a123a;p=public-inbox.git diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm index fabc01f8..d2396fa7 100644 --- a/lib/PublicInbox/LeiExportKw.pm +++ b/lib/PublicInbox/LeiExportKw.pm @@ -7,11 +7,11 @@ use strict; use v5.10.1; use parent qw(PublicInbox::IPC PublicInbox::LeiInput); use Errno qw(EEXIST ENOENT); +use PublicInbox::Syscall qw(rename_noreplace); sub export_kw_md { # LeiMailSync->each_src callback my ($oidbin, $id, $self, $mdir) = @_; - my $oidhex = unpack('H*', $oidbin); - my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return; + my $sto_kw = $self->{lse}->oidbin_keywords($oidbin) or return; my $bn = $$id; my ($md_kw, $unknown, @try); if ($bn =~ s/:2,([a-zA-Z]*)\z//) { @@ -26,62 +26,65 @@ sub export_kw_md { # LeiMailSync->each_src callback } $bn .= ':2,'. PublicInbox::LeiToMail::kw2suffix([keys %$sto_kw], @$unknown); + return if $bn eq $$id; my $dst = "$mdir/cur/$bn"; - my @fail; + my $lei = $self->{lei}; for my $d (@try) { my $src = "$mdir/$d/$$id"; - next if $src eq $dst; - - # we use link(2) + unlink(2) since rename(2) may - # inadvertently clobber if the "uniquefilename" part wasn't - # actually unique. - if (link($src, $dst)) { # success - # unlink(2) may ENOENT from parallel invocation, - # ignore it, but not other serious errors - if (!unlink($src) and $! != ENOENT) { - $self->{lei}->child_error(1, - "E: unlink($src): $!"); - } + if (rename_noreplace($src, $dst)) { # success $self->{lms}->mv_src("maildir:$mdir", - $oidbin, $id, $bn) or die; - return; # success anyways if link(2) worked - } - if ($! == ENOENT && !-e $src) { # some other process moved it - $self->{lms}->clear_src("maildir:$mdir", $id); - next; - } - push @fail, $src if $! != EEXIST; + $oidbin, $id, $bn); + return; # success + } elsif ($! == EEXIST) { # lost race with lei/store? + return; + } elsif ($! != ENOENT) { + $lei->child_error(1, + "E: rename_noreplace($src -> $dst): $!"); + } # else loop @try } - return unless @fail; - # both tries failed my $e = $!; - my $orig = '['.join('|', @fail).']'; - $self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e"); + # both tries failed + my $oidhex = unpack('H*', $oidbin); + my $src = "$mdir/{".join(',', @try)."}/$$id"; + $lei->child_error(1, "rename_noreplace($src -> $dst) ($oidhex): $e"); + for (@try) { return if -e "$mdir/$_/$$id" } + $self->{lms}->clear_src("maildir:$mdir", $id); } sub export_kw_imap { # LeiMailSync->each_src callback my ($oidbin, $id, $self, $mic) = @_; - my $oidhex = unpack('H*', $oidbin); - my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return; + my $sto_kw = $self->{lse}->oidbin_keywords($oidbin) or return; $self->{imap_mod_kw}->($self->{nwr}, $mic, $id, [ keys %$sto_kw ]); } # overrides PublicInbox::LeiInput::input_path_url sub input_path_url { my ($self, $input, @args) = @_; - my $lms = $self->{lms} //= $self->{lse}->lms; - $lms->lms_begin; + $self->{lms}->lms_write_prepare; if ($input =~ /\Amaildir:(.+)/i) { my $mdir = $1; require PublicInbox::LeiToMail; # kw2suffix - $lms->each_src($input, \&export_kw_md, $self, $mdir); + $self->{lms}->each_src($input, \&export_kw_md, $self, $mdir); } elsif ($input =~ m!\Aimaps?://!i) { my $uri = PublicInbox::URIimap->new($input); my $mic = $self->{nwr}->mic_for_folder($uri); - $lms->each_src($$uri, \&export_kw_imap, $self, $mic); - $mic->expunge; + if ($mic && !$self->{nwr}->can_store_flags($mic)) { + my $m = "$input does not support PERMANENTFLAGS"; + if (defined $self->{lei}->{opt}->{all}) { + $self->{lei}->qerr("# $m"); + } else { # set error code if user explicitly requested + $self->{lei}->child_error(0, "E: $m"); + } + return; + } + if ($mic) { + $self->{lms}->each_src($$uri, \&export_kw_imap, + $self, $mic); + $mic->expunge; + } else { + $self->{lei}->child_error(0, "$input unavailable: $@"); + } } else { die "BUG: $input not supported" } - $lms->lms_commit; } sub lei_export_kw { @@ -89,80 +92,23 @@ sub lei_export_kw { my $sto = $lei->_lei_store or return $lei->fail(<search; - my $lms = $lse->lms or return $lei->fail(<lms or return $lei->fail(<{opt}; - my $all = $opt->{all}; - my @all = $lms->folders; - if (defined $all) { # --all= - my %x = map { $_ => $_ } split(/,/, $all); - my @ok = grep(defined, delete(@x{qw(local remote), ''})); - my @no = keys %x; - if (@no) { - @no = (join(',', @no)); - return $lei->fail(<fail("--all=$all not understood"); - } else { - @inc = @all; - } - for (@inc) { - push(@folders, $_) unless $seen{$_}++; - } - } - return $lei->fail(<{opt}->{all})) { # --all= + $lms->group2folders($lei, $all, \@folders) or return; + @folders = grep(/\A(?:maildir|imaps?):/i, @folders); } else { - my %all = map { $_ => 1 } @all; - my @no; - for (@folders) { - next if $all{$_}; # ok - if (-d "$_/new" && -d "$_/cur") { - my $d = 'maildir:'.$lei->abs_path($_); - push(@no, $_) unless $all{$d}; - $_ = $d; - } elsif (m!\Aimaps?://!i) { - my $orig = $_; - my $res = $lms->match_imap_url($orig, $all); - if (ref $res) { - $_ = $$res; - $lei->qerr(<err($res) if defined $res; - push @no, $orig; - } - } else { - push @no, $_; - } - } - my $no = join("\n\t", @no); - return $lei->fail(<arg2folder($lei, \@folders); # may die } - my $self = bless { lse => $lse }, __PACKAGE__; + $lms->lms_pause; + my $self = bless { lse => $sto->search, lms => $lms }, __PACKAGE__; $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs $self->prepare_inputs($lei, \@folders) or return; - my $j = $opt->{jobs} // scalar(@{$self->{inputs}}) || 1; if (my @ro = grep(!/\A(?:maildir|imaps?):/i, @folders)) { return $lei->fail("cannot export to read-only folders: @ro"); } - my $m = $opt->{mode} // 'merge'; + my $m = $lei->{opt}->{mode} // 'merge'; if ($m eq 'merge') { # default $self->{-merge_kw} = 1; } elsif ($m eq 'set') { @@ -176,24 +122,21 @@ EOM $self->{nwr} = bless $net, 'PublicInbox::NetWriter'; $self->{imap_mod_kw} = $net->can($self->{-merge_kw} ? 'imap_add_kw' : 'imap_set_kw'); + $self->{nwr}->{-skip_creat} = 1; } - undef $lms; - my $ops = {}; - $lei->{auth}->op_merge($ops, $self) if $lei->{auth}; - $self->{-wq_nr_workers} = $j // 1; # locked - (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); - $lei->{wq1} = $self; $lei->{-err_type} = 'non-fatal'; - net_merge_all_done($self) unless $lei->{auth}; - $op_c->op_wait_event($ops); # calls net_merge_all_done if $lei->{auth} + $lei->wq1_start($self); } sub _complete_export_kw { my ($lei, @argv) = @_; - my $sto = $lei->_lei_store or return; - my $lms = $sto->search->lms or return; + my $lms = $lei->lms or return (); my $match_cb = $lei->complete_url_prepare(\@argv); - map { $match_cb->($_) } $lms->folders; + # filter-out read-only sources: + my @k = grep(m!(?:maildir|imaps?):!, + $lms->folders($argv[-1] // undef, 1)); + my @m = map { $match_cb->($_) } @k; + @m ? @m : @k; } no warnings 'once'; @@ -201,7 +144,4 @@ no warnings 'once'; *ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child; *net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done; -# the following works even when LeiAuth is lazy-loaded -*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all; - 1;