From b584a53f053a7629e656df00ccf6f168c260af48 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 7 Sep 2021 11:32:10 +0000 Subject: [PATCH] lei up: support --all for IMAP folders Since "lei up" is expected to be a heavily-used command, better support for IMAP seems like a reasonable idea. This is inefficient since we waste an IMAP(S) TCP connection since it dies when an auth-only LeiUp worker process dies, but it's better than not working at all, right now. --- Documentation/lei-up.pod | 15 ++++- lib/PublicInbox/LEI.pm | 19 ++++--- lib/PublicInbox/LeiAuth.pm | 15 +++-- lib/PublicInbox/LeiToMail.pm | 3 +- lib/PublicInbox/LeiUp.pm | 104 ++++++++++++++++++++++------------- xt/net_writer-imap.t | 1 + 6 files changed, 102 insertions(+), 55 deletions(-) diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod index cea0f619..ca4cf4fe 100644 --- a/Documentation/lei-up.pod +++ b/Documentation/lei-up.pod @@ -6,15 +6,24 @@ lei-up - update a saved search lei up [OPTIONS] OUTPUT -lei up [OPTIONS] --all=TYPE +lei up [OPTIONS] --all[=] =head1 DESCRIPTION -Update the saved search at C or all saved searches of C -(currently C is the only supported value). +Update the saved search at C or all saved searches. =head1 OPTIONS +=over + +=item --all[=] + +C<--all> updates all saved searches (listed in L). +C<--all=local> only updates local mailboxes, C<--all=remote> only +updates remote mailboxes (currently C and C). + +=back + The following options, described in L, are supported. =over diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index bd44cfae..a258722e 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -605,16 +605,19 @@ sub incr { $self->{counters}->{$field} += $nr; } +sub pkt_ops { + my ($lei, $ops) = @_; + $ops->{'!'} = [ \&fail_handler, $lei ]; + $ops->{'|'} = [ \&sigpipe_handler, $lei ]; + $ops->{x_it} = [ \&x_it, $lei ]; + $ops->{child_error} = [ \&child_error, $lei ]; + $ops->{incr} = [ \&incr, $lei ]; + $ops; +} + sub workers_start { my ($lei, $wq, $jobs, $ops, $flds) = @_; - $ops = { - '!' => [ \&fail_handler, $lei ], - '|' => [ \&sigpipe_handler, $lei ], - 'x_it' => [ \&x_it, $lei ], - 'child_error' => [ \&child_error, $lei ], - 'incr' => [ \&incr, $lei ], - ($ops ? %$ops : ()), - }; + $ops = pkt_ops($lei, { ($ops ? %$ops : ()) }); $ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ]; my $end = $lei->pkt_op_pair; my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker"; diff --git a/lib/PublicInbox/LeiAuth.pm b/lib/PublicInbox/LeiAuth.pm index 465a2758..9b09cecf 100644 --- a/lib/PublicInbox/LeiAuth.pm +++ b/lib/PublicInbox/LeiAuth.pm @@ -30,10 +30,16 @@ sub do_auth_atfork { # used by IPC WQ workers return if $wq->{-wq_worker_nr} != 0; # only first worker calls this my $lei = $wq->{lei}; my $net = $lei->{net}; + if ($net->{-auth_done}) { # from previous worker... (ugly) + $lei->{pkt_op_p}->pkt_do('net_merge_continue', $net) or + $lei->fail("pkt_do net_merge_continue: $!"); + return; + } eval { # fill auth info (may prompt user or read netrc) my $mics = $net->imap_common_init($lei); my $nn = $net->nntp_common_init($lei); # broadcast successful auth info to lei-daemon: + $net->{-auth_done} = 1; $lei->{pkt_op_p}->pkt_do('net_merge_continue', $net) or die "pkt_do net_merge_continue: $!"; $net->{mics_cached} = $mics if $mics; @@ -51,14 +57,15 @@ sub net_merge_all { # called in wq worker via wq_broadcast # called by top-level lei-daemon when first worker is done with auth # passes updated net auth info to current workers sub net_merge_continue { - my ($wq, $net_new) = @_; + my ($wq, $lei, $net_new) = @_; + $wq->{-net_new} = $net_new; # for "lei up" $wq->wq_broadcast('PublicInbox::LeiAuth::net_merge_all', $net_new); - $wq->net_merge_all_done; # defined per-WQ + $wq->net_merge_all_done($lei); # defined per-WQ } sub op_merge { # prepares PktOp->pair ops - my ($self, $ops, $wq) = @_; - $ops->{net_merge_continue} = [ \&net_merge_continue, $wq ]; + my ($self, $ops, $wq, $lei) = @_; + $ops->{net_merge_continue} = [ \&net_merge_continue, $wq, $lei ]; } sub new { bless \(my $x), __PACKAGE__ } diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index 2c7a92de..01f08384 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -396,7 +396,8 @@ sub new { } elsif ($fmt =~ /\Aimaps?\z/) { require PublicInbox::NetWriter; require PublicInbox::URIimap; - my $net = PublicInbox::NetWriter->new; + # {net} may exist from "lei up" for auth + my $net = $lei->{net} // PublicInbox::NetWriter->new; $net->{quiet} = $lei->{opt}->{quiet}; my $uri = PublicInbox::URIimap->new($dst)->canonical; $net->add_url($$uri); diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm index a39d6047..30358e9d 100644 --- a/lib/PublicInbox/LeiUp.pm +++ b/lib/PublicInbox/LeiUp.pm @@ -5,8 +5,14 @@ package PublicInbox::LeiUp; use strict; use v5.10.1; +# n.b. we use LeiInput to setup IMAP auth +use parent qw(PublicInbox::IPC PublicInbox::LeiInput); use PublicInbox::LeiSavedSearch; -use parent qw(PublicInbox::IPC); +use PublicInbox::DS; +use PublicInbox::PktOp; +use PublicInbox::LeiFinmsg; +use PublicInbox::LEI; +my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP sub up1 ($$) { my ($lei, $out) = @_; @@ -48,75 +54,95 @@ sub up1 ($$) { sub up1_redispatch { my ($lei, $out, $op_p) = @_; - require PublicInbox::LeiFinmsg; - $lei->{fmsg} //= PublicInbox::LeiFinmsg->new($lei->{2}); my $l = bless { %$lei }, ref($lei); $l->{opt} = { %{$l->{opt}} }; - delete $l->{sock}; - $l->{''} = $op_p; # daemon only + delete $l->{sock}; # do not close + $l->{''} = $op_p; # daemon only ($l => $lei => script/lei) # make close($l->{1}) happy in lei->dclose open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!"); + local $PublicInbox::LEI::current_lei = $l; $l->{1} = $fh; eval { $l->qerr("# updating $out"); up1($l, $out); - $l->qerr("# $out done"); }; - $l->child_error(0, $@) if $@; + $lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail() +} + +sub redispatch_all ($$) { + my ($self, $lei) = @_; + # re-dispatch into our event loop w/o creating an extra fork-level + $lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei->{2}); + my ($op_c, $op_p) = PublicInbox::PktOp->pair; + for my $o (@{$self->{local} // []}, @{$self->{remote} // []}) { + PublicInbox::DS::requeue(sub { + up1_redispatch($lei, $o, $op_p); + }); + } + $lei->event_step_init; + $lei->pkt_ops($op_c->{ops} = { '' => [$lei->can('dclose'), $lei] }); } sub lei_up { my ($lei, @outs) = @_; - $lei->{lse} = $lei->_lei_store(1)->search; my $opt = $lei->{opt}; - my @local; - if (defined $opt->{all}) { + my $self = bless { -mail_sync => 1 }, __PACKAGE__; + $lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search; + if (defined(my $all = $opt->{all})) { return $lei->fail("--all and @outs incompatible") if @outs; length($opt->{mua}//'') and return $lei->fail('--all and --mua= are incompatible'); - - # supporting IMAP outputs is more involved due to - # git-credential prompts. TODO: add this in 1.8 - $opt->{all} eq 'local' or return - $lei->fail('only --all=local works at the moment'); - my @all = PublicInbox::LeiSavedSearch::list($lei); - @local = grep(!m!\Aimaps?://!i, @all); + @outs = PublicInbox::LeiSavedSearch::list($lei); + if ($all eq 'local') { + $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ]; + } elsif ($all eq 'remote') { + $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ]; + } elsif ($all eq '') { + $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ]; + $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ]; + } else { + $lei->fail("only --all=$all not understood"); + } } else { - @local = @outs; + $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ]; + $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ]; } - if (scalar(@outs) > 1) { - length($opt->{mua}//'') and return $lei->fail(<{local} // []} + @{$self->{remote} // []}) > 1 && + length($opt->{mua} // '')) and return $lei->fail(<fail(<{remote}) { # setup lei->{auth} + $self->prepare_inputs($lei, $self->{remote}) or return; } - if (scalar(@local) > 1) { - $lei->_lei_store->write_prepare($lei); # share early - # daemon mode, re-dispatch into our event loop w/o - # creating an extra fork-level - require PublicInbox::DS; - require PublicInbox::PktOp; - my ($op_c, $op_p) = PublicInbox::PktOp->pair; - for my $o (@local) { - PublicInbox::DS::requeue(sub { - up1_redispatch($lei, $o, $op_p); - }); - } - $lei->event_step_init; - $op_c->{ops} = { '' => [$lei->can('dclose'), $lei] }; + if ($lei->{auth}) { # start auth worker + require PublicInbox::NetWriter; + bless $lei->{net}, 'PublicInbox::NetWriter'; + $lei->{auth}->op_merge(my $ops = {}, $self, $lei); + (my $op_c, $ops) = $lei->workers_start($self, 1, $ops); + $lei->{wq1} = $self; + $lei->wait_wq_events($op_c, $ops); + # net_merge_all_done will fire when auth is done } else { - up1($lei, $local[0]); + redispatch_all($self, $lei); # see below } } +# called in top-level lei-daemon when LeiAuth is done +sub net_merge_all_done { + my ($self, $lei) = @_; + $lei->{net} = delete($self->{-net_new}) if $self->{-net_new}; + $self->wq_close(1); + redispatch_all($self, $lei); +} + sub _complete_up { my ($lei, @argv) = @_; my $match_cb = $lei->complete_url_prepare(\@argv); map { $match_cb->($_) } PublicInbox::LeiSavedSearch::list($lei); } +no warnings 'once'; +*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child; + 1; diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t index aeed3144..0a4cea68 100644 --- a/xt/net_writer-imap.t +++ b/xt/net_writer-imap.t @@ -242,6 +242,7 @@ EOM lei_ok qw(q m:testmessage --no-external -o), $folder_url; lei_ok qw(up), $folder_url; + lei_ok qw(up --all=remote); }); undef $cleanup; # remove temporary folder -- 2.44.0