1 # Copyright (C) 2021 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
4 # "lei up" - updates the result of "lei q --save"
5 package PublicInbox::LeiUp;
8 # n.b. we use LeiInput to setup IMAP auth
9 use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
10 use PublicInbox::LeiSavedSearch;
12 use PublicInbox::PktOp;
13 use PublicInbox::LeiFinmsg;
14 my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP
18 my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
20 my $mset_opt = $lei->{mset_opt} = { relevance => -2 };
21 my $q = $mset_opt->{q_raw} = $lss->{-cfg}->{'lei.q'} //
22 return $lei->fail("lei.q unset in $f");
23 my $lse = $lei->{lse} // die 'BUG: {lse} missing';
25 $mset_opt->{qstr} = $lse->query_argv_to_string($lse->git, $q);
27 $lse->query_approxidate($lse->git, $mset_opt->{qstr} = $q);
29 # n.b. only a few CLI args are accepted for "up", so //= usually sets
30 for my $k ($lss->ARRAY_FIELDS) {
31 my $v = $lss->{-cfg}->get_all("lei.q.$k") // next;
32 $lei->{opt}->{$k} //= $v;
34 for my $k ($lss->BOOL_FIELDS, $lss->SINGLE_FIELDS) {
35 my $v = $lss->{-cfg}->get_1("lei.q.$k") // next;
36 $lei->{opt}->{$k} //= $v;
38 my $o = $lei->{opt}->{output} // '';
39 return $lei->fail("lei.q.output unset in $f (out=$out)") if $o eq '';
40 $lss->translate_dedupe($lei) or return;
41 $lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd
42 my $lxs = $lei->lxs_prepare or return;
43 $lei->ale->refresh_externals($lxs, $lei);
47 sub redispatch_all ($$) {
48 my ($self, $lei) = @_;
49 my $upq = [ (@{$self->{local} // []}, @{$self->{remote} // []}) ];
50 return up1($lei, $upq->[0]) if @$upq == 1; # just one, may start MUA
52 # FIXME: this is also used per-query, see lei->_start_query
53 my $j = $lei->{opt}->{jobs} || do {
54 my $n = $self->detect_nproc // 1;
57 $j = ($j =~ /\A([0-9]+)/) ? $1 + 0 : 1; # may be --jobs=$x,$m on CLI
58 # re-dispatch into our event loop w/o creating an extra fork-level
59 # $upq will be drained via DESTROY as each query finishes
60 $lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei);
61 my ($op_c, $op_p) = PublicInbox::PktOp->pair;
62 # call lei->dclose when upq is done processing:
63 $op_c->{ops} = { '' => [ $lei->can('dclose'), $lei ] };
64 my @first_batch = splice(@$upq, 0, $j); # initial parallelism
66 $lei->{daemon_pid} = $$;
67 $lei->event_step_init; # wait for client disconnects
68 for my $out (@first_batch) {
69 PublicInbox::DS::requeue(
70 PublicInbox::LeiUp1::nxt($lei, $out, $op_p));
75 my ($lei, @outs) = @_;
76 my $opt = $lei->{opt};
77 my $self = bless { -mail_sync => 1 }, __PACKAGE__;
78 if (defined(my $all = $opt->{all})) {
79 return $lei->fail("--all and @outs incompatible") if @outs;
80 defined($opt->{mua}) and return
81 $lei->fail('--all and --mua= are incompatible');
82 @outs = PublicInbox::LeiSavedSearch::list($lei);
83 if ($all eq 'local') {
84 $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
85 } elsif ($all eq 'remote') {
86 $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
87 } elsif ($all eq '') {
88 $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
89 $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
91 $lei->fail("only --all=$all not understood");
93 } elsif ($lei->{lse}) { # redispatched
94 scalar(@outs) == 1 or die "BUG: lse set w/ >1 out[@outs]";
95 return up1($lei, $outs[0]);
97 $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
98 $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
100 $lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
101 ((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
102 defined($opt->{mua})) and return $lei->fail(<<EOM);
103 multiple outputs and --mua= are incompatible
105 if ($self->{remote}) { # setup lei->{auth}
106 $self->prepare_inputs($lei, $self->{remote}) or return;
108 if ($lei->{auth}) { # start auth worker
109 require PublicInbox::NetWriter;
110 bless $lei->{net}, 'PublicInbox::NetWriter';
111 $lei->{auth}->op_merge(my $ops = {}, $self, $lei);
112 (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
114 $lei->wait_wq_events($op_c, $ops);
115 # net_merge_all_done will fire when auth is done
117 redispatch_all($self, $lei); # see below
121 # called in top-level lei-daemon when LeiAuth is done
122 sub net_merge_all_done {
123 my ($self, $lei) = @_;
124 $lei->{net} = delete($self->{-net_new}) if $self->{-net_new};
126 eval { redispatch_all($self, $lei) };
130 sub _complete_up { # lei__complete hook
131 my ($lei, @argv) = @_;
132 my $match_cb = $lei->complete_url_prepare(\@argv);
133 map { $match_cb->($_) } PublicInbox::LeiSavedSearch::list($lei);
136 sub _wq_done_wait { # dwaitpid callback
137 my ($arg, $pid) = @_;
138 my ($wq, $lei) = @$arg;
139 $lei->child_error($?, 'auth failure') if $?
143 *ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
145 package PublicInbox::LeiUp1; # for redispatch_all
150 my ($lei, $out, $op_p) = @_;
151 bless { lei => $lei, out => $out, op_p => $op_p }, __PACKAGE__;
154 sub event_step { # runs via PublicInbox::DS::requeue
156 my $lei = $self->{lei}; # the original, from lei_up
157 my $l = bless { %$lei }, ref($lei); # per-output copy
158 delete($l->{sock}) or return; # client disconnected if {sock} is gone
159 $l->{opt} = { %{$l->{opt}} }; # deep copy
160 delete $l->{opt}->{all};
161 $l->qerr("# updating $self->{out}");
162 my $o = " (output: $self->{out})"; # add to all warnings
163 my $cb = $SIG{__WARN__} // \&CORE::warn;
164 local $SIG{__WARN__} = sub {
166 push(@m, $o) if !@m || $m[-1] !~ s/\n\z/$o\n/;
169 $l->{-up1} = $self; # for LeiUp1->DESTROY
170 delete @$l{qw(-socks -event_init_done)};
171 my ($op_c, $op_p) = PublicInbox::PktOp->pair;
172 $self->{unref_on_destroy} = $op_c->{sock}; # to cleanup $lei->{-socks}
173 $lei->pkt_ops($op_c->{ops} //= {}); # errors from $l -> script/lei
174 push @{$lei->{-socks}}, $op_c->{sock}; # script/lei signals to $l
175 $l->{sock} = $op_p->{op_p}; # receive signals from op_c->{sock}
176 $op_c = $op_p = undef;
178 eval { $l->dispatch('up', $self->{out}) };
179 $lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
184 my $lei = $self->{lei}; # the original, from lei_up
185 return if $lei->{daemon_pid} != $$;
186 my $sock = delete $self->{unref_on_destroy};
187 my $s = $lei->{-socks} // [];
188 @$s = grep { $_ != $sock } @$s;
189 my $out = shift(@{$lei->{-upq}}) or return;
190 PublicInbox::DS::requeue(nxt($lei, $out, $self->{op_p}));