$size >= $mo->{limit} && (($mo->{offset} += $size) < $mo->{limit});
}
-# $startq will EOF when query_prepare is done augmenting and allow
+# $startq will EOF when do_augment is done augmenting and allow
# query_mset and query_thread_mset to proceed.
sub wait_startq ($) {
my ($lei) = @_;
my $startq = delete $lei->{startq} or return;
while (1) {
- my $n = sysread($startq, my $query_prepare_done, 1);
+ my $n = sysread($startq, my $do_augment_done, 1);
if (defined $n) {
return if $n == 0; # no MUA
- if ($query_prepare_done eq 'q') {
+ if ($do_augment_done eq 'q') {
$lei->{opt}->{quiet} = 1;
delete $lei->{opt}->{verbose};
delete $lei->{-progress};
} else {
- $lei->fail("$$ WTF `$query_prepare_done'");
+ $lei->fail("$$ WTF `$do_augment_done'");
}
return;
}
$lei->{ovv}->ovv_atexit_child($lei);
}
-sub each_eml { # callback for MboxReader->mboxrd
+sub each_remote_eml { # callback for MboxReader->mboxrd
my ($eml, $self, $lei, $each_smsg) = @_;
+ $lei->{sto}->ipc_do('set_eml', $eml) if $lei->{sto}; # --import-remote
my $smsg = bless {}, 'PublicInbox::Smsg';
$smsg->populate($eml);
$smsg->parse_references($eml, mids($eml));
local $0 = "$0 query_remote_mboxrd";
local $SIG{TERM} = sub { exit(0) }; # for DESTROY (File::Temp, $reap)
my $lei = $self->{lei};
- my ($opt, $env) = @$lei{qw(opt env)};
+ my $opt = $lei->{opt};
my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
push(@qform, t => 1) if $opt->{threads};
my $verbose = $opt->{verbose};
$uri->query_form(@qform);
my $cmd = $curl->for_uri($lei, $uri);
$lei->qerr("# $cmd");
- my ($fh, $pid) = popen_rd($cmd, $env, $rdr);
+ my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
$reap_curl = PublicInbox::OnDestroy->new($sigint_reap, $pid);
$fh = IO::Uncompress::Gunzip->new($fh);
- PublicInbox::MboxReader->mboxrd($fh, \&each_eml, $self,
+ PublicInbox::MboxReader->mboxrd($fh, \&each_remote_eml, $self,
$lei, $each_smsg);
my $err = waitpid($pid, 0) == $pid ? undef
: "BUG: waitpid($cmd): $!";
@$reap_curl = (); # cancel OnDestroy
die $err if $err;
+ my $nr = $lei->{-nr_remote_eml};
+ if ($nr && $lei->{sto}) {
+ my $wait = $lei->{sto}->ipc_do('done');
+ }
if ($? == 0) {
- my $nr = $lei->{-nr_remote_eml};
mset_progress($lei, $lei->{-current_url}, $nr, $nr);
next;
}
sub do_post_augment {
my ($lei) = @_;
- my $l2m = $lei->{l2m};
+ my $l2m = $lei->{l2m} or die 'BUG: unexpected do_post_augment';
my $err;
- if ($l2m) {
- eval { $l2m->post_augment($lei) };
- $err = $@;
- if ($err) {
- if (my $lxs = delete $lei->{lxs}) {
- $lxs->wq_kill;
- $lxs->wq_close(0, undef, $lei);
- }
- $lei->fail("$err");
+ eval { $l2m->post_augment($lei) };
+ $err = $@;
+ if ($err) {
+ if (my $lxs = delete $lei->{lxs}) {
+ $lxs->wq_kill;
+ $lxs->wq_close(0, undef, $lei);
}
+ $lei->fail("$err");
}
if (!$err && delete $lei->{early_mua}) { # non-augment case
$lei->start_mua;
close(delete $lei->{au_done}); # triggers wait_startq in lei_xsearch
}
+sub incr_post_augment { # called whenever an l2m shard finishes augment
+ my ($lei) = @_;
+ my $l2m = $lei->{l2m} or die 'BUG: unexpected incr_post_augment';
+ return if ++$lei->{nr_post_augment} != $l2m->{-wq_nr_workers};
+ do_post_augment($lei);
+}
+
my $MAX_PER_HOST = 4;
sub concurrency {
}
sub start_query { # always runs in main (lei-daemon) process
- my ($self, $lei) = @_;
- if ($lei->{opt}->{threads}) {
+ my ($self) = @_;
+ if ($self->{threads}) {
for my $ibxish (locals($self)) {
$self->wq_io_do('query_thread_mset', [], $ibxish);
}
for my $uris (@$q) {
$self->wq_io_do('query_remote_mboxrd', [], $uris);
}
+ $self->wq_close(1); # lei_xsearch workers stop when done
+}
+
+sub incr_start_query { # called whenever an l2m shard starts do_post_auth
+ my ($self, $l2m) = @_;
+ return if ++$self->{nr_start_query} != $l2m->{-wq_nr_workers};
+ start_query($self);
}
sub ipc_atfork_child {
$self->SUPER::ipc_atfork_child;
}
-sub query_prepare { # called by wq_io_do
- my ($self) = @_;
- local $0 = "$0 query_prepare";
- my $lei = $self->{lei};
- eval { $lei->{l2m}->do_augment($lei) };
- $lei->fail($@) if $@;
- pkt_do($lei->{pkt_op_p}, '.') == 1 or die "do_post_augment trigger: $!"
-}
-
sub do_query {
my ($self, $lei) = @_;
+ my $l2m = $lei->{l2m};
my $ops = {
'|' => [ $lei->can('sigpipe_handler'), $lei ],
'!' => [ $lei->can('fail_handler'), $lei ],
'.' => [ \&do_post_augment, $lei ],
+ '+' => [ \&incr_post_augment, $lei ],
'' => [ \&query_done, $lei ],
'mset_progress' => [ \&mset_progress, $lei ],
'x_it' => [ $lei->can('x_it'), $lei ],
'child_error' => [ $lei->can('child_error'), $lei ],
+ 'incr_start_query' => [ \&incr_start_query, $self, $l2m ],
};
+ $lei->{auth}->op_merge($ops, $l2m) if $l2m && $lei->{auth};
($lei->{pkt_op_c}, $lei->{pkt_op_p}) = PublicInbox::PktOp->pair($ops);
$lei->{1}->autoflush(1);
$lei->start_pager if delete $lei->{need_pager};
$lei->{ovv}->ovv_begin($lei);
- my $l2m = $lei->{l2m};
if ($l2m) {
$l2m->pre_augment($lei);
if ($lei->{opt}->{augment} && delete $lei->{early_mua}) {
$lei->start_mua;
}
- $l2m->wq_workers_start('lei2mail', $l2m->{jobs},
+ $l2m->wq_workers_start('lei2mail', undef,
$lei->oldset, { lei => $lei });
pipe($lei->{startq}, $lei->{au_done}) or die "pipe: $!";
# 1031: F_SETPIPE_SZ
# delete until all lei2mail + lei_xsearch workers are reaped
$lei->{git_tmp} = $self->{git_tmp} = git_tmp($self);
}
- $self->wq_workers_start('lei_xsearch', $self->{jobs},
+ $self->wq_workers_start('lei_xsearch', undef,
$lei->oldset, { lei => $lei });
my $op = delete $lei->{pkt_op_c};
delete $lei->{pkt_op_p};
- $l2m->wq_close(1) if $l2m;
+ $self->{threads} = $lei->{opt}->{threads};
+ if ($l2m) {
+ $l2m->net_merge_complete unless $lei->{auth};
+ } else {
+ start_query($self);
+ }
$lei->event_step_init; # wait for shutdowns
- $self->wq_io_do('query_prepare', []) if $l2m; # for augment/dedupe
- start_query($self, $lei);
- $self->wq_close(1); # lei_xsearch workers stop when done
if ($lei->{oneshot}) {
while ($op->{sock}) { $op->event_step }
}