use File::Spec ();
use PublicInbox::Search qw(xap_terms);
use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::MID qw(mids);
sub new {
my ($class) = @_;
my $mo = { %{$lei->{mset_opt}} };
my $mset;
my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $ibxish);
- my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
- $dedupe->prepare_dedupe;
do {
$mset = $srch->mset($mo->{qstr}, $mo);
my $ids = $srch->mset_to_artnums($mset, $mo);
for my $n (@{$ctx->{xids}}) {
my $smsg = $over->get_art($n) or next;
wait_startq($startq) if $startq;
- next if $dedupe->is_smsg_dup($smsg);
my $mitem = delete $n2item{$smsg->{num}};
$each_smsg->($smsg, $mitem);
}
attach_external($self, $loc);
}
my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $self);
- my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
- $dedupe->prepare_dedupe;
do {
$mset = $self->mset($mo->{qstr}, $mo);
for my $mitem ($mset->items) {
my $smsg = smsg_for($self, $mitem) or next;
wait_startq($startq) if $startq;
- next if $dedupe->is_smsg_dup($smsg);
$each_smsg->($smsg, $mitem);
}
} while (_mset_more($mset, $mo));
my ($eml, $self, $lei, $each_smsg) = @_;
my $smsg = bless {}, 'PublicInbox::Smsg';
$smsg->populate($eml);
+ $smsg->parse_references($eml, mids($eml));
$smsg->{$_} //= '' for qw(from to cc ds subject references mid);
delete @$smsg{qw(From Subject -ds -ts)};
if (my $startq = delete($self->{5})) { wait_startq($startq) }
- return if !$lei->{l2m} && $lei->{dedupe}->is_smsg_dup($smsg);
$each_smsg->($smsg, undef, $eml);
}
sub query_remote_mboxrd {
- my ($self, $lei, $uri) = @_;
+ my ($self, $lei, $uris) = @_;
local $0 = "$0 query_remote_mboxrd";
my %sig = $lei->atfork_child_wq($self); # keep $self->{5} startq
local @SIG{keys %sig} = values %sig;
- my $opt = $lei->{opt};
- $uri->query_form(q => $lei->{mset_opt}->{qstr}, x => 'm',
- $opt->{thread} ? (t => 1) : ());
- my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $uri);
- my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
- $dedupe->prepare_dedupe;
- my @cmd = qw(curl -XPOST -sSf);
- my $tor = $opt->{torsocks} //= 'auto';
- if ($tor eq 'auto' && substr($uri->host, -6) eq '.onion' &&
- (($lei->{env}->{LD_PRELOAD}//'') !~ /torsocks/)) {
- unshift @cmd, 'torsocks';
- } elsif (PublicInbox::Config::git_bool($tor)) {
- unshift @cmd, 'torsocks';
- }
+ my ($opt, $env) = @$lei{qw(opt env)};
+ my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
+ push(@qform, t => 1) if $opt->{thread};
+ my @cmd = (qw(curl -sSf -d), '');
my $verbose = $opt->{verbose};
push @cmd, '-v' if $verbose;
- push @cmd, $uri->as_string;
- $lei->err("# @cmd") if $verbose;
- $? = 0;
- my $fh = popen_rd(\@cmd, $lei->{env}, { 2 => $lei->{2} });
- $fh = IO::Uncompress::Gunzip->new($fh);
- eval {
- PublicInbox::MboxReader->mboxrd($fh, \&each_eml,
- $self, $lei, $each_smsg);
- };
- return $lei->fail("E: @cmd: $@") if $@;
- if (($? >> 8) == 22) { # HTTP 404 from curl(1)
- $uri->query_form(q => $lei->{mset_opt}->{qstr});
- $lei->err('# no results from '.$uri->as_string);
- } elsif ($?) {
- $uri->query_form(q => $lei->{mset_opt}->{qstr});
- $lei->err('E: '.$uri->as_string);
- $lei->child_error($?);
+ for my $o ($lei->curl_opt) {
+ $o =~ s/\|[a-z0-9]\b//i; # remove single char short option
+ if ($o =~ s/=[is]@\z//) {
+ my $ary = $opt->{$o} or next;
+ push @cmd, map { ("--$o", $_) } @$ary;
+ } elsif ($o =~ s/=[is]\z//) {
+ my $val = $opt->{$o} // next;
+ push @cmd, "--$o", $val;
+ } elsif ($opt->{$o}) {
+ push @cmd, "--$o";
+ }
+ }
+ $opt->{torsocks} = 'false' if $opt->{'no-torsocks'};
+ my $tor = $opt->{torsocks} //= 'auto';
+ my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
+ for my $uri (@$uris) {
+ $uri->query_form(@qform);
+ my $cmd = [ @cmd, $uri->as_string ];
+ if ($tor eq 'auto' && substr($uri->host, -6) eq '.onion' &&
+ (($env->{LD_PRELOAD}//'') !~ /torsocks/)) {
+ unshift @$cmd, 'torsocks';
+ } elsif (PublicInbox::Config::git_bool($tor)) {
+ unshift @$cmd, 'torsocks';
+ }
+ $lei->err("# @$cmd") if $verbose;
+ $? = 0;
+ my $fh = popen_rd($cmd, $env, { 2 => $lei->{2} });
+ $fh = IO::Uncompress::Gunzip->new($fh);
+ eval {
+ PublicInbox::MboxReader->mboxrd($fh, \&each_eml, $self,
+ $lei, $each_smsg);
+ };
+ return $lei->fail("E: @$cmd: $@") if $@;
+ if (($? >> 8) == 22) { # HTTP 404 from curl(1)
+ $uri->query_form(q => $lei->{mset_opt}->{qstr});
+ $lei->err('# no results from '.$uri->as_string);
+ } elsif ($?) {
+ $uri->query_form(q => $lei->{mset_opt}->{qstr});
+ $lei->err('E: '.$uri->as_string);
+ $lei->child_error($?);
+ }
}
undef $each_smsg;
$lei->{ovv}->ovv_atexit_child($lei);
close $au_done; # triggers wait_startq
}
+my $MAX_PER_HOST = 4;
+sub MAX_PER_HOST { $MAX_PER_HOST }
+
+sub concurrency {
+ my ($self, $opt) = @_;
+ my $nl = $opt->{thread} ? locals($self) : 1;
+ my $nr = remotes($self);
+ $nr = $MAX_PER_HOST if $nr > $MAX_PER_HOST;
+ $nl + $nr;
+}
+
sub start_query { # always runs in main (lei-daemon) process
my ($self, $io, $lei) = @_;
if ($lei->{opt}->{thread}) {
for my $ibxish (locals($self)) {
$self->wq_do('query_thread_mset', $io, $lei, $ibxish);
}
- } else {
+ } elsif (locals($self)) {
$self->wq_do('query_mset', $io, $lei);
}
+ my $i = 0;
+ my $q = [];
for my $uri (remotes($self)) {
- $self->wq_do('query_remote_mboxrd', $io, $lei, $uri);
+ push @{$q->[$i++ % $MAX_PER_HOST]}, $uri;
+ }
+ for my $uris (@$q) {
+ $self->wq_do('query_remote_mboxrd', $io, $lei, $uris);
}
@$io = ();
}
require PublicInbox::MboxReader;
require IO::Uncompress::Gunzip;
}
- # (0: done_wr, 1: stdout|mbox, 2: stderr,
- # 3: sock, 4: $l2m->{-wq_s1}, 5: $startq)
- $self->wq_set_recv_modes(qw[+<&= >&= >&= +<&= +<&= <&=]);
+ # FDS: (0: done_wr, 1: stdout|mbox, 2: stderr,
+ # 3: sock, 4: $l2m->{-wq_s1}, 5: $startq)
$self->SUPER::ipc_atfork_prepare; # PublicInbox::IPC
}