-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Combine any combination of PublicInbox::Search,
$smsg;
}
-sub recent {
- my ($self, $qstr, $opt) = @_;
- $opt //= {};
- $opt->{relevance} //= -2;
- $self->mset($qstr //= 'z:1..', $opt);
-}
-
sub over {}
sub _check_mset_limit ($$$) {
}
sub l2m_progress {
- my ($lei, $nr) = @_;
- $lei->{-nr_write} += $nr;
+ my ($lei, $nr_write, $nr_seen) = @_;
+ $lei->{-nr_write} += $nr_write;
+ $lei->{-nr_seen} += $nr_seen;
}
sub query_one_mset { # for --threads and l2m w/o sort
my ($res, $kw) = $self->{import_sto}->wq_do('add_eml', $eml);
if (ref($res) eq ref($smsg)) { # totally new message
$smsg = $res;
- $self->{-imported} = 1;
+ $self->{-sto_imported} = 1;
}
$smsg->{kw} = $kw; # short-circuit xsmsg_vmd
}
local $SIG{TERM} = sub { exit(0) }; # for DESTROY (File::Temp, $reap)
my $lei = $self->{lei};
my $opt = $lei->{opt};
- my $qstr = $lei->{mset_opt}->{qstr};
+ chomp(my $qstr = $lei->{mset_opt}->{qstr});
$qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
my @qform = (x => 'm');
push(@qform, t => 1) if $opt->{threads};
$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
PublicInbox::MboxReader->mboxrd($fh, \&each_remote_eml, $self,
$lei, $each_smsg);
- if ($self->{import_sto} && delete($self->{-imported})) {
+ if (delete($self->{-sto_imported})) {
my $wait = $self->{import_sto}->wq_do('done');
}
$reap_curl->join;
sub git { $_[0]->{git} // die 'BUG: git uninitialized' }
-sub xsearch_done_wait { # dwaitpid callback
- my ($arg, $pid) = @_;
- my ($wq, $lei) = @$arg;
+sub xsearch_done_wait { # awaitpid cb (via awaitpid_init)
+ my ($pid, $wq, $lei) = @_;
return if !$?;
my $s = $? & 127;
return $lei->child_error($?) if $s == 13 || $s == 15;
}
if ($lei->{-progress}) {
my $tot = $lei->{-mset_total} // 0;
- my $nr = $lei->{-nr_write} // 0;
+ my $nr_w = $lei->{-nr_write} // 0;
+ my $d = ($lei->{-nr_seen} // 0) - $nr_w;
+ my $x = "$tot matches";
+ $x .= ", $d duplicates" if $d;
if ($l2m) {
- my $m = "# $nr written to " .
- "$lei->{ovv}->{dst} ($tot matches)";
- $nr ? $lei->qfin($m) : $lei->qerr($m);
+ my $m = "# $nr_w written to " .
+ "$lei->{ovv}->{dst} ($x)";
+ $nr_w ? $lei->qfin($m) : $lei->qerr($m);
} else {
- $lei->qerr("# $tot matches");
+ $lei->qerr("# $x");
}
}
$lei->start_mua if $l2m && !$l2m->lock_free;
fcntl($b_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
$l2m->{au_peers} = [ $a_r, $a_w, $b_r, $b_w ];
}
+ $l2m->awaitpid_init(\&xsearch_done_wait, $lei);
$l2m->wq_workers_start('lei2mail', undef,
$lei->oldset, { lei => $lei });
- $l2m->wq_wait_async(\&xsearch_done_wait, $lei);
pipe($lei->{startq}, $lei->{au_done}) or die "pipe: $!";
fcntl($lei->{startq}, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
delete $l2m->{au_peers};
}
+ $self->awaitpid_init(\&xsearch_done_wait, $lei);
$self->wq_workers_start('lei_xsearch', undef,
$lei->oldset, { lei => $lei });
- $self->wq_wait_async(\&xsearch_done_wait, $lei);
my $op_c = delete $lei->{pkt_op_c};
delete $lei->{pkt_op_p};
@$end = ();
require IO::Uncompress::Gunzip;
require PublicInbox::LeiCurl;
push @{$self->{remotes}}, $uri;
+ $uri;
} else {
warn "curl missing, ignoring $uri\n";
+ undef;
}
}
+# returns URI or PublicInbox::Inbox-like object
sub prepare_external {
my ($self, $loc, $boost) = @_; # n.b. already ordered by boost
if (ref $loc) { # already a URI, or PublicInbox::Inbox-like object
return add_uri($self, $loc) if $loc->can('scheme');
+ # fall-through on Inbox-like objects
} elsif ($loc =~ m!\Ahttps?://!) {
require URI;
return add_uri($self, URI->new($loc));
- } elsif (-f "$loc/ei.lock") {
+ } elsif (-f "$loc/ei.lock" && -d "$loc/ALL.git/objects") {
require PublicInbox::ExtSearch;
die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
$loc = PublicInbox::ExtSearch->new($loc);
- } elsif (-f "$loc/inbox.lock" || -d "$loc/public-inbox") {
+ } elsif ((-f "$loc/inbox.lock" && -d "$loc/all.git/objects") ||
+ (-d "$loc/public-inbox" && -d "$loc/objects")) {
die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
require PublicInbox::Inbox; # v2, v1
$loc = bless { inboxdir => $loc }, 'PublicInbox::Inbox';
} elsif (!-e $loc) {
warn "W: $loc gone, perhaps run: lei forget-external $loc\n";
- return;
+ return undef;
} else {
warn "W: $loc ignored, unable to determine external type\n";
- return;
+ return undef;
}
push @{$self->{locals}}, $loc;
+ $loc;
}
sub _lcat_i { # LeiMailSync->each_src iterator callback